From 390d975b0454ad033e47ec00b46aef4b7133b797 Mon Sep 17 00:00:00 2001 From: JM <44305062+jmaeng72@users.noreply.github.com> Date: Wed, 4 Dec 2024 12:12:53 -0500 Subject: [PATCH] CMR-10182: Enforce Keyword Validation Rules at CMR Ingest (#2196) * update doc and make cmr-validate-keywords default to true * update int tests --- ingest-app/docs/api.md | 2 +- ingest-app/src/cmr/ingest/api/collections.clj | 2 +- .../src/cmr/system_int_test/data2/core.clj | 2 +- .../utils/virtual_product_util.clj | 5 +- .../system_int_test/admin/cache_api_test.clj | 2 +- .../bootstrap/bulk_index/delete_test.clj | 14 +- .../bootstrap/bulk_index/services_test.clj | 2 +- .../bootstrap/bulk_index/tag_test.clj | 2 +- .../bootstrap/bulk_index/tools_test.clj | 2 +- .../bootstrap/bulk_index/variables_test.clj | 35 +++-- .../bootstrap/fingerprint_test.clj | 18 ++- .../bootstrap/rebalance_collections_test.clj | 47 +++--- .../bootstrap/virtual_products_test.clj | 24 +-- .../ingest/bulk_update/bulk_update_test.clj | 67 ++++---- .../collection/collection_ingest_test.clj | 146 ++++++++++-------- .../collection_misc_validation_test.clj | 17 +- .../collection_umm_spec_validation_test.clj | 30 ++-- .../collection/collection_update_test.clj | 72 +++++---- .../collection_validation_end_point_test.clj | 2 +- .../ingest/granule/granule_ingest_test.clj | 78 ++++++---- .../granule_parent_dependency_test.clj | 34 ++-- .../granule_bulk_update_test.clj | 2 +- .../ingest/misc/generics_test.clj | 2 +- .../ingest_lifecycle_integration_test.clj | 4 +- .../provider_cmr_only_client_id_test.clj | 23 ++- .../provider_ingest_permissions_test.clj | 2 +- .../search/acls/collection_test.clj | 129 ++++++++++------ .../search/acls/granule_test.clj | 12 +- .../acls/temporal_collection_granule_test.clj | 14 +- .../search/acls/variable_association_test.clj | 2 +- .../autocomplete/suggestion_reindex_test.clj | 27 ++-- .../collection_campaign_search_test.clj | 12 +- .../collection_concept_retrieval_test.clj | 56 ++++--- .../collection_consortium_search_test.clj | 9 +- .../collection_directory_pages_test.clj | 18 ++- .../collection/collection_doi_search_test.clj | 15 +- .../collection_humanized_search_test.clj | 54 ++++--- .../collection_identifier_search_test.clj | 11 +- .../collection_keyword_search_test.clj | 137 +++++++++------- ...platform_instrument_sensor_search_test.clj | 81 ++++++---- .../collection/collection_psa_search_test.clj | 46 +++--- ...collection_science_keyword_search_test.clj | 22 +-- .../collection_search_data_format_test.clj | 2 +- .../collection_search_format_test.clj | 119 ++++++++------ .../collection_shapefile_search_test.clj | 3 +- ...n_shapefile_simplification_search_test.clj | 3 +- .../collection_sorting_search_test.clj | 51 ++++-- ...collection_spatial_keyword_search_test.clj | 20 +-- .../collection_spatial_search_test.clj | 3 +- ...ion_spatialkeyword_keyword_search_test.clj | 3 +- .../collection_relevancy_test.clj | 21 ++- .../collection_usage_relevancy_test.clj | 21 ++- .../facets/collection_facets_search_test.clj | 9 +- .../collection_facets_v2_search_test.clj | 87 +++++++---- .../search/facets/facets_util.clj | 3 +- .../granule/granule_campaign_search_test.clj | 16 +- .../granule/granule_counts_search_test.clj | 5 +- .../granule/granule_orbit_search_test.clj | 18 ++- ...platform_instrument_sensor_search_test.clj | 12 +- .../granule/granule_psa_search_test.clj | 32 ++-- .../search/granule/granule_retrieval_test.clj | 3 +- .../granule/granule_search_format_test.clj | 42 +++-- .../granule_search_with_json_query_test.clj | 12 +- .../granule/granule_sorting_search_test.clj | 37 ++--- .../search/humanizer/humanizer_test.clj | 12 +- .../misc/coll_archive_ctr_search_test.clj | 28 ++-- .../search/misc/harvesting_test.clj | 4 +- .../search/misc/provider_holdings_test.clj | 6 +- .../search/misc/sitemaps_test.clj | 6 +- .../search/misc/umm_json_search_test.clj | 15 +- .../collection_service_search_test.clj | 33 ++-- .../service/service_association_test.clj | 18 ++- .../collection_revisions_tag_search_test.clj | 16 +- .../tagging/collection_tag_search_test.clj | 14 +- .../search/tagging/enable_disable_test.clj | 3 +- ..._association_collection_revisions_test.clj | 34 ++-- .../search/tagging/tag_association_test.clj | 38 +++-- .../tool/collection_tool_search_test.clj | 6 +- .../search/tool/tool_association_test.clj | 18 ++- .../collection_variable_search_test.clj | 12 +- .../variable/concept_retrieval_test.clj | 9 +- .../virtual_product/translation_test.clj | 26 ++-- .../virtual_product/virtual_product_test.clj | 39 +++-- .../virtual_product_token_ingest_test.clj | 5 +- 84 files changed, 1260 insertions(+), 885 deletions(-) diff --git a/ingest-app/docs/api.md b/ingest-app/docs/api.md index 86e2ffdd9e..4bdaa549bf 100644 --- a/ingest-app/docs/api.md +++ b/ingest-app/docs/api.md @@ -159,10 +159,10 @@ The following fields are validated: * [Data Format](https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/dataformat?format=csv) - Archival and Distribution File Format, and GetData Format * [ProcessingLevel] (https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/productlevelid?format=csv) - productlevelid +**Note**: cmr-validate-keywords is set to TRUE by default **Note**: that when multiple fields are present the combination of keywords are validated to match a known combination. **Note**: Among the validation fields above, [Platforms], [Instruments], [Projects], [Science Keywords], [Location Keywords] and [Data Centers] are also validated when the `Cmr-Validate-Keywords` header is not set to `true` except that validation errors will be returned to users as warnings. - **Note**: the following fields are always checked: * [Related URL Content Type](https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/rucontenttype?format=csv) - Related URL Content Type, Type, and Subtype are all verified and must belong to the correct set and context. Example: A data center contact URL can have a URL Content Type of "DataContactURL" and a Type of "HOME PAGE" with no Subtype. diff --git a/ingest-app/src/cmr/ingest/api/collections.clj b/ingest-app/src/cmr/ingest/api/collections.clj index 0db4c0da8d..f1cab60af3 100644 --- a/ingest-app/src/cmr/ingest/api/collections.clj +++ b/ingest-app/src/cmr/ingest/api/collections.clj @@ -16,7 +16,7 @@ (defn get-validation-options "Returns a map of validation options with boolean values" [headers] - {:validate-keywords? (= "true" (get headers VALIDATE_KEYWORDS_HEADER)) + {:validate-keywords? (if (= "false" (get headers VALIDATE_KEYWORDS_HEADER)) false true) :validate-umm? (= "true" (get headers ENABLE_UMM_C_VALIDATION_HEADER)) :test-existing-errors? (= "true" (get headers TESTING_EXISTING_ERRORS_HEADER))}) diff --git a/system-int-test/src/cmr/system_int_test/data2/core.clj b/system-int-test/src/cmr/system_int_test/data2/core.clj index e5b1831d7d..2e3419fcb7 100644 --- a/system-int-test/src/cmr/system_int_test/data2/core.clj +++ b/system-int-test/src/cmr/system_int_test/data2/core.clj @@ -225,7 +225,7 @@ :native-id (or native-id "native-id") :metadata metadata :format (or format (mime-types/format->mime-type format-key))} - response (ingest/ingest-concept concept)] + response (ingest/ingest-concept concept {:validate-keywords false})] (if (= concept-type :granule) (merge (umm-legacy/parse-concept context concept) response) (merge (umm-spec/parse-metadata context concept) response)))) diff --git a/system-int-test/src/cmr/system_int_test/utils/virtual_product_util.clj b/system-int-test/src/cmr/system_int_test/utils/virtual_product_util.clj index b9fbaa70dc..e2a19132e1 100644 --- a/system-int-test/src/cmr/system_int_test/utils/virtual_product_util.clj +++ b/system-int-test/src/cmr/system_int_test/utils/virtual_product_util.clj @@ -202,7 +202,8 @@ (dc/collection {:entry-title "ASTER L1A Reconstructed Unprocessed Instrument Data V003" :short-name "AST_L1A"}) - :provider-id "LPDAAC_ECS")]))) + :provider-id "LPDAAC_ECS")] + {:validate-keywords false}))) ;;; Functions for use in assertions @@ -222,7 +223,7 @@ psa2 (dg/psa "SWIR_ObservationMode" ["ON"]) psa3 (dg/psa "VNIR1_ObservationMode" ["ON"]) psa4 (dg/psa "VNIR2_ObservationMode" ["ON"])] - (ingest-virtual-collections [ast-coll]) + (ingest-virtual-collections [ast-coll] {:validate-keywords false}) (are [granule-attrs expected-granule-urs] (let [params {"attribute[]" (format "string,%s,%s" svm/source-granule-ur-additional-attr-name diff --git a/system-int-test/test/cmr/system_int_test/admin/cache_api_test.clj b/system-int-test/test/cmr/system_int_test/admin/cache_api_test.clj index 01721bb40b..0e2706d1bf 100644 --- a/system-int-test/test/cmr/system_int_test/admin/cache_api_test.clj +++ b/system-int-test/test/cmr/system_int_test/admin/cache_api_test.clj @@ -26,7 +26,7 @@ admin-read-token (e/login (s/context) "admin" [admin-read-group-concept-id]) normal-user-token (e/login (s/context) "user") _ (e/grant-group-admin (s/context) admin-read-group-concept-id :read) - _ (d/ingest "PROV1" (dc/collection {:entry-title "coll1"})) + _ (d/ingest "PROV1" (dc/collection {:entry-title "coll1"}) {:validate-keywords false}) _ (refresh-cache (url/refresh-index-names-cache-url) (t-config/echo-system-token))] (testing "list caches" diff --git a/system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/delete_test.clj b/system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/delete_test.clj index b4356c7ee7..2d31f3973e 100644 --- a/system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/delete_test.clj +++ b/system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/delete_test.clj @@ -1,15 +1,8 @@ (ns cmr.system-int-test.bootstrap.bulk-index.delete-test "Integration test for CMR bulk index delete operations." (:require - [clj-time.coerce :as cr] - [clj-time.core :as t] - [clojure.java.jdbc :as j] [clojure.test :refer :all] - [cmr.access-control.test.util :as u] - [cmr.common.date-time-parser :as p] - [cmr.common.util :as util :refer [are3]] - [cmr.mock-echo.client.echo-util :as e] - [cmr.oracle.connection :as oracle] + [cmr.common.util :refer [are3]] [cmr.system-int-test.bootstrap.bulk-index.core :as core] [cmr.system-int-test.data2.collection :as dc] [cmr.system-int-test.data2.core :as d] @@ -21,7 +14,6 @@ [cmr.system-int-test.utils.metadata-db-util :as mdb] [cmr.system-int-test.utils.search-util :as search] [cmr.system-int-test.utils.tag-util :as tags] - [cmr.transmit.access-control :as ac] [cmr.transmit.config :as tc] [cmr.umm.echo10.echo10-core :as echo10])) @@ -124,8 +116,8 @@ :native-id "coll1" :short-name "coll1"}) umm1 (merge umm1 (select-keys coll1 [:concept-id :revision-id])) - ;; coll2 is a regualr collection that is ingested and will be deleted later - coll2 (d/ingest "PROV1" (dc/collection {:short-name "coll2" :entry-title "coll2"})) + ;; coll2 is a regular collection that is ingested and will be deleted later + coll2 (d/ingest "PROV1" (dc/collection {:short-name "coll2" :entry-title "coll2"}) {:validate-keywords false}) ;; coll3 is a collection with an expired delete time umm3 (dc/collection {:short-name "coll3" :entry-title "coll3" :delete-time "2000-01-01T12:00:00Z"}) xml3 (echo10/umm->echo10-xml umm3) diff --git a/system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/services_test.clj b/system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/services_test.clj index 6b407815d8..e72fdb21c3 100644 --- a/system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/services_test.clj +++ b/system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/services_test.clj @@ -167,7 +167,7 @@ (deftest ^:oracle bulk-index-collections-with-service-association-test (s/only-with-real-database - (let [coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1"})) + (let [coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1"}) {:validate-keywords false}) coll1-concept-id (:concept-id coll1) token (e/login (s/context) "user1") {serv1-concept-id :concept-id} (service/ingest-service-with-attrs diff --git a/system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/tag_test.clj b/system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/tag_test.clj index d6c36f5a40..e8be26fb3e 100644 --- a/system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/tag_test.clj +++ b/system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/tag_test.clj @@ -25,7 +25,7 @@ (deftest ^:oracle bulk-index-collections-with-tag-association-test (s/only-with-real-database (let [[coll1 coll2] (for [n (range 1 3)] - (d/ingest "PROV1" (dc/collection {:entry-title (str "coll" n)}))) + (d/ingest "PROV1" (dc/collection {:entry-title (str "coll" n)}) {:validate-keywords false})) ;; Wait until collections are indexed so tags can be associated with them _ (index/wait-until-indexed) user1-token (e/login (s/context) "user1") diff --git a/system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/tools_test.clj b/system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/tools_test.clj index 28589a0c1b..6b6bf5fe08 100644 --- a/system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/tools_test.clj +++ b/system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/tools_test.clj @@ -185,7 +185,7 @@ (deftest bulk-index-collections-with-tool-association-test (system/only-with-real-database - (let [coll1 (data-core/ingest "PROV1" (data-collection/collection {:entry-title "coll1"})) + (let [coll1 (data-core/ingest "PROV1" (data-collection/collection {:entry-title "coll1"}) {:validate-keywords false}) coll1-concept-id (:concept-id coll1) token (echo-util/login (system/context) "user1") {tool1-concept-id :concept-id} (tool/ingest-tool-with-attrs diff --git a/system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/variables_test.clj b/system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/variables_test.clj index 0fc1ca47be..99d3e24cdb 100644 --- a/system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/variables_test.clj +++ b/system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/variables_test.clj @@ -28,12 +28,14 @@ "PROV1" (data2-collection/collection {:entry-title "ET1" :short-name "S1" - :version-id "V1"})) + :version-id "V1"}) + {:validate-keywords false}) coll2 (data2-core/ingest "PROV2" (data2-collection/collection {:entry-title "ET2" :short-name "S2" - :version-id "V2"})) + :version-id "V2"}) + {:validate-keywords false}) _ (index/wait-until-indexed) var1 (variable/make-variable-concept {:Name "Variable1"} @@ -47,7 +49,7 @@ _ (core/disable-automatic-indexing) ;; The following is saved, but not indexed due to the above call {var1-concept-id :concept-id} (variable/ingest-variable-with-association var1) - {var2-concept-id :concept-id} (variable/ingest-variable-with-association var2) + {_ :concept-id} (variable/ingest-variable-with-association var2) {:keys [status errors]} (bootstrap/bulk-index-variables "PROV1" nil)] (is (= [401 ["You do not have permission to perform that action."]] @@ -103,17 +105,20 @@ "PROV1" (data2-collection/collection {:entry-title "ET1" :short-name "S1" - :version-id "V1"})) + :version-id "V1"}) + {:validate-keywords false}) coll2 (data2-core/ingest "PROV2" (data2-collection/collection {:entry-title "ET2" :short-name "S2" - :version-id "V2"})) + :version-id "V2"}) + {:validate-keywords false}) coll3 (data2-core/ingest "PROV3" (data2-collection/collection {:entry-title "ET3" :short-name "S3" - :version-id "V3"})) + :version-id "V3"}) + {:validate-keywords false}) _ (index/wait-until-indexed) var1 (variable/make-variable-concept {:Name "Variable1"} @@ -167,17 +172,20 @@ "PROV1" (data2-collection/collection {:entry-title "ET1" :short-name "S1" - :version-id "V1"})) + :version-id "V1"}) + {:validate-keywords false}) coll2 (data2-core/ingest "PROV2" (data2-collection/collection {:entry-title "ET2" :short-name "S2" - :version-id "V2"})) + :version-id "V2"}) + {:validate-keywords false}) coll3 (data2-core/ingest "PROV3" (data2-collection/collection {:entry-title "ET3" :short-name "S3" - :version-id "V3"})) + :version-id "V3"}) + {:validate-keywords false}) _ (index/wait-until-indexed) var1 (variable/make-variable-concept {:Name "Variable1"} @@ -215,17 +223,20 @@ "PROV1" (data2-collection/collection {:entry-title "ET1" :short-name "S1" - :version-id "V1"})) + :version-id "V1"}) + {:validate-keywords false}) coll2 (data2-core/ingest "PROV2" (data2-collection/collection {:entry-title "ET2" :short-name "S2" - :version-id "V2"})) + :version-id "V2"}) + {:validate-keywords false}) coll3 (data2-core/ingest "PROV3" (data2-collection/collection {:entry-title "ET3" :short-name "S3" - :version-id "V3"})) + :version-id "V3"}) + {:validate-keywords false}) _ (index/wait-until-indexed) ;; Disable message publishing so items are not indexed. _ (core/disable-automatic-indexing) diff --git a/system-int-test/test/cmr/system_int_test/bootstrap/fingerprint_test.clj b/system-int-test/test/cmr/system_int_test/bootstrap/fingerprint_test.clj index 9ec57f36eb..620817c9e6 100644 --- a/system-int-test/test/cmr/system_int_test/bootstrap/fingerprint_test.clj +++ b/system-int-test/test/cmr/system_int_test/bootstrap/fingerprint_test.clj @@ -25,7 +25,8 @@ "PROV1" (data2-collection/collection {:entry-title "ET1" :short-name "S1" - :version-id "V1"})) + :version-id "V1"}) + {:validate-keywords false}) _ (index/wait-until-indexed) var1 (variable/make-variable-concept {:Name "Variable1"} @@ -82,7 +83,7 @@ (get-in latest-revision-concept [:extra-fields :fingerprint])))))) (testing "update fingerprint of a deleted variable" - (let [{:keys [concept-id revision-id]} (ingest/delete-concept var2-concept) + (let [_ (ingest/delete-concept var2-concept) latest-revision-concept (mdb/get-concept var2-concept-id)] (is (= var2-concept-id (:concept-id latest-revision-concept))) (is (= 2 (:revision-id latest-revision-concept))) @@ -110,12 +111,14 @@ "PROV1" (data2-collection/collection {:entry-title "ET1" :short-name "S1" - :version-id "V1"})) + :version-id "V1"}) + {:validate-keywords false}) coll2 (data2-core/ingest "PROV2" (data2-collection/collection {:entry-title "ET2" :short-name "S2" - :version-id "V2"})) + :version-id "V2"}) + {:validate-keywords false}) _ (index/wait-until-indexed) var1 (variable/make-variable-concept {:Name "Variable1"} @@ -137,7 +140,6 @@ var2-concept (mdb/get-concept var2-concept-id) var2-fingerprint (get-in var2-concept [:extra-fields :fingerprint]) var3-concept (mdb/get-concept var3-concept-id) - var3-fingerprint (get-in var3-concept [:extra-fields :fingerprint]) ;; now change the fingerprint of var1 and var3 to a different vaule latest-var1 (-> var1-concept (assoc :revision-id (inc (:revision-id var1-concept))) @@ -202,12 +204,14 @@ "PROV1" (data2-collection/collection {:entry-title "ET1" :short-name "S1" - :version-id "V1"})) + :version-id "V1"}) + {:validate-keywords false}) coll2 (data2-core/ingest "PROV2" (data2-collection/collection {:entry-title "ET2" :short-name "S2" - :version-id "V2"})) + :version-id "V2"}) + {:validate-keywords false}) _ (index/wait-until-indexed) var1 (variable/make-variable-concept {:Name "Variable1"} diff --git a/system-int-test/test/cmr/system_int_test/bootstrap/rebalance_collections_test.clj b/system-int-test/test/cmr/system_int_test/bootstrap/rebalance_collections_test.clj index b36a48adb1..22d6f359cf 100644 --- a/system-int-test/test/cmr/system_int_test/bootstrap/rebalance_collections_test.clj +++ b/system-int-test/test/cmr/system_int_test/bootstrap/rebalance_collections_test.clj @@ -3,7 +3,6 @@ index to separate collection indexes" (:require [clojure.test :refer :all] - [clj-http.client :as client] [cmr.common-app.test.side-api :as side] [cmr.common.time-keeper :as tk] [cmr.system-int-test.data2.collection :as dc] @@ -29,10 +28,10 @@ (deftest ^:oracle rebalance-collection-error-test (s/only-with-real-database - (let [coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1"})) - gran1 (d/ingest "PROV1" (dg/granule coll1 {:granule-ur "gran1"})) - coll2 (d/ingest "PROV1" (dc/collection {:entry-title "coll2"})) - gran2 (d/ingest "PROV1" (dg/granule coll2 {:granule-ur "gran2"}))] + (let [coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1"}) {:validate-keywords false}) + _ (d/ingest "PROV1" (dg/granule coll1 {:granule-ur "gran1"})) + coll2 (d/ingest "PROV1" (dc/collection {:entry-title "coll2"}) {:validate-keywords false}) + _ (d/ingest "PROV1" (dg/granule coll2 {:granule-ur "gran2"}))] (index/wait-until-indexed) (testing "no permission for start-rebalance-collection" (is (= {:status 401 @@ -163,8 +162,8 @@ ;; Rebalances a single collection with a single granule (deftest ^:oracle simple-rebalance-collection-test (s/only-with-real-database - (let [coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1"})) - gran1 (ingest-granule-for-coll coll1 1) + (let [coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1"}) {:validate-keywords false}) + _ (ingest-granule-for-coll coll1 1) expected-provider-holdings (-> coll1 (select-keys [:provider-id :concept-id :entry-title]) (assoc :granule-count 1) @@ -209,11 +208,11 @@ (deftest ^:oracle rebalance-collection-test (s/only-with-real-database - (let [coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1"})) - coll2 (d/ingest "PROV1" (dc/collection {:entry-title "coll2"})) - coll3 (d/ingest "PROV2" (dc/collection {:entry-title "coll3"})) - coll4 (d/ingest "PROV2" (dc/collection {:entry-title "coll4"})) - granules (doall (for [coll [coll1 coll2 coll3 coll4] + (let [coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1"}) {:validate-keywords false}) + coll2 (d/ingest "PROV1" (dc/collection {:entry-title "coll2"}) {:validate-keywords false}) + coll3 (d/ingest "PROV2" (dc/collection {:entry-title "coll3"}) {:validate-keywords false}) + coll4 (d/ingest "PROV2" (dc/collection {:entry-title "coll4"}) {:validate-keywords false}) + _ (doall (for [coll [coll1 coll2 coll3 coll4] n (range 4)] (ingest-granule-for-coll coll n))) @@ -284,11 +283,11 @@ ;; Tests rebalancing multiple collections at the same time. (deftest ^:oracle rebalance-multiple-collections-test (s/only-with-real-database - (let [coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1"})) - coll2 (d/ingest "PROV1" (dc/collection {:entry-title "coll2"})) - coll3 (d/ingest "PROV2" (dc/collection {:entry-title "coll3"})) - coll4 (d/ingest "PROV2" (dc/collection {:entry-title "coll4"})) - granules (doall (for [coll [coll1 coll2 coll3 coll4] + (let [coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1"}) {:validate-keywords false}) + coll2 (d/ingest "PROV1" (dc/collection {:entry-title "coll2"}) {:validate-keywords false}) + coll3 (d/ingest "PROV2" (dc/collection {:entry-title "coll3"}) {:validate-keywords false}) + coll4 (d/ingest "PROV2" (dc/collection {:entry-title "coll4"}) {:validate-keywords false}) + _ (doall (for [coll [coll1 coll2 coll3 coll4] n (range 4)] (ingest-granule-for-coll coll n))) @@ -345,11 +344,11 @@ (deftest ^:oracle rebalance-collection-after-other-collections-test (s/only-with-real-database - (let [coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1"})) - coll2 (d/ingest "PROV1" (dc/collection {:entry-title "coll2"})) - coll3 (d/ingest "PROV2" (dc/collection {:entry-title "coll3"})) - coll4 (d/ingest "PROV2" (dc/collection {:entry-title "coll4"})) - granules (doall (for [coll [coll1 coll2 coll3 coll4] + (let [coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1"}) {:validate-keywords false}) + coll2 (d/ingest "PROV1" (dc/collection {:entry-title "coll2"}) {:validate-keywords false}) + coll3 (d/ingest "PROV2" (dc/collection {:entry-title "coll3"}) {:validate-keywords false}) + coll4 (d/ingest "PROV2" (dc/collection {:entry-title "coll4"}) {:validate-keywords false}) + _ (doall (for [coll [coll1 coll2 coll3 coll4] n (range 4)] (ingest-granule-for-coll coll n))) @@ -474,8 +473,8 @@ (deftest ^:oracle rebalance-old-deleted-collection-back-to-small-collections-test (side/eval-form `(tk/set-time-override! (tk/now))) (s/only-with-real-database - (let [deleted-coll (d/ingest "PROV1" (dc/collection {:entry-title "deleted-coll"})) - granules (doseq [n (range 2)] + (let [deleted-coll (d/ingest "PROV1" (dc/collection {:entry-title "deleted-coll"}) {:validate-keywords false}) + _ (doseq [n (range 2)] (ingest-granule-for-coll deleted-coll n)) deleted-coll-concept (mdb/get-concept (:concept-id deleted-coll))] diff --git a/system-int-test/test/cmr/system_int_test/bootstrap/virtual_products_test.clj b/system-int-test/test/cmr/system_int_test/bootstrap/virtual_products_test.clj index e7dcedf796..c088bb9b4a 100644 --- a/system-int-test/test/cmr/system_int_test/bootstrap/virtual_products_test.clj +++ b/system-int-test/test/cmr/system_int_test/bootstrap/virtual_products_test.clj @@ -11,7 +11,6 @@ [cmr.system-int-test.utils.search-util :as search] [cmr.system-int-test.utils.virtual-product-util :as vp] [cmr.system-int-test.system :as s] - [cmr.umm.umm-granule :as umm-g] [cmr.virtual-product.config] [cmr.virtual-product.data.source-to-virtual-mapping :as svm] [cmr.common.util :refer [are2]])) @@ -103,14 +102,14 @@ ;; collections to test the edge case documented in CMR-2169 coll1 (d/ingest "LPDAAC_ECS" (dc/collection {:entry-title "ASTER L2 Surface Emissivity V003" - :native-id "NID-1"})) + :native-id "NID-1"}) {:validate-keywords false}) _ (ingest/delete-concept (d/item->concept coll1 :echo10)) - source-collections (vp/ingest-source-collections) + source-collections (vp/ingest-source-collections (vp/source-collections) {:validate-keywords false}) ;; Ingest the virtual collections. For each virtual collection associate it with the source ;; collection to use later. vp-colls (reduce (fn [new-colls source-coll] (into new-colls (map #(assoc % :source-collection source-coll) - (vp/ingest-virtual-collections [source-coll])))) + (vp/ingest-virtual-collections [source-coll] {:validate-keywords false})))) [] source-collections) source-granules (doall (for [source-coll source-collections @@ -164,8 +163,9 @@ {:entry-title entry-title :short-name "AST_L1A" :projects (dc/projects "proj1" "proj2" "proj3")}) - :provider-id "LP_ALIAS")]) - vp-colls (vp/ingest-virtual-collections [ast-coll]) + :provider-id "LP_ALIAS")] + {:validate-keywords false}) + _ (vp/ingest-virtual-collections [ast-coll] {:validate-keywords false}) granule-ur "SC:AST_L1A.003:2006227720" ast-l1a-gran (vp/ingest-source-granule "LP_ALIAS" (dg/granule ast-coll {:granule-ur granule-ur @@ -215,8 +215,9 @@ :short-name "AST_L1A" :version-id "003" :projects (dc/projects "proj1" "proj2" "proj3")}) - :provider-id "LPDAAC_ECS")]) - vp-colls (vp/ingest-virtual-collections [ast-coll]) + :provider-id "LPDAAC_ECS")] + {:validate-keywords false}) + _ (vp/ingest-virtual-collections [ast-coll] {:validate-keywords false}) granule-ur "SC:AST_L1A.003:2006227720" ast-l1a-gran (vp/ingest-source-granule "LPDAAC_ECS" (dg/granule ast-coll {:granule-ur granule-ur @@ -254,13 +255,14 @@ (deftest ^:oracle deleted-virtual-granules (s/only-with-real-database (let [ast-coll (vp/ingest-ast-coll) - vp-colls (vp/ingest-virtual-collections [ast-coll]) + vp-colls (vp/ingest-virtual-collections [ast-coll] {:validate-keywords false}) s-granules (doall (for [n (range 10)] (vp/ingest-source-granule "LPDAAC_ECS" - (dg/granule ast-coll {:granule-ur (format "SC:AST_L1A.003:%d" n) - :revision-id 5})))) + (dg/granule ast-coll + {:granule-ur (format "SC:AST_L1A.003:%d" n) + :revision-id 5})))) _ (bootstrap-and-index) v-granules (mapcat #(:refs (search/find-refs :granule diff --git a/system-int-test/test/cmr/system_int_test/ingest/bulk_update/bulk_update_test.clj b/system-int-test/test/cmr/system_int_test/ingest/bulk_update/bulk_update_test.clj index ad6c43d473..6f8c84ab99 100644 --- a/system-int-test/test/cmr/system_int_test/ingest/bulk_update/bulk_update_test.clj +++ b/system-int-test/test/cmr/system_int_test/ingest/bulk_update/bulk_update_test.clj @@ -1,18 +1,13 @@ (ns cmr.system-int-test.ingest.bulk-update.bulk-update-test "CMR bulk update. Test the actual update " (:require - [cheshire.core :as json] [clojure.test :refer :all] [clojure.java.io :as io] [cmr.common-app.test.side-api :as side] - [cmr.common.time-keeper :as time-keeper] - [cmr.common.util :as util :refer [are3]] [cmr.ingest.config :as ingest-config] - [cmr.message-queue.test.queue-broker-side-api :as qb-side-api] [cmr.mock-echo.client.echo-util :as e] [cmr.system-int-test.data2.core :as data2-core] [cmr.system-int-test.data2.umm-spec-collection :as data-umm-c] - [cmr.system-int-test.data2.umm-spec-common :as data-umm-cmn] [cmr.system-int-test.system :as s] [cmr.system-int-test.utils.index-util :as index] [cmr.system-int-test.utils.ingest-util :as ingest] @@ -167,7 +162,7 @@ (data-umm-c/collection x attribs) format)]] (:concept-id (ingest/ingest-concept - (assoc collection :concept-id (generate-concept-id x "PROV1")))))))) + (assoc collection :concept-id (generate-concept-id x "PROV1")) {:validate-keywords false})))))) (defn- ingest-collection-in-umm-json-format "Ingest a collection in UMM Json format and return a list of one concept-id. @@ -179,7 +174,7 @@ (data-umm-c/collection 1 attribs) :umm-json)] [(:concept-id (ingest/ingest-concept - (assoc collection :concept-id (generate-concept-id 1 "PROV1"))))])) + (assoc collection :concept-id (generate-concept-id 1 "PROV1")) {:validate-keywords false}))])) (deftest bulk-update-science-keywords ;; Ingest a collection in each format with science keywords to update @@ -318,11 +313,11 @@ :Term "ENVIRONMENTAL IMPACTS1" :VariableLevel1 "HEAVY METALS CONCENTRATION1"}]}] ;; Initiate bulk update that shouldn't add any duplicates. - (ingest/bulk-update-collections "PROV1" duplicate-body) + (ingest/bulk-update-collections "PROV1" duplicate-body {:validate-keywords false}) ;; Wait for queueing/indexing to catch up (index/wait-until-indexed) ;; Kick off bulk update - (let [response (ingest/bulk-update-collections "PROV1" bulk-update-body)] + (let [response (ingest/bulk-update-collections "PROV1" bulk-update-body {:validate-keywords false})] (is (= 200 (:status response))) ;; Wait for queueing/indexing to catch up (index/wait-until-indexed) @@ -370,7 +365,7 @@ :Term "ENVIRONMENTAL IMPACTS1" :VariableLevel1 "HEAVY METALS CONCENTRATION1"}]}] ;; Kick off bulk update - (let [response (ingest/bulk-update-collections "PROV1" bulk-update-body)] + (let [response (ingest/bulk-update-collections "PROV1" bulk-update-body {:validate-keywords false})] (is (= 200 (:status response))) ;; Wait for queueing/indexing to catch up (index/wait-until-indexed) @@ -416,7 +411,7 @@ :Term "ENVIRONMENTAL IMPACTS1" :VariableLevel1 "HEAVY METALS CONCENTRATION1"}]}] ;; Kick off bulk update - (let [response (ingest/bulk-update-collections "PROV1" bulk-update-body)] + (let [response (ingest/bulk-update-collections "PROV1" bulk-update-body {:validate-keywords false})] (is (= 200 (:status response))) ;; Wait for queueing/indexing to catch up (index/wait-until-indexed) @@ -450,7 +445,7 @@ :find-value {:ShortName "NSID"} :update-value {:ShortName "NSIDC" :Roles ["ORIGINATOR"]}} - task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body))] + task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body {:validate-keywords false}))] (index/wait-until-indexed) (let [collection-response (ingest/bulk-update-task-status "PROV1" task-id)] (is (= "COMPLETE" (:task-status collection-response)))) @@ -481,7 +476,7 @@ :find-value {:ShortName "atm"} :update-value {:ShortName "LVIS" :LongName nil}} - task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body))] + task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body {:validate-keywords false}))] (index/wait-until-indexed) (let [collection-response (ingest/bulk-update-task-status "PROV1" task-id)] (is (= "COMPLETE" (:task-status collection-response)))) @@ -521,7 +516,7 @@ :update-field "INSTRUMENTS" :find-value {:ShortName "atm"} :update-value [{:ShortName "LVIS"}]} - task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body))] + task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body {:validate-keywords false}))] (index/wait-until-indexed) (let [collection-response (ingest/bulk-update-task-status "PROV1" task-id)] (is (= "COMPLETE" (:task-status collection-response)))) @@ -561,7 +556,7 @@ :find-value {:ShortName "a340-600-1"} :update-value {:ShortName "SMAP" :LongName nil}} - task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body))] + task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body {:validate-keywords false}))] (index/wait-until-indexed) (let [collection-response (ingest/bulk-update-task-status "PROV1" task-id)] (is (= "COMPLETE" (:task-status collection-response)))) @@ -602,7 +597,7 @@ :find-value {:ShortName "ShortName"} :update-value {:ShortName "New ShortName" :LongName nil}} - task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body))] + task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body {:validate-keywords false}))] (index/wait-until-indexed) (let [collection-response (ingest/bulk-update-task-status "PROV1" task-id)] (is (= "COMPLETE" (:task-status collection-response)))) @@ -642,7 +637,7 @@ :ContactInformation {:RelatedUrls [{:URLContentType "DataCenterURL" :Type "HOME PAGE" :URL "http://test.org/daac/index.html"}]}}} - task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body))] + task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body {:validate-keywords false}))] (index/wait-until-indexed) (let [collection-response (ingest/bulk-update-task-status "PROV1" task-id)] (is (= "COMPLETE" (:task-status collection-response)))) @@ -691,7 +686,7 @@ :Topic "ATMOSPHERE" :Term "AIR QUALITY" :VariableLevel1 "EMISSIONS"}} - task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body))] + task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body {:validate-keywords false}))] (index/wait-until-indexed) (let [collection-response (ingest/bulk-update-task-status "PROV1" task-id)] (is (= "COMPLETE" (:task-status collection-response)))) @@ -724,7 +719,7 @@ :Topic "ATMOSPHERE" :Term "AIR QUALITY" :VariableLevel1 "CARBON MONOXIDE"}} - task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body))] + task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body {:validate-keywords false}))] (index/wait-until-indexed) (let [collection-response (ingest/bulk-update-task-status "PROV1" task-id)] (is (= "COMPLETE" (:task-status collection-response)))) @@ -758,7 +753,7 @@ :update-type "FIND_AND_REMOVE" :update-field "INSTRUMENTS" :find-value {:ShortName "atm"}} - task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body))] + task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body {:validate-keywords false}))] (index/wait-until-indexed) (let [collection-response (ingest/bulk-update-task-status "PROV1" task-id) collection-status (first (:collection-statuses collection-response))] @@ -797,7 +792,7 @@ :Topic "ATMOSPHERE" :Term "AIR QUALITY" :VariableLevel1 "EMISSIONS"}} - task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body))] + task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body {:validate-keywords false}))] (index/wait-until-indexed) (let [collection-response (ingest/bulk-update-task-status "PROV1" task-id)] (is (= "COMPLETE" (:task-status collection-response)))) @@ -834,7 +829,7 @@ :update-value {:Category "EARTH SCIENCE" :Topic "OCEANS" :Term "MARINE SEDIMENTS"}} - task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body))] + task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body {:validate-keywords false}))] (index/wait-until-indexed) (let [collection-response (ingest/bulk-update-task-status "PROV1" task-id)] (is (= "COMPLETE" (:task-status collection-response)))) @@ -873,7 +868,7 @@ :results :items first)) - task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body)) + task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body {:validate-keywords false})) _ (index/wait-until-indexed) new-concepts (doseq [concept-id concept-ids] (-> (search/find-concepts-umm-json :collection @@ -898,9 +893,11 @@ (deftest bulk-update-update-all-tombstone-test (let [coll1 (data2-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "E1" - :ShortName "S1"})) + :ShortName "S1"}) + {:validate-keywords false}) coll2 (data2-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "E2" - :ShortName "S2"})) + :ShortName "S2"}) + {:validate-keywords false}) _ (index/wait-until-indexed) bulk-update-body1 {:concept-ids ["ALL"] :update-type "FIND_AND_UPDATE" @@ -922,7 +919,7 @@ :VariableLevel1 "EMISSIONS"}}] ;; perform bulk update, verify that both collections are skipped because find-value is not found. (testing "all the non-deleted collections are included in update all case." - (let [response (ingest/bulk-update-collections "PROV1" bulk-update-body1) + (let [response (ingest/bulk-update-collections "PROV1" bulk-update-body1 {:validate-keywords false}) _ (index/wait-until-indexed) collection-response (ingest/bulk-update-task-status "PROV1" (:task-id response))] (is (= "COMPLETE" (:task-status collection-response))) @@ -935,14 +932,14 @@ ;; perform another bulk update, verify that the deleted collection is not included when getting all ;; collections from the provider. (testing "Deleted collection is excluded in update all case." - (let [response (ingest/bulk-update-collections "PROV1" bulk-update-body1) + (let [response (ingest/bulk-update-collections "PROV1" bulk-update-body1 {:validate-keywords false}) _ (index/wait-until-indexed) collection-response (ingest/bulk-update-task-status "PROV1" (:task-id response))] (is (= "Task completed with 1 SKIPPED out of 1 total collection update(s)." (:status-message collection-response))))) ;; perform a bulk update with the deleted collection's concept-id and a non-deleted collection's concept-id ;; The deleted one should fail the update. (testing "Deleted collection is failed in update concept-id case." - (let [response (ingest/bulk-update-collections "PROV1" bulk-update-body2) + (let [response (ingest/bulk-update-collections "PROV1" bulk-update-body2 {:validate-keywords false}) _ (index/wait-until-indexed) collection-response (ingest/bulk-update-task-status "PROV1" (:task-id response)) collection-statuses (:collection-statuses collection-response)] @@ -958,7 +955,7 @@ ;; perform another bulk update, verify that the deleted collections are not included when getting all ;; collections from the provider. (testing "All collections are deleted in update all case." - (let [response (ingest/bulk-update-collections "PROV1" bulk-update-body1) + (let [response (ingest/bulk-update-collections "PROV1" bulk-update-body1 {:validate-keywords false}) _ (index/wait-until-indexed)] (is (= ["There are no collections that have not been deleted for provider [PROV1]."] (:errors response))))))) @@ -974,7 +971,7 @@ :Topic "ATMOSPHERE" :Term "AIR QUALITY" :VariableLevel1 "EMISSIONS"}} - response (ingest/bulk-update-collections "PROV1" bulk-update-body)] + response (ingest/bulk-update-collections "PROV1" bulk-update-body {:validate-keywords false})] (is (= 200 (:status response))) ;; Wait for queueing/indexing to catch up (index/wait-until-indexed) @@ -994,8 +991,8 @@ :Topic "ATMOSPHERE" :Term "AIR QUALITY" :VariableLevel1 "EMISSIONS"}} - response (ingest/bulk-update-collections "PROV1" bulk-update-body) - response2 (ingest/bulk-update-collections "PROV1" bulk-update-body)] + response (ingest/bulk-update-collections "PROV1" bulk-update-body {:validate-keywords false}) + response2 (ingest/bulk-update-collections "PROV1" bulk-update-body {:validate-keywords false})] (is (= 200 (:status response))) (is (= 422 (:status response2))) (is (= ["Error creating bulk update task: Bulk update name needs to be unique within the provider."] @@ -1004,7 +1001,7 @@ (deftest bulk-update-xml-to-umm-failure-test (let [coll-metadata (slurp (io/resource "dif-samples/cmr-4455-collection.xml")) concept (ingest/ingest-concept - (ingest/concept :collection "PROV1" "foo" :dif coll-metadata)) + (ingest/concept :collection "PROV1" "foo" :dif coll-metadata) {:validate-keywords false}) _ (index/wait-until-indexed) bulk-update-body {:concept-ids [(:concept-id concept)] :update-type "ADD_TO_EXISTING" @@ -1014,7 +1011,7 @@ :Term "ENVIRONMENTAL IMPACTS" :VariableLevel1 "HEAVY METALS CONCENTRATION"}}] ;; Kick off bulk update - (let [response (ingest/bulk-update-collections "PROV1" bulk-update-body)] + (let [response (ingest/bulk-update-collections "PROV1" bulk-update-body {:validate-keywords false})] (is (= 200 (:status response))) ;; Wait for queueing/indexing to catch up (index/wait-until-indexed) @@ -1031,7 +1028,7 @@ :Topic "HUMAN DIMENSIONS" :Term "ENVIRONMENTAL IMPACTS" :VariableLevel1 "HEAVY METALS CONCENTRATION"}} - task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body))] + task-id (:task-id (ingest/bulk-update-collections "PROV1" bulk-update-body {:validate-keywords false}))] (index/wait-until-indexed) (let [collection-response (ingest/bulk-update-task-status "PROV1" task-id) collection-status (first (:collection-statuses collection-response))] diff --git a/system-int-test/test/cmr/system_int_test/ingest/collection/collection_ingest_test.clj b/system-int-test/test/cmr/system_int_test/ingest/collection/collection_ingest_test.clj index 017e4783cc..974d2cdd33 100644 --- a/system-int-test/test/cmr/system_int_test/ingest/collection/collection_ingest_test.clj +++ b/system-int-test/test/cmr/system_int_test/ingest/collection/collection_ingest_test.clj @@ -44,13 +44,15 @@ (data-umm-c/collection-concept {:provider-id "PROV3" :StandardProduct true} - :umm-json)) + :umm-json) + {:validate-keywords false}) coll4-wrong-collection-data-type (ingest/ingest-concept (data-umm-c/collection-concept {:provider-id "PROV4" :StandardProduct true :CollectionDataType "NEAR_REAL_TIME"} - :umm-json))] + :umm-json) + {:validate-keywords false})] (is (= ["Standard product validation failed: Standard Product designation is only allowed for NASA data products. This collection is being ingested using a non-NASA provider which means the record is not a NASA record. Please remove the StandardProduct element from the record."] (:errors coll3-non-eosdis-consortium))) (is (= ["Standard product validation failed: Standard Product cannot be true with the CollectionDataType being one of the following values: NEAR_REAL_TIME, LOW_LATENCY, or EXPEDITED. The CollectionDataType is [NEAR_REAL_TIME]."] @@ -60,26 +62,26 @@ (deftest basic-collection-ingest-test (testing "ingest of a new concept" (let [concept (data-umm-c/collection-concept {}) - {:keys [concept-id revision-id]} (ingest/ingest-concept concept)] + {:keys [concept-id revision-id]} (ingest/ingest-concept concept {:validate-keywords false})] (index/wait-until-indexed) (is (mdb/concept-exists-in-mdb? concept-id revision-id)) (is (= 1 revision-id))) ;;testing that when launchpad token is enforced, collection ingest fails without launchpad token. (side/eval-form `(common-config/set-launchpad-token-enforced! true)) (let [concept (data-umm-c/collection-concept {}) - result (ingest/ingest-concept concept)] + result (ingest/ingest-concept concept {:validate-keywords false})] (is (= ["Launchpad token is required. Token [XXX] is not a launchpad token."] (:errors result)))) (side/eval-form `(common-config/set-launchpad-token-enforced! false))) (testing "ingest of a concept with a revision id" (let [concept (data-umm-c/collection-concept {:revision-id 5}) - {:keys [concept-id revision-id]} (ingest/ingest-concept concept)] + {:keys [concept-id revision-id]} (ingest/ingest-concept concept {:validate-keywords false})] (index/wait-until-indexed) (is (= 5 revision-id)) (is (mdb/concept-exists-in-mdb? concept-id 5)))) (testing "ingest of a new concept using a launchpad token" (let [concept (data-umm-c/collection-concept {}) launchpad-token (echo-util/login-with-launchpad-token (system/context) "user1") - {:keys [concept-id revision-id]} (ingest/ingest-concept concept {:token launchpad-token})] + {:keys [concept-id revision-id]} (ingest/ingest-concept concept {:token launchpad-token :validate-keywords false})] (index/wait-until-indexed) (is (mdb/concept-exists-in-mdb? concept-id revision-id)) (is (= 6 revision-id))))) @@ -88,8 +90,8 @@ (deftest collection-ingest-test (testing "ingest of a new concept twice to get two revisions, and reindex revision 1, with ignore_conflict on and off." (let [concept (data-umm-c/collection-concept {}) - ingested-concept1 (ingest/ingest-concept concept) - ingested-concept2 (ingest/ingest-concept concept) + ingested-concept1 (ingest/ingest-concept concept {:validate-keywords false}) + ingested-concept2 (ingest/ingest-concept concept {:validate-keywords false}) concept-id1 (:concept-id ingested-concept1) concept-id2 (:concept-id ingested-concept2) revision-id1 (:revision-id ingested-concept1) @@ -134,10 +136,10 @@ ingest-header3 expected-user-id3 ingest-header4 expected-user-id4] (let [concept (data-umm-c/collection-concept {}) - {:keys [concept-id revision-id]} (ingest/ingest-concept concept ingest-header1)] - (ingest/ingest-concept concept ingest-header2) + {:keys [concept-id revision-id]} (ingest/ingest-concept concept (assoc ingest-header1 :validate-keywords false))] + (ingest/ingest-concept concept (assoc ingest-header2 :validate-keywords false)) (ingest/delete-concept concept ingest-header3) - (ingest/ingest-concept concept ingest-header4) + (ingest/ingest-concept concept (assoc ingest-header4 :validate-keywords false)) (index/wait-until-indexed) (ingest/assert-user-id concept-id revision-id expected-user-id1) (ingest/assert-user-id concept-id (inc revision-id) expected-user-id2) @@ -171,23 +173,23 @@ concept (data-umm-c/collection-concept {:concept-id supplied-concept-id :native-id "Atlantic-1"})] (testing "ingest of a new concept with concept-id present" - (let [{:keys [concept-id revision-id]} (ingest/ingest-concept concept)] + (let [{:keys [concept-id revision-id]} (ingest/ingest-concept concept {:validate-keywords false})] (index/wait-until-indexed) (is (mdb/concept-exists-in-mdb? concept-id revision-id)) (is (= [supplied-concept-id 1] [concept-id revision-id])))) (testing "Update the concept with the concept-id" - (let [{:keys [concept-id revision-id]} (ingest/ingest-concept concept)] + (let [{:keys [concept-id revision-id]} (ingest/ingest-concept concept {:validate-keywords false})] (index/wait-until-indexed) (is (= [supplied-concept-id 2] [concept-id revision-id])))) (testing "update the concept without the concept-id" - (let [{:keys [concept-id revision-id]} (ingest/ingest-concept (dissoc concept :concept-id))] + (let [{:keys [concept-id revision-id]} (ingest/ingest-concept (dissoc concept :concept-id) {:validate-keywords false})] (index/wait-until-indexed) (is (= [supplied-concept-id 3] [concept-id revision-id])))) (testing "update concept with a different concept-id is invalid" - (let [{:keys [status errors]} (ingest/ingest-concept (assoc concept :concept-id "C1111-PROV1"))] + (let [{:keys [status errors]} (ingest/ingest-concept (assoc concept :concept-id "C1111-PROV1") {:validate-keywords false})] (index/wait-until-indexed) (is (= [422 ["Concept-id [C1111-PROV1] does not match the existing concept-id [C1000-PROV1] for native-id [Atlantic-1]"]] [status errors])))))) @@ -198,14 +200,14 @@ (let [concept (data-umm-c/collection-concept {:EntryTitle "E1" :ShortName "S1" :concept-id "C1200000000-PROV1"}) - response (ingest/ingest-concept concept {:accept-format :json :raw? true})] + response (ingest/ingest-concept concept {:accept-format :json :raw? true :validate-keywords false})] (is (= {:concept-id (:concept-id concept) :revision-id 1} (select-keys (ingest/parse-ingest-body :json response) [:concept-id :revision-id]))))) (testing "xml response" (let [concept (data-umm-c/collection-concept {:EntryTitle "E2" :ShortName "S2" :concept-id "C1200000001-PROV1"}) - response (ingest/ingest-concept concept {:accept-format :xml :raw? true})] + response (ingest/ingest-concept concept {:accept-format :xml :raw? true :validate-keywords false})] (is (= {:concept-id (:concept-id concept) :revision-id 1} (select-keys (ingest/parse-ingest-body :xml response) [:concept-id :revision-id])))))) @@ -214,13 +216,13 @@ (testing "json response" (let [concept-with-empty-body (assoc (data-umm-c/collection-concept {}) :metadata "") response (ingest/ingest-concept concept-with-empty-body - {:accept-format :json :raw? true}) + {:accept-format :json :raw? true :validate-keywords false}) {:keys [errors]} (ingest/parse-ingest-body :json response)] (is (re-find #"Request content is too short." (first errors))))) (testing "xml response" (let [concept-with-empty-body (assoc (data-umm-c/collection-concept {}) :metadata "") response (ingest/ingest-concept concept-with-empty-body - {:accept-format :xml :raw? true}) + {:accept-format :xml :raw? true :validate-keywords false}) {:keys [errors]} (ingest/parse-ingest-body :xml response)] (is (re-find #"Request content is too short." (first errors)))))) @@ -229,7 +231,7 @@ (testing "xml response" (let [concept-with-empty-body (assoc (data-umm-c/collection-concept {}) :metadata "") response (ingest/ingest-concept concept-with-empty-body - {:raw? true}) + {:raw? true :validate-keywords false}) {:keys [errors]} (ingest/parse-ingest-body :xml response)] (is (re-find #"Request content is too short." (first errors)))))) @@ -238,7 +240,8 @@ (testing "json response" (let [coll1 (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "E1" - :ShortName "S1"})) + :ShortName "S1"}) + {:validate-keywords false}) response (ingest/delete-concept (data-core/umm-c-collection->concept coll1 :echo10) {:accept-format :json :raw? true})] @@ -247,7 +250,8 @@ (testing "xml response" (let [coll1 (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "E2" - :ShortName "S2"})) + :ShortName "S2"}) + {:validate-keywords false}) _ (index/wait-until-indexed) response (ingest/delete-concept (data-core/umm-c-collection->concept coll1 :echo10) {:accept-format :xml @@ -270,7 +274,7 @@ (let [concept (data-umm-c/collection-concept {:EntryTitle "E1" :ShortName "S1" :concept-id "C1-PROV1"} :echo10) - response (ingest/ingest-concept concept {:raw? true})] + response (ingest/ingest-concept concept {:raw? true :validate-keywords false})] (is (= {:concept-id "C1-PROV1" :revision-id 1} (select-keys (ingest/parse-ingest-body :xml response) [:concept-id :revision-id]))))) (testing "dif" @@ -278,14 +282,14 @@ :ShortName "S2" :concept-id "C2-PROV1"}) :provider-id "PROV1") :dif) - response (ingest/ingest-concept concept {:raw? true})] + response (ingest/ingest-concept concept {:raw? true :validate-keywords false})] (is (= {:concept-id "C2-PROV1" :revision-id 1} (select-keys (ingest/parse-ingest-body :xml response) [:concept-id :revision-id]))))) (testing "iso" (let [concept (data-umm-c/collection-concept {:EntryTitle "E3" :ShortName "S3" :concept-id "C3-PROV1"} :iso-smap) - response (ingest/ingest-concept concept {:raw? true})] + response (ingest/ingest-concept concept {:raw? true :validate-keywords false})] (is (= {:concept-id "C3-PROV1" :revision-id 1} (select-keys (ingest/parse-ingest-body :xml response) [:concept-id :revision-id])))))) @@ -303,22 +307,22 @@ :native-id "NID-3"}) coll1-4 (assoc-in coll1-3 [:ShortName] "EID-4") ;; ingest the collections/granules for test - _ (data-core/ingest-umm-spec-collection "PROV1" coll1-1 {:format :dif}) - coll3 (data-core/ingest-umm-spec-collection "PROV1" coll1-3 {:format :dif}) + _ (data-core/ingest-umm-spec-collection "PROV1" coll1-1 {:format :dif :validate-keywords false}) + coll3 (data-core/ingest-umm-spec-collection "PROV1" coll1-3 {:format :dif :validate-keywords false}) gran1 (data-core/ingest "PROV1" (granule/granule-with-umm-spec-collection coll3 (:concept-id coll3) {:granule-ur "Granule1"})) gran2 (data-core/ingest "PROV1" (granule/granule-with-umm-spec-collection coll3 (:concept-id coll3) {:granule-ur "Granule2"}))] (index/wait-until-indexed) (testing "update the collection with a different entry-id is OK" (let [{:keys [status concept-id revision-id errors]} - (data-core/ingest-umm-spec-collection "PROV1" coll1-2 {:format :dif})] + (data-core/ingest-umm-spec-collection "PROV1" coll1-2 {:format :dif :validate-keywords false})] (is (= ["C1-PROV1" 2 200 nil] [concept-id revision-id status errors])))) (testing "update the collection that has granules with a different entry-id is allowed" ;; For CMR-2403 we decided to temporary allow collection identifiers to be updated even ;; with existing granules for the collection. We will change this with CMR-2485. (let [{:keys [status concept-id revision-id errors]} - (data-core/ingest-umm-spec-collection "PROV1" coll1-4 {:format :dif :allow-failure? true})] + (data-core/ingest-umm-spec-collection "PROV1" coll1-4 {:format :dif :allow-failure? true :validate-keywords false})] (is (= ["C3-PROV1" 2 200 nil] [concept-id revision-id status errors])))) (testing "ingest collection with entry-id used by a different collection latest revision within the same provider is invalid" @@ -328,7 +332,7 @@ :concept-id "C2-PROV1" :native-id "NID-2" :EntryTitle "EID-2") - {:format :dif :allow-failure? true})] + {:format :dif :allow-failure? true :validate-keywords false})] (is (= [409 ["The Short Name [EID-2] and Version Id [V1] combined must be unique. The following concepts with the same Short Name and Version Id were found: [C1-PROV1]."]] [status errors])))) @@ -340,7 +344,7 @@ :concept-id "C2-PROV1" :native-id "NID-2" :EntryTitle "EID-2") - {:format :dif :allow-failure? true})] + {:format :dif :allow-failure? true :validate-keywords false})] (is (= ["C2-PROV1" 1 201 nil] [concept-id revision-id status errors])))) (testing "entry-id and entry-title constraint violations return multiple errors" @@ -349,7 +353,7 @@ (assoc coll1-2 :concept-id "C5-PROV1" :native-id "NID-5") - {:format :dif :allow-failure? true})] + {:format :dif :allow-failure? true :validate-keywords false})] (is (= [409 ["The Entry Title [ET-1] must be unique. The following concepts with the same entry title were found: [C1-PROV1]." "The Short Name [EID-2] and Version Id [V1] combined must be unique. The following concepts with the same Short Name and Version Id were found: [C1-PROV1]."]] [status errors])))) @@ -358,7 +362,7 @@ (let [{:keys [status]} (data-core/ingest-umm-spec-collection "PROV2" (assoc coll1-2 :concept-id "C1-PROV2") - {:format :dif})] + {:format :dif :validate-keywords false})] (is (= 201 status)))))) ;; Ingest same concept N times and verify same concept-id is returned and @@ -394,7 +398,8 @@ :TemporalExtents [(data-umm-cmn/temporal-extent {:beginning-date-time "1965-12-12T12:00:00Z" :ending-date-time "1967-12-12T12:00:00Z"})]}) - {:format coll-format})] + {:format coll-format + :validate-keywords false})] (index/wait-until-indexed) (is (= expected-rev (:revision-id coll))) (is (= 1 (:hits (search/find-refs :collection {:keyword (name coll-format)})))))))) @@ -402,7 +407,7 @@ ;; Verify ingest behaves properly if empty body is presented in the request. (deftest empty-collection-ingest-test (let [concept-with-empty-body (assoc (data-umm-c/collection-concept {}) :metadata "") - {:keys [status errors]} (ingest/ingest-concept concept-with-empty-body)] + {:keys [status errors]} (ingest/ingest-concept concept-with-empty-body {:validate-keywords false})] (index/wait-until-indexed) (is (= status 400)) (is (re-find #"Request content is too short." (first errors))))) @@ -412,7 +417,8 @@ (let [coll (data-umm-c/collection {:DataDates [(umm-cmn/map->DateType {:Date (date-time-parser/parse-datetime "2000-01-01T12:00:00Z") :Type "DELETE"})]}) {:keys [status errors]} (ingest/ingest-concept - (data-core/umm-c-collection->concept (assoc coll :provider-id "PROV1") :echo10))] + (data-core/umm-c-collection->concept (assoc coll :provider-id "PROV1") :echo10) + {:validate-keywords false})] (index/wait-until-indexed) (is (= status 422)) (is (re-find #"DeleteTime 2000-01-01T12:00:00.000Z is before the current time." (first errors))))) @@ -421,7 +427,7 @@ (testing "It should be possible to delete existing concept and the operation without revision id should result in revision id 1 greater than max revision id of the concept prior to the delete" (let [concept (data-umm-c/collection-concept {}) - ingest-result (ingest/ingest-concept concept) + ingest-result (ingest/ingest-concept concept {:validate-keywords false}) delete-result (ingest/delete-concept concept) ingest-revision-id (:revision-id ingest-result) delete-revision-id (:revision-id delete-result)] @@ -429,7 +435,7 @@ (is (= (inc ingest-revision-id) delete-revision-id)))) (testing "Deleting existing concept with a revision-id should respect the revision id" (let [concept (data-umm-c/collection-concept {}) - ingest-result (ingest/ingest-concept concept) + ingest-result (ingest/ingest-concept concept {:validate-keywords false}) delete-result (ingest/delete-concept concept {:revision-id 5}) delete-revision-id (:revision-id delete-result)] (index/wait-until-indexed) @@ -437,12 +443,14 @@ (is (mdb/concept-exists-in-mdb? (:concept-id ingest-result) 5))))) (deftest delete-collection-test - (let [coll1 (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "E1" - :ShortName "S1"})) + (let [coll1 (data-core/ingest-umm-spec-collection "PROV1" + (data-umm-c/collection {:EntryTitle "E1" :ShortName "S1"}) + {:validate-keywords false}) gran1 (data-core/ingest "PROV1" (granule/granule-with-umm-spec-collection coll1 (:concept-id coll1))) gran2 (data-core/ingest "PROV1" (granule/granule-with-umm-spec-collection coll1 (:concept-id coll1))) - coll2 (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "E2" - :ShortName "S2"})) + coll2 (data-core/ingest-umm-spec-collection "PROV1" + (data-umm-c/collection {:EntryTitle "E2":ShortName "S2"}) + {:validate-keywords false}) gran3 (data-core/ingest "PROV1" (granule/granule-with-umm-spec-collection coll2 (:concept-id coll2)))] (index/wait-until-indexed) @@ -479,13 +487,17 @@ ;; Ingest a collection under PROV1. coll1 (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "E1" - :ShortName "S1"})) + :ShortName "S1"}) + {:validate-keywords false}) coll2 (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "E2" - :ShortName "S2"})) + :ShortName "S2"}) + {:validate-keywords false}) coll3 (data-core/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "E3" - :ShortName "S3"})) + :ShortName "S3"}) + {:validate-keywords false}) coll4 (data-core/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "E4" - :ShortName "S4"})) + :ShortName "S4"}) + {:validate-keywords false}) token (echo-util/login-guest (system/context))] ;; wait for the collections to be indexed so that ACLs will be valid @@ -568,7 +580,7 @@ :concept_ids))))))) (deftest delete-deleted-collection-with-new-revision-id-returns-404 - (let [coll (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection)) + (let [coll (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection) {:validate-keywords false}) concept-id (:concept-id coll) native-id (:EntryTitle coll) coll-del1 (ingest/delete-concept (data-core/umm-c-collection->concept coll :echo10) {:revision-id 5}) @@ -588,14 +600,14 @@ (deftest content-type-with-parameter-ingest-test (let [concept (assoc (data-umm-c/collection-concept {}) :format "application/echo10+xml; charset=utf-8") - {:keys [status]} (ingest/ingest-concept concept)] + {:keys [status]} (ingest/ingest-concept concept {:validate-keywords false})] (index/wait-until-indexed) (is (= 201 status)))) ;; Verify ingest behaves properly if request is missing content type. (deftest missing-content-type-ingest-test (let [concept-with-no-content-type (assoc (data-umm-c/collection-concept {}) :format "") - response (ingest/ingest-concept concept-with-no-content-type {:accept-format :json :raw? true}) + response (ingest/ingest-concept concept-with-no-content-type {:accept-format :json :raw? true :validate-keywords false}) status (:status response) {:keys [errors]} (ingest/parse-ingest-body :json response)] (index/wait-until-indexed) @@ -605,7 +617,7 @@ ;; Verify ingest behaves properly if request contains invalid content type. (deftest invalid-content-type-ingest-test (let [concept (assoc (data-umm-c/collection-concept {}) :format "blah") - response (ingest/ingest-concept concept {:accept-format :json :raw? true}) + response (ingest/ingest-concept concept {:accept-format :json :raw? true :validate-keywords false}) status (:status response) {:keys [errors]} (ingest/parse-ingest-body :json response)] (index/wait-until-indexed) @@ -616,7 +628,7 @@ (deftest ingest-collection-with-slash-in-native-id-test (let [crazy-id "`1234567890-=qwertyuiop[]\\asdfghjkl;'zxcvbnm,./ ~!@#$%^&*()_+QWERTYUIOP{}ASDFGHJKL:\"ZXCVBNM<>?" collection (data-umm-c/collection-concept {:EntryTitle crazy-id}) - {:keys [concept-id revision-id] :as response} (ingest/ingest-concept collection) + {:keys [concept-id revision-id] :as response} (ingest/ingest-concept collection {:validate-keywords false}) ingested-concept (mdb/get-concept concept-id)] (index/wait-until-indexed) (is (= 201 (:status response))) @@ -644,7 +656,8 @@ ;; this is to cause validation error for iso19115 format (string/replace "fileIdentifier" "XXXX") ;; this is to cause validation error for iso-smap format - (string/replace "gmd:DS_Series" "XXXX"))))] + (string/replace "gmd:DS_Series" "XXXX"))) + {:validate-keywords false})] (index/wait-until-indexed) (= [400 validation-errors] [status errors])) @@ -675,7 +688,7 @@ ;; assumes the current version :format "application/vnd.nasa.cmr.umm+json" :metadata json} - response (ingest/ingest-concept coll-map)] + response (ingest/ingest-concept coll-map {:validate-keywords false})] (is (= 201 (:status response))) (is (= nil (:errors response))) (index/wait-until-indexed) @@ -686,7 +699,7 @@ (is (= 1 (count (:refs (search/find-refs :collection {"entry-title" "The entry title V5"})))))) (testing "Updating a UMM-JSON collection" - (let [response (ingest/ingest-concept (assoc coll-map :revision-id "2"))] + (let [response (ingest/ingest-concept (assoc coll-map :revision-id "2") {:validate-keywords false})] (is (= 200 (:status response))) (index/wait-until-indexed) (is (mdb/concept-exists-in-mdb? (:concept-id response) 2)) @@ -705,7 +718,7 @@ :concept-type :collection :format "application/vnd.nasa.cmr.umm+json" :metadata json} - response (ingest/ingest-concept concept-map {:accept-format :json})] + response (ingest/ingest-concept concept-map {:accept-format :json :validate-keywords false})] (is (= ["#/DataDates/0/Date: [invalid date] is not a valid date-time. Expected [yyyy-MM-dd'T'HH:mm:ssZ, yyyy-MM-dd'T'HH:mm:ss.[0-9]{1,9}Z, yyyy-MM-dd'T'HH:mm:ss[+-]HH:mm, yyyy-MM-dd'T'HH:mm:ss.[0-9]{1,9}[+-]HH:mm]"] (:errors response))) (is (= 400 (:status response)))))) @@ -716,7 +729,7 @@ :concept-type :collection :format "application/vnd.nasa.cmr.umm+json;version=1.0" :metadata json} - response (ingest/ingest-concept coll-map {:accept-format :json})] + response (ingest/ingest-concept coll-map {:accept-format :json :validate-keywords false})] (is (= 201 (:status response))))) (deftest ingest-higher-than-accepted-umm-version @@ -727,7 +740,7 @@ :concept-type :collection :format "application/vnd.nasa.cmr.umm+json;version=1.9" :metadata "{\"foo\":\"bar\"}"} - response (ingest/ingest-concept coll-map {:accept-format :json}) + response (ingest/ingest-concept coll-map {:accept-format :json :validate-keywords false}) _ (side/eval-form `(common-config/set-collection-umm-version! ~accepted-version))] (is (= 400 (:status response))) (is (= [(str "UMM JSON version 1.8 or lower can be ingested. Any version above that is considered in-development and cannot be ingested at this time.")] @@ -739,7 +752,7 @@ :concept-type :collection :format "application/vnd.nasa.cmr.umm+json;version=1.0009" :metadata "{\"foo\":\"bar\"}"} - response (ingest/ingest-concept coll-map {:accept-format :json})] + response (ingest/ingest-concept coll-map {:accept-format :json :validate-keywords false})] (is (= 400 (:status response))) (is (= ["Invalid UMM JSON schema version: 1.0009"] (:errors response))))) @@ -750,7 +763,7 @@ :concept-type :collection :format "application/vnd.nasa.cmr.umm+json;version=\"1.1\"" :metadata "{\"foo\":\"bar\"}"} - response (ingest/ingest-concept coll-map {:accept-format :json})] + response (ingest/ingest-concept coll-map {:accept-format :json :validate-keywords false})] (is (= 400 (:status response))) (is (= ["Invalid UMM JSON schema version: \"1.1\""] (:errors response))))) @@ -762,7 +775,7 @@ " in a schema validation error")) concept (assoc (data-umm-c/collection-concept {:projects [project]}) :format "application/echo10+xml; charset=utf-8") - {:keys [status]} (ingest/ingest-concept concept)] + {:keys [status]} (ingest/ingest-concept concept {:validate-keywords false})] (index/wait-until-indexed) (is (= 201 status)))) @@ -771,14 +784,16 @@ defaults to collections by default during ingest." (let [coll-metadata (slurp (io/resource "example-data/dif/C1214305813-AU_AADC.xml")) {:keys [status]} (ingest/ingest-concept - (ingest/concept :collection "PROV1" "foo" :dif coll-metadata))] + (ingest/concept :collection "PROV1" "foo" :dif coll-metadata) + {:validate-keywords false})] (is (= 201 status))))) (deftest invalid-mimetype-ingest (testing "Ingest of a json collection with invalid mimetypes. This verifies that we validate mimetypes against kms." (let [coll-metadata (slurp (io/resource "CMR-7647/CMR-7647.json")) {:keys [status errors]} (ingest/ingest-concept - (ingest/concept :collection "PROV1" "foo" :umm-json coll-metadata))] + (ingest/concept :collection "PROV1" "foo" :umm-json coll-metadata) + {:validate-keywords false})] (is (= 422 status)) (is (= [["Related URL Content Type, Type, and Subtype [ContactPerson1: BadURLContentType1>HOME PAGE>] are not a valid set together."] ["Related URL Content Type, Type, and Subtype [ContactPerson1: BadURLContentType2>HOME PAGE>] are not a valid set together."] @@ -797,7 +812,7 @@ :DOI {:MissingReason "Not Applicable" :Explanation "Explanation String"} :concept-id "C1-PROV1"} :echo10) - response (ingest/ingest-concept concept {:raw? true})] + response (ingest/ingest-concept concept {:raw? true :validate-keywords false})] (is (= {:concept-id "C1-PROV1" :revision-id 1} (select-keys (ingest/parse-ingest-body :xml response) [:concept-id :revision-id])))))) @@ -807,7 +822,7 @@ :ShortName "S2" :CollectionProgress "DEPRECATED" :concept-id "C2-PROV1"} :umm-json) - response (ingest/ingest-concept concept {:raw? true})] + response (ingest/ingest-concept concept {:raw? true :validate-keywords false})] (is (= {:concept-id "C2-PROV1" :revision-id 1} (select-keys (ingest/parse-ingest-body :xml response) [:concept-id :revision-id])))))) @@ -815,6 +830,7 @@ (testing "Ingest ECHO10 collection with GET DATA RelatedUrl type and invalid KMS subtype is OK" (let [coll-metadata (-> "CMR-8657/coll_invalid_kms_getdata_subtypes.xml" io/resource slurp) {:keys [status errors]} (ingest/ingest-concept - (ingest/concept :collection "PROV1" "CMR-8657-GET-DATA-subtype" :echo10 coll-metadata))] + (ingest/concept :collection "PROV1" "CMR-8657-GET-DATA-subtype" :echo10 coll-metadata) + {:validate-keywords false})] (is (= 201 status)) (is (= nil errors))))) diff --git a/system-int-test/test/cmr/system_int_test/ingest/collection/collection_misc_validation_test.clj b/system-int-test/test/cmr/system_int_test/ingest/collection/collection_misc_validation_test.clj index ed5b0a9e8e..73b92f1e6c 100644 --- a/system-int-test/test/cmr/system_int_test/ingest/collection/collection_misc_validation_test.clj +++ b/system-int-test/test/cmr/system_int_test/ingest/collection/collection_misc_validation_test.clj @@ -22,7 +22,7 @@ (defn assert-conflict [coll-attributes errors] (let [collection (assoc (data-umm-c/collection coll-attributes) :native-id (:native-id coll-attributes)) - response (data-core/ingest "PROV1" collection {:allow-failure? true})] + response (data-core/ingest "PROV1" collection {:allow-failure? true :validate-keywords false})] (is (= {:status 409 :errors errors} (select-keys response [:status :errors]))))) @@ -36,7 +36,7 @@ (let [bad-metadata (slurp (io/resource "iso-samples/iso-spatial-data-missing-coordinate-system.iso19115")) {:keys [status warnings]} - (ingest/ingest-concept (ingest/concept :collection "PROV1" "foo" :iso19115 bad-metadata))] + (ingest/ingest-concept (ingest/concept :collection "PROV1" "foo" :iso19115 bad-metadata) {:validate-keywords false})] (is (= 201 status)) (is (string/includes? warnings @@ -57,7 +57,7 @@ (deftest nil-version-test (testing "Collections with nil versions are rejected" (let [concept (data-umm-c/collection-concept {:Version nil} :iso19115) - response (ingest/ingest-concept concept)] + response (ingest/ingest-concept concept {:validate-keywords false})] (is (= 201 (:status response)))))) @@ -68,7 +68,7 @@ :Purpose (apply str (repeat 12000 "y")) :ProcessingLevel {:Id "1"} :CollectionProgress "COMPLETE"}) - ingest-response (data-core/ingest "PROV1" collection {:format :dif10}) + ingest-response (data-core/ingest "PROV1" collection {:format :dif10 :validate-keywords false}) validation-response (ingest/validate-concept (data-umm-c/collection-concept collection :dif10))] (is (some? (re-find #"#/Platforms/0/ShortName: expected maxLength: 80, actual: 81" (:warnings ingest-response)))) (is (some? (re-find #"#/Platforms/0/ShortName: expected maxLength: 80, actual: 81" (:warnings validation-response)))) @@ -96,7 +96,7 @@ :DetailedVariable "" :Topic ""})]})] (are3 [collection] - (let [ingest-response (data-core/ingest-umm-spec-collection "PROV1" collection {:format :umm-json :allow-failure? true}) + (let [ingest-response (data-core/ingest-umm-spec-collection "PROV1" collection {:format :umm-json :allow-failure? true :validate-keywords false}) validation-response (ingest/validate-concept (data-umm-c/collection-concept collection :umm-json))] (is (nil? (first (map #(re-find #"(ECMA|regex)" %) (:errors ingest-response))))) (is (nil? (first (map #(re-find #"(ECMA|regex)" %) (:errors validation-response))))) @@ -111,7 +111,7 @@ "Invalid Science Keywords" science-keyword-collection) (testing "Explicitly test error messages - ShorName and LongName should fail the regex" - (let [ingest-response (data-core/ingest-umm-spec-collection "PROV1" platform-collection {:format :umm-json :allow-failure? true}) + (let [ingest-response (data-core/ingest-umm-spec-collection "PROV1" platform-collection {:format :umm-json :allow-failure? true :validate-keywords false}) validation-response (ingest/validate-concept (data-umm-c/collection-concept platform-collection :umm-json))] (is (= (:errors ingest-response) ["#/Platforms/0/LongName: expected minLength: 1, actual: 0" @@ -127,7 +127,7 @@ :RelatedUrls [{:URL "htp://www.x.com" :URLContentType "DistributionURL" :Type "GET DATA"}]}) - ingest-response (data-core/ingest "PROV1" collection) + ingest-response (data-core/ingest "PROV1" collection {:validate-keywords false}) validation-response (ingest/validate-concept (data-umm-c/collection-concept collection))] (is (some? (re-find #"#: required key \[DataCenters\] not found" (:warnings ingest-response)))) (is (some? (re-find #"#: required key \[DataCenters\] not found" (:warnings validation-response)))) @@ -150,6 +150,7 @@ (future (do (let [response (ingest/ingest-concept (data-umm-c/collection-concept {:concept-id "C1-PROV1" - :native-id "Same Native ID"}))] + :native-id "Same Native ID"}) + {:validate-keywords false})] (when (= 409 (:status response)) (println "409 returned, Errors:" (:errors response))))))))) diff --git a/system-int-test/test/cmr/system_int_test/ingest/collection/collection_umm_spec_validation_test.clj b/system-int-test/test/cmr/system_int_test/ingest/collection/collection_umm_spec_validation_test.clj index 0b2346dd8e..41a59730d8 100644 --- a/system-int-test/test/cmr/system_int_test/ingest/collection/collection_umm_spec_validation_test.clj +++ b/system-int-test/test/cmr/system_int_test/ingest/collection/collection_umm_spec_validation_test.clj @@ -129,7 +129,7 @@ {:AdditionalAttributes [(data-umm-cmn/additional-attribute {:Name "bool1" :DataType "BOOLEAN" :Value true}) (data-umm-cmn/additional-attribute {:Name "bool2" :DataType "BOOLEAN" :Value true})]}) - {:allow-failure? true})] + {:allow-failure? true :validate-keywords false})] (is (= {:status 422 :errors ["#: required key [DataCenters] not found" @@ -143,7 +143,8 @@ (testing "schema validation errors not returned" (side/eval-form `(ingest-config/set-return-umm-json-validation-errors! false)) (assert-valid {:AdditionalAttributes [(data-umm-cmn/additional-attribute {:Name "bool1" :DataType "BOOLEAN" :Value true}) - (data-umm-cmn/additional-attribute {:Name "bool2" :DataType "BOOLEAN" :Value true})]}))) + (data-umm-cmn/additional-attribute {:Name "bool2" :DataType "BOOLEAN" :Value true})]} + {:validate-keywords false}))) (testing "UMM-C JSON-Schema validation through Cmr-Validate-Umm-C header" (testing "schema validation errors returned when Cmr-Validate-Umm-C header is true" @@ -207,7 +208,7 @@ coll-attr ["AdditionalAttributes"] ["Additional Attributes must be unique. This contains duplicates named [bool]."] - {:validate-umm-c true}))) + {:validate-umm-c true :validate-keywords false}))) (testing "Additional Attribute validation" (assert-invalid @@ -243,7 +244,8 @@ {:Platforms [(data-umm-c/platform {:Instruments [(data-umm-c/instrument {:ShortName "I1"}) (data-umm-c/instrument {:ShortName "I1"})]})]} ["Platforms" 0 "Instruments"] - ["Instruments must be unique. This contains duplicates named [I1]."])) + ["Instruments must be unique. This contains duplicates named [I1]."] + {:validate-keywords false})) (testing "Spatial validation" (testing "Spatial enums" @@ -363,14 +365,15 @@ ["Spatial validation error: The bounding rectangle north value [45] was less than the south value [46]"])))) (deftest umm-spec-validation-warnings - ;; By default the config return-umm-spec-validation-errors is false, so warnings are returned with the collection. + ;; By default, the config return-umm-spec-validation-errors is true, + ;; so need to set it as false so we can get a successful 200 return status of the collection with warnings instead of errors. (testing "Ingest and Ingest Validation with warning messages for all formats" (are3 [format collection warning-message] (do - (let [response (data-core/ingest-umm-spec-collection "PROV1" collection {:format format})] + (let [response (data-core/ingest-umm-spec-collection "PROV1" collection {:format format :validate-keywords false})] (is (#{200 201} (:status response))) (is (= warning-message (:warnings response)))) - (let [response (ingest/validate-concept (data-umm-c/collection-concept collection format))] + (let [response (ingest/validate-concept (data-umm-c/collection-concept collection format) {:validate-keywords false})] (is (= 200 (:status response))) (is (= warning-message (:warnings response))))) @@ -381,12 +384,15 @@ "umm-json Ingest and Ingest Validation for Invalid data date ranges" :umm-json (collection-invalid-data-date-ranges) "After translating item to UMM-C the metadata had the following issue(s): [:MetadataDates] latest UPDATE date value: [2049-01-01T00:00:00.000Z] should be in the past. earliest REVIEW date value: [2011-01-01T00:00:00.000Z] should be in the future. DELETE date value: [2049-01-01T00:00:00.000Z] should be equal or later than latest REVIEW date value: [2050-01-01T00:00:00.000Z].;; [:DataDates] CREATE date value: [2050-01-01T00:00:00.000Z] should be in the past. latest UPDATE date value: [2049-01-01T00:00:00.000Z] should be in the past. earliest REVIEW date value: [2011-01-01T00:00:00.000Z] should be in the future. Earliest UPDATE date value: [2011-01-01T00:00:00.000Z] should be equal or later than CREATE date value: [2050-01-01T00:00:00.000Z].;; [:Platforms 0] Platform short name [A340-600], long name [Airbus A340-600], and type [Aircraft] was not a valid keyword combination." + "DIF10 Ingest and Ingest Validation" :dif10 (data-umm-c/collection-missing-properties-dif10 {}) "After translating item to UMM-C the metadata had the following issue(s): #: required key [ProcessingLevel] not found;; #: required key [CollectionProgress] not found;; [:Platforms 0] Platform short name [A340-600], long name [Airbus A340-600], and type [] was not a valid keyword combination.;; [:Platforms 0 :Instruments 0] Instrument short name [Not provided] and long name [] was not a valid keyword combination.;; [:Projects 0] Project short name [Not provided] and long name [] was not a valid keyword combination." + "DIF9 Ingest and Ingest Validation" :dif (data-umm-c/collection-missing-properties-dif {}) "After translating item to UMM-C the metadata had the following issue(s): #: required key [ProcessingLevel] not found;; #: required key [TemporalExtents] not found;; #: required key [SpatialExtent] not found;; #: required key [Platforms] not found;; #: required key [CollectionProgress] not found" + "ISO19115 Ingest and Ingest Validation" :iso19115 (data-umm-c/collection-missing-properties {}) "After translating item to UMM-C the metadata had the following issue(s): #: required key [DataCenters] not found;; #: required key [ProcessingLevel] not found;; #: required key [ScienceKeywords] not found;; #: required key [TemporalExtents] not found;; #: required key [SpatialExtent] not found;; #: required key [Platforms] not found;; #: required key [CollectionProgress] not found" @@ -397,7 +403,9 @@ "DIF9 with no version - has warnings, but passes ingest" :dif (assoc (data-umm-c/collection-missing-properties-dif {}) :Version nil) - "After translating item to UMM-C the metadata had the following issue(s): #: required key [Version] not found;; #: required key [ProcessingLevel] not found;; #: required key [TemporalExtents] not found;; #: required key [SpatialExtent] not found;; #: required key [Platforms] not found;; #: required key [CollectionProgress] not found"))) + "After translating item to UMM-C the metadata had the following issue(s): #: required key [Version] not found;; #: required key [ProcessingLevel] not found;; #: required key [TemporalExtents] not found;; #: required key [SpatialExtent] not found;; #: required key [Platforms] not found;; #: required key [CollectionProgress] not found")) + + ) (deftest umm-spec-temporal-validation (testing "Invalid temporal extents" @@ -428,7 +436,8 @@ (let [response (data-core/ingest-umm-spec-collection "PROV1" collection {:format format - :allow-failure? true})] + :allow-failure? true + :validate-keywords false})] (is (= 422 (:status response))) (is (= error-message (first (:errors response))))) @@ -482,7 +491,8 @@ collection {:format format :allow-failure? true - :validate-umm-c true})] + :validate-umm-c true + :validate-keywords false})] (is (= error-code (:status response))) (is (re-find error-matcher (prn-str (:errors response))))) diff --git a/system-int-test/test/cmr/system_int_test/ingest/collection/collection_update_test.clj b/system-int-test/test/cmr/system_int_test/ingest/collection/collection_update_test.clj index 24e9ffaabf..5098519b08 100644 --- a/system-int-test/test/cmr/system_int_test/ingest/collection/collection_update_test.clj +++ b/system-int-test/test/cmr/system_int_test/ingest/collection/collection_update_test.clj @@ -453,14 +453,17 @@ {:EntryTitle "parent-collection" :ShortName "S1" :Version "V1" - :Projects (data-umm-cmn/projects "p1" "p2" "p3" "p4")})) + :Projects (data-umm-cmn/projects "p1" "p2" "p3" "p4")}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "parent-collection2" - :Projects (data-umm-cmn/projects "p4")})) + :Projects (data-umm-cmn/projects "p4")}) + {:validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "parent-collection3" :ShortName "S3" - :Projects (data-umm-cmn/projects "USGS_SOFIA")})) + :Projects (data-umm-cmn/projects "USGS_SOFIA")}) + {:validate-keywords false}) _ (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll "C1-PROV1" {:project-refs ["p1"]})) _ (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll "C1-PROV1" {:project-refs ["p2" "p3"]})) _ (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll "C1-PROV1" {:project-refs ["p3"]})) @@ -477,7 +480,8 @@ {:EntryTitle "parent-collection" :ShortName "S1" :Version "V1" - :Projects (apply data-umm-cmn/projects projects)})) + :Projects (apply data-umm-cmn/projects projects)}) + {:validate-keywords false}) {:keys [status errors]} response] (is (= [200 nil] [status errors]))) @@ -495,7 +499,8 @@ :ShortName "S1" :Version "V1" :Projects (apply data-umm-cmn/projects projects)}) - {:allow-failure? true}) + {:allow-failure? true + :validate-keywords false}) {:keys [status errors]} response] (is (= [422 expected-errors] [status errors]))) @@ -510,7 +515,8 @@ {:EntryTitle "parent-collection3" :ShortName "S3" :Projects (apply data-umm-cmn/projects projects)}) - {:allow-failure? true}) + {:allow-failure? true + :validate-keywords false}) {:keys [status errors]} response] (is (= [200 nil] [status errors]))) @@ -520,10 +526,11 @@ (deftest collection-update-granule-spatial-representation-test (let [make-coll (fn [entry-title spatial-params] - (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle entry-title - :ShortName (d/unique-str "short-name") - :SpatialExtent (when spatial-params - (data-umm-c/spatial spatial-params))}))) + (d/ingest-umm-spec-collection "PROV1" + (data-umm-c/collection {:EntryTitle entry-title + :ShortName (d/unique-str "short-name") + :SpatialExtent (when spatial-params (data-umm-c/spatial spatial-params))}) + {:validate-keywords false})) make-gran (fn [coll spatial] (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll "C1-PROV1" {:spatial-coverage (when spatial (dg/spatial spatial))}))) @@ -559,7 +566,7 @@ updated-coll (assoc updated-coll :SpatialExtent (when new-spatial-params (data-umm-c/spatial new-spatial-params)))] - (d/ingest-umm-spec-collection "PROV1" updated-coll {:allow-failure? true})))] + (d/ingest-umm-spec-collection "PROV1" updated-coll {:allow-failure? true :validate-keywords false})))] (index/wait-until-indexed) (testing "Updates allowed with no granules" @@ -774,12 +781,14 @@ {:EntryTitle "parent-collection" :ShortName "S1" :Version "V1" - :Platforms (data-umm-cmn/platforms "p1" "p2" "AM-1" "p4")})) + :Platforms (data-umm-cmn/platforms "p1" "p2" "AM-1" "p4")}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "parent-collection2" :ShortName "S2" :Version "V2" - :Platforms (data-umm-cmn/platforms "p4" "Terra")}))] + :Platforms (data-umm-cmn/platforms "p4" "Terra")}) + {:validate-keywords false})] ;; CMR-3926 We need to make sure granules with no platfrom ref do not inherit their parent collection's instrument (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll "C1-PROV1")) (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll "C1-PROV1" {:platform-refs (dg/platform-refs "p1")})) @@ -797,7 +806,8 @@ {:EntryTitle "parent-collection" :ShortName "S1" :Version "V1" - :Platforms (apply data-umm-cmn/platforms platforms)})) + :Platforms (apply data-umm-cmn/platforms platforms)}) + {:validate-keywords false}) {:keys [status errors]} response] (is (= [200 nil] [status errors]))) @@ -818,7 +828,8 @@ :ShortName "S2" :Version "V2" :Platforms (apply data-umm-cmn/platforms platforms)}) - {:allow-failure? true}) + {:allow-failure? true + :validate-keywords false}) {:keys [status errors]} response] (is (= [422 expected-errors] [status errors]))) @@ -836,7 +847,8 @@ {:EntryTitle "parent-collection" :ShortName "S1" :Version "V1" - :TilingIdentificationSystems (data-umm-c/tiling-identification-systems "CALIPSO" "MISR" "WRS-1" "WRS-2")}))] + :TilingIdentificationSystems (data-umm-c/tiling-identification-systems "CALIPSO" "MISR" "WRS-1" "WRS-2")}) + {:validate-keywords false})] (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll "C1-PROV1" {:two-d-coordinate-system (dg/two-d "MISR")})) (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll "C1-PROV1" {:two-d-coordinate-system (dg/two-d "MISR")})) (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll "C1-PROV1" {:two-d-coordinate-system (dg/two-d "CALIPSO")})) @@ -848,7 +860,8 @@ {:EntryTitle "parent-collection" :ShortName "S1" :Version "V1" - :TilingIdentificationSystems (apply data-umm-c/tiling-identification-systems tile-names)})) + :TilingIdentificationSystems (apply data-umm-c/tiling-identification-systems tile-names)}) + {:validate-keywords false}) {:keys [status errors]} response] (is (= [200 nil] [status errors]))) @@ -869,7 +882,8 @@ :ShortName "S2" :Version "V2" :TilingIdentificationSystems (apply data-umm-c/tiling-identification-systems tile-names)}) - {:allow-failure? true}) + {:allow-failure? true + :validate-keywords false}) {:keys [status errors]} response] (is (= [422 expected-errors] [status errors]))) @@ -888,12 +902,14 @@ :ShortName "S1" :Version "V1" :Platforms [(data-umm-cmn/platform-with-instruments "p1-1" "i1" "i2" "GPS" "i4") - (data-umm-cmn/platform-with-instruments "p1-2" "i1" "i2" "GPS" "i4")]})) + (data-umm-cmn/platform-with-instruments "p1-2" "i1" "i2" "GPS" "i4")]}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "parent-collection2" :ShortName "S2" :Version "V2" - :Platforms [(data-umm-cmn/platform-with-instrument-and-childinstruments "p2" "i2" "s1" "GPS RECEIVERS")]}))] + :Platforms [(data-umm-cmn/platform-with-instrument-and-childinstruments "p2" "i2" "s1" "GPS RECEIVERS")]}) + {:validate-keywords false})] ;; CMR-3926 We need to make sure granules with no instrument ref or sensor ref do not inherit their parent collection's instrument or sensor (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll "C1-PROV1")) (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll "C1-PROV1" {:platform-refs [(dg/platform-ref-with-instrument-refs "p1-1" "i1")]})) @@ -911,7 +927,8 @@ :ShortName "S1" :Version "V1" :Platforms [(apply data-umm-cmn/platform-with-instruments plat-instruments-1) - (apply data-umm-cmn/platform-with-instruments plat-instruments-2)]})) + (apply data-umm-cmn/platform-with-instruments plat-instruments-2)]}) + {:validate-keywords false}) {:keys [status errors]} response] (is (= [200 nil] [status errors]))) @@ -940,7 +957,8 @@ :ShortName "S2" :Version "V2" :Platforms [(apply data-umm-cmn/platform-with-instrument-and-childinstruments plat-instr-sensors)]}) - {:allow-failure? true}) + {:allow-failure? true + :validate-keywords false}) {:keys [status errors]} response] (is (= [422 expected-errors] [status errors]))) @@ -981,12 +999,12 @@ (dg/psa "TIME" ["01:02:03Z"]) (dg/psa "DTS" ["2012-01-01T01:02:03Z"])]} - coll (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection collection-map)) + coll (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection collection-map) {:validate-keywords false}) gran (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll "C1-PROV1" granule-map))] (are3 [coll-map gran-map] - (let [response (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection coll-map)) + (let [response (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection coll-map) {:validate-keywords false}) response2 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll "C1-PROV1" gran-map)) {:keys [status errors]} response] (is (= [200 nil] [status errors])) @@ -1027,7 +1045,8 @@ updated-coll-metadata (-> "iso-samples/cmr-5871-coll-updated.xml" io/resource slurp) gran-metadata (-> "iso-samples/cmr-5871-gran.xml" io/resource slurp)] (ingest/ingest-concept - (ingest/concept :collection "PROV1" "coll1" :iso19115 coll-metadata)) + (ingest/concept :collection "PROV1" "coll1" :iso19115 coll-metadata) + {:validate-keywords false}) (let [{:keys [status]} (ingest/ingest-concept (ingest/concept :granule "PROV1" "gran1" :iso-smap gran-metadata))] @@ -1036,5 +1055,6 @@ (testing "Update collection child instrument name that not referenced in its granule is OK" (let [{:keys [status errors]} (ingest/ingest-concept (ingest/concept - :collection "PROV1" "coll1" :iso19115 updated-coll-metadata))] + :collection "PROV1" "coll1" :iso19115 updated-coll-metadata) + {:validate-keywords false})] (is (= [200 nil] [status errors])))))) diff --git a/system-int-test/test/cmr/system_int_test/ingest/collection/collection_validation_end_point_test.clj b/system-int-test/test/cmr/system_int_test/ingest/collection/collection_validation_end_point_test.clj index 608f91a4ad..8739381d88 100644 --- a/system-int-test/test/cmr/system_int_test/ingest/collection/collection_validation_end_point_test.clj +++ b/system-int-test/test/cmr/system_int_test/ingest/collection/collection_validation_end_point_test.clj @@ -27,7 +27,7 @@ (are [accept warnings] (let [collection (data-umm-c/collection {}) concept (data-umm-c/collection-concept collection :dif10) - response-map (select-keys (ingest/validate-concept concept {:accept-format accept :raw? true}) + response-map (select-keys (ingest/validate-concept concept {:accept-format accept :raw? true :validate-keywords false}) [:status :body])] (= {:status 200 :body warnings} response-map)) :json diff --git a/system-int-test/test/cmr/system_int_test/ingest/granule/granule_ingest_test.clj b/system-int-test/test/cmr/system_int_test/ingest/granule/granule_ingest_test.clj index acd4d855f4..878abe39c3 100644 --- a/system-int-test/test/cmr/system_int_test/ingest/granule/granule_ingest_test.clj +++ b/system-int-test/test/cmr/system_int_test/ingest/granule/granule_ingest_test.clj @@ -40,13 +40,13 @@ (deftest granule-referencing-collection-with-changing-concept-id-test (let [common-fields {:EntryTitle "coll1" :ShortName "short1" :Version "V1"} orig-coll (data-umm-c/collection-concept (assoc common-fields :native-id "native1")) - _ (ingest/ingest-concept orig-coll) + _ (ingest/ingest-concept orig-coll {:validate-keywords false}) ;; delete the collection deleted-response (ingest/delete-concept orig-coll) ;; Create collection again with same details but a different native id - new-coll (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection (assoc common-fields :native-id "native2"))) + new-coll (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection (assoc common-fields :native-id "native2")) {:validate-keywords false}) ;; Create granules associated with the collection fields. gran1 (data-core/ingest "PROV1" (update-in (granule/granule-with-umm-spec-collection new-coll (:concept-id new-coll)) @@ -74,12 +74,14 @@ "PROV1" (data-umm-c/collection {:EntryTitle "coll1" :ShortName "short1" :Version "V1" - :native-id "native1"})) + :native-id "native1"}) + {:validate-keywords false}) coll2 (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "coll2" :ShortName "short2" :Version "V2" - :native-id "native2"})) + :native-id "native2"}) + {:validate-keywords false}) gran1 (data-core/item->concept (granule/granule-with-umm-spec-collection coll1 @@ -127,14 +129,14 @@ ;; Verify a new granule is ingested successfully. (deftest granule-ingest-test (testing "ingest of a new granule" - (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {})) + (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {}) {:validate-keywords false}) granule (data-core/item->concept (granule/granule-with-umm-spec-collection collection (:concept-id collection))) {:keys [concept-id revision-id]} (ingest/ingest-concept granule)] (index/wait-until-indexed) (is (mdb/concept-exists-in-mdb? concept-id revision-id)) (is (= 1 revision-id)))) (testing "ingest of a new granule with a revision id" - (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {})) + (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {}) {:validate-keywords false}) granule (assoc (data-core/item->concept (granule/granule-with-umm-spec-collection collection (:concept-id collection))) :revision-id 5) {:keys [concept-id revision-id]} (ingest/ingest-concept granule)] (index/wait-until-indexed) @@ -144,7 +146,7 @@ ;; Verify a new granule with concept-id is ingested successfully. (deftest granule-w-concept-id-ingest-test (testing "ingest of a new granule with concept-id present" - (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {})) + (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {}) {:validate-keywords false}) supplied-concept-id "G1-PROV1" granule (data-core/item->concept (granule/granule-with-umm-spec-collection collection (:concept-id collection) {:concept-id supplied-concept-id})) @@ -158,7 +160,7 @@ ;; revision id is 1 greater on each subsequent ingest (deftest repeat-same-granule-ingest-test (testing "ingest same granule n times ..." - (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {})) + (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {}) {:validate-keywords false}) n 4 granule (data-core/item->concept (granule/granule-with-umm-spec-collection collection (:concept-id collection) {:concept-id "G1-PROV1"})) created-granules (doall (take n (repeatedly n #(ingest/ingest-concept granule))))] @@ -168,7 +170,7 @@ ;; Verify ingest behaves properly if empty body is presented in the request. (deftest empty-granule-ingest-test - (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {})) + (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {}) {:validate-keywords false}) granule (data-core/item->concept (granule/granule-with-umm-spec-collection collection (:concept-id collection))) granule-with-empty-body (assoc granule :metadata "") {:keys [status errors]} (ingest/ingest-concept granule-with-empty-body)] @@ -178,7 +180,7 @@ ;; Verify that the accept header works (deftest granule-ingest-accept-header-response-test - (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {}))] + (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {}) {:validate-keywords false})] (testing "json response" (let [granule (data-core/item->concept (granule/granule-with-umm-spec-collection collection (:concept-id collection))) @@ -222,7 +224,7 @@ ;; Verify that the accept header works with deletions (deftest delete-granule-with-accept-header-test - (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {}))] + (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {}) {:validate-keywords false})] (testing "json response" (let [granule (data-core/item->concept (granule/granule-with-umm-spec-collection collection (:concept-id collection) {:concept-id "G1-PROV1"})) _ (ingest/ingest-concept granule) @@ -241,7 +243,7 @@ (deftest delete-granule-test (testing "It should be possible to delete existing concept and the operation without revision id should result in revision id 1 greater than max revision id of the concept prior to the delete" - (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {})) + (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {}) {:validate-keywords false}) granule (data-core/item->concept (granule/granule-with-umm-spec-collection collection (:concept-id collection) {:concept-id "G1-PROV1"})) ingest-result (ingest/ingest-concept granule) delete-result (ingest/delete-concept granule) @@ -250,7 +252,7 @@ (index/wait-until-indexed) (is (= 1 (- delete-revision-id ingest-revision-id))))) (testing "Deleting existing concept with a revision-id should respect the revision id" - (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {})) + (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {}) {:validate-keywords false}) granule (data-core/item->concept (granule/granule-with-umm-spec-collection collection (:concept-id collection) {:concept-id "G2-PROV1"})) _ (ingest/ingest-concept granule) delete-result (ingest/delete-concept granule {:revision-id 5}) @@ -262,7 +264,7 @@ ;; Verify deleting non-existent concepts returns good error messages (deftest delete-non-existing-concept-gives-good-error-message-test (testing "granule" - (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {})) + (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {}) {:validate-keywords false}) granule (data-core/item->concept (granule/granule-with-umm-spec-collection collection (:concept-id collection) {:concept-id "G1-PROV1"})) response (ingest/delete-concept granule {:raw? true}) {:keys [errors]} (ingest/parse-ingest-body :xml response)] @@ -271,7 +273,7 @@ ;; Verify ingest is successful for request with content type that has parameters (deftest content-type-with-parameter-ingest-test - (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {})) + (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {}) {:validate-keywords false}) granule (data-core/item->concept (assoc (granule/granule-with-umm-spec-collection collection (:concept-id collection)) :format "application/echo10+xml; charset=utf-8")) @@ -281,7 +283,7 @@ ;; Verify ingest behaves properly if request is missing content type. (deftest missing-content-type-ingest-test - (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {})) + (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {}) {:validate-keywords false}) granule (data-core/item->concept (granule/granule-with-umm-spec-collection collection (:concept-id collection))) response (ingest/ingest-concept (assoc granule :format "") {:accept-format :json :raw? true}) status (:status response) @@ -292,7 +294,7 @@ ;; Verify ingest behaves properly if request contains invalid content type. (deftest invalid-content-type-ingest-test - (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {})) + (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {}) {:validate-keywords false}) granule (data-core/item->concept (granule/granule-with-umm-spec-collection collection (:concept-id collection))) response (ingest/ingest-concept (assoc granule :format "blah") {:accept-format :json :raw? true}) status (:status response) @@ -303,7 +305,7 @@ ;; Verify deleting same granule twice returns a 404 (deftest delete-same-granule-twice-test - (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {})) + (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {}) {:validate-keywords false}) granule (data-core/item->concept (granule/granule-with-umm-spec-collection collection (:concept-id collection) {:concept-id "G1-PROV1"})) ingest-result (ingest/ingest-concept granule) @@ -319,7 +321,7 @@ ;; Verify that attempts to ingest a granule whose parent does not exist result in a 422 error (deftest ingest-orphan-granule-test - (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "Coll1"})) + (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "Coll1"}) {:validate-keywords false}) umm-granule (granule/granule-with-umm-spec-collection collection (:concept-id collection) {:concept-id "G1-PROV1" :granule-ur "Gran1"}) granule (data-core/item->concept umm-granule) @@ -332,7 +334,7 @@ ;; Verify that granules with embedded / (%2F) in the native-id are handled correctly (deftest ingest-granule-with-slash-in-native-id-test - (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {})) + (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {}) {:validate-keywords false}) umm-granule (granule/granule-with-umm-spec-collection collection (:concept-id collection) {:native-id "Name/With/Slashes"}) granule (data-core/item->concept umm-granule) {:keys [concept-id revision-id] :as response} (ingest/ingest-concept granule) @@ -345,7 +347,7 @@ (deftest granule-schema-validation-test (are3 [concept-format validation-errors] - (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {})) + (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {}) {:validate-keywords false}) concept (data-core/item->concept (granule/granule-with-umm-spec-collection collection (:concept-id collection) {:beginning-date-time "2010-12-12T12:00:00Z"}) concept-format) @@ -372,7 +374,8 @@ (deftest ingest-smap-iso-granule-test (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "correct" :ShortName "S1" - :Version "V1"}))] + :Version "V1"}) + {:validate-keywords false})] (testing "Valid SMAP ISO granule with collection-ref attributes" (are3 [attrs] (let [granule (-> (granule/granule-with-umm-spec-collection collection (:concept-id collection) {:granule-ur "Gran1"}) @@ -437,7 +440,7 @@ :concept-id coll-concept-id :format metadata-format :metadata cddis-umm} - ingest-collection-response (ingest/ingest-concept coll-map {:accept-format :json}) + ingest-collection-response (ingest/ingest-concept coll-map {:accept-format :json :validate-keywords false}) granule (data-core/item->concept (granule/granule-with-umm-spec-collection (json/parse-string cddis-umm true) coll-concept-id @@ -464,7 +467,8 @@ valid-gran-metadata (-> "iso-samples/valid-CMR-5129-gran.xml" io/resource slurp)] (testing "Invalid orbit" (ingest/ingest-concept - (ingest/concept :collection "PROV1" "foo" :iso19115 coll-metadata)) + (ingest/concept :collection "PROV1" "foo" :iso19115 coll-metadata) + {:validate-keywords false}) (let [{:keys [status]} (ingest/ingest-concept (ingest/concept :granule "PROV1" "foo" :iso-smap invalid-gran-metadata))] (is (= 422 status)))) @@ -478,7 +482,8 @@ invalid-gran-metadata (-> "iso-samples/invalid-CMR-5226-gran.xml" io/resource slurp)] (testing "Invalid Geometries for ORBIT granule." (ingest/ingest-concept - (ingest/concept :collection "PROV1" "foo" :iso19115 coll-metadata)) + (ingest/concept :collection "PROV1" "foo" :iso19115 coll-metadata) + {:validate-keywords false}) (let [{:keys [status errors]} (ingest/ingest-concept (ingest/concept :granule "PROV1" "foo" :iso-smap invalid-gran-metadata))] (is (= 422 status)) @@ -491,7 +496,8 @@ invalid-gran-metadata (-> "iso-samples/invalid-CMR-5226-gran.xml" io/resource slurp)] (testing "Invalid geographic description " (ingest/ingest-concept - (ingest/concept :collection "PROV1" "foo" :iso19115 coll-metadata)) + (ingest/concept :collection "PROV1" "foo" :iso19115 coll-metadata) + {:validate-keywords false}) (let [{:keys [status errors]} (ingest/ingest-concept (ingest/concept :granule "PROV1" "foo" :iso-smap invalid-gran-metadata))] (is (= 422 status)) @@ -510,7 +516,7 @@ :path ["OrbitCalculatedSpatialDomains" 1]}]] (testing "Invalid orbit calculated spatial domain" (ingest/ingest-concept - (ingest/concept :collection "PROV1" "foo" :iso19115 coll-metadata)) + (ingest/concept :collection "PROV1" "foo" :iso19115 coll-metadata) {:validate-keywords false}) (let [{:keys [status errors]} (ingest/ingest-concept (ingest/concept :granule "PROV1" "foo" :iso-smap invalid-gran-metadata))] (is (= 422 status)) @@ -521,7 +527,8 @@ valid-gran-metadata (-> "iso-samples/5216_Valid_IsoSmap_Granule.xml" io/resource slurp)] (testing "Invalid orbit calculated spatial domain" (ingest/ingest-concept - (ingest/concept :collection "PROV1" "foo" :iso19115 coll-metadata)) + (ingest/concept :collection "PROV1" "foo" :iso19115 coll-metadata) + {:validate-keywords false}) (let [{:keys [status errors]} (ingest/ingest-concept (ingest/concept :granule "PROV1" "foo" :iso-smap valid-gran-metadata))] (is (= 201 status)) @@ -538,7 +545,8 @@ :path ["OrbitCalculatedSpatialDomains" 0]}]] (testing "Invalid orbit calculated spatial domain" (ingest/ingest-concept - (ingest/concept :collection "PROV1" "foo" :iso19115 coll-metadata)) + (ingest/concept :collection "PROV1" "foo" :iso19115 coll-metadata) + {:validate-keywords false}) (let [{:keys [status errors]} (ingest/ingest-concept (ingest/concept :granule "PROV1" "foo" :echo10 invalid-gran-metadata))] (is (= 422 status)) @@ -549,7 +557,8 @@ invalid-gran-metadata (-> "5216_Valid_Echo10_Granule.xml" io/resource slurp)] (testing "Invalid orbit calculated spatial domain" (ingest/ingest-concept - (ingest/concept :collection "PROV1" "foo" :iso19115 coll-metadata)) + (ingest/concept :collection "PROV1" "foo" :iso19115 coll-metadata) + {:validate-keywords false}) (let [{:keys [status errors]} (ingest/ingest-concept (ingest/concept :granule "PROV1" "foo" :echo10 invalid-gran-metadata))] (is (= 201 status)) @@ -559,7 +568,8 @@ (s/only-with-real-database (let [collection (data-core/ingest-umm-spec-collection "PROV1" - (data-umm-c/collection 1 {}))] + (data-umm-c/collection 1 {}) + {:validate-keywords false})] (testing "DeleteTime in past results in validation error" (let [granule (granule/granule-with-umm-spec-collection collection @@ -570,7 +580,8 @@ "PROV1" granule {:format :umm-json - :allow-failure? true})] + :allow-failure? true + :validate-keywords false})] (is (= 422 status)) (is (= ["DeleteTime 2000-01-01T00:00:00.000Z is before the current time."] errors)))) @@ -597,7 +608,8 @@ (let [collection (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "correct" :ShortName "S1" - :Version "V1"}))] + :Version "V1"}) + {:validate-keywords false})] (testing "Valid UMM-G granule with collection-ref attributes, default UMM-G version" (are3 [attrs] (let [granule (-> (granule/granule-with-umm-spec-collection diff --git a/system-int-test/test/cmr/system_int_test/ingest/granule/granule_parent_dependency_test.clj b/system-int-test/test/cmr/system_int_test/ingest/granule/granule_parent_dependency_test.clj index 7e3b917516..f989fcdbb2 100644 --- a/system-int-test/test/cmr/system_int_test/ingest/granule/granule_parent_dependency_test.clj +++ b/system-int-test/test/cmr/system_int_test/ingest/granule/granule_parent_dependency_test.clj @@ -168,9 +168,9 @@ :ending-date-time "1967-10-12T07:00:00.000-05:00"} echo10-coll1 (data-umm-c/collection coll-data1) - _ (d/ingest-umm-spec-collection "PROV1" echo10-coll1 {:format :echo10}) + _ (d/ingest-umm-spec-collection "PROV1" echo10-coll1 {:format :echo10 :validate-keywords false}) echo10-coll2 (data-umm-c/collection coll-data2) - _ (d/ingest-umm-spec-collection "PROV1" echo10-coll2 {:format :echo10}) + _ (d/ingest-umm-spec-collection "PROV1" echo10-coll2 {:format :echo10 :validate-keywords false}) gran-Terra-coll1 (dg/granule-with-umm-spec-collection echo10-coll1 (:concept-id echo10-coll1) gran-data1) gran-AM-1-coll1 (dg/granule-with-umm-spec-collection echo10-coll1 (:concept-id echo10-coll1) gran-data5) @@ -322,9 +322,9 @@ :beginning-date-time "1966-12-12T07:00:00.000-05:00" :ending-date-time "1967-10-12T07:00:00.000-05:00"} echo10-coll-A (data-umm-c/collection coll-data-A) - _ (d/ingest-umm-spec-collection "PROV1" echo10-coll-A {:format :echo10}) + _ (d/ingest-umm-spec-collection "PROV1" echo10-coll-A {:format :echo10 :validate-keywords false}) echo10-coll-B (data-umm-c/collection coll-data-B) - _ (d/ingest-umm-spec-collection "PROV1" echo10-coll-B {:format :echo10}) + _ (d/ingest-umm-spec-collection "PROV1" echo10-coll-B {:format :echo10 :validate-keywords false}) gran-A-for-echo10-coll-A (dg/granule-with-umm-spec-collection echo10-coll-A (:concept-id echo10-coll-A) gran-data-A) gran-B-for-echo10-coll-A (dg/granule-with-umm-spec-collection echo10-coll-A (:concept-id echo10-coll-A) gran-data-B) gran-A-for-echo10-coll-B (dg/granule-with-umm-spec-collection echo10-coll-B (:concept-id echo10-coll-B) gran-data-A) @@ -447,9 +447,9 @@ :beginning-date-time "1966-12-12T07:00:00.000-05:00" :ending-date-time "1967-10-12T07:00:00.000-05:00"} echo10-coll-A (data-umm-c/collection coll-data-A) - _ (d/ingest-umm-spec-collection "PROV1" echo10-coll-A {:format :echo10}) + _ (d/ingest-umm-spec-collection "PROV1" echo10-coll-A {:format :echo10 :validate-keywords false}) echo10-coll-B (data-umm-c/collection coll-data-B) - _ (d/ingest-umm-spec-collection "PROV1" echo10-coll-B {:format :echo10}) + _ (d/ingest-umm-spec-collection "PROV1" echo10-coll-B {:format :echo10 :validate-keywords false}) gran-A-for-echo10-coll-A (dg/granule-with-umm-spec-collection echo10-coll-A (:concept-id echo10-coll-A) gran-data-A) gran-B-for-echo10-coll-A (dg/granule-with-umm-spec-collection echo10-coll-A (:concept-id echo10-coll-A) gran-data-B) gran-A-for-echo10-coll-B (dg/granule-with-umm-spec-collection echo10-coll-B (:concept-id echo10-coll-B) gran-data-A) @@ -526,19 +526,19 @@ :beginning-date-time "1966-12-12T07:00:00.000-05:00" :ending-date-time "1967-10-12T07:00:00.000-05:00"} echo10-coll (data-umm-c/collection coll-data) - _ (d/ingest-umm-spec-collection "PROV1" echo10-coll {:format :echo10}) + _ (d/ingest-umm-spec-collection "PROV1" echo10-coll {:format :echo10 :validate-keywords false}) dif-coll (data-umm-c/collection (assoc coll-data :EntryTitle "short_name2_version" :ShortName "short_name2")) - _ (d/ingest-umm-spec-collection "PROV1" dif-coll {:format :dif}) + _ (d/ingest-umm-spec-collection "PROV1" dif-coll {:format :dif :validate-keywords false}) dif10-coll (data-umm-c/collection (assoc coll-data :EntryTitle "short_name3_version" :ShortName "short_name3")) - _ (d/ingest-umm-spec-collection "PROV1" dif10-coll {:format :dif10}) + _ (d/ingest-umm-spec-collection "PROV1" dif10-coll {:format :dif10 :validate-keywords false}) iso19115-coll (data-umm-c/collection (assoc coll-data :EntryTitle "short_name4_version" :ShortName "short_name4")) - _ (d/ingest-umm-spec-collection "PROV1" iso19115-coll {:format :iso19115}) + _ (d/ingest-umm-spec-collection "PROV1" iso19115-coll {:format :iso19115 :validate-keywords false}) iso-smap-coll (data-umm-c/collection (assoc coll-data :EntryTitle "short_name5_version" :ShortName "short_name5")) - _ (d/ingest-umm-spec-collection "PROV1" iso-smap-coll {:format :iso-smap}) + _ (d/ingest-umm-spec-collection "PROV1" iso-smap-coll {:format :iso-smap :validate-keywords false}) gran-for-echo10-coll (dg/granule-with-umm-spec-collection echo10-coll (:concept-id echo10-coll) gran-data) gran-for-dif-coll (dg/granule-with-umm-spec-collection dif-coll (:concept-id dif-coll) gran-data) gran-for-dif10-coll (dg/granule-with-umm-spec-collection dif10-coll (:concept-id dif-coll) gran-data) @@ -620,7 +620,7 @@ :beginning-date-time "1966-12-12T07:00:00.000-05:00" :ending-date-time "1967-10-12T07:00:00.000-05:00"} coll (data-umm-c/collection coll-data) - _ (d/ingest-umm-spec-collection "PROV1" coll {:format :echo10}) + _ (d/ingest-umm-spec-collection "PROV1" coll {:format :echo10 :validate-keywords false}) gran (dg/granule-with-umm-spec-collection coll (:concept-id coll) gran-data) _ (d/ingest "PROV1" gran {:format :echo10 :allow-failure? true})] (index/wait-until-indexed) @@ -628,7 +628,7 @@ (are [exp-errors metadata-format] (= exp-errors (flatten (:errors (d/ingest-umm-spec-collection "PROV1" coll {:format metadata-format - :allow-failure? true})))) + :allow-failure? true :validate-keywords false})))) ["Collection TilingIdentificationSystemName [misr] is referenced by existing granules, cannot be removed. Found 1 granules."] :dif @@ -647,7 +647,8 @@ works correctly when the parent collection is in ISO19115 format" (let [coll-metadata (slurp (io/resource "iso-samples/nsidc-cmr-3177-iso-collection.xml")) _ (ingest/ingest-concept - (ingest/concept :collection "PROV1" "foo" :iso19115 coll-metadata)) + (ingest/concept :collection "PROV1" "foo" :iso19115 coll-metadata) + {:validate-keywords false}) gran-metadata (slurp (io/resource "iso-samples/nsidc-cmr-3177-granule.xml")) {:keys [status]} (ingest/ingest-concept (ingest/concept :granule "PROV1" "foo" :echo10 gran-metadata))] @@ -658,7 +659,8 @@ works correctly when the parent collection is in ISO SMAP format" (let [coll-metadata (slurp (io/resource "iso-samples/cmr-4125-iso-collection.xml")) _ (ingest/ingest-concept - (ingest/concept :collection "PROV1" "foo" :iso-smap coll-metadata)) + (ingest/concept :collection "PROV1" "foo" :iso-smap coll-metadata) + {:validate-keywords false}) gran-metadata (slurp (io/resource "iso-samples/cmr-4125-granule.xml")) {:keys [status]} (ingest/ingest-concept (ingest/concept :granule "PROV1" "foo" :echo10 gran-metadata))] @@ -693,7 +695,7 @@ :ending-date-time "1967-10-12T07:00:00.000-05:00"} coll1 (data-umm-c/collection coll-data1) - _ (d/ingest-umm-spec-collection "PROV1" coll1 {:format :iso19115}) + _ (d/ingest-umm-spec-collection "PROV1" coll1 {:format :iso19115 :validate-keywords false}) gran1 (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) gran-data1) granule-result (d/ingest "PROV1" gran1 {:format :echo10 :allow-failure? true})] ;; The test is checking that the exception does not occur diff --git a/system-int-test/test/cmr/system_int_test/ingest/granule_bulk_update/granule_bulk_update_test.clj b/system-int-test/test/cmr/system_int_test/ingest/granule_bulk_update/granule_bulk_update_test.clj index fdefdca9cc..77df64f27b 100644 --- a/system-int-test/test/cmr/system_int_test/ingest/granule_bulk_update/granule_bulk_update_test.clj +++ b/system-int-test/test/cmr/system_int_test/ingest/granule_bulk_update/granule_bulk_update_test.clj @@ -18,7 +18,7 @@ (use-fixtures :each (ingest/reset-fixture {"provguid1" "PROV1"})) (deftest bulk-granule-update-test - (let [bulk-update-options {:token (echo-util/login (system/context) "user1")} + (let [bulk-update-options {:token (echo-util/login (system/context) "user1") :validate-keywords false} coll1 (data-core/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "coll1" :ShortName "short1" diff --git a/system-int-test/test/cmr/system_int_test/ingest/misc/generics_test.clj b/system-int-test/test/cmr/system_int_test/ingest/misc/generics_test.clj index ac9cc6aeac..c06e41ce86 100644 --- a/system-int-test/test/cmr/system_int_test/ingest/misc/generics_test.clj +++ b/system-int-test/test/cmr/system_int_test/ingest/misc/generics_test.clj @@ -274,7 +274,7 @@ nil "PROV1" cd-native-id :collection-draft gen-util/collection-draft :post) cd-concept-id (:concept-id coll-draft) coll-published (ingest/publish-draft - cd-concept-id coll-native-id {:format "application/vnd.nasa.cmr.umm+json"}) + cd-concept-id coll-native-id {:format "application/vnd.nasa.cmr.umm+json" :validate-keywords false}) coll-concept-id (:concept-id coll-published) _ (index/wait-until-indexed) diff --git a/system-int-test/test/cmr/system_int_test/ingest/misc/ingest_lifecycle_integration_test.clj b/system-int-test/test/cmr/system_int_test/ingest/misc/ingest_lifecycle_integration_test.clj index a074c953f8..037fbfd2c8 100644 --- a/system-int-test/test/cmr/system_int_test/ingest/misc/ingest_lifecycle_integration_test.clj +++ b/system-int-test/test/cmr/system_int_test/ingest/misc/ingest_lifecycle_integration_test.clj @@ -51,7 +51,7 @@ (defn assert-valid-umm-spec-collection [umm-spec-collection] - (let [response (ingest/validate-concept (d/umm-c-collection->concept umm-spec-collection))] + (let [response (ingest/validate-concept (d/umm-c-collection->concept umm-spec-collection) {:validate-keywords false})] (is (= {:status 200} (select-keys response [:status :errors]))))) (defn assert-granule-with-parent-collection-valid @@ -67,7 +67,7 @@ rather than passing one in." [coll] (assert-valid-umm-spec-collection coll) - (let [response (d/ingest-umm-spec-collection "PROV1" (dissoc coll :revision-id))] + (let [response (d/ingest-umm-spec-collection "PROV1" (dissoc coll :revision-id) {:validate-keywords false})] (is (#{200 201} (:status response))) response)) diff --git a/system-int-test/test/cmr/system_int_test/ingest/provider/provider_cmr_only_client_id_test.clj b/system-int-test/test/cmr/system_int_test/ingest/provider/provider_cmr_only_client_id_test.clj index 2ba724b26f..f73bf5579c 100644 --- a/system-int-test/test/cmr/system_int_test/ingest/provider/provider_cmr_only_client_id_test.clj +++ b/system-int-test/test/cmr/system_int_test/ingest/provider/provider_cmr_only_client_id_test.clj @@ -7,8 +7,7 @@ [cmr.system-int-test.data2.umm-spec-collection :as data-umm-c] [cmr.system-int-test.utils.index-util :as index] [cmr.system-int-test.utils.ingest-util :as ingest] - [cmr.system-int-test.utils.search-util :as search] - [cmr.umm.umm-granule :as umm-g])) + [cmr.system-int-test.utils.search-util :as search])) (use-fixtures :each (ingest/reset-fixture {"provguid1" "PROV1" "provguid2" "GES_DISC"})) @@ -25,17 +24,17 @@ (deftest collection-cmr-only-client-id-test (testing "ingest operations on CMR-ONLY provider can be submitted by client Echo" - (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "ET1"})) + (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "ET1"}) {:validate-keywords false}) concept (dissoc (d/item->concept coll1) :revision-id)] (doseq [func ingest-functions-to-test] (assert-ingest-success func concept "ECHO")))) (testing "ingest operations on CMR-ONLY provider can be submitted by client other than Echo" - (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "ET1"})) + (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "ET1"}) {:validate-keywords false}) concept (dissoc (d/item->concept coll1) :revision-id)] (doseq [func ingest-functions-to-test] (assert-ingest-success func concept "any")))) (testing "ingest operations on non CMR-ONLY provider can be submitted by client Echo" - (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "ET1"})) + (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "ET1"}) {:validate-keywords false}) concept (dissoc (d/item->concept coll1) :revision-id)] (ingest/update-ingest-provider {:provider-id "PROV1" :short-name "PROV1" @@ -45,7 +44,7 @@ (doseq [func ingest-functions-to-test] (assert-ingest-success func concept "ECHO")))) (testing "ingest operations on non CMR-ONLY provider can be submitted by client other than Echo" - (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "ET1"})) + (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "ET1"}) {:validate-keywords false}) concept (dissoc (d/item->concept coll1) :revision-id)] (ingest/update-ingest-provider {:provider-id "PROV1" :short-name "PROV1" @@ -57,19 +56,19 @@ (deftest granule-cmr-only-client-id-test (testing "ingest operations on CMR-ONLY provider can be submitted by client Echo" - (let [collection (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "ET1"})) + (let [collection (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "ET1"}) {:validate-keywords false}) concept (d/item->concept (dg/granule-with-umm-spec-collection collection "C1-PROV1"))] (ingest/ingest-concept concept) (doseq [func ingest-functions-to-test] (assert-ingest-success func concept "ECHO")))) (testing "ingest operations on CMR-ONLY provider can be submitted by client other than Echo" - (let [collection (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "ET1"})) + (let [collection (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "ET1"}) {:validate-keywords false}) concept (d/item->concept (dg/granule-with-umm-spec-collection collection "C1-PROV1"))] (ingest/ingest-concept concept) (doseq [func ingest-functions-to-test] (assert-ingest-success func concept "any")))) (testing "ingest operations on non CMR-ONLY provider can be submitted by client Echo" - (let [collection (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "ET1"})) + (let [collection (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "ET1"}) {:validate-keywords false}) concept (d/item->concept (dg/granule-with-umm-spec-collection collection "C1-PROV1"))] (ingest/ingest-concept concept) (ingest/update-ingest-provider {:provider-id "PROV1" @@ -80,7 +79,7 @@ (doseq [func ingest-functions-to-test] (assert-ingest-success func concept "ECHO")))) (testing "ingest operations on non CMR-ONLY provider can be submitted by client other than Echo" - (let [collection (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "ET1"})) + (let [collection (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "ET1"}) {:validate-keywords false}) concept (d/item->concept (dg/granule-with-umm-spec-collection collection "C1-PROV1"))] (ingest/ingest-concept concept) (ingest/update-ingest-provider {:provider-id "PROV1" @@ -93,7 +92,7 @@ (deftest granule-virtual-product-service-ingest-test (testing "ingest with Virtual-Product-Service as client-id should succeed for cmr-only provider" - (let [collection (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "ET1"})) + (let [collection (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "ET1"}) {:validate-keywords false}) granule (dg/granule-with-umm-spec-collection collection "C1-PROV1") concept (d/item->concept granule)] (assert-ingest-success #'ingest/validate-concept concept "Virtual-Product-Service") @@ -106,7 +105,7 @@ (is (= 0 (:hits (search/find-refs :granule {:granule-ur (:granule-ur granule) :page-size 50})))))) (testing "ingest with Virtual-Product-Service as client-id should succeed for non cmr-only provider" - (let [collection (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "ET1"})) + (let [collection (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "ET1"}) {:validate-keywords false}) granule (dg/granule-with-umm-spec-collection collection "C1-PROV1") concept (d/item->concept granule)] (ingest/update-ingest-provider {:provider-id "PROV1" diff --git a/system-int-test/test/cmr/system_int_test/ingest/provider/provider_ingest_permissions_test.clj b/system-int-test/test/cmr/system_int_test/ingest/provider/provider_ingest_permissions_test.clj index 0d16e4c0fc..f3e75b1f0f 100644 --- a/system-int-test/test/cmr/system_int_test/ingest/provider/provider_ingest_permissions_test.clj +++ b/system-int-test/test/cmr/system_int_test/ingest/provider/provider_ingest_permissions_test.clj @@ -444,7 +444,7 @@ "umm-var-guid2" "umm-var-user2") {update-token :token} (variable-util/setup-update-acl (s/context) "PROV1") - coll-concept-id (->> {:token update-token} + coll-concept-id (->> {:token update-token :validate-keywords false} (d/ingest "PROV1" (dc/collection)) :concept-id) _ (index/wait-until-indexed) diff --git a/system-int-test/test/cmr/system_int_test/search/acls/collection_test.clj b/system-int-test/test/cmr/system_int_test/search/acls/collection_test.clj index d566fed639..b2972ed341 100644 --- a/system-int-test/test/cmr/system_int_test/search/acls/collection_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/acls/collection_test.clj @@ -40,19 +40,24 @@ (let [guest-token (e/login-guest (s/context)) c1-echo (d/ingest "PROV1" (dc/collection {:entry-title "c1-echo" :access-value 1}) - {:format :echo10}) + {:format :echo10 + :validate-keywords false}) c1-dif (d/ingest "PROV1" (dc/collection-dif {:entry-title "c1-dif" :access-value 1}) - {:format :dif}) + {:format :dif + :validate-keywords false}) c1-dif10 (d/ingest "PROV1" (dc/collection-dif10 {:entry-title "c1-dif10" :access-value 1}) - {:format :dif10}) + {:format :dif10 + :validate-keywords false}) c1-iso (d/ingest "PROV1" (dc/collection {:entry-title "c1-iso" :access-value 1}) - {:format :iso19115}) + {:format :iso19115 + :validate-keywords false}) c1-smap (d/ingest "PROV1" (dc/collection {:entry-title "c1-smap" :access-value 1}) - {:format :iso-smap})] + {:format :iso-smap + :validate-keywords false})] (index/wait-until-indexed) ;;;;system token sees everything @@ -65,33 +70,42 @@ (let [guest-token (e/login-guest (s/context)) c1-echo (d/ingest "PROV1" (dc/collection {:entry-title "c1-echo" :access-value 1}) - {:format :echo10}) + {:format :echo10 + :validate-keywords false}) c2-echo (d/ingest "PROV1" (dc/collection {:entry-title "c2-echo" :access-value 0}) - {:format :echo10}) + {:format :echo10 + :validate-keywords false}) c1-dif (d/ingest "PROV1" (dc/collection-dif {:entry-title "c1-dif" :access-value 1}) - {:format :dif}) + {:format :dif + :validate-keywords false}) c2-dif (d/ingest "PROV1" (dc/collection-dif {:entry-title "c2-dif" :access-value 0}) - {:format :dif}) + {:format :dif + :validate-keywords false}) c1-dif10 (d/ingest "PROV1" (dc/collection-dif10 {:entry-title "c1-dif10" :access-value 1}) - {:format :dif10}) + {:format :dif10 + :validate-keywords false}) c2-dif10 (d/ingest "PROV2" (dc/collection-dif10 {:entry-title "c2-dif10" :access-value 0}) - {:format :dif10}) + {:format :dif10 + :validate-keywords false}) c1-iso (d/ingest "PROV1" (dc/collection {:entry-title "c1-iso" :access-value 1}) - {:format :iso19115}) + {:format :iso19115 + :validate-keywords false}) c2-iso (d/ingest "PROV1" (dc/collection {:entry-title "c2-iso" :access-value 0}) - {:format :iso19115}) + {:format :iso19115 + :validate-keywords false}) ;; access-value is not supported in ISO-SMAP, so it won't be found c1-smap (d/ingest "PROV1" (dc/collection {:entry-title "c1-smap" :access-value 1}) - {:format :iso-smap}) - coll3 (d/ingest "PROV1" (dc/collection {:entry-title "coll3"}))] + {:format :iso-smap + :validate-keywords false}) + coll3 (d/ingest "PROV1" (dc/collection {:entry-title "coll3"}) {:validate-keywords false})] (index/wait-until-indexed) ;; grant restriction flag acl @@ -120,18 +134,20 @@ :variable-level-3 "Level1-3" :detailed-variable "Detail1"}) - coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1" :science-keywords [sk1]})) - coll2 (d/ingest "PROV1" (dc/collection {:entry-title "coll2"})) - coll3 (d/ingest "PROV1" (dc/collection {:entry-title "coll3"})) - coll4 (d/ingest "PROV1" (dc/collection {:entry-title "coll4" - :science-keywords [sk1] - :access-value 5.0})) + coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1" :science-keywords [sk1]}) {:validate-keywords false}) + coll2 (d/ingest "PROV1" (dc/collection {:entry-title "coll2"}) {:validate-keywords false}) + coll3 (d/ingest "PROV1" (dc/collection {:entry-title "coll3"}) {:validate-keywords false}) + coll4 (d/ingest "PROV1" + (dc/collection {:entry-title "coll4" + :science-keywords [sk1] + :access-value 5.0}) + {:validate-keywords false}) ;; no permission granted on coll5 - coll5 (d/ingest "PROV1" (dc/collection {:entry-title "coll5"})) + coll5 (d/ingest "PROV1" (dc/collection {:entry-title "coll5"}) {:validate-keywords false}) ;; PROV2 - coll6 (d/ingest "PROV2" (dc/collection {:entry-title "coll6" :science-keywords [sk1]})) - coll7 (d/ingest "PROV2" (dc/collection {:entry-title "coll7" :science-keywords [sk1]})) + coll6 (d/ingest "PROV2" (dc/collection {:entry-title "coll6" :science-keywords [sk1]}) {:validate-keywords false}) + coll7 (d/ingest "PROV2" (dc/collection {:entry-title "coll7" :science-keywords [sk1]}) {:validate-keywords false}) ;; A dif collection coll8 (d/ingest "PROV2" (dc/collection-dif {:entry-title "coll8" @@ -139,36 +155,47 @@ :short-name "S8" :version-id "V8" :long-name "coll8"}) - {:format :dif}) + {:format :dif + :validate-keywords false}) ;; added for atom results coll8 (assoc coll8 :original-format "DIF") ;; PROV3 - coll9 (d/ingest "PROV3" (dc/collection {:entry-title "coll9" :science-keywords [sk1]})) - coll10 (d/ingest "PROV3" (dc/collection {:entry-title "coll10" - :access-value 12.0})) + coll9 (d/ingest "PROV3" (dc/collection {:entry-title "coll9" :science-keywords [sk1]}) {:validate-keywords false}) + coll10 (d/ingest "PROV3" + (dc/collection {:entry-title "coll10" + :access-value 12.0}) + {:validate-keywords false}) ;; PROV4 ;; group3 has permission to read this collection revision - coll11-1 (d/ingest "PROV4" (dc/collection {:entry-title "coll11" - :native-id "coll11" - :access-value 32.0})) + coll11-1 (d/ingest "PROV4" + (dc/collection {:entry-title "coll11" + :native-id "coll11" + :access-value 32.0}) + {:validate-keywords false}) ;; tombstone coll11-2 (assoc (ingest/delete-concept (d/item->concept coll11-1) {:token (tc/echo-system-token)}) :entry-title "coll11" :deleted true :revision-id 2) ;; no permissions to read this revision since entry-title has changed - coll11-3 (d/ingest "PROV4" (dc/collection {:entry-title "coll11" - :native-id "coll11" - :access-value 34.0})) + coll11-3 (d/ingest "PROV4" + (dc/collection {:entry-title "coll11" + :native-id "coll11" + :access-value 34.0}) + {:validate-keywords false}) ;; group 3 has permission to read this collection revision - coll12-1 (d/ingest "PROV4" (dc/collection {:entry-title "coll12" - :access-value 32.0 - :native-id "coll12"})) + coll12-1 (d/ingest "PROV4" + (dc/collection {:entry-title "coll12" + :access-value 32.0 + :native-id "coll12"}) + {:validate-keywords false}) ;; no permissions to read this collection since entry-title has changed - coll12-2 (d/ingest "PROV4" (dc/collection {:entry-title "coll12" - :access-value 34.0 - :native-id "coll12"})) + coll12-2 (d/ingest "PROV4" + (dc/collection {:entry-title "coll12" + :access-value 34.0 + :native-id "coll12"}) + {:validate-keywords false}) ;; no permision to see this tombstone since it has same entry-title as coll12-2 coll12-3 (assoc (ingest/delete-concept (d/item->concept coll12-2)) :deleted true @@ -360,14 +387,14 @@ ;; This tests that when acls change after collections have been indexed that collections will be ;; reindexed when ingest detects the acl hash has change. (deftest acl-change-test - (let [coll1 (d/ingest "PROV1" (dc/collection-dif10 {:entry-title "coll1"}) {:format :dif10}) + (let [coll1 (d/ingest "PROV1" (dc/collection-dif10 {:entry-title "coll1"}) {:format :dif10 :validate-keywords false}) coll2-umm (dc/collection {:entry-title "coll2" :short-name "short1"}) - coll2-1 (d/ingest "PROV1" coll2-umm) + coll2-1 (d/ingest "PROV1" coll2-umm {:validate-keywords false}) ;; 2 versions of collection 2 will allow us to test the force reindex option after we ;; force delete the latest version of coll2-2 - coll2-2 (d/ingest "PROV1" (assoc-in coll2-umm [:product :short-name] "short2")) - coll3 (d/ingest "PROV2" (dc/collection-dif10 {:entry-title "coll3"}) {:format :dif10}) - coll4 (d/ingest "PROV2" (dc/collection {:entry-title "coll4"})) + coll2-2 (d/ingest "PROV1" (assoc-in coll2-umm [:product :short-name] "short2") {:validate-keywords false}) + coll3 (d/ingest "PROV2" (dc/collection-dif10 {:entry-title "coll3"}) {:format :dif10 :validate-keywords false}) + coll4 (d/ingest "PROV2" (dc/collection {:entry-title "coll4"}) {:validate-keywords false}) _ (index/wait-until-indexed) acl1 (e/grant-guest (s/context) (e/coll-catalog-item-id "PROV1" (e/coll-id ["coll1"]))) @@ -414,7 +441,7 @@ ;; Verifies that tokens are cached by checking that a logged out token still works after it was ;; used. This isn't the desired behavior. It's just a side effect that shows it's working. (deftest cache-token-test - (let [coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1"})) + (let [coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1"}) {:validate-keywords false}) acl1 (e/grant-registered-users (s/context) (e/coll-catalog-item-id "PROV1" (e/coll-id ["coll1"]))) user1-token (e/login (s/context) "user1") @@ -441,10 +468,12 @@ guest-token (e/login-guest (s/context)) user1-token (e/login (s/context) "user1" [group1-concept-id]) user2-token (e/login (s/context) "user2") - coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1" - :native-id "coll1"})) - coll2 (d/ingest "PROV1" (dc/collection {:entry-title "coll2" - :native-id "coll2"})) + coll1 (d/ingest "PROV1" + (dc/collection {:entry-title "coll1" :native-id "coll1"}) + {:validate-keywords false}) + coll2 (d/ingest "PROV1" + (dc/collection {:entry-title "coll2" :native-id "coll2"}) + {:validate-keywords false}) coll1-edited (-> coll1 (assoc :entry-title "coll1-edited") (assoc :revision-id 2))] diff --git a/system-int-test/test/cmr/system_int_test/search/acls/granule_test.clj b/system-int-test/test/cmr/system_int_test/search/acls/granule_test.clj index 7198a816df..273690b09f 100644 --- a/system-int-test/test/cmr/system_int_test/search/acls/granule_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/acls/granule_test.clj @@ -30,7 +30,8 @@ (d/ingest prov (dc/collection (merge {:entry-title (str "coll" n)} - attribs))))) + attribs)) + {:validate-keywords false}))) (defn make-gran ([n coll] @@ -343,10 +344,8 @@ guest-token (e/login-guest (s/context)) user1-token (e/login (s/context) "user1" [group1-concept-id]) user2-token (e/login (s/context) "user2") - coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1" - :native-id "coll1"})) - coll2 (d/ingest "PROV1" (dc/collection {:entry-title "coll2" - :native-id "coll2"})) + coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1":native-id "coll1"}) {:validate-keywords false}) + coll2 (d/ingest "PROV1" (dc/collection {:entry-title "coll2" :native-id "coll2"}) {:validate-keywords false}) gran1 (make-gran 1 coll1) gran2 (make-gran 2 coll1) gran3 (make-gran 3 coll2) @@ -365,8 +364,7 @@ (d/assert-refs-match [] (search/find-refs :granule {:token guest-token :concept-id (map :concept-id all-colls)})) ;; Update entry-title in collection. - (d/ingest "PROV1" (dc/collection {:entry-title "coll1-edited" - :native-id "coll1"})) + (d/ingest "PROV1" (dc/collection {:entry-title "coll1-edited" :native-id "coll1"}) {:validate-keywords false}) (index/wait-until-indexed) (dev-sys-util/clear-caches) diff --git a/system-int-test/test/cmr/system_int_test/search/acls/temporal_collection_granule_test.clj b/system-int-test/test/cmr/system_int_test/search/acls/temporal_collection_granule_test.clj index 2b303225c5..e443c8b70a 100644 --- a/system-int-test/test/cmr/system_int_test/search/acls/temporal_collection_granule_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/acls/temporal_collection_granule_test.clj @@ -63,7 +63,8 @@ "PROV1" (dc/collection {:entry-title (str "coll" (swap! coll-num inc)) :single-date-time (tu/n->date-time-string n)}) - {:format metadata-format})) + {:format metadata-format + :validate-keywords false})) range-date-coll (fn [begin end metadata-format] (d/ingest "PROV1" @@ -71,7 +72,8 @@ :version-id "V1" :beginning-date-time (tu/n->date-time-string begin) :ending-date-time (tu/n->date-time-string end)}) - {:format metadata-format}))] + {:format metadata-format + :validate-keywords false}))] ;; Set current time (dev-sys-util/freeze-time! (tu/n->date-time-string now-n)) @@ -208,7 +210,7 @@ group2-concept-id (e/get-or-create-group (s/context) "group2") group3-concept-id (e/get-or-create-group (s/context) "group3") group4-concept-id (e/get-or-create-group (s/context) "group4") - collection (d/ingest "PROV1" (dc/collection {:beginning-date-time (tu/n->date-time-string 0)})) + collection (d/ingest "PROV1" (dc/collection {:beginning-date-time (tu/n->date-time-string 0)}) {:validate-keywords false}) gran-num (atom 0) single-date-gran (fn [n metadata-format] (d/ingest @@ -216,7 +218,8 @@ (dg/granule collection {:granule-ur (str "gran" (swap! gran-num inc)) :single-date-time (tu/n->date-time-string n)}) - {:format metadata-format})) + {:format metadata-format + :validate-keywords false})) range-date-gran (fn [begin end metadata-format] (d/ingest "PROV1" @@ -224,7 +227,8 @@ {:granule-ur (str "gran" (swap! gran-num inc)) :beginning-date-time (tu/n->date-time-string begin) :ending-date-time (tu/n->date-time-string end)}) - {:format metadata-format}))] + {:format metadata-format + :validate-keywords false}))] ;; Set current time (dev-sys-util/freeze-time! (tu/n->date-time-string now-n)) diff --git a/system-int-test/test/cmr/system_int_test/search/acls/variable_association_test.clj b/system-int-test/test/cmr/system_int_test/search/acls/variable_association_test.clj index 6b7f6dcbef..8702501475 100644 --- a/system-int-test/test/cmr/system_int_test/search/acls/variable_association_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/acls/variable_association_test.clj @@ -26,7 +26,7 @@ (s/context) "PROV1") {create-token :token} (variable-util/setup-update-acl (s/context) "PROV1" :create) - coll-concept-id (->> {:token update-token} + coll-concept-id (->> {:token update-token :validate-keywords false} (d/ingest "PROV1" (dc/collection)) :concept-id) _ (index/wait-until-indexed) diff --git a/system-int-test/test/cmr/system_int_test/search/autocomplete/suggestion_reindex_test.clj b/system-int-test/test/cmr/system_int_test/search/autocomplete/suggestion_reindex_test.clj index 0c323bcd49..5ebef2e54b 100644 --- a/system-int-test/test/cmr/system_int_test/search/autocomplete/suggestion_reindex_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/autocomplete/suggestion_reindex_test.clj @@ -109,7 +109,8 @@ (fu/processing-level-id "PL1") (fu/projects "proj1" "PROJ2") (fu/platforms fu/FROM_KMS 2 2 1) - (fu/science-keywords sk1 sk2)})) + (fu/science-keywords sk1 sk2)}) + {:validate-keywords false}) coll2 (fu/make-coll 2 "PROV1" (fu/science-keywords sk1 sk3) (fu/projects "proj1" "PROJ2") @@ -132,7 +133,8 @@ :ScienceKeywords (:ScienceKeywords (fu/science-keywords sk12)) :AccessConstraints (data-umm-spec/access-constraints {:Value 1 :Description "Those files are for British eyes only."})}) - {:format :umm-json}) + {:format :umm-json + :validate-keywords false}) coll6 (d/ingest-umm-spec-collection "PROV2" (data-umm-spec/collection @@ -140,10 +142,12 @@ :EntryTitle "Registered Collection" :Projects (:Projects (fu/projects "DMSP 5B/F3")) :Platforms (:Platforms (fu/platforms fu/FROM_KMS 2 2 1))}) - {:format :umm-json}) + {:format :umm-json + :validate-keywords false}) c1-echo (d/ingest "PROV1" (dc/collection {:entry-title "c1-echo" :access-value 1}) - {:format :echo10}) + {:format :echo10 + :validate-keywords false}) group1-concept-id (e/get-or-create-group (s/context) "group1") group2-concept-id (e/get-or-create-group (s/context) "group2") group3-concept-id (e/get-or-create-group (s/context) "group3") @@ -238,7 +242,8 @@ {:ShortName "This one is old and should be cleaned up" :EntryTitle "Oldie" :Projects (:Projects (fu/projects "OLD")) - :Platforms (:Platforms (fu/platforms "STALE" 2 2 1))})) + :Platforms (:Platforms (fu/platforms "STALE" 2 2 1))}) + {:validate-keywords false}) _ (dev-sys-util/freeze-time! (time/yesterday)) coll8 (d/ingest-umm-spec-collection @@ -246,7 +251,8 @@ (data-umm-spec/collection {:ShortName "Yesterday's news" :EntryTitle "Also an Oldie" - :Platforms (:Platforms (fu/platforms "old AND stale" 2 1 1))})) + :Platforms (:Platforms (fu/platforms "old AND stale" 2 1 1))}) + {:validate-keywords false}) _ (index/wait-until-indexed) _ (dev-sys-util/clear-current-time!) @@ -270,7 +276,8 @@ (data-umm-spec/collection {:EntryTitle "Secret Collection" :ScienceKeywords (:ScienceKeywords (fu/science-keywords gap-sk))}) - {:format :umm-json}) + {:format :umm-json + :validate-keywords false}) (d/ingest-umm-spec-collection "PROV1" @@ -279,7 +286,8 @@ :EntryTitle "Terra Test Collection" :Projects (:Projects (fu/projects "Terra")) :Platforms [{:ShortName "AM-1"}]}) - {:format :umm-json}) + {:format :umm-json + :validate-keywords false}) (index/wait-until-indexed) (index/reindex-suggestions) @@ -307,7 +315,8 @@ "PROV1" (data-umm-spec/collection {:EntryTitle "A boring collection"}) - {:format :umm-json}) + {:format :umm-json + :validate-keywords false}) (index/wait-until-indexed) (try diff --git a/system-int-test/test/cmr/system_int_test/search/collection/collection_campaign_search_test.clj b/system-int-test/test/cmr/system_int_test/search/collection/collection_campaign_search_test.clj index 6034691bbc..f1b1d6cf94 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection/collection_campaign_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection/collection_campaign_search_test.clj @@ -13,13 +13,13 @@ (use-fixtures :each (ingest/reset-fixture {"provguid1" "PROV1" "provguid2" "PROV2"})) (deftest search-by-campaign-short-names - (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "C1" :ShortName "S1"})) - coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:Projects [] :EntryTitle "C2" :ShortName "S2"})) - coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:Projects (data-umm-cmn/projects "ESI") :EntryTitle "C3" :ShortName "S3"})) + (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "C1" :ShortName "S1"}) {:validate-keywords false}) + coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:Projects [] :EntryTitle "C2" :ShortName "S2"}) {:validate-keywords false}) + coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:Projects (data-umm-cmn/projects "ESI") :EntryTitle "C3" :ShortName "S3"}) {:validate-keywords false}) - coll4 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Projects (data-umm-cmn/projects "ESI" "Esi") :EntryTitle "C4" :ShortName "S4"})) - coll5 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Projects (data-umm-cmn/projects "EVI" "EPI") :EntryTitle "C5" :ShortName "S5"})) - coll6 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Projects (data-umm-cmn/projects "ESI" "EVI" "EPI") :EntryTitle "C6" :ShortName "S6"}))] + coll4 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Projects (data-umm-cmn/projects "ESI" "Esi") :EntryTitle "C4" :ShortName "S4"}) {:validate-keywords false}) + coll5 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Projects (data-umm-cmn/projects "EVI" "EPI") :EntryTitle "C5" :ShortName "S5"}) {:validate-keywords false}) + coll6 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Projects (data-umm-cmn/projects "ESI" "EVI" "EPI") :EntryTitle "C6" :ShortName "S6"}) {:validate-keywords false})] (index/wait-until-indexed) diff --git a/system-int-test/test/cmr/system_int_test/search/collection/collection_concept_retrieval_test.clj b/system-int-test/test/cmr/system_int_test/search/collection/collection_concept_retrieval_test.clj index f276362bc4..79661e5b90 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection/collection_concept_retrieval_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection/collection_concept_retrieval_test.clj @@ -78,15 +78,17 @@ ;; Ingest 2 early versions of coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1" - :projects (dc/projects "ESI_1")})) + :projects (dc/projects "ESI_1")}) + {:validate-keywords false}) (d/ingest "PROV1" (dc/collection {:entry-title "coll1" - :projects (dc/projects "ESI_2")})) + :projects (dc/projects "ESI_2")}) + {:validate-keywords false}) (let [umm-coll (dc/collection {:entry-title "coll1" :projects (dc/projects "ESI_3")}) - coll1 (d/ingest "PROV1" umm-coll) - coll2 (d/ingest "PROV1" (dc/collection {:entry-title "Dataset2"})) - del-coll (d/ingest "PROV1" (dc/collection)) + coll1 (d/ingest "PROV1" umm-coll {:validate-keywords false}) + coll2 (d/ingest "PROV1" (dc/collection {:entry-title "Dataset2"}) {:validate-keywords false}) + del-coll (d/ingest "PROV1" (dc/collection) {:validate-keywords false}) ;; tokens guest-token (e/login-guest (s/context)) user1-token (e/login (s/context) "user1")] @@ -226,42 +228,51 @@ (let [c1-echo (d/ingest "PROV1" (dc/collection {:short-name "S1" :version-id "V1" :entry-title "ET1"}) - {:format :echo10}) + {:format :echo10 + :validate-keywords false}) c2-echo (d/ingest "PROV2" (dc/collection {:short-name "S2" :version-id "V2" :entry-title "ET2"}) - {:format :echo10}) + {:format :echo10 + :validate-keywords false}) c3-dif (d/ingest "PROV1" (dc/collection-dif {:short-name "S3" :version-id "V3" :entry-title "ET3" :long-name "ET3"}) - {:format :dif}) + {:format :dif + :validate-keywords false}) c4-dif (d/ingest "PROV2" (dc/collection-dif {:short-name "S4" :version-id "V4" :entry-title "ET4" :long-name "ET4"}) - {:format :dif}) + {:format :dif + :validate-keywords false}) c5-iso (d/ingest "PROV1" (dc/collection {:short-name "S5" :entry-id "S5" :version-id "V5"}) - {:format :iso19115}) + {:format :iso19115 + :validate-keywords false}) c6-iso (d/ingest "PROV2" (dc/collection {:short-name "S6" :entry-id "S6" :version-id "V6"}) - {:format :iso19115}) + {:format :iso19115 + :validate-keywords false}) c7-smap (d/ingest "PROV1" (dc/collection {:short-name "S7" :version-id "V7"}) - {:format :iso-smap}) + {:format :iso-smap + :validate-keywords false}) c8-dif10 (d/ingest "PROV1" (dc/collection-dif10 {:short-name "S8" :version-id "V8" :entry-title "ET8" :long-name "ET8"}) - {:format :dif10}) + {:format :dif10 + :validate-keywords false}) c9-dif10 (d/ingest "PROV2" (dc/collection-dif10 {:short-name "S9" :version-id "V9" :entry-title "ET9" :long-name "ET9"}) - {:format :dif10}) + {:format :dif10 + :validate-keywords false}) all-colls [c1-echo c2-echo c3-dif c4-dif c5-iso c6-iso c7-smap c8-dif10 c9-dif10]] (index/wait-until-indexed) @@ -368,18 +379,19 @@ ;; to vars to make it easier to see what is being ingested. ;; Ingest a collection twice. - coll1-1 (d/ingest "PROV1" umm-coll1-1) - coll1-2 (d/ingest "PROV1" umm-coll1-2) + coll1-1 (d/ingest "PROV1" umm-coll1-1 {:validate-keywords false}) + coll1-2 (d/ingest "PROV1" umm-coll1-2 {:validate-keywords false}) ;; Ingest collection once, delete, then ingest again. - coll2-1 (d/ingest "PROV1" umm-coll2-1) + coll2-1 (d/ingest "PROV1" umm-coll2-1 {:validate-keywords false}) _ (ingest/delete-concept (d/item->concept coll2-1)) - coll2-3 (d/ingest "PROV1" umm-coll2-3) + coll2-3 (d/ingest "PROV1" umm-coll2-3 {:validate-keywords false}) ;; Ingest a collection for PROV2 that is not visible to guests. coll3 (d/ingest "PROV2" (dc/collection {:entry-title "et1" :version-id "v1" - :short-name "s1"}))] + :short-name "s1"}) + {:validate-keywords false})] (index/wait-until-indexed) (testing "retrieve metadata from search by concept-id/revision-id" @@ -486,10 +498,12 @@ (e/grant-registered-users (s/context) (e/gran-catalog-item-id "PROV3")) (let [coll1 (d/ingest "PROV3" (dc/collection {:short-name "S1" :version-id "V1" - :entry-title "ET1"})) + :entry-title "ET1"}) + {:validate-keywords false}) coll2 (d/ingest "PROV4" (dc/collection {:short-name "S2" :version-id "V2" - :entry-title "ET2"})) + :entry-title "ET2"}) + {:validate-keywords false}) guest-token (e/login-guest (s/context)) user1-token (e/login (s/context) "user1")] diff --git a/system-int-test/test/cmr/system_int_test/search/collection/collection_consortium_search_test.clj b/system-int-test/test/cmr/system_int_test/search/collection/collection_consortium_search_test.clj index aac3931389..a43c078d39 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection/collection_consortium_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection/collection_consortium_search_test.clj @@ -33,18 +33,21 @@ (umm-c/collection 1 {:CollectionCitations [{:Creator "K. Hilburn"} {:Creator "Nelkin"}]}) - {:format :umm-json}) + {:format :umm-json + :validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV2" (umm-c/collection 2 {:CollectionCitations [{:Creator "K. Hilburn, J. Ardizzone, and S. Gao" :OtherCitationDetails "Sounder PEATE (Product Evaluation and Test Element) Team/Ruth Monarrez, JPL"}]}) - {:format :umm-json}) + {:format :umm-json + :validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV3" (umm-c/collection 3 {:UseConstraints {:Description "PROV4" :FreeAndOpenData false}}) - {:format :umm-json}) + {:format :umm-json + :validate-keywords false}) id1 (:concept-id coll1) id2 (:concept-id coll2) id3 (:concept-id coll3)] diff --git a/system-int-test/test/cmr/system_int_test/search/collection/collection_directory_pages_test.clj b/system-int-test/test/cmr/system_int_test/search/collection/collection_directory_pages_test.clj index 14c9206e84..70b9735f06 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection/collection_directory_pages_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection/collection_directory_pages_test.clj @@ -132,7 +132,8 @@ :ShortName (str "s" n) :EntryTitle (str "Collection Item " n)) {:format :umm-json - :accept-format :json}))) + :accept-format :json + :validate-keywords false}))) _ (index/wait-until-indexed) [c1-p3 c2-p3 c3-p3] (doall (for [n (range 101 104)] (d/ingest-umm-spec-collection @@ -144,7 +145,8 @@ {:DOI (str "doi" n) :Authority (str "auth" n)})) {:format :umm-json - :accept-format :json}))) + :accept-format :json + :validate-keywords false}))) _ (index/wait-until-indexed) ;; Let's create another collection that will put the total over the ;; default of 10 values so that we can ensure the :unlimited option @@ -156,7 +158,8 @@ :ShortName (str "s" n) :EntryTitle (str "Collection Item " n)) {:format :umm-json - :accept-format :json}))) + :accept-format :json + :validate-keywords false}))) [admin-1 admin-2] (doall (for [n (range 110 113)] (d/ingest-umm-spec-collection "ONLYADMIN" @@ -167,7 +170,8 @@ {:DOI (str "doi" n) :Authority (str "auth" n)})) {:format :umm-json - :accept-format :json}))) + :accept-format :json + :validate-keywords false}))) someadmin-guest-colls (doall (for [n (range 120 123)] (d/ingest-umm-spec-collection "SOMEADMIN" @@ -178,7 +182,8 @@ {:DOI (str "doi" n) :Authority (str "auth" n)})) {:format :umm-json - :accept-format :json}))) + :accept-format :json + :validate-keywords false}))) someadmin-invisible-colls (doall (for [n (range 130 133)] (d/ingest-umm-spec-collection "SOMEADMIN" @@ -189,7 +194,8 @@ {:DOI (str "doi" n) :Authority (str "auth" n)})) {:format :umm-json - :accept-format :json})))] + :accept-format :json + :validate-keywords false})))] (reset! test-collections {"PROV1" (sort (map :concept-id [c1-p1 c2-p1 c3-p1])) diff --git a/system-int-test/test/cmr/system_int_test/search/collection/collection_doi_search_test.clj b/system-int-test/test/cmr/system_int_test/search/collection/collection_doi_search_test.clj index 6b78934d42..2953f09158 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection/collection_doi_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection/collection_doi_search_test.clj @@ -21,7 +21,8 @@ (assoc :DOI (cm/map->DoiType {:DOI "doi1" :Authority "auth1"}))) {:format :umm-json - :accept-format :json}) + :accept-format :json + :validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (-> exp-conv/curr-ingest-ver-example-collection-record @@ -30,7 +31,8 @@ (assoc :DOI (cm/map->DoiType {:DOI "doi2" :Authority "auth2"}))) {:format :umm-json - :accept-format :json}) + :accept-format :json + :validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV1" (-> exp-conv/curr-ingest-ver-example-collection-record @@ -40,7 +42,8 @@ {:DOI "Not provided" :Authority "auth3"})) (assoc :AssociatedDOIs nil)) {:format :umm-json - :accept-format :json}) + :accept-format :json + :validate-keywords false}) no-doi (d/ingest-umm-spec-collection "PROV1" (-> exp-conv/curr-ingest-ver-example-collection-record @@ -48,7 +51,8 @@ (assoc :EntryTitle "These Are Not The DOIs You're Looking For") (assoc :AssociatedDOIs nil)) {:format :umm-json - :accept-format :json})] + :accept-format :json + :validate-keywords false})] (index/wait-until-indexed) (testing "search collections by doi" @@ -110,7 +114,8 @@ (assoc :ShortName "CMR3674SN1") (assoc :EntryTitle "CMR3674ET1")) {:format :umm-json - :accept-format :json})] + :accept-format :json + :validate-keywords false})] (index/wait-until-indexed) (testing "search collections by associated doi" (are3 [items doi options] diff --git a/system-int-test/test/cmr/system_int_test/search/collection/collection_humanized_search_test.clj b/system-int-test/test/cmr/system_int_test/search/collection/collection_humanized_search_test.clj index 369547a6e4..02ea22f9bf 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection/collection_humanized_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection/collection_humanized_search_test.clj @@ -66,13 +66,15 @@ :Platforms [(data-umm-cmn/platform {:ShortName "TERRA" :Instruments - [(data-umm-cmn/instrument {:ShortName "GPS RECEIVERS"})]})]})) + [(data-umm-cmn/instrument {:ShortName "GPS RECEIVERS"})]})]}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:ShortName "B" :Version "V2" - :Platforms [(data-umm-cmn/platform {:ShortName "AM-1"})]})) + :Platforms [(data-umm-cmn/platform {:ShortName "AM-1"})]}) + {:validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 3 @@ -84,7 +86,8 @@ :Term "Term1"} {:Category "Bio sphere" :Topic "Topic2" - :Term "Term2"}]}))] + :Term "Term2"}]}) + {:validate-keywords false})] (index/wait-until-indexed) ;; Refresh the metadata cache @@ -102,9 +105,9 @@ (is (= report (get-cached-report)))))))) (deftest search-by-platform-humanized - (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 1 {:Platforms [(data-umm-cmn/platform {:ShortName "TERRA"})]})) - coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:Platforms [(data-umm-cmn/platform {:ShortName "AM-1"})]})) - coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 3 {:Platforms [(data-umm-cmn/platform {:ShortName "Aqua"})]}))] + (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 1 {:Platforms [(data-umm-cmn/platform {:ShortName "TERRA"})]}) {:validate-keywords false}) + coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:Platforms [(data-umm-cmn/platform {:ShortName "AM-1"})]}) {:validate-keywords false}) + coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 3 {:Platforms [(data-umm-cmn/platform {:ShortName "Aqua"})]}) {:validate-keywords false})] (index/wait-until-indexed) (testing "search collections by humanized platform" (is (d/refs-match? [coll1 coll2] @@ -125,18 +128,18 @@ p2 (data-umm-cmn/platform {:ShortName "platform_2" :Instruments [i2]}) p3 (data-umm-cmn/platform {:ShortName "platform_3" :Instruments [i3]}) - coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 1 {:Platforms [p1]})) - coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:Platforms [p2]})) - coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 3 {:Platforms [p3]}))] + coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 1 {:Platforms [p1]}) {:validate-keywords false}) + coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:Platforms [p2]}) {:validate-keywords false}) + coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 3 {:Platforms [p3]}) {:validate-keywords false})] (index/wait-until-indexed) (testing "search collections by humanized instrument" (is (d/refs-match? [coll1 coll2] (search/find-refs :collection {:instrument-h "GPS Receivers"})))))) (deftest search-by-project-humanized - (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 1 {:Projects (data-umm-cmn/projects "USGS SOFIA")})) - coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:Projects (data-umm-cmn/projects "USGS_SOFIA")})) - coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 3 {:Projects (data-umm-cmn/projects "OPENDAP")}))] + (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 1 {:Projects (data-umm-cmn/projects "USGS SOFIA")}) {:validate-keywords false}) + coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:Projects (data-umm-cmn/projects "USGS_SOFIA")}) {:validate-keywords false}) + coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 3 {:Projects (data-umm-cmn/projects "OPENDAP")}) {:validate-keywords false})] (index/wait-until-indexed) (testing "search collections by humanized project" (is (d/refs-match? [coll1 coll2] @@ -146,24 +149,27 @@ (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 1 {:DataCenters [(data-umm-cmn/data-center {:Roles ["ARCHIVER"] - :ShortName "NSIDC"})]})) + :ShortName "NSIDC"})]}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:DataCenters [(data-umm-cmn/data-center {:Roles ["ARCHIVER"] - :ShortName "NASA/NSIDC_DAAC"})]})) + :ShortName "NASA/NSIDC_DAAC"})]}) + {:validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 3 {:DataCenters [(data-umm-cmn/data-center {:Roles ["ARCHIVER"] - :ShortName "ASF"})]}))] + :ShortName "ASF"})]}) + {:validate-keywords false})] (index/wait-until-indexed) (testing "search collections by humanized organization" (is (d/refs-match? [coll1 coll2] (search/find-refs :collection {:data-center-h "NSIDC"})))))) (deftest search-by-processing-level-id-humanized - (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 1 {:ProcessingLevel {:Id "1T"}})) - coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:ProcessingLevel {:Id "L1T"}})) - coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 3 {:ProcessingLevel {:Id "3"}}))] + (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 1 {:ProcessingLevel {:Id "1T"}}) {:validate-keywords false}) + coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:ProcessingLevel {:Id "L1T"}}) {:validate-keywords false}) + coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 3 {:ProcessingLevel {:Id "3"}}) {:validate-keywords false})] (index/wait-until-indexed) (testing "search collections by humanized processing-level-id" (is (d/refs-match? [coll1 coll2] @@ -185,11 +191,11 @@ sk5 (data-umm-cmn/science-keyword {:Category "category1" :Topic "topic1" :Term "term1"}) - coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 1 {:ScienceKeywords [sk1]})) - coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:ScienceKeywords [sk2]})) - coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 3 {:ScienceKeywords [sk3]})) - coll4 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 4 {:ScienceKeywords [sk4]})) - coll5 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 5 {:ScienceKeywords [sk5]}))] + coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 1 {:ScienceKeywords [sk1]}) {:validate-keywords false}) + coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:ScienceKeywords [sk2]}) {:validate-keywords false}) + coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 3 {:ScienceKeywords [sk3]}) {:validate-keywords false}) + coll4 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 4 {:ScienceKeywords [sk4]}) {:validate-keywords false}) + coll5 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 5 {:ScienceKeywords [sk5]}) {:validate-keywords false})] (index/wait-until-indexed) (testing "search collections by humanized science keyword" (is (d/refs-match? [coll1 coll3] @@ -213,7 +219,7 @@ :AverageFileSizeUnit "MB" :Fees "None currently" :Format "NetCDF-3"}]}} - coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection aadi1))] + coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection aadi1) {:validate-keywords false})] (index/wait-until-indexed) (testing "search collections by humanized granule data format" diff --git a/system-int-test/test/cmr/system_int_test/search/collection/collection_identifier_search_test.clj b/system-int-test/test/cmr/system_int_test/search/collection/collection_identifier_search_test.clj index 63445dc4ba..c924750539 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection/collection_identifier_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection/collection_identifier_search_test.clj @@ -470,13 +470,14 @@ ;; Create 2 collection sets of which only 1 set has processing-level-id (deftest processing-level-search-test (let [[c1-p1 c2-p1 c3-p1 c4-p1] (doall (for [n (range 1 5)] - (d/ingest-umm-spec-collection - "PROV1" (data-umm-c/collection n {})))) + (d/ingest-umm-spec-collection "PROV1" + (data-umm-c/collection n {}) + {:validate-keywords false}))) ;; include processing level id [c1-p2 c2-p2 c3-p2 c4-p2] (doall (for [n (range 1 5)] - (d/ingest-umm-spec-collection - "PROV2" (data-umm-c/collection n - {:ProcessingLevel (umm-c/map->ProcessingLevelType {:Id (str n "B")})})))) + (d/ingest-umm-spec-collection "PROV2" + (data-umm-c/collection n {:ProcessingLevel (umm-c/map->ProcessingLevelType {:Id (str n "B")})}) + {:validate-keywords false}))) all-prov2-colls [c1-p2 c2-p2 c3-p2 c4-p2]] (index/wait-until-indexed) (testing "processing level search" diff --git a/system-int-test/test/cmr/system_int_test/search/collection/collection_keyword_search_test.clj b/system-int-test/test/cmr/system_int_test/search/collection/collection_keyword_search_test.clj index 35099a859b..61a787c6ae 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection/collection_keyword_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection/collection_keyword_search_test.clj @@ -113,59 +113,67 @@ :Type "PROJECT HOME PAGE" :Description "Home page of National Snow and Ice Data Center"}) coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "coll1" :ShortName "S1" - :VersionDescription "VersionDescription"})) + :VersionDescription "VersionDescription"}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "Mitch made a (merry-go-round)" - :ShortName "ABC!XYZ" :Version "V001"})) - coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "coll3" :ShortName "S3" :CollectionDataType "OTHER"})) - coll4 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll4" :ShortName "S4" :CollectionDataType "OTHER"})) - coll5 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll5" :ShortName "Space!Laser"})) + :ShortName "ABC!XYZ" :Version "V001"}) + {:validate-keywords false}) + coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "coll3" :ShortName "S3" :CollectionDataType "OTHER"}) {:validate-keywords false}) + coll4 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll4" :ShortName "S4" :CollectionDataType "OTHER"}) {:validate-keywords false}) + coll5 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll5" :ShortName "Space!Laser"}) {:validate-keywords false}) coll6 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll6" :ShortName "S6" :DataCenters [org] :Projects pr2 - :Platforms [p7]})) - coll7 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll7" :ShortName "S7" :Version "Laser"})) - coll8 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll8" :ShortName "S8" :ProcessingLevel {:Id "PDQ123"}})) + :Platforms [p7]}) + {:validate-keywords false}) + coll7 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll7" :ShortName "S7" :Version "Laser"}) {:validate-keywords false}) + coll8 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll8" :ShortName "S8" :ProcessingLevel {:Id "PDQ123"}}) {:validate-keywords false}) - coll9 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll09" :ShortName "S9" :ScienceKeywords [sk1 sk2]})) + coll9 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll09" :ShortName "S9" :ScienceKeywords [sk1 sk2]}) {:validate-keywords false}) coll10 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll10" :ShortName "S10" :SpatialKeywords ["in out"] - :ScienceKeywords [sk3]})) + :ScienceKeywords [sk3]}) + {:validate-keywords false}) coll11 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll11" :ShortName "S11" :Platforms [p2 p3 p5 p6] - :AdditionalAttributes [psa5]})) + :AdditionalAttributes [psa5]}) + {:validate-keywords false}) coll12 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll12" :ShortName "S12" - :AdditionalAttributes [psa1 psa2 psa3 psa4]})) + :AdditionalAttributes [psa1 psa2 psa3 psa4]}) + {:validate-keywords false}) coll13 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll13" :ShortName "S13" - :TilingIdentificationSystems [tdcs1 tdcs2]})) - coll14 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll14" :ShortName "spoonA laser"})) + :TilingIdentificationSystems [tdcs1 tdcs2]}) + {:validate-keywords false}) + coll14 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll14" :ShortName "spoonA laser"}) {:validate-keywords false}) coll15 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll15" :ShortName "S15" :ProcessingLevel {:Id "plid1"} :CollectionDataType "SCIENCE_QUALITY" :Platforms [p1] - :Abstract "summary" :TemporalKeywords ["tk1" "tk2"]})) - coll16 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll16" :ShortName "entryid4"}) {:format :dif}) + :Abstract "summary" :TemporalKeywords ["tk1" "tk2"]}) + {:validate-keywords false}) + coll16 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll16" :ShortName "entryid4"}) {:format :dif :validate-keywords false}) ;;coll17 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:associated-difs ["DIF-1" "DIF-2"]})) - coll18 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll18" :ShortName "SNFoobar"})) - coll20 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Projects pr1 :EntryTitle "Mixed" :ShortName "S20"})) - coll21 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll21" :ShortName "Laser"})) + coll18 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll18" :ShortName "SNFoobar"}) {:validate-keywords false}) + coll20 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Projects pr1 :EntryTitle "Mixed" :ShortName "S20"}) {:validate-keywords false}) + coll21 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll21" :ShortName "Laser"}) {:validate-keywords false}) coll22 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll22" :CollectionDataType "NEAR_REAL_TIME" - :ShortName "Mixed"})) - coll23 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "coll23" :ShortName "\"Quoted\" collection"})) - coll24 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll24" :ShortName "coll24" :Platforms [p4]})) + :ShortName "Mixed"}) {:validate-keywords false}) + coll23 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "coll23" :ShortName "\"Quoted\" collection"}) {:validate-keywords false}) + coll24 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll24" :ShortName "coll24" :Platforms [p4]}) {:validate-keywords false}) ;; Adding personnel here to test keyword search using DataCenter contacts coll25 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "coll25" :ShortName "S25" - :RelatedUrls [url1 url2] :ContactPersons [personnel3]})) + :RelatedUrls [url1 url2] :ContactPersons [personnel3]}) {:validate-keywords false}) coll-boost (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "boost" :ShortName "boost" :Platforms [pboost] - :ScienceKeywords [skboost]})) + :ScienceKeywords [skboost]}) {:validate-keywords false}) coll26 (d/ingest-umm-spec-collection "PROV4" (data-umm-c/collection {:EntryTitle "coll26" :ShortName "S26" - :ContactPersons [personnel1]}) {:format :dif10}) + :ContactPersons [personnel1]}) {:format :dif10 :validate-keywords false}) coll26-1 (d/ingest-umm-spec-collection "PROV4" (data-umm-c/collection {:EntryTitle "coll26 one" :ShortName "S26 (sname one) \"sname one\"" - :ContactPersons [personnel1]}) {:format :dif10}) - coll27 (d/ingest-umm-spec-collection "PROV5" (data-umm-c/collection {:EntryTitle "coll27" :ShortName "S27" :ContactPersons [personnel2]}) {:format :dif10})] + :ContactPersons [personnel1]}) {:format :dif10 :validate-keywords false}) + coll27 (d/ingest-umm-spec-collection "PROV5" (data-umm-c/collection {:EntryTitle "coll27" :ShortName "S27" :ContactPersons [personnel2]}) {:format :dif10 :validate-keywords false})] (index/wait-until-indexed) @@ -712,7 +720,8 @@ :Subregion3 "GAZA STRIP" :DetailedLocation "Testing Detailed Location"})])) {:format :umm-json - :accept-format :json}) + :accept-format :json + :validate-keywords false}) coll2 (d/ingest "PROV1" (-> exp-conv/curr-ingest-ver-example-collection-record (assoc :AncillaryKeywords ["CMR2652AKW3" "CMR2652AKW4"]) @@ -723,7 +732,8 @@ (assoc :ShortName "CMR2652SN2") (assoc :EntryTitle "CMR2652ET2")) {:format :umm-json - :accept-format :json}) + :accept-format :json + :validate-keywords false}) coll3 (d/ingest-concept-with-metadata-file "data/iso_mends/no_spatial_iso_collection.xml" {:provider-id "PROV1" :concept-type :collection @@ -834,43 +844,52 @@ ;; sorting the results and then if short name is the same version is used for sorting the results (deftest search-by-keywords-relevancy-sorting-includes-short-name-and-version (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection - {:EntryTitle "MODIS/Terra Aerosol 5-Min L2 Swath 10km V5.1", - :ShortName "MOD04_L2", - :Version "5.1"})) + {:EntryTitle "MODIS/Terra Aerosol 5-Min L2 Swath 10km V5.1", + :ShortName "MOD04_L2", + :Version "5.1"}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection - {:EntryTitle "MODIS/Aqua Aerosol 5-Min L2 Swath 10km V5.1", - :ShortName "MYD04_L2", - :Version "5.1"})) + {:EntryTitle "MODIS/Aqua Aerosol 5-Min L2 Swath 10km V5.1", + :ShortName "MYD04_L2", + :Version "5.1"}) + {:validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection - {:EntryTitle "MODIS/Aqua Aerosol 5-Min L2 Swath 10km V006", - :ShortName "MYD04_L2", - :Version "6"})) + {:EntryTitle "MODIS/Aqua Aerosol 5-Min L2 Swath 10km V006", + :ShortName "MYD04_L2", + :Version "6"}) + {:validate-keywords false}) coll4 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection - {:EntryTitle "MODIS/Terra Aerosol 5-Min L2 Swath 10km V006", - :ShortName "MOD04_L2", - :Version "6"})) + {:EntryTitle "MODIS/Terra Aerosol 5-Min L2 Swath 10km V006", + :ShortName "MOD04_L2", + :Version "6"}) + {:validate-keywords false}) coll5 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection - {:EntryTitle "MODIS/Aqua Total Precipitable Water Aerosol 5-Min L2 Swath 1km and 5km V5.1", - :ShortName "MYD05_L2", - :Version "5.1"})) + {:EntryTitle "MODIS/Aqua Total Precipitable Water Aerosol 5-Min L2 Swath 1km and 5km V5.1", + :ShortName "MYD05_L2", + :Version "5.1"}) + {:validate-keywords false}) coll6 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection - {:EntryTitle "MODIS/Terra Total Precipitable Water Aerosol 5-Min L2 Swath 1km and 5km V5.1", - :ShortName "MOD05_L2", - :Version "5.1"})) + {:EntryTitle "MODIS/Terra Total Precipitable Water Aerosol 5-Min L2 Swath 1km and 5km V5.1", + :ShortName "MOD05_L2", + :Version "5.1"}) + {:validate-keywords false}) coll7 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection - {:EntryTitle "MODIS/Aqua Total Precipitable Water Aerosol 5-Min L2 Swath 1km and 5km V006", - :ShortName "MYD05_L2", - :Version "6"})) + {:EntryTitle "MODIS/Aqua Total Precipitable Water Aerosol 5-Min L2 Swath 1km and 5km V006", + :ShortName "MYD05_L2", + :Version "6"}) + {:validate-keywords false}) coll8 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection - {:EntryTitle "MODIS/Terra Total Precipitable Water Aerosol 5-Min L2 Swath 1km and 5km V006", - :ShortName "MOD05_L2", - :Version "6"})) + {:EntryTitle "MODIS/Terra Total Precipitable Water Aerosol 5-Min L2 Swath 1km and 5km V006", + :ShortName "MOD05_L2", + :Version "6"}) + {:validate-keywords false}) coll9 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection - {:EntryTitle "MODIS Aerosol Other", - :Projects [(data-umm-cmn/project "MODIS" "ignored")] - :Platforms [(data-umm-cmn/platform {:ShortName "MODIS"})] - :ShortName "Other", - :Version "1"}))] + {:EntryTitle "MODIS Aerosol Other", + :Projects [(data-umm-cmn/project "MODIS" "ignored")] + :Platforms [(data-umm-cmn/platform {:ShortName "MODIS"})] + :ShortName "Other", + :Version "1"}) + {:validate-keywords false})] (index/wait-until-indexed) (let [refs (search/find-refs :collection {:keyword "modis aerosol"}) expected-order [coll9 ;; higher score @@ -945,7 +964,7 @@ ["coll49" "choco or taco"] ["coll50" "begin*end"]] colls (doall (for [[coll summary] coll-data] - (d/ingest-umm-spec-collection "PROV3" (data-umm-c/collection (d/unique-num) {:EntryTitle coll :Abstract summary}))))] + (d/ingest-umm-spec-collection "PROV3" (data-umm-c/collection (d/unique-num) {:EntryTitle coll :Abstract summary}) {:validate-keywords false})))] (index/wait-until-indexed) (are [keyword-str indexes] (let [refs (search/find-refs :collection {:keyword keyword-str :page_size 51}) diff --git a/system-int-test/test/cmr/system_int_test/search/collection/collection_platform_instrument_sensor_search_test.clj b/system-int-test/test/cmr/system_int_test/search/collection/collection_platform_instrument_sensor_search_test.clj index 12a1fc2796..814cc06435 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection/collection_platform_instrument_sensor_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection/collection_platform_instrument_sensor_search_test.clj @@ -27,34 +27,42 @@ coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:Platforms [p1 p7] :EntryTitle "E1" :ShortName "S1" - :Version "V1"})) + :Version "V1"}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:Platforms [p1 p2 p8] :EntryTitle "E2" :ShortName "S2" - :Version "V2"})) + :Version "V2"}) + {:validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:Platforms [p2] :EntryTitle "E3" :ShortName "S3" - :Version "V3"})) + :Version "V3"}) + {:validate-keywords false}) coll4 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Platforms [p3] :EntryTitle "E4" :ShortName "S4" - :Version "V4"})) + :Version "V4"}) + {:validate-keywords false}) coll5 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Platforms [p4] :EntryTitle "E5" :ShortName "S5" - :Version "V5"})) + :Version "V5"}) + {:validate-keywords false}) coll6 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Platforms [p5] :EntryTitle "E6" :ShortName "S6" - :Version "V6"})) + :Version "V6"}) + {:validate-keywords false}) coll7 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Platforms [p6] :EntryTitle "E7" :ShortName "S7" - :Version "V7"})) + :Version "V7"}) + {:validate-keywords false}) coll8 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "E8" :ShortName "S8" - :Version "V8"})) + :Version "V8"}) + {:validate-keywords false}) ;; Added to test SMAP ISO platform and instrument support - note that this collection is ;; found in KMS with a category of "Earth Observation Satellites" coll9 (d/ingest-concept-with-metadata-file "example-data/iso-smap/SMAPExample.xml" @@ -195,38 +203,47 @@ coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:Platforms [p1] :EntryTitle "E1" :ShortName "S1" - :Version "V1"})) + :Version "V1"}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:Platforms [p1 p2] :EntryTitle "E2" :ShortName "S2" - :Version "V2"})) + :Version "V2"}) + {:validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:Platforms [p2] :EntryTitle "E3" :ShortName "S3" - :Version "V3"})) + :Version "V3"}) + {:validate-keywords false}) coll4 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Platforms [p3] :EntryTitle "E4" :ShortName "S4" - :Version "V4"})) + :Version "V4"}) + {:validate-keywords false}) coll5 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Platforms [p4] :EntryTitle "E5" :ShortName "S5" - :Version "V5"})) + :Version "V5"}) + {:validate-keywords false}) coll6 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Platforms [p5] :EntryTitle "E6" :ShortName "S6" - :Version "V6"})) + :Version "V6"}) + {:validate-keywords false}) coll7 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Platforms [p6] :EntryTitle "E7" :ShortName "S7" - :Version "V7"})) + :Version "V7"}) + {:validate-keywords false}) coll8 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Platforms [p7] :EntryTitle "E8" :ShortName "S8" - :Version "V8"})) + :Version "V8"}) + {:validate-keywords false}) coll9 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "E9" :ShortName "S9" - :Version "V9"})) + :Version "V9"}) + {:validate-keywords false}) ;; Added to test SMAP ISO platform and instrument support coll10 (d/ingest-concept-with-metadata-file "example-data/iso-smap/SMAPExample.xml" {:provider-id "PROV1" @@ -382,42 +399,52 @@ coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:Platforms [p1] :EntryTitle "E1" :ShortName "S1" - :Version "V1"})) + :Version "V1"}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:Platforms [p1 p2] :EntryTitle "E2" :ShortName "S2" - :Version "V2"})) + :Version "V2"}) + {:validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:Platforms [p2] :EntryTitle "E3" :ShortName "S3" - :Version "V3"})) + :Version "V3"}) + {:validate-keywords false}) coll4 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Platforms [p3] :EntryTitle "E4" :ShortName "S4" - :Version "V4"})) + :Version "V4"}) + {:validate-keywords false}) coll5 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Platforms [p4] :EntryTitle "E5" :ShortName "S5" - :Version "V5"})) + :Version "V5"}) + {:validate-keywords false}) coll6 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Platforms [p5] :EntryTitle "E6" :ShortName "S6" - :Version "V6"})) + :Version "V6"}) + {:validate-keywords false}) coll7 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Platforms [p6] :EntryTitle "E7" :ShortName "S7" - :Version "V7"})) + :Version "V7"}) + {:validate-keywords false}) coll8 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Platforms [p7] :EntryTitle "E8" :ShortName "S8" - :Version "V8"})) + :Version "V8"}) + {:validate-keywords false}) coll9 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:Platforms [p8] :EntryTitle "E9" :ShortName "S9" - :Version "V9"})) + :Version "V9"}) + {:validate-keywords false}) coll10 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "E10" :ShortName "S10" - :Version "V10"}))] + :Version "V10"}) + {:validate-keywords false})] (index/wait-until-indexed) diff --git a/system-int-test/test/cmr/system_int_test/search/collection/collection_psa_search_test.clj b/system-int-test/test/cmr/system_int_test/search/collection/collection_psa_search_test.clj index 1f43aede3b..d18a011c01 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection/collection_psa_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection/collection_psa_search_test.clj @@ -35,15 +35,15 @@ coll1 (data-core/ingest "PROV1" (collection/collection-dif {:product-specific-attributes [psa1 psa2 psa3]}) - {:format :dif}) + {:format :dif :validate-keywords false}) coll2 (data-core/ingest "PROV1" (collection/collection-dif {:product-specific-attributes [psa2 psa3]}) - {:format :dif}) + {:format :dif :validate-keywords false}) coll3 (data-core/ingest "PROV1" (collection/collection-dif {:product-specific-attributes [psa3 psa4]}) - {:format :dif})] + {:format :dif :validate-keywords false})] (index/wait-until-indexed) (are [v items] (data-core/refs-match? items (search/find-refs :collection {"attribute[]" v})) @@ -73,15 +73,15 @@ coll1 (data-core/ingest "PROV1" (collection/collection-dif {:product-specific-attributes [psa1 psa2]}) - {:format :dif}) + {:format :dif :validate-keywords false}) coll2 (data-core/ingest "PROV1" (collection/collection-dif {:product-specific-attributes [psa2 psa3]}) - {:format :dif}) + {:format :dif :validate-keywords false}) coll3 (data-core/ingest "PROV1" (collection/collection-dif {:product-specific-attributes [psa4]}) - {:format :dif})] + {:format :dif :validate-keywords false})] (index/wait-until-indexed) (testing "search by value" (are [v items] @@ -289,11 +289,13 @@ coll1 (data-core/ingest "PROV1" (collection/collection - {:product-specific-attributes [psa1 psa2]})) + {:product-specific-attributes [psa1 psa2]}) + {:validate-keywords false}) coll2 (data-core/ingest "PROV1" (collection/collection - {:product-specific-attributes [psa2 psa3]}))] + {:product-specific-attributes [psa2 psa3]}) + {:validate-keywords false})] (index/wait-until-indexed) (testing "search by value" @@ -418,11 +420,13 @@ coll1 (data-core/ingest "PROV1" (collection/collection - {:product-specific-attributes [psa1 psa2]})) + {:product-specific-attributes [psa1 psa2]}) + {:validate-keywords false}) coll2 (data-core/ingest "PROV1" (collection/collection - {:product-specific-attributes [psa2 psa3]}))] + {:product-specific-attributes [psa2 psa3]}) + {:validate-keywords false})] (index/wait-until-indexed) (testing "search by value" @@ -546,11 +550,13 @@ coll1 (data-core/ingest "PROV1" (collection/collection - {:product-specific-attributes [psa1 psa2]})) + {:product-specific-attributes [psa1 psa2]}) + {:validate-keywords false}) coll2 (data-core/ingest "PROV1" (collection/collection - {:product-specific-attributes [psa2 psa3]}))] + {:product-specific-attributes [psa2 psa3]}) + {:validate-keywords false})] (index/wait-until-indexed) (testing "search by value" @@ -699,11 +705,13 @@ coll1 (data-core/ingest "PROV1" (collection/collection - {:product-specific-attributes [psa1 psa2]})) + {:product-specific-attributes [psa1 psa2]}) + {:validate-keywords false}) coll2 (data-core/ingest "PROV1" (collection/collection - {:product-specific-attributes [psa2 psa3]}))] + {:product-specific-attributes [psa2 psa3]}) + {:validate-keywords false})] (index/wait-until-indexed) (testing "search by value" @@ -855,11 +863,13 @@ coll1 (data-core/ingest "PROV1" (collection/collection - {:product-specific-attributes [psa1 psa2]})) + {:product-specific-attributes [psa1 psa2]}) + {:validate-keywords false}) coll2 (data-core/ingest "PROV1" (collection/collection - {:product-specific-attributes [psa2 psa3]}))] + {:product-specific-attributes [psa2 psa3]}) + {:validate-keywords false})] (index/wait-until-indexed) @@ -1047,11 +1057,11 @@ coll1 (data-core/ingest "PROV1" (collection/collection-dif {:product-specific-attributes [psa1]}) - {:format :dif}) + {:format :dif :validate-keywords false}) coll2 (data-core/ingest "PROV1" (collection/collection-dif {:product-specific-attributes [psa2]}) - {:format :dif})] + {:format :dif :validate-keywords false})] (index/wait-until-indexed) (are [matches a-name group pattern?] (data-core/refs-match? matches diff --git a/system-int-test/test/cmr/system_int_test/search/collection/collection_science_keyword_search_test.clj b/system-int-test/test/cmr/system_int_test/search/collection/collection_science_keyword_search_test.clj index 2f7c400ec8..b8136e4237 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection/collection_science_keyword_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection/collection_science_keyword_search_test.clj @@ -52,17 +52,17 @@ :variable-level-1 "sedimentary rocks" :variable-level-2 "sedimentary rock physical/optical properties" :variable-level-3 "luminescence"}) - coll1 (d/ingest "PROV1" (dc/collection {:science-keywords [sk1]})) - coll2 (d/ingest "PROV1" (dc/collection {:science-keywords [sk2]})) - coll3 (d/ingest "PROV1" (dc/collection {:science-keywords [sk3]})) - coll4 (d/ingest "PROV1" (dc/collection {:science-keywords [sk4]})) - coll5 (d/ingest "PROV1" (dc/collection {:science-keywords [sk5]})) - coll6 (d/ingest "PROV1" (dc/collection {:science-keywords [sk3 sk5]})) - coll7 (d/ingest "PROV2" (dc/collection {:science-keywords [sk4 sk5]})) - coll8 (d/ingest "PROV2" (dc/collection {:science-keywords [sk6]})) - coll9 (d/ingest "PROV2" (dc/collection {:science-keywords [sk7]})) - coll10 (d/ingest "PROV2" (dc/collection {})) - coll11 (d/ingest "PROV1" (dc/collection {:science-keywords [sk8 sk9]}))] + coll1 (d/ingest "PROV1" (dc/collection {:science-keywords [sk1]}) {:validate-keywords false}) + coll2 (d/ingest "PROV1" (dc/collection {:science-keywords [sk2]}) {:validate-keywords false}) + coll3 (d/ingest "PROV1" (dc/collection {:science-keywords [sk3]}) {:validate-keywords false}) + coll4 (d/ingest "PROV1" (dc/collection {:science-keywords [sk4]}) {:validate-keywords false}) + coll5 (d/ingest "PROV1" (dc/collection {:science-keywords [sk5]}) {:validate-keywords false}) + coll6 (d/ingest "PROV1" (dc/collection {:science-keywords [sk3 sk5]}) {:validate-keywords false}) + coll7 (d/ingest "PROV2" (dc/collection {:science-keywords [sk4 sk5]}) {:validate-keywords false}) + coll8 (d/ingest "PROV2" (dc/collection {:science-keywords [sk6]}) {:validate-keywords false}) + coll9 (d/ingest "PROV2" (dc/collection {:science-keywords [sk7]}) {:validate-keywords false}) + coll10 (d/ingest "PROV2" (dc/collection {}) {:validate-keywords false}) + coll11 (d/ingest "PROV1" (dc/collection {:science-keywords [sk8 sk9]}) {:validate-keywords false})] (index/wait-until-indexed) diff --git a/system-int-test/test/cmr/system_int_test/search/collection/collection_search_data_format_test.clj b/system-int-test/test/cmr/system_int_test/search/collection/collection_search_data_format_test.clj index 38fbc49ebe..993c5148d8 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection/collection_search_data_format_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection/collection_search_data_format_test.clj @@ -45,7 +45,7 @@ (testing "ingest and search by data format of a collection with data formats." (let [concept (umm-spec-collection/collection-concept sample-data-format-collection-test) - {:keys [concept-id revision-id]} (ingest/ingest-concept concept)] + {:keys [concept-id revision-id]} (ingest/ingest-concept concept {:validate-keywords false})] (index/wait-until-indexed) (cache-util/refresh-cache (url/refresh-collection-metadata-cache-url) (transmit-config/echo-system-token)) (is (= 1 diff --git a/system-int-test/test/cmr/system_int_test/search/collection/collection_search_format_test.clj b/system-int-test/test/cmr/system_int_test/search/collection/collection_search_format_test.clj index 8046e94539..3190f42266 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection/collection_search_format_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection/collection_search_format_test.clj @@ -53,7 +53,7 @@ ;; Whitespace here but not stripped out for expected ;; results. It will be present in metadata. :entry-title " ET1 "}) - {:format :echo10})] + {:format :echo10 :validate-keywords false})] (index/wait-until-indexed) (let [params {:concept-id (:concept-id c1-echo)} options {:accept nil @@ -94,30 +94,31 @@ _ (side/eval-form `(common-config/set-collection-umm-version! umm-version/current-collection-version)) c1-echo (d/ingest "PROV1" (dc/collection {:entry-title "c1-echo"}) - {:format :echo10}) + {:format :echo10 :validate-keywords false}) c2-echo (d/ingest "PROV2" (dc/collection {:entry-title "c2-echo"}) - {:format :echo10}) + {:format :echo10 :validate-keywords false}) c3-dif (d/ingest "PROV1" (dc/collection-dif {:entry-title "c3-dif" :long-name "c3-dif"}) - {:format :dif}) + {:format :dif :validate-keywords false}) c5-iso (d/ingest "PROV1" (dc/collection {:entry-title "c5-iso"}) - {:format :iso19115}) + {:format :iso19115 :validate-keywords false}) c7-smap (d/ingest "PROV1" (dc/collection-smap {:entry-title "c7-smap"}) - {:format :iso-smap}) + {:format :iso-smap :validate-keywords false}) c8-dif10 (d/ingest "PROV1" (dc/collection-dif10 {:entry-title "c8-dif10" :long-name "c8-dif10"}) - {:format :dif10}) + {:format :dif10 :validate-keywords false}) c10-umm-json (d/ingest "PROV1" exp-conv/example-collection-record {:format :umm-json - :accept-format :json}) + :accept-format :json + :validate-keywords false}) c10-umm-json (dissoc c10-umm-json :warnings) ;; An item ingested with and XML preprocessing line to ensure this is tested item (assoc (dc/collection {:entry-title "c11-echo"}) :provider-id "PROV1") concept (-> (d/item->concept item :echo10) (update :metadata #(str "" %))) - response (ingest/ingest-concept concept) + response (ingest/ingest-concept concept {:validate-keywords false}) _ (is (#{200 201} (:status response))) c11-echo (assoc item :concept-id (:concept-id response) @@ -157,7 +158,7 @@ (testing "Ingesting newer metadata (not cached) is successfully retrieved" (let [c1-r2-echo (d/ingest "PROV1" (dc/collection {:entry-title "c1-echo" :description "updated"}) - {:format :echo10}) + {:format :echo10 :validate-keywords false}) all-colls [c1-r2-echo c2-echo c3-dif c5-iso c7-smap c8-dif10 c10-umm-json c11-echo]] (index/wait-until-indexed) (assert-found-by-format [c1-r2-echo c3-dif] :echo10 mt/echo10) @@ -211,16 +212,17 @@ _ (side/eval-form `(common-config/set-collection-umm-version! umm-version/current-collection-version)) c1-r1-echo (d/ingest "PROV1" (du/umm-spec-collection {:entry-title "c1-echo"}) - {:format :echo10}) + {:format :echo10 :validate-keywords false}) c1-r2-echo (d/ingest "PROV1" (du/umm-spec-collection {:entry-title "c1-echo" :description "updated"}) - {:format :echo10}) + {:format :echo10 :validate-keywords false}) c2-echo (d/ingest "PROV2" (du/umm-spec-collection {:entry-title "c2-echo"}) - {:format :echo10}) + {:format :echo10 :validate-keywords false}) c10-umm-json (d/ingest "PROV1" exp-conv/example-collection-record {:format :umm-json - :accept-format :json}) + :accept-format :json + :validate-keywords false}) c10-umm-json (dissoc c10-umm-json :warnings) latest-umm-format {:format :umm-json :version umm-version/current-collection-version}] (index/wait-until-indexed) @@ -264,45 +266,46 @@ ;; Whitespace here but not stripped out for expected ;; results. It will be present in metadata. :entry-title " ET1 "}) - {:format :echo10}) + {:format :echo10 :validate-keywords false}) c2-echo (d/ingest "PROV2" (dc/collection {:short-name "S2" :version-id "V2" :entry-title "ET2"}) - {:format :echo10}) + {:format :echo10 :validate-keywords false}) c3-dif (d/ingest "PROV1" (dc/collection-dif {:short-name "S3" :version-id "V3" :entry-title "ET3" :long-name "ET3"}) - {:format :dif}) + {:format :dif :validate-keywords false}) c4-dif (d/ingest "PROV2" (dc/collection-dif {:short-name "S4" :version-id "V4" :entry-title "ET4" :long-name "ET4"}) - {:format :dif}) + {:format :dif :validate-keywords false}) c5-iso (d/ingest "PROV1" (dc/collection {:short-name "S5" :version-id "V50"}) - {:format :iso19115}) + {:format :iso19115 :validate-keywords false}) c6-iso (d/ingest "PROV2" (dc/collection {:short-name "S6" :version-id "V6"}) - {:format :iso19115}) + {:format :iso19115 :validate-keywords false}) c7-smap (d/ingest "PROV1" (dc/collection-smap {:short-name "S7" :version-id "V7"}) - {:format :iso-smap}) + {:format :iso-smap :validate-keywords false}) c8-dif10 (d/ingest "PROV1" (dc/collection-dif10 {:short-name "S8" :version-id "V8" :entry-title "ET8" :long-name "ET8"}) - {:format :dif10}) + {:format :dif10 :validate-keywords false}) c9-dif10 (d/ingest "PROV2" (dc/collection-dif10 {:short-name "S9" :version-id "V9" :entry-title "ET9" :long-name "ET9"}) - {:format :dif10}) + {:format :dif10 :validate-keywords false}) c10-umm-json (d/ingest "PROV1" exp-conv/example-collection-record {:format :umm-json - :accept-format :json}) + :accept-format :json + :validate-keywords false}) c10-umm-json (dissoc c10-umm-json :warnings) all-colls [c1-echo c2-echo c3-dif c4-dif c5-iso c6-iso c7-smap c8-dif10 c9-dif10 c10-umm-json]] @@ -426,12 +429,12 @@ ; Tests that we can ingest and find difs with spatial and that granules in the dif can also be ; ingested and found (deftest dif-with-spatial - (let [c1 (d/ingest "PROV1" (dc/collection-dif {:spatial-coverage nil}) {:format :dif}) + (let [c1 (d/ingest "PROV1" (dc/collection-dif {:spatial-coverage nil}) {:format :dif :validate-keywords false}) g1 (d/ingest "PROV1" (dg/granule c1)) ;; A collection with a granule spatial representation c2 (d/ingest "PROV1" (dc/collection-dif {:spatial-coverage (dc/spatial {:gsr :geodetic})}) - {:format :dif}) + {:format :dif :validate-keywords false}) g2 (d/ingest "PROV1" (dg/granule c2 {:spatial-coverage (dg/spatial (m/mbr -160 45 -150 35))})) @@ -441,7 +444,7 @@ {:spatial-coverage (dc/spatial {:gsr :geodetic :sr :geodetic :geometries [(m/mbr -10 9 0 -10)]})}) - {:format :dif}) + {:format :dif :validate-keywords false}) g3 (d/ingest "PROV1" (dg/granule c3 {:spatial-coverage (dg/spatial m/whole-world)}))] (index/wait-until-indexed) @@ -532,7 +535,8 @@ (l/ords->line-string nil [0 0, 0 1, 0 -90, 180 0]) (l/ords->line-string nil [1 2, 3 4, 5 6, 7 8]) (m/mbr -180 90 180 -90) - (m/mbr -10 20 30 -40)]})})) + (m/mbr -10 20 30 -40)]})}) + {:validate-keywords false}) (update :entry-title string/trim)) coll2 (d/ingest "PROV1" (dc/collection {:entry-title "Dataset2" @@ -553,18 +557,20 @@ :spatial-coverage (dc/spatial {:sr :cartesian :gsr :cartesian - :geometries [polygon-without-holes]})})) + :geometries [polygon-without-holes]})}) + {:validate-keywords false}) coll3 (d/ingest "PROV1" (dc/collection {:entry-title "Dataset3" :personnel [p3] :science-keywords [sk1 sk2] :spatial-coverage (dc/spatial {:gsr :orbit - :orbit op1})})) + :orbit op1})}) + {:validate-keywords false}) coll5 (d/ingest "PROV1" (dc/collection-dif {:entry-title "Dataset5" :science-keywords [sk1 sk2]}) - {:format :dif}) + {:format :dif :validate-keywords false}) coll6 (d/ingest "PROV1" (dc/collection {:entry-title "Dataset6" :short-name "ShortName#6" :version-id "Version6" @@ -579,7 +585,8 @@ (dc/spatial {:sr :cartesian :gsr :cartesian :geometries [(p/point 1 2) - (p/point -179.9 89.4)]})})) + (p/point -179.9 89.4)]})}) + {:validate-keywords false}) coll7 (d/ingest "PROV1" (dc/collection {:entry-title "Dataset7" :short-name "ShortName#7" :version-id "Version7" @@ -593,7 +600,8 @@ (dc/spatial {:sr :cartesian :gsr :cartesian :geometries [(l/ords->line-string nil [0 0, 0 1, 0 -90, 180 0]) - (l/ords->line-string nil [1 2, 3 4, 5 6, 7 8])]})})) + (l/ords->line-string nil [1 2, 3 4, 5 6, 7 8])]})}) + {:validate-keywords false}) coll8 (d/ingest "USGS_EROS" (dc/collection {:entry-title "Dataset8" :short-name "ShortName#8" :version-id "Version8" @@ -606,11 +614,12 @@ (dc/spatial {:sr :cartesian :gsr :cartesian :geometries [(m/mbr -180 90 180 -90) - (m/mbr -10 20 30 -40)]})})) + (m/mbr -10 20 30 -40)]})}) + {:validate-keywords false}) coll9 (d/ingest "PROV1" (dc/collection-dif10 {:entry-title "Dataset9" :science-keywords [sk1 sk2]}) - {:format :dif10}) + {:format :dif10 :validate-keywords false}) _ (index/wait-until-indexed) ;; coll5's revision-date is needed to populate "modified" field in opendata. umm-json-coll5 (search/find-concepts-umm-json :collection {:concept_id (:concept-id coll5)}) @@ -701,7 +710,8 @@ :beginning-date-time "1970-01-01T12:00:00Z" :platforms [{:short-name "platform #1" :long-name "platform #1" - :type "t"}]})) + :type "t"}]}) + {:validate-keywords false}) c2 (d/ingest "PROV2" (dc/collection {:short-name "world's shortest name: so short you won't believe your eyes!" :version-id "V2" :long-name "world's longest name" @@ -713,7 +723,8 @@ :type "good platform"} {:short-name "platform #32" :long-name "platform #32" - :type "very good platform"}]}))]) + :type "very good platform"}]}) + {:validate-keywords false})]) (index/wait-until-indexed) (testing "csv" (let [response (search/find-concepts-csv :collection {})] @@ -773,7 +784,7 @@ umm-attributes (merge related-urls data-centers) concept-umm (-> (umm-spec-collection/collection 1 umm-attributes) (umm-spec-collection/collection-concept :umm-json) - ingest/ingest-concept) + (ingest/ingest-concept {:validate-keywords false})) _ (index/wait-until-indexed) atom-json-links (->> (search/find-concepts-json :collection {}) :results @@ -858,10 +869,10 @@ umm-attributes (merge related-urls data-centers) concept-umm (-> (umm-spec-collection/collection 1 umm-attributes) (umm-spec-collection/collection-concept :umm-json) - ingest/ingest-concept) + (ingest/ingest-concept {:validate-keywords false})) concept-citation (-> (umm-spec-collection/collection 2 collection-citations) (umm-spec-collection/collection-concept :umm-json) - ingest/ingest-concept) + (ingest/ingest-concept {:validate-keywords false})) _ (index/wait-until-indexed) opendata-5138-1 (search/find-concepts-opendata :collection {:concept_id (:concept-id concept-5138-1)}) opendata-5138-2 (search/find-concepts-opendata :collection {:concept_id (:concept-id concept-5138-2)}) @@ -1013,7 +1024,7 @@ (:landingPage opendata-coll-5138-2)))))))) (deftest formats-have-scores-test - (let [coll1 (d/ingest "PROV1" (dc/collection {:short-name "ABC!XYZ" :entry-title "Foo"}))] + (let [coll1 (d/ingest "PROV1" (dc/collection {:short-name "ABC!XYZ" :entry-title "Foo"}) {:validate-keywords false})] (index/wait-until-indexed) (testing "XML references" (testing "XML has score for keyword search." @@ -1099,35 +1110,41 @@ (dc/collection-dif {:short-name "S-DIF9" :organizations [distribution-org distribution-org-1]}) - {:format :dif}) + {:format :dif + :validate-keywords false}) (d/ingest "PROV1" (dc/collection-dif10 {:short-name "S-DIF10" :organizations [processing-org originating-org distribution-org archive-org]}) - {:format :dif10}) + {:format :dif10 + :validate-keywords false}) (d/ingest "PROV1" (dc/collection {:short-name "S-ECHO10" - :organizations [processing-org archive-org]})) + :organizations [processing-org archive-org]}) + {:validate-keywords false}) (d/ingest "PROV1" (dc/collection {:short-name "S-ISO-SMAP" :organizations [processing-org archive-org]}) - {:format :iso-smap}) + {:format :iso-smap + :validate-keywords false}) (d/ingest "PROV1" (dc/collection {:short-name "S-ISO19115" :organizations [archive-org]}) - {:format :iso19115}) + {:format :iso19115 + :validate-keywords false}) (d/ingest "PROV1" (assoc exp-conv/example-collection-record :ShortName "S-UMM-JSON") {:format :umm-json - :accept-format :json}) + :accept-format :json + :validate-keywords false}) (index/wait-until-indexed) (are3 [short-name expected-orgs] @@ -1160,7 +1177,8 @@ (deftest collection-concept-stac-retrieval-test (let [coll-metadata (slurp (io/resource "stac-test/C1299783579-LPDAAC_ECS.xml")) {coll-concept-id :concept-id} (ingest/ingest-concept - (ingest/concept :collection "PROV1" "foo" :echo10 coll-metadata)) + (ingest/concept :collection "PROV1" "foo" :echo10 coll-metadata) + {:validate-keywords false}) expected (-> "stac-test/collection_stac.json" io/resource slurp @@ -1204,7 +1222,8 @@ (let [coll1 (d/ingest "PROV1" (dc/collection {:entry-title "Dataset1" :beginning-date-time "1970-01-01T12:00:00Z" - :spatial-coverage (dc/spatial {:gsr :geodetic})})) + :spatial-coverage (dc/spatial {:gsr :geodetic})}) + {:validate-keywords false}) coll-concept-id (:concept-id coll1) _ (index/wait-until-indexed) response (search/find-concepts-stac :collection {:concept-id coll-concept-id}) @@ -1219,7 +1238,7 @@ (:errors ext-response)))))) (deftest search-with-empty-json - (let [coll1 (d/ingest "PROV1" (dc/collection)) + (let [coll1 (d/ingest "PROV1" (dc/collection) {:validate-keywords false}) coll-concept-id (:concept-id coll1) _ (index/wait-until-indexed) response1 (client/get diff --git a/system-int-test/test/cmr/system_int_test/search/collection/collection_shapefile_search_test.clj b/system-int-test/test/cmr/system_int_test/search/collection/collection_shapefile_search_test.clj index 220193e53f..f3661cd90e 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection/collection_shapefile_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection/collection_shapefile_search_test.clj @@ -42,7 +42,8 @@ {:entry-title et :spatial-coverage (dc/spatial {:gsr coord-sys :sr coord-sys - :geometries shapes})})))) + :geometries shapes})}) + {:validate-keywords false}))) (deftest collection-shapefile-search-test (let [_ (side/eval-form `(shapefile/set-enable-shapefile-parameter-flag! true)) diff --git a/system-int-test/test/cmr/system_int_test/search/collection/collection_shapefile_simplification_search_test.clj b/system-int-test/test/cmr/system_int_test/search/collection/collection_shapefile_simplification_search_test.clj index e40e79c599..2b1e40a999 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection/collection_shapefile_simplification_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection/collection_shapefile_simplification_search_test.clj @@ -44,7 +44,8 @@ {:entry-title et :spatial-coverage (dc/spatial {:gsr coord-sys :sr coord-sys - :geometries shapes})})))) + :geometries shapes})}) + {:validate-keywords false}))) (deftest collection-shapefile-simplification-search-test (let [_ (side/eval-form `(shapefile/set-enable-shapefile-parameter-flag! true)) diff --git a/system-int-test/test/cmr/system_int_test/search/collection/collection_sorting_search_test.clj b/system-int-test/test/cmr/system_int_test/search/collection/collection_sorting_search_test.clj index 27f829f95d..3b5fced6e9 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection/collection_sorting_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection/collection_sorting_search_test.clj @@ -57,7 +57,8 @@ :short-name entry-title :version-id "1" :beginning-date-time (d/make-datetime begin) - :ending-date-time (d/make-datetime end)}))) + :ending-date-time (d/make-datetime end)}) + {:validate-keywords false})) (defn delete-coll "Deletes the collection and returns a deleted version of it for sorting comparison." [coll] @@ -162,7 +163,8 @@ "Makes a minimal collection with a shortname and ingests it." [shortname] (d/ingest "PROV1" - (dc/collection {:short-name shortname}))) + (dc/collection {:short-name shortname}) + {:validate-keywords false})) (deftest shortname-sorting-test (let [c1 (make-coll-with-sn "Bob2") @@ -287,19 +289,22 @@ ;; 1 will have the least relevance coll1 (d/ingest "PROV1" (dc/collection (merge common-attribs - {:entry-title "coll1"}))) + {:entry-title "coll1"})) + {:validate-keywords false}) ;; 2 has the most by having both spatial and temporal keywords that match the search term coll2 (d/ingest "PROV1" (dc/collection (merge common-attribs {:entry-title "coll2" :spatial-keywords ["wood"] - :temporal-keywords ["wood"]}))) + :temporal-keywords ["wood"]})) + {:validate-keywords false}) ;; 3 has more than 1 but less than 2. It has spatial keywords that match the search term but ;; no temporal keywords coll3 (d/ingest "PROV1" (dc/collection (merge common-attribs {:entry-title "coll3" - :spatial-keywords ["wood"]})))] + :spatial-keywords ["wood"]})) + {:validate-keywords false})] (index/wait-until-indexed) (testing "Keyword searching is by score" @@ -352,7 +357,8 @@ (let [make-collection (fn [& platforms] (d/ingest "PROV1" (dc/collection - {:platforms (map #(dc/platform {:short-name %}) platforms)}))) + {:platforms (map #(dc/platform {:short-name %}) platforms)}) + {:validate-keywords false})) c1 (make-collection "c10" "c41") c2 (make-collection "c20" "c51") c3 (make-collection "c30") @@ -374,7 +380,8 @@ (dc/collection {:platforms [(dc/platform {:instruments (map #(dc/instrument {:short-name %}) - instruments)})]}))) + instruments)})]}) + {:validate-keywords false})) c1 (make-collection "c10" "c41") c2 (make-collection "c20" "c51") c3 (make-collection "c30") @@ -398,7 +405,8 @@ {:instruments [(dc/instrument {:short-name (d/unique-str "instrument") - :sensors (map #(dc/sensor {:short-name %}) sensors)})]})]}))) + :sensors (map #(dc/sensor {:short-name %}) sensors)})]})]}) + {:validate-keywords false})) c1 (make-collection "c10" "c41") c2 (make-collection "c20" "c51") c3 (make-collection "c30") @@ -430,42 +438,49 @@ (data-umm-c/collection 1 {:EntryTitle "Dataset1" :TemporalExtents [(data-umm-cmn/temporal-extent {:beginning-date-time "2010-01-01T12:00:00Z" - :ending-date-time year-in-past})]})) + :ending-date-time year-in-past})]}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:EntryTitle "Dataset2" :TemporalExtents [(data-umm-cmn/temporal-extent {:beginning-date-time "2010-01-01T12:00:00Z" - :ending-date-time year-in-future})]})) + :ending-date-time year-in-future})]}) + {:validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 3 {:EntryTitle "Dataset3" :TemporalExtents [(data-umm-cmn/temporal-extent {:beginning-date-time "2010-01-01T12:00:00Z" - :ending-date-time outside-ongoing})]})) + :ending-date-time outside-ongoing})]}) + {:validate-keywords false}) coll4 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 4 {:EntryTitle "Dataset4" :TemporalExtents [(data-umm-cmn/temporal-extent {:beginning-date-time "2010-01-01T12:00:00Z" - :ending-date-time inside-ongoing})]})) + :ending-date-time inside-ongoing})]}) + {:validate-keywords false}) coll5 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 5 {:EntryTitle "Dataset5" :TemporalExtents [(data-umm-cmn/temporal-extent {:beginning-date-time "2010-01-01T12:00:00Z" - :ending-date-time today})]})) + :ending-date-time today})]}) + {:validate-keywords false}) coll6 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 6 {:EntryTitle "Dataset6" :TemporalExtents [(data-umm-cmn/temporal-extent - {:beginning-date-time "2010-01-01T12:00:00Z"})]})) + {:beginning-date-time "2010-01-01T12:00:00Z"})]}) + {:validate-keywords false}) coll7 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 7 {:EntryTitle "Dataset7" :TemporalExtents [(data-umm-cmn/temporal-extent {:beginning-date-time "2010-01-01T12:00:00Z" - :ending-date-time today})]}))] + :ending-date-time today})]}) + {:validate-keywords false})] (index/wait-until-indexed) (are [sort-key items] (sort-order-correct? items sort-key) @@ -482,7 +497,8 @@ :short-name cname :long-name (str cname "-long") :version-id "1"} - attribs)))) + attribs)) + {:validate-keywords false})) ;; community usage values assigned to the following collections alpha_1_10 (make-named-collection "alpha_a" {:short-name "alpha"}) @@ -552,7 +568,8 @@ :short-name cname :long-name (str cname "-long") :version-id "1"} - attribs)))) + attribs)) + {:validate-keywords false})) _ (humanizer-util/ingest-community-usage-metrics non-version-usage-csv) diff --git a/system-int-test/test/cmr/system_int_test/search/collection/collection_spatial_keyword_search_test.clj b/system-int-test/test/cmr/system_int_test/search/collection/collection_spatial_keyword_search_test.clj index fc97823c46..0caa717d26 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection/collection_spatial_keyword_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection/collection_spatial_keyword_search_test.clj @@ -10,17 +10,17 @@ (use-fixtures :each (ingest/reset-fixture {"provguid1" "PROV1" "provguid2" "PROV2"})) (deftest search-by-spatial-keywords - (let [coll1 (d/ingest "PROV1" (dc/collection {})) - coll2 (d/ingest "PROV1" (dc/collection {:spatial-keywords []})) - coll3 (d/ingest "PROV1" (dc/collection {:spatial-keywords ["DC"]})) + (let [coll1 (d/ingest "PROV1" (dc/collection {}) {:validate-keywords false}) + coll2 (d/ingest "PROV1" (dc/collection {:spatial-keywords []}) {:validate-keywords false}) + coll3 (d/ingest "PROV1" (dc/collection {:spatial-keywords ["DC"]}) {:validate-keywords false}) - coll4 (d/ingest "PROV2" (dc/collection {:spatial-keywords ["DC" "LA"]})) - coll5 (d/ingest "PROV2" (dc/collection {:spatial-keywords ["Detroit"]})) - coll6 (d/ingest "PROV2" (dc/collection {:spatial-keywords ["LL"]})) - coll7 (d/ingest "PROV2" (dc/collection {:spatial-keywords ["detroit"]})) - coll-angola (d/ingest "PROV2" (dc/collection {:spatial-keywords ["ANGOLA"]})) - coll-c-africa (d/ingest "PROV2" (dc/collection {:spatial-keywords ["central africa"]})) - coll-gaza (d/ingest "PROV2" (dc/collection {:spatial-keywords ["Gaza Strip"]})) + coll4 (d/ingest "PROV2" (dc/collection {:spatial-keywords ["DC" "LA"]}) {:validate-keywords false}) + coll5 (d/ingest "PROV2" (dc/collection {:spatial-keywords ["Detroit"]}) {:validate-keywords false}) + coll6 (d/ingest "PROV2" (dc/collection {:spatial-keywords ["LL"]}) {:validate-keywords false}) + coll7 (d/ingest "PROV2" (dc/collection {:spatial-keywords ["detroit"]}) {:validate-keywords false}) + coll-angola (d/ingest "PROV2" (dc/collection {:spatial-keywords ["ANGOLA"]}) {:validate-keywords false}) + coll-c-africa (d/ingest "PROV2" (dc/collection {:spatial-keywords ["central africa"]}) {:validate-keywords false}) + coll-gaza (d/ingest "PROV2" (dc/collection {:spatial-keywords ["Gaza Strip"]}) {:validate-keywords false}) coll-iso-sample (d/ingest-concept-with-metadata-file "iso-samples/cmr-4192-iso-collection.xml" {:provider-id "PROV2" :concept-type :collection diff --git a/system-int-test/test/cmr/system_int_test/search/collection/collection_spatial_search_test.clj b/system-int-test/test/cmr/system_int_test/search/collection/collection_spatial_search_test.clj index da373c82bd..be3725ef3b 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection/collection_spatial_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection/collection_spatial_search_test.clj @@ -37,7 +37,8 @@ {:entry-title et :spatial-coverage (dc/spatial {:gsr coord-sys :sr coord-sys - :geometries shapes})})))) + :geometries shapes})}) + {:validate-keywords false}))) (deftest excessive-points-line-spatial-search-test (let [whole-world (make-coll :geodetic "whole-world" (m/mbr -180 90 180 -90)) diff --git a/system-int-test/test/cmr/system_int_test/search/collection/collection_spatialkeyword_keyword_search_test.clj b/system-int-test/test/cmr/system_int_test/search/collection/collection_spatialkeyword_keyword_search_test.clj index f59a3542ec..163eb9382d 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection/collection_spatialkeyword_keyword_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection/collection_spatialkeyword_keyword_search_test.clj @@ -15,7 +15,8 @@ (deftest search-by-keywords (let [coll1 (d/ingest "PROV2" (dc/collection {:entry-title "coll10" - :spatial-keywords ["in out"]}))] + :spatial-keywords ["in out"]}) + {:validate-keywords false})] (index/wait-until-indexed) diff --git a/system-int-test/test/cmr/system_int_test/search/collection_relevancy/collection_relevancy_test.clj b/system-int-test/test/cmr/system_int_test/search/collection_relevancy/collection_relevancy_test.clj index 44dcfdb114..2c95b10733 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection_relevancy/collection_relevancy_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection_relevancy/collection_relevancy_test.clj @@ -39,7 +39,8 @@ :Platforms [(data-umm-c/platform {:ShortName "Usage"})] :TemporalExtents [(data-common/temporal-extent {:beginning-date-time "2003-08-01T00:00:00Z" - :ending-date-time "2005-10-01T00:00:00Z"})]})) + :ending-date-time "2005-10-01T00:00:00Z"})]}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection @@ -48,7 +49,8 @@ :EntryTitle "Elevation coll2" :TemporalExtents [(data-common/temporal-extent {:beginning-date-time "2001-08-01T00:00:00Z" - :ending-date-time "2010-10-01T00:00:00Z"})]})) + :ending-date-time "2010-10-01T00:00:00Z"})]}) + {:validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection @@ -57,7 +59,8 @@ :EntryTitle "Elevation coll3" :TemporalExtents [(data-common/temporal-extent {:beginning-date-time "2002-10-15T12:00:00Z" - :ends-at-present? true})]}))] + :ends-at-present? true})]}) + {:validate-keywords false})] (index/wait-until-indexed) (testing "Keyword and temporal" @@ -202,25 +205,29 @@ :Version "2" :EntryTitle "Elevation coll1" ;; Will get humanized to 1T - :ProcessingLevel {:Id "L1T"}})) + :ProcessingLevel {:Id "L1T"}}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:ShortName "Elevation L1" :Version "1" :EntryTitle "Elevation coll2" - :ProcessingLevel {:Id "2"}})) + :ProcessingLevel {:Id "2"}}) + {:validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:ShortName "Elevation L4" :Version "4" :EntryTitle "Elevation coll3" - :ProcessingLevel {:Id "4"}})) + :ProcessingLevel {:Id "4"}}) + {:validate-keywords false}) coll4 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:ShortName "Elevation NP" :Version "5" :EntryTitle "Elevation coll4" - :ProcessingLevel {:Id "Not provided"}}))] + :ProcessingLevel {:Id "Not provided"}}) + {:validate-keywords false})] (index/wait-until-indexed) (is (d/refs-match-order? [coll3 coll2 coll1 coll4] (search/find-refs :collection {:keyword "Elevation"}))) diff --git a/system-int-test/test/cmr/system_int_test/search/collection_relevancy/collection_usage_relevancy_test.clj b/system-int-test/test/cmr/system_int_test/search/collection_relevancy/collection_usage_relevancy_test.clj index bb84521458..d15f5fc5a0 100644 --- a/system-int-test/test/cmr/system_int_test/search/collection_relevancy/collection_usage_relevancy_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/collection_relevancy/collection_usage_relevancy_test.clj @@ -73,22 +73,26 @@ (data-umm-c/collection {:ShortName "AMSR-L1A" :EntryTitle "Relevancy 1" :Version "3" - :Platforms [(data-umm-c/platform {:ShortName "Relevancy"})]})) + :Platforms [(data-umm-c/platform {:ShortName "Relevancy"})]}) + {:validate-keywords false}) (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:ShortName "AST_05" :EntryId "Relevancy" :EntryTitle "AST_05" :Version "B" :Projects (data-umm-c/projects "Relevancy") - :Platforms [(data-umm-c/platform {:ShortName "Relevancy"})]})) + :Platforms [(data-umm-c/platform {:ShortName "Relevancy"})]}) + {:validate-keywords false}) (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:ShortName "AG_VIRTUAL" :EntryTitle "Relevancy 2" - :Version "1"})) + :Version "1"}) + {:validate-keywords false}) (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:ShortName "AG_MAPSS" :EntryTitle "Relevancy 3" - :Version "2"})) + :Version "2"}) + {:validate-keywords false}) (index/wait-until-indexed) (let [results (:refs (search/find-refs :collection {:keyword "Relevancy"}))] (is (= ["Relevancy 1" "Relevancy 2" "Relevancy 3" "AST_05"] (map :name results)))) @@ -149,16 +153,19 @@ (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:ShortName "AMSR-L1A" ;10 :EntryTitle "Relevancy 1" - :Version "3"})) + :Version "3"}) + {:validate-keywords false}) (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:ShortName "AG_VIRTUAL" ; 100 :EntryTitle "Relevancy 2" - :Version "1"})) + :Version "1"}) + {:validate-keywords false}) (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:ShortName "AG_MAPSS" ;30 :EntryTitle "Relevancy 3" :Version "2" - :Platforms [(data-umm-c/platform {:ShortName "Relevancy"})]})) + :Platforms [(data-umm-c/platform {:ShortName "Relevancy"})]}) + {:validate-keywords false}) (index/wait-until-indexed) (testing "Sort by usage ascending" (let [results (:refs (search/find-refs :collection {:sort-key "usage-score"}))] diff --git a/system-int-test/test/cmr/system_int_test/search/facets/collection_facets_search_test.clj b/system-int-test/test/cmr/system_int_test/search/facets/collection_facets_search_test.clj index c72ffd3e67..2d12a2f96c 100644 --- a/system-int-test/test/cmr/system_int_test/search/facets/collection_facets_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/facets/collection_facets_search_test.clj @@ -24,7 +24,8 @@ [n prov & attribs] (d/ingest-umm-spec-collection prov (data-umm-spec/collection-missing-properties-dif (apply merge {:EntryTitle (str "coll" n)} attribs)) - {:format :dif})) + {:format :dif + :validate-keywords false})) (defn- grant-permissions "Grant permissions to all collections in PROV1 and a subset of collections in PROV2" @@ -885,12 +886,14 @@ :platforms [(dc/platform {:short-name "smap" :instruments [(dc/instrument {:short-name "atm"})]})] - :organizations [(dc/org :archive-center "OR-STATE/eoarc")]})) + :organizations [(dc/org :archive-center "OR-STATE/eoarc")]}) + {:validate-keywords false}) coll2 (d/ingest "PROV1" (dc/collection {:entry-title "coll2" :platforms [(dc/platform {:short-name "sMaP" :instruments [(dc/instrument {:short-name "aTM"})]})] - :organizations [(dc/org :archive-center "or-state/EOARC")]})) + :organizations [(dc/org :archive-center "or-state/EOARC")]}) + {:validate-keywords false}) _ (index/wait-until-indexed) facet-results (:facets (search/find-refs :collection {:include-facets true}))] (are [value-counts field] diff --git a/system-int-test/test/cmr/system_int_test/search/facets/collection_facets_v2_search_test.clj b/system-int-test/test/cmr/system_int_test/search/facets/collection_facets_v2_search_test.clj index 4f6c411705..1211b1bf75 100644 --- a/system-int-test/test/cmr/system_int_test/search/facets/collection_facets_v2_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/facets/collection_facets_v2_search_test.clj @@ -551,14 +551,16 @@ :ShortName "S1" :VersionId "V1" :Platforms (data-umm-spec/platforms - (get-in sample-platforms [:diadem :short-name]))})) + (get-in sample-platforms [:diadem :short-name]))}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-spec/collection {:EntryTitle "coll2" :ShortName "S2" :VersionId "V2" :Platforms (data-umm-spec/platforms (get-in sample-platforms [:diadem :short-name]) - (get-in sample-platforms [:dmsp :short-name]))})) + (get-in sample-platforms [:dmsp :short-name]))}) + {:validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-spec/collection {:EntryTitle "coll3" :ShortName "S3" @@ -566,7 +568,8 @@ :Platforms [(data-umm-spec/platform {:ShortName (get-in sample-platforms [:dmsp :short-name]) :Instruments [(data-umm-spec/instrument {:ShortName "I3"})]})] - :Projects (umm-spec-common/projects "proj3")})) + :Projects (umm-spec-common/projects "proj3")}) + {:validate-keywords false}) coll4 (d/ingest-umm-spec-collection "PROV1" (data-umm-spec/collection {:EntryTitle "coll4" :ShortName "S4" @@ -574,7 +577,8 @@ :Platforms [(data-umm-spec/platform {:ShortName (get-in sample-platforms [:smap :short-name]) :Instruments [(data-umm-spec/instrument {:ShortName "I4"})]})] - :Projects (umm-spec-common/projects "proj4")}))] + :Projects (umm-spec-common/projects "proj4")}) + {:validate-keywords false})] (testing "search by platform parameter filters the other facets, but not platforms facets" (let [facets-result (search-and-return-v2-facets {:platforms-h {:0 (sample-platform-full :smap)}}) @@ -660,7 +664,8 @@ :Topic "Topic1" :Term "Term1" :VariableLevel1 "Level3-1" - :DetailedVariable "Detail3"})]}))] + :DetailedVariable "Detail3"})]}) + {:validate-keywords false})] (testing "Testing that a sub-category and two short name platform facets exist" (let [facets-result (search-and-return-v2-facets {:platforms-h @@ -696,7 +701,8 @@ :VariableLevel1 "Level1-1" :VariableLevel2 "Level1-2" :VariableLevel3 "Level1-3" - :DetailedVariable "Detail1"})]})) + :DetailedVariable "Detail1"})]}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-spec/collection {:EntryTitle "coll2" :ShortName "S2" @@ -707,7 +713,8 @@ :Term "Term2" :VariableLevel1 "Level2-1" :VariableLevel2 "Level2-2" - :DetailedVariable "Detail2"})]})) + :DetailedVariable "Detail2"})]}) + {:validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-spec/collection {:EntryTitle "coll3" :ShortName "S3" @@ -717,7 +724,8 @@ :Topic "Topic3" :Term "Term3" :VariableLevel1 "Level3-1" - :DetailedVariable "Detail3"})]})) + :DetailedVariable "Detail3"})]}) + {:validate-keywords false}) coll4 (d/ingest-umm-spec-collection "PROV1" (data-umm-spec/collection {:EntryTitle "coll4" :ShortName "S4" @@ -726,7 +734,8 @@ {:Category "Earth Science" :Topic "Topic4" :Term "Term4" - :DetailedVariable "Detail4"})]}))] + :DetailedVariable "Detail4"})]}) + {:validate-keywords false})] (are3 [query detailed-variable] (let [facets-result (search-and-return-v2-facets query)] @@ -772,7 +781,8 @@ :Platforms [(umm-spec-common/platform {:ShortName "NASA S-3B VIKING" :LongName "NASA S-3B VIKING" - :Type "Jet"})]})) + :Type "Jet"})]}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-spec/collection {:EntryTitle "coll2" :ShortName "S2" @@ -780,7 +790,8 @@ :Platforms [(umm-spec-common/platform {:ShortName "AEROS-1" :LongName "AEROS-1" - :Type "Earth Observation Satellites"})]})) + :Type "Earth Observation Satellites"})]}) + {:validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-spec/collection {:EntryTitle "coll3" :ShortName "S3" @@ -788,7 +799,8 @@ :Platforms [(umm-spec-common/platform {:ShortName "Aqua" :LongName "Earth Observing System, Aqua" - :Type "Earth Observation Satellites"})]})) + :Type "Earth Observation Satellites"})]}) + {:validate-keywords false}) coll4 (d/ingest-umm-spec-collection "PROV1" (data-umm-spec/collection {:EntryTitle "coll4" :ShortName "S4" @@ -796,7 +808,8 @@ :Platforms [(umm-spec-common/platform {:ShortName "ISS" :LongName "International Space Station" - :Type "Space Stations/Crewed Spacecraft"})]}))] + :Type "Space Stations/Crewed Spacecraft"})]}) + {:validate-keywords false})] (are3 [query short-name] @@ -842,7 +855,8 @@ :VariableLevel1 "Level1-1" :VariableLevel2 "Level1-2" :VariableLevel3 "Level1-3" - :DetailedVariable "Detail1"})]})) + :DetailedVariable "Detail1"})]}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-spec/collection {:EntryTitle "coll2" :ShortName "S2" @@ -854,7 +868,8 @@ :Topic "Topic2" :Term "Term2" :VariableLevel1 "Level2-1" - :VariableLevel2 "Level2-2"})]})) + :VariableLevel2 "Level2-2"})]}) + {:validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-spec/collection {:EntryTitle "coll3" :ShortName "S3" @@ -868,7 +883,8 @@ (umm-spec-common/science-keyword {:Category "Earth Science" :Topic "Topic2" - :Term "Term3"})]}))] + :Term "Term3"})]}) + {:validate-keywords false})] ;; We only check the topic level science keywords facet for convenience since the whole ;; hierarchical structure of science keywords facet has been covered in all facets test. (testing "search by science-keywords param filters the other facets, but not science-keywords facets" @@ -926,24 +942,28 @@ {:EntryTitle "coll1" :ShortName "S1" :VersionId "V1" - :DataCenters [org1]})) + :DataCenters [org1]}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-spec/collection {:EntryTitle "coll2" :ShortName "S2" :VersionId "V2" - :DataCenters [org2]})) + :DataCenters [org2]}) + {:validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-spec/collection {:EntryTitle "coll3" :ShortName "S3" :VersionId "V3" :DataCenters [org1 org2] - :Projects (data-umm-spec/projects "proj3")})) + :Projects (data-umm-spec/projects "proj3")}) + {:validate-keywords false}) coll4 (d/ingest-umm-spec-collection "PROV1" (data-umm-spec/collection {:EntryTitle "coll4" :ShortName "S4" :VersionId "V4" :DataCenters [org3] - :Projects (data-umm-spec/projects "proj4")}))] + :Projects (data-umm-spec/projects "proj4")}) + {:validate-keywords false})] (testing "search by data-center parameter filters the other facets, but not Organizations facets" (let [facets-result (search-and-return-v2-facets {:data-center-h ["NSIDC"]})] @@ -996,21 +1016,25 @@ {:EntryTitle "coll1" :ShortName "S1" :VersionId "V1" - :Platforms (data-umm-spec/platforms "P1")})) + :Platforms (data-umm-spec/platforms "P1")}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-spec/collection {:EntryTitle "coll2" :ShortName "S2" :VersionId "V2" - :Platforms (data-umm-spec/platforms "P2")})) + :Platforms (data-umm-spec/platforms "P2")}) + {:validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-spec/collection {:EntryTitle "coll3" :ShortName "S3" :VersionId "V3" - :Platforms (data-umm-spec/platforms "P3")})) + :Platforms (data-umm-spec/platforms "P3")}) + {:validate-keywords false}) coll4 (d/ingest-umm-spec-collection "PROV1" (data-umm-spec/collection {:EntryTitle "coll4" :ShortName "S4" - :VersionId "V4"})) + :VersionId "V4"}) + {:validate-keywords false}) ;; index the collections so that they can be found during variable association _ (index/wait-until-indexed) ;; create variables @@ -1128,19 +1152,22 @@ :SpatialExtent spatial :CollectionDataType "NEAR_REAL_TIME" :Projects [{:ShortName "Proj1" - :LongName "Proj1 Long Name"}]})) + :LongName "Proj1 Long Name"}]}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-spec/collection {:EntryTitle "coll2" :ShortName "S2" :CollectionDataType "LOW_LATENCY" :Projects [{:ShortName "Proj2" - :LongName "Proj2 Long Name"}]})) + :LongName "Proj2 Long Name"}]}) + {:validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-spec/collection {:EntryTitle "coll3" :ShortName "S3" :CollectionDataType "EXPEDITED" :Projects [{:ShortName "Proj3" - :LongName "Proj3 Long Name"}]}))] + :LongName "Proj3 Long Name"}]}) + {:validate-keywords false})] (testing "Latency v2 facets" (let [;; search for collections without latency parameter. @@ -1217,13 +1244,15 @@ :SpatialExtent spatial :Projects [{:ShortName "Proj4" :LongName "Proj4 Long Name"}]}) - {:format :umm-json}) + {:format :umm-json + :validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-spec/collection {:EntryTitle "coll2" :ShortName "S2" :Projects [{:ShortName "Proj3" :LongName "Proj3 Long Name"}]}) - {:format :umm-json})] + {:format :umm-json + :validate-keywords false})] (testing "search by horizontal data resolution. Parameter filters the other facets, but not this facet. This also tests the edge case where the record contains the value of diff --git a/system-int-test/test/cmr/system_int_test/search/facets/facets_util.clj b/system-int-test/test/cmr/system_int_test/search/facets/facets_util.clj index 448db174d2..22069b759d 100644 --- a/system-int-test/test/cmr/system_int_test/search/facets/facets_util.clj +++ b/system-int-test/test/cmr/system_int_test/search/facets/facets_util.clj @@ -15,7 +15,8 @@ (d/ingest-umm-spec-collection prov (data-umm-spec/collection-without-minimal-attribs - n (apply merge {:EntryTitle (str "coll" n)} attribs)))) + n (apply merge {:EntryTitle (str "coll" n)} attribs)) + {:validate-keywords false})) (defn projects [& project-names] diff --git a/system-int-test/test/cmr/system_int_test/search/granule/granule_campaign_search_test.clj b/system-int-test/test/cmr/system_int_test/search/granule/granule_campaign_search_test.clj index 9b3916dfe2..11afadc746 100644 --- a/system-int-test/test/cmr/system_int_test/search/granule/granule_campaign_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/granule/granule_campaign_search_test.clj @@ -13,13 +13,15 @@ (use-fixtures :each (ingest/reset-fixture {"provguid1" "PROV1"})) (deftest search-by-campaign - (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:Projects (data-umm-cmn/projects "ABC" "XYZ" "PDQ" "RST")})) - gran1 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule1" - :project-refs ["ABC"]})) - gran2 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule2" - :project-refs ["ABC" "XYZ"]})) - gran3 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule3" - :project-refs ["PDQ" "RST"]}))] + (let [coll1 (d/ingest-umm-spec-collection "PROV1" + (data-umm-c/collection {:Projects (data-umm-cmn/projects "ABC" "XYZ" "PDQ" "RST")}) + {:validate-keywords false}) + gran1 (d/ingest "PROV1" + (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule1" :project-refs ["ABC"]})) + gran2 (d/ingest "PROV1" + (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule2" :project-refs ["ABC" "XYZ"]})) + gran3 (d/ingest "PROV1" + (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule3" :project-refs ["PDQ" "RST"]}))] (index/wait-until-indexed) (testing "search by campaign" diff --git a/system-int-test/test/cmr/system_int_test/search/granule/granule_counts_search_test.clj b/system-int-test/test/cmr/system_int_test/search/granule/granule_counts_search_test.clj index d68ea7aaca..1c936a95cc 100644 --- a/system-int-test/test/cmr/system_int_test/search/granule/granule_counts_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/granule/granule_counts_search_test.clj @@ -60,7 +60,7 @@ spatial-attribs temporal-attribs other-attribs)] - (d/ingest provider (dc/collection coll-attribs))))) + (d/ingest provider (dc/collection coll-attribs) {:validate-keywords false})))) (defn- polygon "Creates a single ring polygon with the given ordinates. Points must be in counter clockwise order. @@ -124,7 +124,8 @@ :geometries [m/whole-world] :gsr :orbit :orbit orbit-parameters})} - no-match-temporal))) + no-match-temporal)) + {:validate-keywords false}) all-colls [coll1 coll2 coll3 coll4 coll5 coll6 orbit-coll]] diff --git a/system-int-test/test/cmr/system_int_test/search/granule/granule_orbit_search_test.clj b/system-int-test/test/cmr/system_int_test/search/granule/granule_orbit_search_test.clj index 782a2dbd4e..d1c6aae214 100644 --- a/system-int-test/test/cmr/system_int_test/search/granule/granule_orbit_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/granule/granule_orbit_search_test.clj @@ -169,7 +169,8 @@ (dc/collection {:entry-title "orbit-params1" :spatial-coverage (dc/spatial {:gsr :orbit - :orbit orbit-parameters})})) + :orbit orbit-parameters})}) + {:validate-keywords false}) g1 (make-gran coll1 "gran1" 70.80471 50.0 :asc 50.0 :desc) g2 (make-gran coll1 "gran2" 70.80471 50.0 :desc -50.0 :desc) g3 (make-gran coll1 "gran3" 70.80471 -50.0 :desc -50.0 :asc) @@ -259,17 +260,20 @@ (dc/collection {:entry-title "orbit-params1" :spatial-coverage (dc/spatial {:gsr :orbit - :orbit op1})})) + :orbit op1})}) + {:validate-keywords false}) coll2 (d/ingest "PROV1" (dc/collection {:entry-title "orbit-params2" :spatial-coverage (dc/spatial {:gsr :orbit - :orbit op2})})) + :orbit op2})}) + {:validate-keywords false}) coll3 (d/ingest "PROV1" (dc/collection {:entry-title "orbit-params3" :spatial-coverage (dc/spatial {:gsr :orbit - :orbit op3-bad})})) + :orbit op3-bad})}) + {:validate-keywords false}) g1 (make-gran coll1 "gran1" -158.1 81.8 :desc -81.8 :desc) g2 (make-gran coll1 "gran2" 177.16 -81.8 :asc 81.8 :asc) g3 (make-gran coll1 "gran3" 127.73 81.8 :desc -81.8 :desc) @@ -380,12 +384,14 @@ (dc/collection {:entry-title "orbit-params1" :spatial-coverage (dc/spatial {:gsr :orbit - :orbit op1})})) + :orbit op1})}) + {:validate-keywords false}) coll2 (d/ingest "PROV1" (dc/collection {:entry-title "orbit-params2" :spatial-coverage (dc/spatial {:gsr :orbit - :orbit op2})})) + :orbit op2})}) + {:validate-keywords false}) g1 (make-gran coll1 "gran1" 104.0852 50 :asc 50 :asc) g2 (make-gran coll2 "gran2" 31.946 70.113955 :asc -71.344289 :desc)] (index/wait-until-indexed) diff --git a/system-int-test/test/cmr/system_int_test/search/granule/granule_platform_instrument_sensor_search_test.clj b/system-int-test/test/cmr/system_int_test/search/granule/granule_platform_instrument_sensor_search_test.clj index 86637f232f..d284b205c4 100644 --- a/system-int-test/test/cmr/system_int_test/search/granule/granule_platform_instrument_sensor_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/granule/granule_platform_instrument_sensor_search_test.clj @@ -28,8 +28,8 @@ pr5 (dg/platform-ref {:short-name "platform-ONE"}) pr6 (dg/platform-ref {:short-name "platform-x"}) pr7 (dg/platform-ref {:short-name "PLATform-X"}) - coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 1 {:Platforms [p1 p2 p3 p4]})) - coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:Platforms [p0 p5 p6 p7]})) + coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 1 {:Platforms [p1 p2 p3 p4]}) {:validate-keywords false}) + coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:Platforms [p0 p5 p6 p7]}) {:validate-keywords false}) gran1 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:platform-refs [pr1]})) gran2 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:platform-refs [pr1 pr2]})) gran3 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:platform-refs [pr2]})) @@ -130,8 +130,8 @@ pr6 (dg/platform-ref {:short-name "platform-6" :instrument-refs [ir5]}) pr7 (dg/platform-ref {:short-name "platform-7" :instrument-refs [ir6]}) pr8 (dg/platform-ref {:short-name "platform-8" :instrument-refs [ir7]}) - coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 1 {:ShortName "SHORT1" :Platforms [p1 p2 p3 p4 p5]})) - coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:Platforms [p0 p6 p7 p8]})) + coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 1 {:ShortName "SHORT1" :Platforms [p1 p2 p3 p4 p5]}) {:validate-keywords false}) + coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:Platforms [p0 p6 p7 p8]}) {:validate-keywords false}) gran1 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "gran1" :platform-refs [pr1]})) gran2 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "gran2" :platform-refs [pr1 pr2]})) gran3 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "gran3" :platform-refs [pr2]})) @@ -265,8 +265,8 @@ pr7 (dg/platform-ref {:short-name "platform-7" :instrument-refs [ir6]}) pr8 (dg/platform-ref {:short-name "platform-8" :instrument-refs [ir7]}) pr9 (dg/platform-ref {:short-name "platform-9" :instrument-refs [ir8]}) - coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 1 {:Platforms [p1 p2 p3 p4 p5 p6]})) - coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:Platforms [p0 p7 p8 p9]})) + coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 1 {:Platforms [p1 p2 p3 p4 p5 p6]}) {:validate-keywords false}) + coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:Platforms [p0 p7 p8 p9]}) {:validate-keywords false}) gran1 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:platform-refs [pr1]})) gran2 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:platform-refs [pr1 pr2]})) gran3 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:platform-refs [pr2]})) diff --git a/system-int-test/test/cmr/system_int_test/search/granule/granule_psa_search_test.clj b/system-int-test/test/cmr/system_int_test/search/granule/granule_psa_search_test.clj index 36544e9fd9..5885f2a8de 100644 --- a/system-int-test/test/cmr/system_int_test/search/granule/granule_psa_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/granule/granule_psa_search_test.clj @@ -62,7 +62,7 @@ psa2 (dc/psa {:name "dts" :data-type :datetime-string}) psa3 (dc/psa {:name "ts" :data-type :time-string}) psa4 (dc/psa {:name "ds" :data-type :date-string}) - coll (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2 psa3 psa4]})) + coll (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2 psa3 psa4]}) {:validate-keywords false}) gran1 (d/ingest "PROV1" (dg/granule coll {:product-specific-attributes [(dg/psa "bool" [true])]})) gran2 (d/ingest "PROV1" (dg/granule coll {:product-specific-attributes [(dg/psa "bool" [false])]})) @@ -103,16 +103,16 @@ psa3 (dc/psa {:name "charlie" :data-type :string :value "foo"}) psa4 (dc/psa {:name "case" :data-type :string}) - coll1 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2]})) + coll1 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2]}) {:validate-keywords false}) gran1 (d/ingest "PROV1" (dg/granule coll1 {:product-specific-attributes [(dg/psa "alpha" ["ab" "bc"]) (dg/psa "bravo" ["cd" "bf"])]})) gran2 (d/ingest "PROV1" (dg/granule coll1 {:product-specific-attributes [(dg/psa "bravo" ["ab"])]})) - coll2 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa2 psa3]})) + coll2 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa2 psa3]}) {:validate-keywords false}) gran3 (d/ingest "PROV1" (dg/granule coll2 {:product-specific-attributes [(dg/psa "bravo" ["aa" "bf"])]})) gran4 (d/ingest "PROV1" (dg/granule coll2 {:product-specific-attributes [(dg/psa "charlie" ["az"])]})) - coll3 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa4]})) + coll3 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa4]}) {:validate-keywords false}) gran5 (d/ingest "PROV1" (dg/granule coll3 {:product-specific-attributes [(dg/psa "case" ["UP"])]}))] (index/wait-until-indexed) (testing "search by value" @@ -275,12 +275,12 @@ (let [psa1 (dc/psa {:name "alpha" :data-type :float}) psa2 (dc/psa {:name "bravo" :data-type :float}) psa3 (dc/psa {:name "charlie" :data-type :float :value 45}) - coll1 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2]})) + coll1 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2]}) {:validate-keywords false}) gran1 (d/ingest "PROV1" (dg/granule coll1 {:product-specific-attributes [(dg/psa "alpha" [10.5 123]) (dg/psa "bravo" [-12])]})) gran2 (d/ingest "PROV1" (dg/granule coll1 {:product-specific-attributes [(dg/psa "bravo" [10.5 123])]})) - coll2 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2 psa3]})) + coll2 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2 psa3]}) {:validate-keywords false}) gran3 (d/ingest "PROV1" (dg/granule coll2 {:product-specific-attributes [(dg/psa "alpha" [14]) (dg/psa "bravo" [13.7 123])]})) gran4 (d/ingest "PROV1" (dg/granule coll2 {:product-specific-attributes [(dg/psa "charlie" [14])]}))] @@ -363,12 +363,12 @@ (let [psa1 (dc/psa {:name "alpha" :data-type :int}) psa2 (dc/psa {:name "bravo" :data-type :int}) psa3 (dc/psa {:name "charlie" :data-type :int :value 45}) - coll1 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2]})) + coll1 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2]}) {:validate-keywords false}) gran1 (d/ingest "PROV1" (dg/granule coll1 {:product-specific-attributes [(dg/psa "alpha" [10 123]) (dg/psa "bravo" [-12])]})) gran2 (d/ingest "PROV1" (dg/granule coll1 {:product-specific-attributes [(dg/psa "bravo" [10 123])]})) - coll2 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2 psa3]})) + coll2 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2 psa3]}) {:validate-keywords false}) gran3 (d/ingest "PROV1" (dg/granule coll2 {:product-specific-attributes [(dg/psa "alpha" [14]) (dg/psa "bravo" [13 123])]})) gran4 (d/ingest "PROV1" (dg/granule coll2 {:product-specific-attributes [(dg/psa "charlie" [14])]}))] @@ -459,7 +459,7 @@ (let [psa1 (dc/psa {:name "alpha" :data-type :datetime}) psa2 (dc/psa {:name "bravo" :data-type :datetime}) psa3 (dc/psa {:name "charlie" :data-type :datetime :value (d/make-datetime 45 false)}) - coll1 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2]})) + coll1 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2]}) {:validate-keywords false}) gran1 (d/ingest "PROV1" (dg/granule coll1 {:product-specific-attributes [(dg/psa "alpha" [(d/make-datetime 10) (d/make-datetime 123)]) @@ -469,7 +469,7 @@ [(dg/psa "bravo" [(d/make-datetime 10) (d/make-datetime 123)])]})) - coll2 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2 psa3]})) + coll2 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2 psa3]}) {:validate-keywords false}) gran3 (d/ingest "PROV1" (dg/granule coll2 {:product-specific-attributes [(dg/psa "alpha" [(d/make-datetime 14)]) @@ -572,7 +572,7 @@ (let [psa1 (dc/psa {:name "alpha" :data-type :time}) psa2 (dc/psa {:name "bravo" :data-type :time}) psa3 (dc/psa {:name "charlie" :data-type :time :value (d/make-time 45 false)}) - coll1 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2]})) + coll1 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2]}) {:validate-keywords false}) gran1 (d/ingest "PROV1" (dg/granule coll1 {:product-specific-attributes [(dg/psa "alpha" [(d/make-time 10) (d/make-time 23)]) @@ -582,7 +582,7 @@ [(dg/psa "bravo" [(d/make-time 10) (d/make-time 23)])]})) - coll2 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2 psa3]})) + coll2 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2 psa3]}) {:validate-keywords false}) gran3 (d/ingest "PROV1" (dg/granule coll2 {:product-specific-attributes [(dg/psa "alpha" [(d/make-time 14)]) @@ -685,7 +685,7 @@ (let [psa1 (dc/psa {:name "alpha" :data-type :date}) psa2 (dc/psa {:name "bravo" :data-type :date}) psa3 (dc/psa {:name "charlie" :data-type :date :value (d/make-date 45 false)}) - coll1 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2]})) + coll1 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2]}) {:validate-keywords false}) gran1 (d/ingest "PROV1" (dg/granule coll1 {:product-specific-attributes [(dg/psa "alpha" [(d/make-date 10) (d/make-date 23)]) @@ -695,7 +695,7 @@ [(dg/psa "bravo" [(d/make-date 10) (d/make-date 23)])]})) - coll2 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2 psa3]})) + coll2 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2 psa3]}) {:validate-keywords false}) gran3 (d/ingest "PROV1" (dg/granule coll2 {:product-specific-attributes [(dg/psa "alpha" [(d/make-date 14)]) @@ -801,8 +801,8 @@ psa4 (dc/psa {:name "delta" :data-type :string}) psa5 (dc/psa {:name "alpha" :data-type :time}) psa6 (dc/psa {:name "sigma" :data-type :string}) - coll1 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2 psa3]})) - coll2 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa4 psa5 psa6]})) + coll1 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa1 psa2 psa3]}) {:validate-keywords false}) + coll2 (d/ingest "PROV1" (dc/collection {:product-specific-attributes [psa4 psa5 psa6]}) {:validate-keywords false}) gran1 (d/ingest "PROV1" (dg/granule coll1 {:product-specific-attributes [(dg/psa "alpha" ["atlantic"])]})) gran2 (d/ingest "PROV1" (dg/granule coll1 {:product-specific-attributes [(dg/psa "beta" ["wind"])]})) gran3 (d/ingest "PROV1" (dg/granule coll1 diff --git a/system-int-test/test/cmr/system_int_test/search/granule/granule_retrieval_test.clj b/system-int-test/test/cmr/system_int_test/search/granule/granule_retrieval_test.clj index 1d1364312c..e27ff0e4ee 100644 --- a/system-int-test/test/cmr/system_int_test/search/granule/granule_retrieval_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/granule/granule_retrieval_test.clj @@ -21,7 +21,8 @@ (deftest retrieve-granule-by-cmr-concept-id (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection - {:Projects (data-umm-cmn/projects "ABC" "KLM" "XYZ")})) + {:Projects (data-umm-cmn/projects "ABC" "KLM" "XYZ")}) + {:validate-keywords false}) gran1 (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll1 (:concept-id coll1) {:granule-ur "Granule1" diff --git a/system-int-test/test/cmr/system_int_test/search/granule/granule_search_format_test.clj b/system-int-test/test/cmr/system_int_test/search/granule/granule_search_format_test.clj index b75a756fd1..764a9620d3 100644 --- a/system-int-test/test/cmr/system_int_test/search/granule/granule_search_format_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/granule/granule_search_format_test.clj @@ -6,7 +6,6 @@ [clojure.java.io :as io] [clojure.string :as string] [clojure.test :refer :all] - [cmr.common.concepts :as cu] [cmr.common.mime-types :as mt] [cmr.common.util :as util] [cmr.spatial.line-string :as l] @@ -64,7 +63,7 @@ (is (= (:concept-id (first (:items response))) (:concept-id gran-7340)))))) (deftest simple-search-test - (let [c1-echo (d/ingest "PROV1" (dc/collection) {:format :echo10}) + (let [c1-echo (d/ingest "PROV1" (dc/collection) {:format :echo10 :validate-keywords false}) g1-echo (d/ingest "PROV1" (dg/granule c1-echo {:granule-ur "g1" :producer-gran-id "p1"}) {:format :echo10})] @@ -104,7 +103,8 @@ "PROV1" (data-umm-c/collection {:EntryTitle "UMM-G-Test" :ShortName "U-G-T" - :Version "V1"})) + :Version "V1"}) + {:validate-keywords false}) granule (dg/granule-with-umm-spec-collection collection (:concept-id collection) @@ -149,8 +149,8 @@ :native umm-g-gran-concept-id umm-g-gran nil "native")))) (deftest search-granules-in-xml-metadata - (let [c1-echo (d/ingest "PROV1" (dc/collection) {:format :echo10}) - c2-smap (d/ingest "PROV2" (dc/collection) {:format :iso-smap}) + (let [c1-echo (d/ingest "PROV1" (dc/collection) {:format :echo10 :validate-keywords false}) + c2-smap (d/ingest "PROV2" (dc/collection) {:format :iso-smap :validate-keywords false}) g1-echo (d/ingest "PROV1" (dg/granule c1-echo {:granule-ur "g1" :producer-gran-id "p1"}) {:format :echo10}) g2-echo (d/ingest "PROV1" (dg/granule c1-echo {:granule-ur "g2" @@ -165,7 +165,7 @@ :provider-id "PROV1") concept (-> (d/item->concept item :echo10) (update :metadata #(str "" %))) - response (ingest/ingest-concept concept) + response (ingest/ingest-concept concept {:validate-keywords false}) _ (is (= 201 (:status response))) g5-echo (assoc item :concept-id (:concept-id response) @@ -387,8 +387,8 @@ (let [ru1 (dc/related-url {:type "GET DATA" :url "http://example.com"}) ru2 (dc/related-url {:type "GET DATA" :url "http://example2.com"}) ru3 (dc/related-url {:type "GET RELATED VISUALIZATION" :url "http://example.com/browse"}) - coll1 (d/ingest "PROV1" (dc/collection {:beginning-date-time "1970-01-01T12:00:00Z"})) - coll2 (d/ingest "PROV1" (dc/collection {:beginning-date-time "1970-01-01T12:00:00Z"})) + coll1 (d/ingest "PROV1" (dc/collection {:beginning-date-time "1970-01-01T12:00:00Z"}) {:validate-keywords false}) + coll2 (d/ingest "PROV1" (dc/collection {:beginning-date-time "1970-01-01T12:00:00Z"}) {:validate-keywords false}) gran1 (d/ingest "PROV1" (dg/granule coll1 {:granule-ur "Granule1" :beginning-date-time "2010-01-01T12:00:00Z" :ending-date-time "2010-01-11T12:00:00Z" @@ -452,10 +452,12 @@ :url "https://ghrc.nsstc.nasa.gov/opendap/ssmis/f16/daily/data/"}) coll1 (d/ingest "PROV1" (dc/collection {:entry-title "Dataset1" :beginning-date-time "1970-01-01T12:00:00Z" - :spatial-coverage (dc/spatial {:gsr :geodetic})})) + :spatial-coverage (dc/spatial {:gsr :geodetic})}) + {:validate-keywords false}) coll2 (d/ingest "PROV1" (dc/collection {:entry-title "Dataset2" :beginning-date-time "1970-01-01T12:00:00Z" - :related-urls [ru4 ru5]})) + :related-urls [ru4 ru5]}) + {:validate-keywords false}) coll3 (d/ingest "PROV1" (dc/collection {:entry-title "OrbitDataset" :beginning-date-time "1970-01-01T12:00:00Z" :spatial-coverage (dc/spatial {:gsr :orbit @@ -463,10 +465,11 @@ :period 97.87 :swath-width 390.0 :start-circular-latitude -90.0 - :number-of-orbits 1.0}})})) + :number-of-orbits 1.0}})}) + {:validate-keywords false}) make-gran (fn [coll attribs] - (d/ingest "PROV1" (dg/granule coll attribs))) + (d/ingest "PROV1" (dg/granule coll attribs) {:validate-keywords false})) ;; polygon with holes outer (umm-s/ords->ring -5.26,-2.59, 11.56,-2.77, 10.47,8.71, -5.86,8.63, -5.26,-2.59) @@ -624,7 +627,8 @@ ru3 (dc/related-url {:type "GET RELATED VISUALIZATION" :url "http://example.com/browse"}) coll1 (d/ingest "PROV1" (dc/collection {:entry-title "Dataset1" :beginning-date-time "1970-01-01T12:00:00Z" - :spatial-coverage (dc/spatial {:gsr :no-spatial})})) + :spatial-coverage (dc/spatial {:gsr :no-spatial})}) + {:validate-keywords false}) make-gran (fn [coll attribs] (d/ingest "PROV1" (dg/granule coll attribs))) @@ -708,7 +712,8 @@ (deftest search-granule-stac (let [coll1 (d/ingest "PROV1" (dc/collection {:entry-title "Dataset1" :beginning-date-time "1970-01-01T12:00:00Z" - :spatial-coverage (dc/spatial {:gsr :geodetic})})) + :spatial-coverage (dc/spatial {:gsr :geodetic})}) + {:validate-keywords false}) coll-concept-id (:concept-id coll1) gran-spatial (dg/spatial (m/mbr 10 30 20 0)) gran1 (d/ingest "PROV1" (dg/granule coll1 {:granule-ur "Granule1" @@ -981,7 +986,8 @@ (deftest granule-concept-stac-retrieval-test (let [coll-metadata (slurp (io/resource "stac-test/C1299783579-LPDAAC_ECS.xml")) {coll-concept-id :concept-id} (ingest/ingest-concept - (ingest/concept :collection "PROV1" "foo" :echo10 coll-metadata)) + (ingest/concept :collection "PROV1" "foo" :echo10 coll-metadata) + {:validate-keywords false}) gran-metadata (slurp (io/resource "stac-test/G1327299284-LPDAAC_ECS.xml")) {gran-concept-id :concept-id} (ingest/ingest-concept (ingest/concept :granule "PROV1" "foo" :echo10 gran-metadata)) @@ -1026,7 +1032,8 @@ (deftest search-no-spatial-concept-stac (let [coll1 (d/ingest "PROV1" (dc/collection {:entry-title "Dataset1" - :beginning-date-time "1970-01-01T12:00:00Z"})) + :beginning-date-time "1970-01-01T12:00:00Z"}) + {:validate-keywords false}) coll-concept-id (:concept-id coll1) gran-spatial (dg/spatial (m/mbr 10 30 20 0)) gran1 (d/ingest "PROV1" (dg/granule coll1 {:granule-ur "Granule1" @@ -1095,7 +1102,8 @@ "PROV1" "foo" {:format :umm-json :version "1.17.0"} - coll-metadata)) + coll-metadata) + {:validate-keywords false}) {aa-gran-concept-id :concept-id} (ingest/ingest-concept (ingest/concept :granule "PROV1" diff --git a/system-int-test/test/cmr/system_int_test/search/granule/granule_search_with_json_query_test.clj b/system-int-test/test/cmr/system_int_test/search/granule/granule_search_with_json_query_test.clj index 36bc788374..f3b6eda5d1 100644 --- a/system-int-test/test/cmr/system_int_test/search/granule/granule_search_with_json_query_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/granule/granule_search_with_json_query_test.clj @@ -28,11 +28,13 @@ :TemporalExtents [(data-umm-cmn/temporal-extent {:beginning-date-time "2012-01-01T00:00:00Z" - :ending-date-time "2012-02-14T12:00:00Z"})]})) + :ending-date-time "2012-02-14T12:00:00Z"})]}) + {:validate-keywords false}) coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:EntryTitle "AnotherCollectionV1" :ShortName "S2" :Version "V2" - :Projects (data-umm-cmn/projects "ABC" "KLM" "XYZ")})) + :Projects (data-umm-cmn/projects "ABC" "KLM" "XYZ")}) + {:validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "OneCollectionV1" :ShortName "S3" :Version "V3" @@ -41,12 +43,14 @@ :Coordinate1 {:MinimumValue 100 :MaximumValue 200} :Coordinate2 {:MinimumValue 300 - :MaximumValue 400}}]})) + :MaximumValue 400}}]}) + {:validate-keywords false}) coll4 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection {:EntryTitle "OtherCollectionV1" :ShortName "S4" :Version "V4" :SpatialExtent (data-umm-c/spatial - {:gsr "GEODETIC"})})) + {:gsr "GEODETIC"})}) + {:validate-keywords false}) coll1-cid (get-in coll1 [:concept-id]) coll2-cid (get-in coll2 [:concept-id]) diff --git a/system-int-test/test/cmr/system_int_test/search/granule/granule_sorting_search_test.clj b/system-int-test/test/cmr/system_int_test/search/granule/granule_sorting_search_test.clj index 804feaad4d..36ff31a784 100644 --- a/system-int-test/test/cmr/system_int_test/search/granule/granule_sorting_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/granule/granule_sorting_search_test.clj @@ -67,8 +67,9 @@ "-revision_date" (reverse [g1 g2 g3 g4 g5]))))) (deftest granule-campaign-sorting-test - (let [coll (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection - {:Projects (data-umm-cmn/projects "c10" "c20" "c30" "c40" "c50" "c41" "c51")})) + (let [coll (d/ingest-umm-spec-collection "PROV1" + (data-umm-c/collection {:Projects (data-umm-cmn/projects "c10" "c20" "c30" "c40" "c50" "c41" "c51")}) + {:validate-keywords false}) make-gran (fn [& campaigns] (d/ingest "PROV1" (dg/granule-with-umm-spec-collection coll (:concept-id coll) {:project-refs campaigns}))) g1 (make-gran "c10" "c41") @@ -197,9 +198,9 @@ "-end_date" [g8 g4 g3 g7 g6 g2 g1 g5 g11])))) (deftest granule-platform-sorting-test - (let [coll (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:Platforms - (map #(data-umm-cmn/platform {:ShortName %}) - ["c10" "c41" "c20" "c51" "c30" "c40" "c50"])})) + (let [coll (d/ingest-umm-spec-collection "PROV1" + (data-umm-c/collection {:Platforms (map #(data-umm-cmn/platform {:ShortName %}) ["c10" "c41" "c20" "c51" "c30" "c40" "c50"])}) + {:validate-keywords false}) make-gran (fn [& platforms] (d/ingest "PROV1" (dg/granule-with-umm-spec-collection @@ -221,11 +222,9 @@ "-platform" [g2 g5 g1 g4 g3]))) (deftest granule-instrument-sorting-test - (let [coll (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection {:Platforms - [(data-umm-cmn/platform - {:ShortName "platform" - :Instruments (map #(data-umm-cmn/instrument {:ShortName %}) - ["c10" "c41" "c20" "c51" "c30" "c40" "c50"])})]})) + (let [coll (d/ingest-umm-spec-collection "PROV1" + (data-umm-c/collection {:Platforms [(data-umm-cmn/platform {:ShortName "platform" :Instruments (map #(data-umm-cmn/instrument {:ShortName %}) ["c10" "c41" "c20" "c51" "c30" "c40" "c50"])})]}) + {:validate-keywords false}) make-gran (fn [& instruments] (d/ingest "PROV1" (dg/granule-with-umm-spec-collection @@ -252,14 +251,16 @@ "-instrument" [g2 g5 g1 g4 g3]))) (deftest granule-sensor-sorting-test - (let [coll (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection - {:Platforms - [(data-umm-cmn/platform - {:ShortName "platform" - :Instruments [(data-umm-cmn/instrument - {:ShortName "instrument" - :ComposedOf (map #(data-umm-cmn/instrument {:ShortName %}) - ["c10" "c41" "c20" "c51" "c30" "c40" "c50"])})]})]})) + (let [coll (d/ingest-umm-spec-collection "PROV1" + (data-umm-c/collection + {:Platforms + [(data-umm-cmn/platform + {:ShortName "platform" + :Instruments [(data-umm-cmn/instrument + {:ShortName "instrument" + :ComposedOf (map #(data-umm-cmn/instrument {:ShortName %}) + ["c10" "c41" "c20" "c51" "c30" "c40" "c50"])})]})]}) + {:validate-keywords false}) make-gran (fn [& sensors] (d/ingest "PROV1" (dg/granule-with-umm-spec-collection diff --git a/system-int-test/test/cmr/system_int_test/search/humanizer/humanizer_test.clj b/system-int-test/test/cmr/system_int_test/search/humanizer/humanizer_test.clj index 68d601fc51..6030f6885c 100644 --- a/system-int-test/test/cmr/system_int_test/search/humanizer/humanizer_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/humanizer/humanizer_test.clj @@ -207,7 +207,8 @@ :Term "Term1"}] :concept-id "C1-PROV1") {:format :umm-json - :accept-format :json}) + :accept-format :json + :validate-keywords false}) _ (index/wait-until-indexed) ;; Humanizers use the cached collection metadata - clear to make sure we have the latest @@ -243,7 +244,8 @@ :concept-id "C2-PROV1") {:format :umm-json - :accept-format :json}) + :accept-format :json + :validate-keywords false}) _ (index/wait-until-indexed) _ (cache-util/refresh-cache (url/refresh-collection-metadata-cache-url) (transmit-config/echo-system-token)) @@ -273,7 +275,8 @@ :Term "Term1"}] :concept-id "C1-PROV1") {:format :umm-json - :accept-format :json}) + :accept-format :json + :validate-keywords false}) (index/wait-until-indexed) (cache-util/refresh-cache (url/refresh-collection-metadata-cache-url) (transmit-config/echo-system-token)) (is (= 200 (:status (search/get-humanizers-report-raw))))) @@ -294,7 +297,8 @@ :Term "Term1"}] :concept-id "C1-PROV1") {:format :umm-json - :accept-format :json}) + :accept-format :json + :validate-keywords false}) (index/wait-until-indexed) (cache-util/refresh-cache (url/refresh-collection-metadata-cache-url) (transmit-config/echo-system-token)) (is (= 200 (:status (search/get-humanizers-report-raw {:regenerate true diff --git a/system-int-test/test/cmr/system_int_test/search/misc/coll_archive_ctr_search_test.clj b/system-int-test/test/cmr/system_int_test/search/misc/coll_archive_ctr_search_test.clj index e737000821..28f79c385d 100644 --- a/system-int-test/test/cmr/system_int_test/search/misc/coll_archive_ctr_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/misc/coll_archive_ctr_search_test.clj @@ -13,43 +13,51 @@ (use-fixtures :each (ingest/reset-fixture {"provguid1" "PROV1" "provguid2" "PROV2"})) (deftest search-colls-by-archive-center-names - (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 1 {})) - coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:DataCenters []})) + (let [coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 1 {}) {:validate-keywords false}) + coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 2 {:DataCenters []}) {:validate-keywords false}) coll3 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 3 {:DataCenters [(data-umm-cmn/data-center {:Roles ["PROCESSOR"] - :ShortName "Larc"})]})) + :ShortName "Larc"})]}) + {:validate-keywords false}) coll4 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 4 {:DataCenters [(data-umm-cmn/data-center {:Roles ["ARCHIVER"] - :ShortName "Larc"})]})) + :ShortName "Larc"})]}) + {:validate-keywords false}) coll5 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection 5 {:DataCenters [(data-umm-cmn/data-center {:Roles ["ARCHIVER"] :ShortName "SEDAC AC"}) (data-umm-cmn/data-center {:Roles ["PROCESSOR"] - :ShortName "SEDAC PC"})]})) + :ShortName "SEDAC PC"})]}) + {:validate-keywords false}) coll6 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection 6 {:DataCenters [(data-umm-cmn/data-center {:Roles ["ARCHIVER"] - :ShortName "Larc"})]})) + :ShortName "Larc"})]}) + {:validate-keywords false}) coll7 (d/ingest-umm-spec-collection "PROV2" (data-umm-c/collection 7 {:DataCenters [(data-umm-cmn/data-center {:Roles ["ARCHIVER"] :ShortName "Sedac AC"}) (data-umm-cmn/data-center {:Roles ["PROCESSOR"] - :ShortName "Sedac"})]})) + :ShortName "Sedac"})]}) + {:validate-keywords false}) ;; KMS collections, but with different case for the short-names kms-coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 8 {:DataCenters [(data-umm-cmn/data-center {:Roles ["ARCHIVER"] - :ShortName "Ucar/ncar/eoL/ceoPdm"})]})) + :ShortName "Ucar/ncar/eoL/ceoPdm"})]}) + {:validate-keywords false}) kms-coll2 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 9 {:DataCenters [(data-umm-cmn/data-center {:Roles ["ARCHIVER"] - :ShortName "Doi/uSGs/Cmg/wHSc"})]})) + :ShortName "Doi/uSGs/Cmg/wHSc"})]}) + {:validate-keywords false}) ;; DIF collections support distribution-center data centers dif-coll1 (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection 10 {:DataCenters [(data-umm-cmn/data-center {:Roles ["DISTRIBUTOR"] :ShortName "Dist center"})]}) - {:format :dif})] + {:format :dif + :validate-keywords false})] (index/wait-until-indexed) diff --git a/system-int-test/test/cmr/system_int_test/search/misc/harvesting_test.clj b/system-int-test/test/cmr/system_int_test/search/misc/harvesting_test.clj index 758e23ef23..99fdbc63ed 100644 --- a/system-int-test/test/cmr/system_int_test/search/misc/harvesting_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/misc/harvesting_test.clj @@ -18,11 +18,11 @@ (deftest harvest-granules (let [format-key :echo10 _ (dev-system-util/freeze-time! "2010-01-01T10:00:00Z") - coll1-echo (data2-core/ingest "PROV1" (data2-collection/collection) {:format :echo10}) + coll1-echo (data2-core/ingest "PROV1" (data2-collection/collection) {:format :echo10 :validate-keywords false}) coll1-concept-id (:concept-id coll1-echo) _ (dev-system-util/freeze-time! "2012-01-01T10:00:00Z") - coll2-echo (data2-core/ingest "PROV1" (data2-collection/collection) {:format :echo10}) + coll2-echo (data2-core/ingest "PROV1" (data2-collection/collection) {:format :echo10 :validate-keywords false}) coll2-concept-id (:concept-id coll2-echo) _ (dev-system-util/freeze-time! "2010-01-01T10:00:00Z") diff --git a/system-int-test/test/cmr/system_int_test/search/misc/provider_holdings_test.clj b/system-int-test/test/cmr/system_int_test/search/misc/provider_holdings_test.clj index 8d63c50192..b81d2ce8b6 100644 --- a/system-int-test/test/cmr/system_int_test/search/misc/provider_holdings_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/misc/provider_holdings_test.clj @@ -52,19 +52,19 @@ [] (let [;; Provider 1 prov1-colls (doall (for [_ (range 0 prov1-collection-count)] - (d/ingest "PROV1" (dc/collection)))) + (d/ingest "PROV1" (dc/collection) {:validate-keywords false}))) prov1-granule-counts (map #(* prov1-grans-increment-count %) (range 1 (inc prov1-collection-count))) prov1-holdings (map (partial collection-holding "PROV1") prov1-colls prov1-granule-counts) ;; Provider 2 prov2-colls (doall (for [_ (range 0 prov2-collection-count)] - (d/ingest "PROV2" (dc/collection)))) + (d/ingest "PROV2" (dc/collection) {:validate-keywords false}))) prov2-granule-counts (map #(* prov2-grans-increment-count %) (range 1 (inc prov2-collection-count))) prov2-holdings (map (partial collection-holding "PROV2") prov2-colls prov2-granule-counts) ;; Provider 3 prov3-colls (doall (for [n (range 0 prov3-collection-count)] - (d/ingest "PROV3" (dc/collection)))) + (d/ingest "PROV3" (dc/collection) {:validate-keywords false}))) prov3-holdings (doall (map #(collection-holding "PROV3" % 0) prov3-colls))] ;; Create provider 1 granules diff --git a/system-int-test/test/cmr/system_int_test/search/misc/sitemaps_test.clj b/system-int-test/test/cmr/system_int_test/search/misc/sitemaps_test.clj index 5bd9df9211..193b23c108 100644 --- a/system-int-test/test/cmr/system_int_test/search/misc/sitemaps_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/misc/sitemaps_test.clj @@ -44,7 +44,8 @@ (assoc :ShortName (str "s" n)) (assoc :EntryTitle (str "Collection Item " n))) {:format :umm-json - :accept-format :json}))) + :accept-format :json + :validate-keywords false}))) _ (index/wait-until-indexed) [c1-p3 c2-p3 c3-p3] (doall (for [n (range 4 7)] (d/ingest-umm-spec-collection @@ -56,7 +57,8 @@ {:DOI (str "doi" n) :Authority (str "auth" n)}))) {:format :umm-json - :accept-format :json})))] + :accept-format :json + :validate-keywords false})))] (reset! test-collections {"PROV1" (map :concept-id [c1-p1 c2-p1 c3-p1]) "PROV2" (map :concept-id [c1-p2 c2-p2 c3-p2]) diff --git a/system-int-test/test/cmr/system_int_test/search/misc/umm_json_search_test.clj b/system-int-test/test/cmr/system_int_test/search/misc/umm_json_search_test.clj index 7e0a9c7eff..f928987fc7 100644 --- a/system-int-test/test/cmr/system_int_test/search/misc/umm_json_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/misc/umm_json_search_test.clj @@ -21,7 +21,8 @@ (let [coll1-1 (d/ingest "PROV1" (du/umm-spec-collection {:entry-title "et1" :version-id "v1" :short-name "s1"}) - {:user-id "user1"}) + {:user-id "user1" + :validate-keywords false}) concept1 {:provider-id "PROV1" :concept-type :collection :native-id (:entry-title coll1-1)} @@ -30,14 +31,17 @@ (ingest/delete-concept concept1 {:user-id "user2"})) coll1-3 (d/ingest "PROV1" (du/umm-spec-collection {:entry-title "et1" :version-id "v2" - :short-name "s1"})) + :short-name "s1"}) + {:validate-keywords false}) coll2-1 (d/ingest "PROV1" (du/umm-spec-collection {:entry-title "et2" :version-id "v1" - :short-name "s2"})) + :short-name "s2"}) + {:validate-keywords false}) coll2-2 (d/ingest "PROV1" (du/umm-spec-collection {:entry-title "et2" :version-id "v2" - :short-name "s2"})) + :short-name "s2"}) + {:validate-keywords false}) concept2 {:provider-id "PROV1" :concept-type :collection :native-id (:entry-title coll2-2)} @@ -46,7 +50,8 @@ coll3 (d/ingest "PROV2" (du/umm-spec-collection {:entry-title "et3" :version-id "v4" :short-name "s1"}) - {:user-id "user3"})] + {:user-id "user3" + :validate-keywords false})] (index/wait-until-indexed) (testing "find collections in legacy UMM JSON format" (are3 [url-extension collections params] diff --git a/system-int-test/test/cmr/system_int_test/search/service/collection_service_search_test.clj b/system-int-test/test/cmr/system_int_test/search/service/collection_service_search_test.clj index 9184cf8f3a..fc60aa37f2 100644 --- a/system-int-test/test/cmr/system_int_test/search/service/collection_service_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/service/collection_service_search_test.clj @@ -25,10 +25,12 @@ (let [token (e/login (s/context) "user1") coll1 (d/ingest "PROV1" (dc/collection {:entry-title "ET1" :short-name "S1" - :version-id "V1"})) + :version-id "V1"}) + {:validate-keywords false}) coll2 (d/ingest "PROV1" (dc/collection {:entry-title "ET2" :short-name "S2" - :version-id "V2"})) + :version-id "V2"}) + {:validate-keywords false}) ;; create services {serv1-concept-id :concept-id} (service-util/ingest-service-with-attrs {:native-id "serv1" @@ -185,10 +187,12 @@ (let [token (e/login (s/context) "user1") coll1 (d/ingest "PROV1" (dc/collection {:entry-title "ET1" :short-name "S1" - :version-id "V1"})) + :version-id "V1"}) + {:validate-keywords false}) coll2 (d/ingest "PROV1" (dc/collection {:entry-title "ET2" :short-name "S2" - :version-id "V2"})) + :version-id "V2"}) + {:validate-keywords false}) ;; create services {serv1-concept-id :concept-id} (service-util/ingest-service-with-attrs @@ -255,13 +259,16 @@ (let [token (e/login (s/context) "user1") coll3 (d/ingest "PROV1" (dc/collection {:entry-title "ET3" :short-name "S3" - :version-id "V3"})) + :version-id "V3"}) + {:validate-keywords false}) coll4 (d/ingest "PROV1" (dc/collection {:entry-title "ET4" :short-name "S4" - :version-id "V4"})) + :version-id "V4"}) + {:validate-keywords false}) coll5 (d/ingest "PROV1" (dc/collection {:entry-title "ET5" :short-name "S5" - :version-id "V5"})) + :version-id "V5"}) + {:validate-keywords false}) ;; create services {serv8-concept-id :concept-id} (service-util/ingest-service-with-attrs @@ -395,7 +402,8 @@ (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection n {}) - {:token token}))) + {:token token + :validate-keywords false}))) ;; index the collections so that they can be found during service association _ (index/wait-until-indexed) ;; create services @@ -504,13 +512,16 @@ (let [token (e/login (s/context) "user1") coll1 (d/ingest "PROV1" (dc/collection {:entry-title "ET1" :short-name "S1" - :version-id "V1"})) + :version-id "V1"}) + {:validate-keywords false}) coll2 (d/ingest "PROV1" (dc/collection {:entry-title "ET2" :short-name "S2" - :version-id "V2"})) + :version-id "V2"}) + {:validate-keywords false}) coll3 (d/ingest "PROV1" (dc/collection {:entry-title "ET3" :short-name "S3" - :version-id "V3"})) + :version-id "V3"}) + {:validate-keywords false}) ;; create services {serv1-concept-id :concept-id} (service-util/ingest-service-with-attrs diff --git a/system-int-test/test/cmr/system_int_test/search/service/service_association_test.clj b/system-int-test/test/cmr/system_int_test/search/service/service_association_test.clj index 324b2e6533..d0e7175bd9 100644 --- a/system-int-test/test/cmr/system_int_test/search/service/service_association_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/service/service_association_test.clj @@ -40,7 +40,8 @@ (collection/collection {:short-name (str "S" n) :version-id (str "V" n) - :entry-title (str "ET" n)}))))) + :entry-title (str "ET" n)}) + {:validate-keywords false})))) all-prov1-colls [c1-p1 c2-p1 c3-p1 c4-p1] all-prov2-colls [c1-p2 c2-p2 c3-p2 c4-p2] token (echo-util/login (system/context) "user1") @@ -118,7 +119,7 @@ :Name "service1" :provider-id "PROV1"}) {:keys [concept-id revision-id]} (service-util/ingest-service serv-concept) - coll-concept-id (:concept-id (data-core/ingest "PROV1" (collection/collection)))] + coll-concept-id (:concept-id (data-core/ingest "PROV1" (collection/collection) {:validate-keywords false}))] (testing "Associate service using query sent with invalid content type" (are [associate-service-fn request-json] (= {:status 400, @@ -174,7 +175,8 @@ (data-core/ingest p (collection/collection {:short-name (str "S" n) :version-id (str "V" n) - :entry-title (str "ET" n)})))) + :entry-title (str "ET" n)}) + {:validate-keywords false}))) all-prov1-colls [c1-p1 c2-p1 c3-p1 c4-p1] all-prov2-colls [c1-p2 c2-p2 c3-p2 c4-p2] all-prov3-colls [c1-p3 c2-p3 c3-p3 c4-p3] @@ -236,7 +238,7 @@ token (echo-util/login (system/context) "user1") serv-concept (service-util/make-service-concept {:Name service-name}) {:keys [concept-id revision-id]} (service-util/ingest-service serv-concept) - coll-concept-id (:concept-id (data-core/ingest "PROV1" (collection/collection)))] + coll-concept-id (:concept-id (data-core/ingest "PROV1" (collection/collection) {:validate-keywords false}))] (testing "Dissociate service using query sent with invalid content type" (are [dissociate-service-fn request-json] @@ -278,9 +280,9 @@ (echo-util/grant-registered-users (system/context) (echo-util/coll-catalog-item-id "PROV1")) (testing "dissociate service with mixed success and failure response" - (let [coll1 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET1"})) - coll2 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET2"})) - coll3 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET3"})) + (let [coll1 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET1"}) {:validate-keywords false}) + coll2 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET2"}) {:validate-keywords false}) + coll3 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET3"}) {:validate-keywords false}) token (echo-util/login (system/context) "user1") service-name "service1" {:keys [concept-id]} (service-util/ingest-service-with-attrs {:Name service-name}) @@ -317,7 +319,7 @@ (echo-util/grant-registered-users (system/context) (echo-util/coll-catalog-item-id "PROV1")) (let [[coll1 coll2 coll3] (doall (for [n (range 1 4)] - (data-core/ingest "PROV1" (collection/collection)))) + (data-core/ingest "PROV1" (collection/collection) {:validate-keywords false}))) [coll1-id coll2-id coll3-id] (map :concept-id [coll1 coll2 coll3]) token (echo-util/login (system/context) "user1") {service1-concept-id :concept-id} (service-util/ingest-service-with-attrs diff --git a/system-int-test/test/cmr/system_int_test/search/tagging/collection_revisions_tag_search_test.clj b/system-int-test/test/cmr/system_int_test/search/tagging/collection_revisions_tag_search_test.clj index 41618836b3..ab2d830cf8 100644 --- a/system-int-test/test/cmr/system_int_test/search/tagging/collection_revisions_tag_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/tagging/collection_revisions_tag_search_test.clj @@ -22,18 +22,18 @@ (search/find-refs :collection params {:snake-kebab? false}))) (deftest search-collection-revisions-by-tag-test - (let [coll1-1 (d/ingest "PROV1" (dc/collection {:entry-title "et1"})) + (let [coll1-1 (d/ingest "PROV1" (dc/collection {:entry-title "et1"}) {:validate-keywords false}) concept1 {:provider-id "PROV1" :concept-type :collection :native-id (:entry-title coll1-1)} coll1-2-tombstone (merge (ingest/delete-concept concept1) concept1 {:deleted true}) - coll1-3 (d/ingest "PROV1" (dc/collection {:entry-title "et1"})) + coll1-3 (d/ingest "PROV1" (dc/collection {:entry-title "et1"}) {:validate-keywords false}) - coll2-1 (d/ingest "PROV1" (dc/collection {:entry-title "et2"})) - coll2-2 (d/ingest "PROV1" (dc/collection {:entry-title "et2"})) + coll2-1 (d/ingest "PROV1" (dc/collection {:entry-title "et2"}) {:validate-keywords false}) + coll2-2 (d/ingest "PROV1" (dc/collection {:entry-title "et2"}) {:validate-keywords false}) - coll3 (d/ingest "PROV2" (dc/collection {})) - coll4-1 (d/ingest "PROV2" (dc/collection {:entry-title "et4"})) + coll3 (d/ingest "PROV2" (dc/collection {}) {:validate-keywords false}) + coll4-1 (d/ingest "PROV2" (dc/collection {:entry-title "et4"}) {:validate-keywords false}) token (e/login (s/context) "user1") tag1 (tags/save-tag token (tags/make-tag {:tag-key "tag1"})) @@ -85,7 +85,7 @@ (assert-collection-refs-found [coll4-1] {:tag-data {"tag2" "land"} :all-revisions true}) ;; update the collection - (let [coll4-2 (d/ingest "PROV2" (dc/collection {:entry-title "et4"}))] + (let [coll4-2 (d/ingest "PROV2" (dc/collection {:entry-title "et4"}) {:validate-keywords false})] (index/wait-until-indexed) ;; Found the latest collection revision only without all-revisions true @@ -100,7 +100,7 @@ (assert-collection-refs-found [coll2-2] {:tag-data {"tag2" "cloud"} :all-revisions true}) ;; update the collection - (let [coll2-3 (d/ingest "PROV1" (dc/collection {:entry-title "et2"}))] + (let [coll2-3 (d/ingest "PROV1" (dc/collection {:entry-title "et2"}) {:validate-keywords false})] (index/wait-until-indexed) ;; Found no collections without all-revisions true diff --git a/system-int-test/test/cmr/system_int_test/search/tagging/collection_tag_search_test.clj b/system-int-test/test/cmr/system_int_test/search/tagging/collection_tag_search_test.clj index 693fbf1374..a4e0f0d4cb 100644 --- a/system-int-test/test/cmr/system_int_test/search/tagging/collection_tag_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/tagging/collection_tag_search_test.clj @@ -49,7 +49,8 @@ c1-p2 c2-p2 c3-p2] (for [p ["PROV1" "PROV2"] n (range 1 4)] (data-core/ingest p (collection/collection - {:entry-title (str "coll" n)}))) + {:entry-title (str "coll" n)}) + {:validate-keywords false})) ;; Wait until collections are indexed so tags can be associated with them _ (index/wait-until-indexed) @@ -202,8 +203,9 @@ (let [[coll1 coll2 coll3 coll4] (for [n (range 1 5)] (data-core/ingest "PROV1" (collection/collection - {:entry-title (str "coll" n)}))) - coll5 (data-core/ingest "PROV2" (collection/collection)) + {:entry-title (str "coll" n)}) + {:validate-keywords false})) + coll5 (data-core/ingest "PROV2" (collection/collection) {:validate-keywords false}) all-prov1-colls [coll1 coll2 coll3 coll4] user1-token (echo-util/login (system/context) "user1") @@ -370,7 +372,8 @@ (let [[coll1 coll2 coll3 coll4] (for [n (range 1 5)] (data-core/ingest "PROV1" (collection/collection - {:entry-title (str "coll" n)}))) + {:entry-title (str "coll" n)}) + {:validate-keywords false})) all-prov1-colls [coll1 coll2 coll3 coll4] user1-token (echo-util/login (system/context) "user1") tag1 (tags/save-tag @@ -429,7 +432,8 @@ (let [[coll1 coll2 coll3 coll4] (for [n (range 1 5)] (data-core/ingest "PROV1" (collection/collection - {:entry-title (str "coll" n)}))) + {:entry-title (str "coll" n)}) + {:validate-keywords false})) all-prov1-colls [coll1 coll2 coll3 coll4] user1-token (echo-util/login (system/context) "user1") tag1 (tags/save-tag diff --git a/system-int-test/test/cmr/system_int_test/search/tagging/enable_disable_test.clj b/system-int-test/test/cmr/system_int_test/search/tagging/enable_disable_test.clj index 20ac24f104..5e04841e5c 100644 --- a/system-int-test/test/cmr/system_int_test/search/tagging/enable_disable_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/tagging/enable_disable_test.clj @@ -87,7 +87,8 @@ :entry-title "coll1" :entry-id "coll1" :version-id "V1" - :short-name "short1"})) + :short-name "short1"}) + {:validate-keywords false}) tag1-key "tag1" tag1 (tags/make-tag {:tag-key tag1-key}) tag2-key "tag2" diff --git a/system-int-test/test/cmr/system_int_test/search/tagging/tag_association_collection_revisions_test.clj b/system-int-test/test/cmr/system_int_test/search/tagging/tag_association_collection_revisions_test.clj index 56e5e2df7a..ce7eb4e157 100644 --- a/system-int-test/test/cmr/system_int_test/search/tagging/tag_association_collection_revisions_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/tagging/tag_association_collection_revisions_test.clj @@ -32,22 +32,22 @@ (e/grant-registered-users (s/context) (e/coll-catalog-item-id "PROV1")) (e/grant-registered-users (s/context) (e/coll-catalog-item-id "PROV2")) - (let [coll1-1 (d/ingest "PROV1" (dc/collection {:entry-title "et1"})) + (let [coll1-1 (d/ingest "PROV1" (dc/collection {:entry-title "et1"}) {:validate-keywords false}) concept1 {:provider-id "PROV1" :concept-type :collection :native-id (:entry-title coll1-1)} coll1-2-tombstone (merge (ingest/delete-concept concept1) concept1 {:deleted true}) - coll1-3 (d/ingest "PROV1" (dc/collection {:entry-title "et1"})) + coll1-3 (d/ingest "PROV1" (dc/collection {:entry-title "et1"}) {:validate-keywords false}) - coll2-1 (d/ingest "PROV1" (dc/collection {:entry-title "et2"})) - coll2-2 (d/ingest "PROV1" (dc/collection {:entry-title "et2"})) + coll2-1 (d/ingest "PROV1" (dc/collection {:entry-title "et2"}) {:validate-keywords false}) + coll2-2 (d/ingest "PROV1" (dc/collection {:entry-title "et2"}) {:validate-keywords false}) concept2 {:provider-id "PROV1" :concept-type :collection :native-id (:entry-title coll2-2)} coll2-3-tombstone (merge (ingest/delete-concept concept2) concept2 {:deleted true}) - coll3 (d/ingest "PROV2" (dc/collection {})) - coll4 (d/ingest "PROV3" (dc/collection {})) + coll3 (d/ingest "PROV2" (dc/collection {}) {:validate-keywords false}) + coll4 (d/ingest "PROV3" (dc/collection {}) {:validate-keywords false}) token (e/login (s/context) "user1") tag-key "tag1"] @@ -164,22 +164,22 @@ (e/grant-registered-users (s/context) (e/coll-catalog-item-id "PROV1")) (e/grant-registered-users (s/context) (e/coll-catalog-item-id "PROV2")) - (let [coll1-1 (d/ingest "PROV1" (dc/collection {:entry-title "et1"})) + (let [coll1-1 (d/ingest "PROV1" (dc/collection {:entry-title "et1"}) {:validate-keywords false}) concept1 {:provider-id "PROV1" :concept-type :collection :native-id (:entry-title coll1-1)} coll1-2-tombstone (merge (ingest/delete-concept concept1) concept1 {:deleted true}) - coll1-3 (d/ingest "PROV1" (dc/collection {:entry-title "et1"})) + coll1-3 (d/ingest "PROV1" (dc/collection {:entry-title "et1"}) {:validate-keywords false}) - coll2-1 (d/ingest "PROV1" (dc/collection {:entry-title "et2"})) - coll2-2 (d/ingest "PROV1" (dc/collection {:entry-title "et2"})) + coll2-1 (d/ingest "PROV1" (dc/collection {:entry-title "et2"}) {:validate-keywords false}) + coll2-2 (d/ingest "PROV1" (dc/collection {:entry-title "et2"}) {:validate-keywords false}) concept2 {:provider-id "PROV1" :concept-type :collection :native-id (:entry-title coll2-2)} coll2-3-tombstone (merge (ingest/delete-concept concept2) concept2 {:deleted true}) - coll3 (d/ingest "PROV2" (dc/collection {})) - coll4 (d/ingest "PROV3" (dc/collection {})) + coll3 (d/ingest "PROV2" (dc/collection {}) {:validate-keywords false}) + coll4 (d/ingest "PROV3" (dc/collection {}) {:validate-keywords false}) token (e/login (s/context) "user1") tag-key "tag1"] @@ -300,11 +300,11 @@ (deftest associate-dissociate-tag-with-collection-revisions-test ;; Grant all collections in PROV1 (e/grant-registered-users (s/context) (e/coll-catalog-item-id "PROV1")) - (let [coll1-1 (d/ingest "PROV1" (dc/collection {:entry-title "et1"})) - coll1-2 (d/ingest "PROV1" (dc/collection {:entry-title "et1"})) - coll1-3 (d/ingest "PROV1" (dc/collection {:entry-title "et1"})) - coll2-1 (d/ingest "PROV1" (dc/collection {:entry-title "et2"})) - coll2-2 (d/ingest "PROV1" (dc/collection {:entry-title "et2"})) + (let [coll1-1 (d/ingest "PROV1" (dc/collection {:entry-title "et1"}) {:validate-keywords false}) + coll1-2 (d/ingest "PROV1" (dc/collection {:entry-title "et1"}) {:validate-keywords false}) + coll1-3 (d/ingest "PROV1" (dc/collection {:entry-title "et1"}) {:validate-keywords false}) + coll2-1 (d/ingest "PROV1" (dc/collection {:entry-title "et2"}) {:validate-keywords false}) + coll2-2 (d/ingest "PROV1" (dc/collection {:entry-title "et2"}) {:validate-keywords false}) token (e/login (s/context) "user1")] (tags/create-tag token (tags/make-tag {:tag-key "tag1"})) (tags/create-tag token (tags/make-tag {:tag-key "tag2"})) diff --git a/system-int-test/test/cmr/system_int_test/search/tagging/tag_association_test.clj b/system-int-test/test/cmr/system_int_test/search/tagging/tag_association_test.clj index e2653a80cc..854ed2e1e0 100644 --- a/system-int-test/test/cmr/system_int_test/search/tagging/tag_association_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/tagging/tag_association_test.clj @@ -39,7 +39,8 @@ (collection/collection {:short-name (str "S" n) :version-id (str "V" n) - :entry-title (str "ET" n)})))) + :entry-title (str "ET" n)}) + {:validate-keywords false}))) all-prov1-colls [c1-p1 c2-p1 c3-p1 c4-p1] all-prov2-colls [c1-p2 c2-p2 c3-p2 c4-p2] tag (tags/make-tag) @@ -100,7 +101,8 @@ (collection/collection {:short-name (str "S" n) :version-id (str "V" n) - :entry-title (str "ET" n)})))) + :entry-title (str "ET" n)}) + {:validate-keywords false}))) all-prov1-colls [c1-p1 c2-p1 c3-p1 c4-p1] all-prov2-colls [c1-p2 c2-p2 c3-p2 c4-p2] tag-key "tag1" @@ -181,7 +183,8 @@ tag (assoc tag :originator-id "user1") coll-concept-id (:concept-id (data-core/ingest "PROV1" - (collection/collection)))] + (collection/collection) + {:validate-keywords false}))] (testing "Associate tag using query sent with invalid content type" (are [associate-tag-fn request-json] (= {:status 400 @@ -243,7 +246,8 @@ (collection/collection {:short-name (str "S" n) :version-id (str "V" n) - :entry-title (str "ET" n)}))) + :entry-title (str "ET" n)}) + {:validate-keywords false})) all-prov1-colls [c1-p1 c2-p1 c3-p1 c4-p1] all-prov2-colls [c1-p2 c2-p2 c3-p2 c4-p2] all-prov3-colls [c1-p3 c2-p3 c3-p3 c4-p3] @@ -306,7 +310,8 @@ (collection/collection {:short-name (str "S" n) :version-id (str "V" n) - :entry-title (str "ET" n)}))) + :entry-title (str "ET" n)}) + {:validate-keywords false})) all-prov1-colls [c1-p1 c2-p1 c3-p1 c4-p1] all-prov2-colls [c1-p2 c2-p2 c3-p2 c4-p2] all-prov3-colls [c1-p3 c2-p3 c3-p3 c4-p3] @@ -374,7 +379,8 @@ tag (assoc tag :originator-id "user1") coll-concept-id (:concept-id (data-core/ingest "PROV1" - (collection/collection)))] + (collection/collection) + {:validate-keywords false}))] (testing "Dissociate tag using query sent with invalid content type" (are [dissociate-tag-fn request-json] @@ -417,8 +423,8 @@ (echo-util/grant-registered-users (system/context) (echo-util/coll-catalog-item-id "PROV1")) (testing "dissociate tag with only some of the collections matching the query are associated with the tag is OK" - (let [coll1 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET1"})) - coll2 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET2"})) + (let [coll1 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET1"}) {:validate-keywords false}) + coll2 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET2"}) {:validate-keywords false}) token (echo-util/login (system/context) "user1") _ (index/wait-until-indexed) tag (tags/save-tag token (tags/make-tag {:tag-key "tag1"}) [coll1]) @@ -432,9 +438,9 @@ (echo-util/grant-registered-users (system/context) (echo-util/coll-catalog-item-id "PROV1")) (testing "dissociate tag with mixed success and failure response" - (let [coll1 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET1"})) - coll2 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET2"})) - coll3 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET3"})) + (let [coll1 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET1"}) {:validate-keywords false}) + coll2 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET2"}) {:validate-keywords false}) + coll3 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET3"}) {:validate-keywords false}) token (echo-util/login (system/context) "user1") tag-key "tag1" assert-tag-associated (partial tags/assert-tag-associated-with-query token {:tag-key "tag1"})] @@ -463,7 +469,7 @@ (deftest association-retention-test (echo-util/grant-all (system/context) (echo-util/coll-catalog-item-id "PROV1")) - (let [coll (data-core/ingest "PROV1" (collection/collection)) + (let [coll (data-core/ingest "PROV1" (collection/collection) {:validate-keywords false}) token (echo-util/login (system/context) "user1") _ (index/wait-until-indexed) tag (tags/save-tag token (tags/make-tag {:tag-key "tag1"}) [coll]) @@ -478,14 +484,14 @@ (assert-tag-associated [coll])) (testing "Tag still associated with collection after updating collection" - (let [updated-coll (data-core/ingest "PROV1" (dissoc coll :revision-id))] + (let [updated-coll (data-core/ingest "PROV1" (dissoc coll :revision-id) {:validate-keywords false})] (is (= 200 (:status updated-coll))) (index/wait-until-indexed) (assert-tag-associated [updated-coll]))) (testing "Tag still associated with collection after deleting and recreating the collection" (is (= 200 (:status (ingest/delete-concept (data-core/item->concept coll))))) - (let [recreated-coll (data-core/ingest "PROV1" (dissoc coll :revision-id))] + (let [recreated-coll (data-core/ingest "PROV1" (dissoc coll :revision-id) {:validate-keywords false})] (is (= 200 (:status recreated-coll))) (index/wait-until-indexed) (assert-tag-associated [recreated-coll]))) @@ -524,7 +530,7 @@ (echo-util/grant-registered-users (system/context) (echo-util/coll-catalog-item-id "PROV1")) (let [[coll1 coll2 coll3] (for [n (range 1 4)] - (data-core/ingest "PROV1" (collection/collection))) + (data-core/ingest "PROV1" (collection/collection) {:validate-keywords false})) [coll1-id coll2-id coll3-id] (map :concept-id [coll1 coll2 coll3]) token (echo-util/login (system/context) "user1")] (tags/create-tag token (tags/make-tag {:tag-key "tag1"})) @@ -581,7 +587,7 @@ (deftest associate-tags-with-data-test (echo-util/grant-all (system/context) (echo-util/coll-catalog-item-id "PROV1")) - (let [coll (data-core/ingest "PROV1" (collection/collection)) + (let [coll (data-core/ingest "PROV1" (collection/collection) {:validate-keywords false}) coll-concept-id (:concept-id coll) token (echo-util/login (system/context) "user1") tag-key "tag1"] diff --git a/system-int-test/test/cmr/system_int_test/search/tool/collection_tool_search_test.clj b/system-int-test/test/cmr/system_int_test/search/tool/collection_tool_search_test.clj index 23ebc91b13..808c3d2e2a 100644 --- a/system-int-test/test/cmr/system_int_test/search/tool/collection_tool_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/tool/collection_tool_search_test.clj @@ -135,11 +135,13 @@ (let [token (e/login (s/context) "user1") coll1 (d/ingest "PROV1" (data2-collection/collection {:entry-title "ET1" :short-name "S1" - :version-id "V1"})) + :version-id "V1"}) + {:validate-keywords false}) coll2 (d/ingest "PROV1" (data2-collection/collection {:entry-title "ET2" :short-name "S2" - :version-id "V2"})) + :version-id "V2"}) + {:validate-keywords false}) ;; create tool {tool1-concept-id :concept-id} (tool-util/ingest-tool-with-attrs {:native-id "tl1" diff --git a/system-int-test/test/cmr/system_int_test/search/tool/tool_association_test.clj b/system-int-test/test/cmr/system_int_test/search/tool/tool_association_test.clj index 1f4459c6ae..15e899eb62 100644 --- a/system-int-test/test/cmr/system_int_test/search/tool/tool_association_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/tool/tool_association_test.clj @@ -39,7 +39,8 @@ (collection/collection {:short-name (str "S" n) :version-id (str "V" n) - :entry-title (str "ET" n)}))))) + :entry-title (str "ET" n)}) + {:validate-keywords false})))) all-prov1-colls [c1-p1 c2-p1 c3-p1 c4-p1] all-prov2-colls [c1-p2 c2-p2 c3-p2 c4-p2] token (echo-util/login (system/context) "user1") @@ -117,7 +118,7 @@ :Name "tool1" :provider-id "PROV1"}) {:keys [concept-id revision-id]} (tool-util/ingest-tool tool-concept) - coll-concept-id (:concept-id (data-core/ingest "PROV1" (collection/collection)))] + coll-concept-id (:concept-id (data-core/ingest "PROV1" (collection/collection) {:validate-keywords false}))] (testing "Associate tool using query sent with invalid content type" (are [associate-tool-fn request-json] (= {:status 400, @@ -173,7 +174,8 @@ (data-core/ingest p (collection/collection {:short-name (str "S" n) :version-id (str "V" n) - :entry-title (str "ET" n)})))) + :entry-title (str "ET" n)}) + {:validate-keywords false}))) all-prov1-colls [c1-p1 c2-p1 c3-p1 c4-p1] all-prov2-colls [c1-p2 c2-p2 c3-p2 c4-p2] all-prov3-colls [c1-p3 c2-p3 c3-p3 c4-p3] @@ -235,7 +237,7 @@ token (echo-util/login (system/context) "user1") tool-concept (tool-util/make-tool-concept {:Name tool-name}) {:keys [concept-id revision-id]} (tool-util/ingest-tool tool-concept) - coll-concept-id (:concept-id (data-core/ingest "PROV1" (collection/collection)))] + coll-concept-id (:concept-id (data-core/ingest "PROV1" (collection/collection) {:validate-keywords false}))] (testing "Dissociate tool using query sent with invalid content type" (are [dissociate-tool-fn request-json] @@ -277,9 +279,9 @@ (echo-util/grant-registered-users (system/context) (echo-util/coll-catalog-item-id "PROV1")) (testing "dissociate tool with mixed success and failure response" - (let [coll1 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET1"})) - coll2 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET2"})) - coll3 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET3"})) + (let [coll1 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET1"}) {:validate-keywords false}) + coll2 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET2"}) {:validate-keywords false}) + coll3 (data-core/ingest "PROV1" (collection/collection {:entry-title "ET3"}) {:validate-keywords false}) token (echo-util/login (system/context) "user1") tool-name "tool1" {:keys [concept-id]} (tool-util/ingest-tool-with-attrs {:Name tool-name}) @@ -316,7 +318,7 @@ (echo-util/grant-registered-users (system/context) (echo-util/coll-catalog-item-id "PROV1")) (let [[coll1 coll2 coll3] (doall (for [n (range 1 4)] - (data-core/ingest "PROV1" (collection/collection)))) + (data-core/ingest "PROV1" (collection/collection) {:validate-keywords false}))) [coll1-id coll2-id coll3-id] (map :concept-id [coll1 coll2 coll3]) token (echo-util/login (system/context) "user1") {tool1-concept-id :concept-id} (tool-util/ingest-tool-with-attrs diff --git a/system-int-test/test/cmr/system_int_test/search/variable/collection_variable_search_test.clj b/system-int-test/test/cmr/system_int_test/search/variable/collection_variable_search_test.clj index 77f97f0072..1d3b0c219e 100644 --- a/system-int-test/test/cmr/system_int_test/search/variable/collection_variable_search_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/variable/collection_variable_search_test.clj @@ -334,16 +334,20 @@ (let [token (e/login (s/context) "user1") coll1 (d/ingest "PROV1" (dc/collection {:entry-title "ET1" :short-name "S1" - :version-id "V1"})) + :version-id "V1"}) + {:validate-keywords false}) coll2 (d/ingest "PROV1" (dc/collection {:entry-title "ET2" :short-name "S2" - :version-id "V2"})) + :version-id "V2"}) + {:validate-keywords false}) coll3 (d/ingest "PROV1" (dc/collection {:entry-title "ET3" :short-name "S3" - :version-id "V3"})) + :version-id "V3"}) + {:validate-keywords false}) coll4 (d/ingest "PROV1" (dc/collection {:entry-title "ET4" :short-name "S4" - :version-id "V4"})) + :version-id "V4"}) + {:validate-keywords false}) ;; index the collections so that they can be found during variable association _ (index/wait-until-indexed) diff --git a/system-int-test/test/cmr/system_int_test/search/variable/concept_retrieval_test.clj b/system-int-test/test/cmr/system_int_test/search/variable/concept_retrieval_test.clj index 6fc5299ea1..ebecf4a6ba 100644 --- a/system-int-test/test/cmr/system_int_test/search/variable/concept_retrieval_test.clj +++ b/system-int-test/test/cmr/system_int_test/search/variable/concept_retrieval_test.clj @@ -34,7 +34,8 @@ (let [coll1 (data2-core/ingest "PROV1" (data2-coll/collection {:entry-title "ET1" :short-name "S1" - :version-id "V1"})) + :version-id "V1"}) + {:validate-keywords false}) _ (index/wait-until-indexed) var-concept (variable/make-variable-concept {:Name "var-to-be-deleted"} @@ -58,7 +59,8 @@ (let [coll1 (data2-core/ingest "PROV1" (data2-coll/collection {:entry-title "ET1" :short-name "S1" - :version-id "V1"})) + :version-id "V1"}) + {:validate-keywords false}) _ (index/wait-until-indexed) var1-name "variable1" var1-long-name "variable1-long-name" @@ -136,7 +138,8 @@ (let [coll1 (data2-core/ingest "PROV1" (data2-coll/collection {:entry-title "ET1" :short-name "S1" - :version-id "V1"})) + :version-id "V1"}) + {:validate-keywords false}) _ (index/wait-until-indexed) var1-name "variable1" var1-long-name "var1-long-name" diff --git a/system-int-test/test/cmr/system_int_test/virtual_product/translation_test.clj b/system-int-test/test/cmr/system_int_test/virtual_product/translation_test.clj index 6f2f87cb20..c0c466bbf4 100644 --- a/system-int-test/test/cmr/system_int_test/virtual_product/translation_test.clj +++ b/system-int-test/test/cmr/system_int_test/virtual_product/translation_test.clj @@ -50,15 +50,17 @@ (collection/collection {:entry-title ast-entry-title :short-name "AST_L1A"}) - :provider-id "LPDAAC_ECS")]) + :provider-id "LPDAAC_ECS")] + {:validate-keywords false}) [alias-ast-coll] (virtual-product-util/ingest-source-collections [(assoc (collection/collection {:entry-title ast-entry-title :short-name "AST_L1A"}) - :provider-id "LP_ALIAS")]) - vp-colls (virtual-product-util/ingest-virtual-collections [ast-coll]) - alias-vp-colls (virtual-product-util/ingest-virtual-collections [alias-ast-coll]) + :provider-id "LP_ALIAS")] + {:validate-keywords false}) + vp-colls (virtual-product-util/ingest-virtual-collections [ast-coll] {:validate-keywords false}) + alias-vp-colls (virtual-product-util/ingest-virtual-collections [alias-ast-coll] {:validate-keywords false}) ast-gran (virtual-product-util/ingest-source-granule "LPDAAC_ECS" (granule/granule @@ -71,7 +73,8 @@ {:granule-ur "SC:AST_L1A.003:2006227720"})) prov-ast-coll (data-core/ingest "PROV" (collection/collection - {:entry-title ast-entry-title})) + {:entry-title ast-entry-title}) + {:validate-keywords false}) prov-ast-gran (data-core/ingest "PROV" (granule/granule prov-ast-coll @@ -79,14 +82,16 @@ lpdaac-non-ast-coll (data-core/ingest "LPDAAC_ECS" (collection/collection - {:entry-title "non virtual entry title"})) + {:entry-title "non virtual entry title"}) + {:validate-keywords false}) lpdaac-non-ast-gran (data-core/ingest "LPDAAC_ECS" (granule/granule lpdaac-non-ast-coll {:granule-ur "granule-ur2"})) prov-coll (data-core/ingest "PROV" (collection/collection - {:entry-title "some other entry title"})) + {:entry-title "some other entry title"}) + {:validate-keywords false}) prov-gran1 (data-core/ingest "PROV" (granule/granule prov-coll {:granule-ur "granule-ur3"})) @@ -251,13 +256,13 @@ ;; granules in the config file. All the virtual granule entries should be translated to corresponding ;; source entries by the end-point. (deftest all-virtual-granules-translate-entries-test - (let [source-collections (virtual-product-util/ingest-source-collections) + (let [source-collections (virtual-product-util/ingest-source-collections (virtual-product-util/source-collections) {:validate-keywords false}) ;; Ingest the virtual collections. For each virtual collection associate it with the source ;; collection to use later. vp-colls (reduce (fn [new-colls source-coll] (into new-colls (map #(assoc % :source-collection source-coll) (virtual-product-util/ingest-virtual-collections - [source-coll])))) + [source-coll] {:validate-keywords false})))) [] source-collections) _ (index/wait-until-indexed) @@ -294,7 +299,8 @@ ;; 10 (default search page size) granules on a single collection during the translation (deftest translate-granule-entries-more-than-default-page-size-test (let [coll (data-core/ingest "PROV" - (collection/collection {:entry-title "MISR Level 2 Aerosol parameters V002"})) + (collection/collection {:entry-title "MISR Level 2 Aerosol parameters V002"}) + {:validate-keywords false}) grans (make-grans coll 12) gran->entry (fn [g] {:concept-id (:concept-id g) diff --git a/system-int-test/test/cmr/system_int_test/virtual_product/virtual_product_test.clj b/system-int-test/test/cmr/system_int_test/virtual_product/virtual_product_test.clj index 59637a5575..bcfc6b19d9 100644 --- a/system-int-test/test/cmr/system_int_test/virtual_product/virtual_product_test.clj +++ b/system-int-test/test/cmr/system_int_test/virtual_product/virtual_product_test.clj @@ -42,8 +42,9 @@ {:entry-title "ASTER L1A Reconstructed Unprocessed Instrument Data V003" :short-name "AST_L1A" :projects (dc/projects "proj1" "proj2" "proj3")}) - :provider-id "LPDAAC_ECS")]) - vp-colls (vp/ingest-virtual-collections [ast-coll]) + :provider-id "LPDAAC_ECS")] + {:validate-keywords false}) + vp-colls (vp/ingest-virtual-collections [ast-coll] {:validate-keywords false}) granule-ur "SC:AST_L1A.003:2006227720" ast-l1a-gran (vp/ingest-source-granule "LPDAAC_ECS" (dg/granule ast-coll {:granule-ur granule-ur @@ -112,12 +113,12 @@ (search/find-refs :granule {:page-size 50}))))))) (deftest all-granules-in-virtual-product-test - (let [source-collections (vp/ingest-source-collections) + (let [source-collections (vp/ingest-source-collections (vp/source-collections) {:validate-keywords false}) ;; Ingest the virtual collections. For each virtual collection associate it with the source ;; collection to use later. vp-colls (reduce (fn [new-colls source-coll] (into new-colls (map #(assoc % :source-collection source-coll) - (vp/ingest-virtual-collections [source-coll])))) + (vp/ingest-virtual-collections [source-coll] {:validate-keywords false})))) [] source-collections) _ (index/wait-until-indexed) @@ -172,8 +173,9 @@ (dc/collection {:entry-title "ASTER L1A Reconstructed Unprocessed Instrument Data V003" :short-name "AST_L1A"}) - :provider-id "LPDAAC_ECS")]) - vp-colls (vp/ingest-virtual-collections [ast-coll]) + :provider-id "LPDAAC_ECS")] + {:validate-keywords false}) + vp-colls (vp/ingest-virtual-collections [ast-coll] {:validate-keywords false}) granule-ur "SC:AST_L1A.003:2006227720" ast-l1a-gran (dg/granule ast-coll {:granule-ur granule-ur}) ingest-result (vp/ingest-source-granule "LPDAAC_ECS" (assoc ast-l1a-gran :revision-id 5)) @@ -202,8 +204,9 @@ (dc/collection {:entry-title ast-entry-title :short-name "AST_L1A"}) - :provider-id "LP_ALIAS")]) - vp-colls (vp/ingest-virtual-collections [ast-coll]) + :provider-id "LP_ALIAS")] + {:validate-keywords false}) + vp-colls (vp/ingest-virtual-collections [ast-coll] {:validate-keywords false}) granule-ur "SC:AST_L1A.003:2006227710" ast-l1a-gran (vp/ingest-source-granule "LP_ALIAS" (dg/granule ast-coll {:granule-ur granule-ur})) @@ -237,8 +240,10 @@ (dc/collection {:entry-title ast-entry-title :short-name "AST_L1A"}) - :provider-id "LPDAAC_ECS")] {:client-id "ECHO"}) - vp-colls (vp/ingest-virtual-collections [ast-coll] {:client-id "ECHO"}) + :provider-id "LPDAAC_ECS")] + {:client-id "ECHO" + :validate-keywords false}) + vp-colls (vp/ingest-virtual-collections [ast-coll] {:client-id "ECHO" :validate-keywords false}) granule-ur "SC:AST_L1A.003:2006227720" ast-l1a-gran (vp/ingest-source-granule "LPDAAC_ECS" (dg/granule ast-coll {:granule-ur granule-ur}) @@ -266,8 +271,9 @@ {:entry-title (str "OMI/Aura Surface UVB Irradiance and Erythemal" " Dose Daily L3 Global 1.0x1.0 deg Grid V003 (OMUVBd) at GES DISC") :short-name "OMUVBd"}) - :provider-id "GES_DISC")]) - vp-colls (vp/ingest-virtual-collections [omi-coll]) + :provider-id "GES_DISC")] + {:validate-keywords false}) + vp-colls (vp/ingest-virtual-collections [omi-coll] {:validate-keywords false}) granule-ur "OMUVBd.003:OMI-Aura_L3-OMUVBd_2015m0103_v003-2015m0107t093002.he5" [ur-prefix ur-suffix] (string/split granule-ur #":") opendap-dir-path "http://acdisc.gsfc.nasa.gov/opendap/HDF-EOS5//Aura_OMI_Level3/OMUVBd.003/2015/" @@ -321,14 +327,14 @@ (deftest virtual-products-disabled-source-collections-test (vp/with-disabled-source-collections disabled-source-colls - (let [source-collections (vp/ingest-source-collections) + (let [source-collections (vp/ingest-source-collections (vp/source-collections) {:validate-keywords false}) enabled-source-collections (remove #(contains? disabled-source-colls (:entry-title %)) source-collections) disabled-source-collections (filter #(contains? disabled-source-colls (:entry-title %)) source-collections) ingest-colls-fn (fn [new-colls source-coll] (into new-colls (map #(assoc % :source-collection source-coll) - (vp/ingest-virtual-collections [source-coll])))) + (vp/ingest-virtual-collections [source-coll] {:validate-keywords false})))) enabled-vp-colls (reduce ingest-colls-fn [] enabled-source-collections) @@ -383,8 +389,9 @@ {:entry-title "ASTER L1A Reconstructed Unprocessed Instrument Data V003" :short-name "AST_L1A" :projects (dc/projects "proj1" "proj2" "proj3" umm-spec-util/not-provided)}) - :provider-id "LPDAAC_ECS")]) - vp-colls (vp/ingest-virtual-collections [ast-coll]) + :provider-id "LPDAAC_ECS")] + {:validate-keywords false}) + vp-colls (vp/ingest-virtual-collections [ast-coll] {:validate-keywords false}) granule-ur "SC:AST_L1A.003:2006227720" granule (vp/add-granule-attributes "LPDAAC_ECS" (dg/granule ast-coll {:granule-ur granule-ur diff --git a/system-int-test/test/cmr/system_int_test/virtual_product/virtual_product_token_ingest_test.clj b/system-int-test/test/cmr/system_int_test/virtual_product/virtual_product_token_ingest_test.clj index 1b818a1b4f..cc2bdaa5d9 100644 --- a/system-int-test/test/cmr/system_int_test/virtual_product/virtual_product_token_ingest_test.clj +++ b/system-int-test/test/cmr/system_int_test/virtual_product/virtual_product_token_ingest_test.clj @@ -27,8 +27,9 @@ {:entry-title "ASTER L1A Reconstructed Unprocessed Instrument Data V003" :short-name "AST_L1A"}) :provider-id "LPDAAC_ECS")] - {:token provider-admin-update-token}) - vp-colls (vp/ingest-virtual-collections [ast-coll] {:token provider-admin-update-token}) + {:token provider-admin-update-token + :validate-keywords false}) + vp-colls (vp/ingest-virtual-collections [ast-coll] {:token provider-admin-update-token :validate-keywords false}) granule-ur "SC:AST_L1A.003:2006227720" ast-l1a-gran (vp/ingest-source-granule "LPDAAC_ECS" (dg/granule ast-coll {:granule-ur granule-ur})