Skip to content

Commit

Permalink
Merge branch 'master' of https://github.com/nasa/Common-Metadata-Repo…
Browse files Browse the repository at this point in the history 
…sitory into CMR-9848-Association-Status
  • Loading branch information
@jmaeng72
jmaeng72 committed Dec 11, 2024
2 parents af94d29 + 390d975 commit 01fa43b
Show file tree
Hide file tree
Showing 84 changed files with 1,260 additions and 885 deletions.
2 changes: 1 addition & 1 deletion ingest-app/docs/api.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,10 +159,10 @@ The following fields are validated:
* [Data Format](https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/dataformat?format=csv) - Archival and Distribution File Format, and GetData Format
* [ProcessingLevel] (https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/productlevelid?format=csv) - productlevelid

**Note**: cmr-validate-keywords is set to TRUE by default
**Note**: that when multiple fields are present the combination of keywords are validated to match a known combination.
**Note**: Among the validation fields above, [Platforms], [Instruments], [Projects], [Science Keywords], [Location Keywords] and [Data Centers] are also validated when the `Cmr-Validate-Keywords` header is not set to `true` except that validation errors will be returned to users as warnings.


**Note**: the following fields are always checked:

* [Related URL Content Type](https://gcmd.earthdata.nasa.gov/kms/concepts/concept_scheme/rucontenttype?format=csv) - Related URL Content Type, Type, and Subtype are all verified and must belong to the correct set and context. Example: A data center contact URL can have a URL Content Type of "DataContactURL" and a Type of "HOME PAGE" with no Subtype.
Expand Down
2 changes: 1 addition & 1 deletion ingest-app/src/cmr/ingest/api/collections.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
(defn get-validation-options
"Returns a map of validation options with boolean values"
[headers]
{:validate-keywords? (= "true" (get headers VALIDATE_KEYWORDS_HEADER))
{:validate-keywords? (if (= "false" (get headers VALIDATE_KEYWORDS_HEADER)) false true)
:validate-umm? (= "true" (get headers ENABLE_UMM_C_VALIDATION_HEADER))
:test-existing-errors? (= "true" (get headers TESTING_EXISTING_ERRORS_HEADER))})

Expand Down
2 changes: 1 addition & 1 deletion system-int-test/src/cmr/system_int_test/data2/core.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@
:native-id (or native-id "native-id")
:metadata metadata
:format (or format (mime-types/format->mime-type format-key))}
response (ingest/ingest-concept concept)]
response (ingest/ingest-concept concept {:validate-keywords false})]
(if (= concept-type :granule)
(merge (umm-legacy/parse-concept context concept) response)
(merge (umm-spec/parse-metadata context concept) response))))
Expand Down
5 changes: 3 additions & 2 deletions system-int-test/src/cmr/system_int_test/utils/virtual_product_util.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,8 @@
(dc/collection
{:entry-title "ASTER L1A Reconstructed Unprocessed Instrument Data V003"
:short-name "AST_L1A"})
:provider-id "LPDAAC_ECS")])))
:provider-id "LPDAAC_ECS")]
{:validate-keywords false})))

;;; Functions for use in assertions

Expand All @@ -222,7 +223,7 @@
psa2 (dg/psa "SWIR_ObservationMode" ["ON"])
psa3 (dg/psa "VNIR1_ObservationMode" ["ON"])
psa4 (dg/psa "VNIR2_ObservationMode" ["ON"])]
(ingest-virtual-collections [ast-coll])
(ingest-virtual-collections [ast-coll] {:validate-keywords false})
(are [granule-attrs expected-granule-urs]
(let [params {"attribute[]" (format "string,%s,%s"
svm/source-granule-ur-additional-attr-name
Expand Down
2 changes: 1 addition & 1 deletion system-int-test/test/cmr/system_int_test/admin/cache_api_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
admin-read-token (e/login (s/context) "admin" [admin-read-group-concept-id])
normal-user-token (e/login (s/context) "user")
_ (e/grant-group-admin (s/context) admin-read-group-concept-id :read)
_ (d/ingest "PROV1" (dc/collection {:entry-title "coll1"}))
_ (d/ingest "PROV1" (dc/collection {:entry-title "coll1"}) {:validate-keywords false})
_ (refresh-cache (url/refresh-index-names-cache-url) (t-config/echo-system-token))]

(testing "list caches"
Expand Down
14 changes: 3 additions & 11 deletions system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/delete_test.clj
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,8 @@
(ns cmr.system-int-test.bootstrap.bulk-index.delete-test
"Integration test for CMR bulk index delete operations."
(:require
[clj-time.coerce :as cr]
[clj-time.core :as t]
[clojure.java.jdbc :as j]
[clojure.test :refer :all]
[cmr.access-control.test.util :as u]
[cmr.common.date-time-parser :as p]
[cmr.common.util :as util :refer [are3]]
[cmr.mock-echo.client.echo-util :as e]
[cmr.oracle.connection :as oracle]
[cmr.common.util :refer [are3]]
[cmr.system-int-test.bootstrap.bulk-index.core :as core]
[cmr.system-int-test.data2.collection :as dc]
[cmr.system-int-test.data2.core :as d]
Expand All @@ -21,7 +14,6 @@
[cmr.system-int-test.utils.metadata-db-util :as mdb]
[cmr.system-int-test.utils.search-util :as search]
[cmr.system-int-test.utils.tag-util :as tags]
[cmr.transmit.access-control :as ac]
[cmr.transmit.config :as tc]
[cmr.umm.echo10.echo10-core :as echo10]))

Expand Down Expand Up @@ -124,8 +116,8 @@
:native-id "coll1"
:short-name "coll1"})
umm1 (merge umm1 (select-keys coll1 [:concept-id :revision-id]))
;; coll2 is a regualr collection that is ingested and will be deleted later
coll2 (d/ingest "PROV1" (dc/collection {:short-name "coll2" :entry-title "coll2"}))
;; coll2 is a regular collection that is ingested and will be deleted later
coll2 (d/ingest "PROV1" (dc/collection {:short-name "coll2" :entry-title "coll2"}) {:validate-keywords false})
;; coll3 is a collection with an expired delete time
umm3 (dc/collection {:short-name "coll3" :entry-title "coll3" :delete-time "2000-01-01T12:00:00Z"})
xml3 (echo10/umm->echo10-xml umm3)
Expand Down
2 changes: 1 addition & 1 deletion system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/services_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@

(deftest ^:oracle bulk-index-collections-with-service-association-test
(s/only-with-real-database
(let [coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1"}))
(let [coll1 (d/ingest "PROV1" (dc/collection {:entry-title "coll1"}) {:validate-keywords false})
coll1-concept-id (:concept-id coll1)
token (e/login (s/context) "user1")
{serv1-concept-id :concept-id} (service/ingest-service-with-attrs
Expand Down
2 changes: 1 addition & 1 deletion system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/tag_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
(deftest ^:oracle bulk-index-collections-with-tag-association-test
(s/only-with-real-database
(let [[coll1 coll2] (for [n (range 1 3)]
(d/ingest "PROV1" (dc/collection {:entry-title (str "coll" n)})))
(d/ingest "PROV1" (dc/collection {:entry-title (str "coll" n)}) {:validate-keywords false}))
;; Wait until collections are indexed so tags can be associated with them
_ (index/wait-until-indexed)
user1-token (e/login (s/context) "user1")
Expand Down
2 changes: 1 addition & 1 deletion system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/tools_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,7 @@

(deftest bulk-index-collections-with-tool-association-test
(system/only-with-real-database
(let [coll1 (data-core/ingest "PROV1" (data-collection/collection {:entry-title "coll1"}))
(let [coll1 (data-core/ingest "PROV1" (data-collection/collection {:entry-title "coll1"}) {:validate-keywords false})
coll1-concept-id (:concept-id coll1)
token (echo-util/login (system/context) "user1")
{tool1-concept-id :concept-id} (tool/ingest-tool-with-attrs
Expand Down
35 changes: 23 additions & 12 deletions system-int-test/test/cmr/system_int_test/bootstrap/bulk_index/variables_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,14 @@
"PROV1"
(data2-collection/collection {:entry-title "ET1"
:short-name "S1"
:version-id "V1"}))
:version-id "V1"})
{:validate-keywords false})
coll2 (data2-core/ingest
"PROV2"
(data2-collection/collection {:entry-title "ET2"
:short-name "S2"
:version-id "V2"}))
:version-id "V2"})
{:validate-keywords false})
_ (index/wait-until-indexed)
var1 (variable/make-variable-concept
{:Name "Variable1"}
Expand All @@ -47,7 +49,7 @@
_ (core/disable-automatic-indexing)
;; The following is saved, but not indexed due to the above call
{var1-concept-id :concept-id} (variable/ingest-variable-with-association var1)
{var2-concept-id :concept-id} (variable/ingest-variable-with-association var2)
{_ :concept-id} (variable/ingest-variable-with-association var2)
{:keys [status errors]} (bootstrap/bulk-index-variables "PROV1" nil)]

(is (= [401 ["You do not have permission to perform that action."]]
Expand Down Expand Up @@ -103,17 +105,20 @@
"PROV1"
(data2-collection/collection {:entry-title "ET1"
:short-name "S1"
:version-id "V1"}))
:version-id "V1"})
{:validate-keywords false})
coll2 (data2-core/ingest
"PROV2"
(data2-collection/collection {:entry-title "ET2"
:short-name "S2"
:version-id "V2"}))
:version-id "V2"})
{:validate-keywords false})
coll3 (data2-core/ingest
"PROV3"
(data2-collection/collection {:entry-title "ET3"
:short-name "S3"
:version-id "V3"}))
:version-id "V3"})
{:validate-keywords false})
_ (index/wait-until-indexed)
var1 (variable/make-variable-concept
{:Name "Variable1"}
Expand Down Expand Up @@ -167,17 +172,20 @@
"PROV1"
(data2-collection/collection {:entry-title "ET1"
:short-name "S1"
:version-id "V1"}))
:version-id "V1"})
{:validate-keywords false})
coll2 (data2-core/ingest
"PROV2"
(data2-collection/collection {:entry-title "ET2"
:short-name "S2"
:version-id "V2"}))
:version-id "V2"})
{:validate-keywords false})
coll3 (data2-core/ingest
"PROV3"
(data2-collection/collection {:entry-title "ET3"
:short-name "S3"
:version-id "V3"}))
:version-id "V3"})
{:validate-keywords false})
_ (index/wait-until-indexed)
var1 (variable/make-variable-concept
{:Name "Variable1"}
Expand Down Expand Up @@ -215,17 +223,20 @@
"PROV1"
(data2-collection/collection {:entry-title "ET1"
:short-name "S1"
:version-id "V1"}))
:version-id "V1"})
{:validate-keywords false})
coll2 (data2-core/ingest
"PROV2"
(data2-collection/collection {:entry-title "ET2"
:short-name "S2"
:version-id "V2"}))
:version-id "V2"})
{:validate-keywords false})
coll3 (data2-core/ingest
"PROV3"
(data2-collection/collection {:entry-title "ET3"
:short-name "S3"
:version-id "V3"}))
:version-id "V3"})
{:validate-keywords false})
_ (index/wait-until-indexed)
;; Disable message publishing so items are not indexed.
_ (core/disable-automatic-indexing)
Expand Down
18 changes: 11 additions & 7 deletions system-int-test/test/cmr/system_int_test/bootstrap/fingerprint_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,8 @@
"PROV1"
(data2-collection/collection {:entry-title "ET1"
:short-name "S1"
:version-id "V1"}))
:version-id "V1"})
{:validate-keywords false})
_ (index/wait-until-indexed)
var1 (variable/make-variable-concept
{:Name "Variable1"}
Expand Down Expand Up @@ -82,7 +83,7 @@
(get-in latest-revision-concept [:extra-fields :fingerprint]))))))

(testing "update fingerprint of a deleted variable"
(let [{:keys [concept-id revision-id]} (ingest/delete-concept var2-concept)
(let [_ (ingest/delete-concept var2-concept)
latest-revision-concept (mdb/get-concept var2-concept-id)]
(is (= var2-concept-id (:concept-id latest-revision-concept)))
(is (= 2 (:revision-id latest-revision-concept)))
Expand Down Expand Up @@ -110,12 +111,14 @@
"PROV1"
(data2-collection/collection {:entry-title "ET1"
:short-name "S1"
:version-id "V1"}))
:version-id "V1"})
{:validate-keywords false})
coll2 (data2-core/ingest
"PROV2"
(data2-collection/collection {:entry-title "ET2"
:short-name "S2"
:version-id "V2"}))
:version-id "V2"})
{:validate-keywords false})
_ (index/wait-until-indexed)
var1 (variable/make-variable-concept
{:Name "Variable1"}
Expand All @@ -137,7 +140,6 @@
var2-concept (mdb/get-concept var2-concept-id)
var2-fingerprint (get-in var2-concept [:extra-fields :fingerprint])
var3-concept (mdb/get-concept var3-concept-id)
var3-fingerprint (get-in var3-concept [:extra-fields :fingerprint])
;; now change the fingerprint of var1 and var3 to a different vaule
latest-var1 (-> var1-concept
(assoc :revision-id (inc (:revision-id var1-concept)))
Expand Down Expand Up @@ -202,12 +204,14 @@
"PROV1"
(data2-collection/collection {:entry-title "ET1"
:short-name "S1"
:version-id "V1"}))
:version-id "V1"})
{:validate-keywords false})
coll2 (data2-core/ingest
"PROV2"
(data2-collection/collection {:entry-title "ET2"
:short-name "S2"
:version-id "V2"}))
:version-id "V2"})
{:validate-keywords false})
_ (index/wait-until-indexed)
var1 (variable/make-variable-concept
{:Name "Variable1"}
Expand Down
Loading

0 comments on commit 01fa43b

Please sign in to comment.