From f19732e5d2faa6bb5cef02f644acb9e63b271365 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 21 Sep 2023 13:25:35 -0400 Subject: [PATCH 001/220] [BI-1847] - added constant --- .../imports/experimentObservation/ExperimentObservation.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 727e34caf..d9a1a2fe7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -400,7 +400,8 @@ private void addReference(List refs, UUID uuid, String r public static final class Columns { public static final String GERMPLASM_NAME = "Germplasm Name"; public static final String GERMPLASM_GID = "Germplasm GID"; - public static final String TEST_CHECK = "Test (T) or Check (C )"; + public static final String TEST_CHECK = "Test (T) or Check (C)"; + public static final String OLD_TEST_CHECK = "Test (T) or Check (C )"; public static final String EXP_TITLE = "Exp Title"; public static final String EXP_DESCRIPTION = "Exp Description"; public static final String EXP_UNIT = "Exp Unit"; From ec215c2a31d65f29857c533a14a5757ad7b3e378 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 25 Sep 2023 12:38:36 -0400 Subject: [PATCH 002/220] [BI-1847] - fixed notice comment in SQL migrations changed Java comments to SQL comments --- .../V0.5.0__create-base-entities.sql | 32 +++++++++---------- .../V0.5.10__create_ontology_tables.sql | 32 +++++++++---------- .../V0.5.11__create-upload-table.sql | 32 +++++++++---------- .../V0.5.12__add-another-test-user.sql | 32 +++++++++---------- .../V0.5.13__add-trait-abbreviation.sql | 32 +++++++++---------- .../migration/V0.5.14__set-orcid-unique.sql | 32 +++++++++---------- .../V0.5.15__create-inventory-tables.sql | 32 +++++++++---------- .../V0.5.16__add-user-account-token.sql | 32 +++++++++---------- .../V0.5.17__remove_unused_trait_columns.sql | 32 +++++++++---------- .../V0.5.18__remove_code_data_type.sql | 32 +++++++++---------- .../migration/V0.5.19__set_trait_active.sql | 32 +++++++++---------- .../migration/V0.5.1__create-user-tables.sql | 32 +++++++++---------- .../V0.5.20__add-program-brapi-url.sql | 32 +++++++++---------- .../V0.5.21__create-brapi-import-tables.sql | 32 +++++++++---------- .../V0.5.22__rename-trait-name-column.sql | 32 +++++++++---------- ...3__create-system-level-import-mappings.sql | 32 +++++++++---------- .../db/migration/V0.5.24__add-species.sql | 32 +++++++++---------- .../V0.5.25__create-breeding-method-table.sql | 32 +++++++++---------- .../db/migration/V0.5.26__add-program-key.sql | 32 +++++++++---------- ..._add_germplasm_template_system_mapping.sql | 32 +++++++++---------- .../V0.5.28__remove_trait_abbreviations.sql | 32 +++++++++---------- .../migration/V0.5.2__populate-user-data.sql | 32 +++++++++---------- .../V0.5.30__create_germplasm_sequence.sql | 32 +++++++++---------- ...2__germplasm_import_change_dbid_to_gid.sql | 32 +++++++++---------- .../V0.5.33__alter-breeding-methods.sql | 32 +++++++++---------- ...0.5.34__created_shared_ontology_tables.sql | 32 +++++++++---------- ...V0.5.35__germplasm_import_add_synonyms.sql | 32 +++++++++---------- .../V0.5.3__create-program-tables.sql | 32 +++++++++---------- .../migration/V0.5.4__create-program-data.sql | 32 +++++++++---------- .../V0.5.5__create-user-role-tables.sql | 32 +++++++++---------- .../V0.5.6__create-new-test-user.sql | 32 +++++++++---------- .../V0.5.7__add-user-active-flag.sql | 32 +++++++++---------- .../V0.5.8__add-inactive-test-user.sql | 32 +++++++++---------- .../V0.5.9__add-program-user-active-flag.sql | 32 +++++++++---------- .../V0.7.1__add_breeding_methods.sql | 32 +++++++++---------- .../migration/V0_5_29__add_program_keys.sql | 32 +++++++++---------- .../migration/V1.0.10__add_onto_term_type.sql | 32 +++++++++---------- ....0.11__create_program_breeding_methods.sql | 32 +++++++++---------- ....12__create_genotypedataimport_mapping.sql | 32 +++++++++---------- .../V1.0.14__germplasm_import_add_GID.sql | 32 +++++++++---------- ...add_experiment_template_system_mapping.sql | 32 +++++++++---------- ...ate_experiment_template_system_mapping.sql | 32 +++++++++---------- .../V1.0.3__create_obs_unit_sequence.sql | 32 +++++++++---------- .../V1.0.4__create_experiment_sequence.sql | 32 +++++++++---------- .../V1.0.5__experiment_template_obsUnitID.sql | 32 +++++++++---------- .../V1.0.6__create_environment_sequence.sql | 32 +++++++++---------- .../V1.0.7__delete_obs_unit_sequence.sql | 32 +++++++++---------- ...eriment_obs_id_template_system_mapping.sql | 32 +++++++++---------- .../V1.0.9__add_dynamic_column_storage.sql | 32 +++++++++---------- .../resources/db/migration/beforeClean.sql | 32 +++++++++---------- 50 files changed, 800 insertions(+), 800 deletions(-) diff --git a/src/main/resources/db/migration/V0.5.0__create-base-entities.sql b/src/main/resources/db/migration/V0.5.0__create-base-entities.sql index 5c28fc61e..28a68474a 100644 --- a/src/main/resources/db/migration/V0.5.0__create-base-entities.sql +++ b/src/main/resources/db/migration/V0.5.0__create-base-entities.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- -- We need the public schema so we can create the uuid object in it create schema if not exists public; diff --git a/src/main/resources/db/migration/V0.5.10__create_ontology_tables.sql b/src/main/resources/db/migration/V0.5.10__create_ontology_tables.sql index 2108c9975..66c2e0765 100644 --- a/src/main/resources/db/migration/V0.5.10__create_ontology_tables.sql +++ b/src/main/resources/db/migration/V0.5.10__create_ontology_tables.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- CREATE TYPE "data_type" AS ENUM ( 'CODE', diff --git a/src/main/resources/db/migration/V0.5.11__create-upload-table.sql b/src/main/resources/db/migration/V0.5.11__create-upload-table.sql index 5f0f84d71..0e3f6bb03 100644 --- a/src/main/resources/db/migration/V0.5.11__create-upload-table.sql +++ b/src/main/resources/db/migration/V0.5.11__create-upload-table.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- CREATE TYPE "upload_type" AS ENUM ( 'TRAIT', diff --git a/src/main/resources/db/migration/V0.5.12__add-another-test-user.sql b/src/main/resources/db/migration/V0.5.12__add-another-test-user.sql index 8083ddcc1..6767ab0b7 100644 --- a/src/main/resources/db/migration/V0.5.12__add-another-test-user.sql +++ b/src/main/resources/db/migration/V0.5.12__add-another-test-user.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- DO $$ DECLARE diff --git a/src/main/resources/db/migration/V0.5.13__add-trait-abbreviation.sql b/src/main/resources/db/migration/V0.5.13__add-trait-abbreviation.sql index 33c610135..f2edeacb4 100644 --- a/src/main/resources/db/migration/V0.5.13__add-trait-abbreviation.sql +++ b/src/main/resources/db/migration/V0.5.13__add-trait-abbreviation.sql @@ -1,18 +1,18 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- ALTER TABLE trait ADD COLUMN abbreviations text[] DEFAULT '{}'; \ No newline at end of file diff --git a/src/main/resources/db/migration/V0.5.14__set-orcid-unique.sql b/src/main/resources/db/migration/V0.5.14__set-orcid-unique.sql index ddbd23380..9a027a40b 100644 --- a/src/main/resources/db/migration/V0.5.14__set-orcid-unique.sql +++ b/src/main/resources/db/migration/V0.5.14__set-orcid-unique.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- ALTER TABLE bi_user ADD CONSTRAINT orcid_unique UNIQUE (orcid); \ No newline at end of file diff --git a/src/main/resources/db/migration/V0.5.15__create-inventory-tables.sql b/src/main/resources/db/migration/V0.5.15__create-inventory-tables.sql index 36dabe7bf..ed7824d41 100644 --- a/src/main/resources/db/migration/V0.5.15__create-inventory-tables.sql +++ b/src/main/resources/db/migration/V0.5.15__create-inventory-tables.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- CREATE TYPE "entity_level" AS ENUM ( 'POPULATION', diff --git a/src/main/resources/db/migration/V0.5.16__add-user-account-token.sql b/src/main/resources/db/migration/V0.5.16__add-user-account-token.sql index f6d1f3e66..3afa712f3 100644 --- a/src/main/resources/db/migration/V0.5.16__add-user-account-token.sql +++ b/src/main/resources/db/migration/V0.5.16__add-user-account-token.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- ALTER TABLE bi_user ADD COLUMN account_token text; diff --git a/src/main/resources/db/migration/V0.5.17__remove_unused_trait_columns.sql b/src/main/resources/db/migration/V0.5.17__remove_unused_trait_columns.sql index e55e4dff5..ceb7eacf5 100644 --- a/src/main/resources/db/migration/V0.5.17__remove_unused_trait_columns.sql +++ b/src/main/resources/db/migration/V0.5.17__remove_unused_trait_columns.sql @@ -1,18 +1,18 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- ALTER TABLE method DROP COLUMN method_name; \ No newline at end of file diff --git a/src/main/resources/db/migration/V0.5.18__remove_code_data_type.sql b/src/main/resources/db/migration/V0.5.18__remove_code_data_type.sql index 672d4a1ca..c94558ce0 100644 --- a/src/main/resources/db/migration/V0.5.18__remove_code_data_type.sql +++ b/src/main/resources/db/migration/V0.5.18__remove_code_data_type.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- ALTER TYPE data_type RENAME TO data_type_old; diff --git a/src/main/resources/db/migration/V0.5.19__set_trait_active.sql b/src/main/resources/db/migration/V0.5.19__set_trait_active.sql index d73c27432..f60313540 100644 --- a/src/main/resources/db/migration/V0.5.19__set_trait_active.sql +++ b/src/main/resources/db/migration/V0.5.19__set_trait_active.sql @@ -1,18 +1,18 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- UPDATE trait set active = true where active is NULL; \ No newline at end of file diff --git a/src/main/resources/db/migration/V0.5.1__create-user-tables.sql b/src/main/resources/db/migration/V0.5.1__create-user-tables.sql index 38d14d49e..afbfb539f 100644 --- a/src/main/resources/db/migration/V0.5.1__create-user-tables.sql +++ b/src/main/resources/db/migration/V0.5.1__create-user-tables.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- CREATE TABLE bi_user ( like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, diff --git a/src/main/resources/db/migration/V0.5.20__add-program-brapi-url.sql b/src/main/resources/db/migration/V0.5.20__add-program-brapi-url.sql index 9a6ffb28d..017d7a92b 100644 --- a/src/main/resources/db/migration/V0.5.20__add-program-brapi-url.sql +++ b/src/main/resources/db/migration/V0.5.20__add-program-brapi-url.sql @@ -1,18 +1,18 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- ALTER TABLE program ADD COLUMN brapi_url text; diff --git a/src/main/resources/db/migration/V0.5.21__create-brapi-import-tables.sql b/src/main/resources/db/migration/V0.5.21__create-brapi-import-tables.sql index ac5726fe4..a88405a01 100644 --- a/src/main/resources/db/migration/V0.5.21__create-brapi-import-tables.sql +++ b/src/main/resources/db/migration/V0.5.21__create-brapi-import-tables.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- create table importer_mapping ( like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, diff --git a/src/main/resources/db/migration/V0.5.22__rename-trait-name-column.sql b/src/main/resources/db/migration/V0.5.22__rename-trait-name-column.sql index a3979b5f7..5e9c6e3c3 100644 --- a/src/main/resources/db/migration/V0.5.22__rename-trait-name-column.sql +++ b/src/main/resources/db/migration/V0.5.22__rename-trait-name-column.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- DO $$ diff --git a/src/main/resources/db/migration/V0.5.23__create-system-level-import-mappings.sql b/src/main/resources/db/migration/V0.5.23__create-system-level-import-mappings.sql index b15accfdb..c7402f408 100644 --- a/src/main/resources/db/migration/V0.5.23__create-system-level-import-mappings.sql +++ b/src/main/resources/db/migration/V0.5.23__create-system-level-import-mappings.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- -- Create new importer_mapping_program create table importer_mapping_program ( diff --git a/src/main/resources/db/migration/V0.5.24__add-species.sql b/src/main/resources/db/migration/V0.5.24__add-species.sql index 39f470a05..d040e19ac 100644 --- a/src/main/resources/db/migration/V0.5.24__add-species.sql +++ b/src/main/resources/db/migration/V0.5.24__add-species.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- DO $$ DECLARE diff --git a/src/main/resources/db/migration/V0.5.25__create-breeding-method-table.sql b/src/main/resources/db/migration/V0.5.25__create-breeding-method-table.sql index 535d2115c..45716d4c3 100644 --- a/src/main/resources/db/migration/V0.5.25__create-breeding-method-table.sql +++ b/src/main/resources/db/migration/V0.5.25__create-breeding-method-table.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- create table breeding_method ( like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, diff --git a/src/main/resources/db/migration/V0.5.26__add-program-key.sql b/src/main/resources/db/migration/V0.5.26__add-program-key.sql index 859ea8ad4..4e2fb0d90 100644 --- a/src/main/resources/db/migration/V0.5.26__add-program-key.sql +++ b/src/main/resources/db/migration/V0.5.26__add-program-key.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- --Add key column to programs ALTER TABLE program ADD COLUMN key text; \ No newline at end of file diff --git a/src/main/resources/db/migration/V0.5.27__add_germplasm_template_system_mapping.sql b/src/main/resources/db/migration/V0.5.27__add_germplasm_template_system_mapping.sql index c7ff87677..caa34b50a 100644 --- a/src/main/resources/db/migration/V0.5.27__add_germplasm_template_system_mapping.sql +++ b/src/main/resources/db/migration/V0.5.27__add_germplasm_template_system_mapping.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- DO $$ DECLARE diff --git a/src/main/resources/db/migration/V0.5.28__remove_trait_abbreviations.sql b/src/main/resources/db/migration/V0.5.28__remove_trait_abbreviations.sql index 1d3901baf..a41b74bc1 100644 --- a/src/main/resources/db/migration/V0.5.28__remove_trait_abbreviations.sql +++ b/src/main/resources/db/migration/V0.5.28__remove_trait_abbreviations.sql @@ -1,18 +1,18 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- ALTER TABLE trait DROP COLUMN abbreviations ; \ No newline at end of file diff --git a/src/main/resources/db/migration/V0.5.2__populate-user-data.sql b/src/main/resources/db/migration/V0.5.2__populate-user-data.sql index c7fe7e5aa..7abc34a37 100644 --- a/src/main/resources/db/migration/V0.5.2__populate-user-data.sql +++ b/src/main/resources/db/migration/V0.5.2__populate-user-data.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- insert into bi_user (id, name, created_by, updated_by) values diff --git a/src/main/resources/db/migration/V0.5.30__create_germplasm_sequence.sql b/src/main/resources/db/migration/V0.5.30__create_germplasm_sequence.sql index 60c022582..d127b0fe3 100644 --- a/src/main/resources/db/migration/V0.5.30__create_germplasm_sequence.sql +++ b/src/main/resources/db/migration/V0.5.30__create_germplasm_sequence.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- alter table program add column germplasm_sequence text; diff --git a/src/main/resources/db/migration/V0.5.32__germplasm_import_change_dbid_to_gid.sql b/src/main/resources/db/migration/V0.5.32__germplasm_import_change_dbid_to_gid.sql index 32cd0e5fc..d943bfc62 100644 --- a/src/main/resources/db/migration/V0.5.32__germplasm_import_change_dbid_to_gid.sql +++ b/src/main/resources/db/migration/V0.5.32__germplasm_import_change_dbid_to_gid.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- update importer_mapping set mapping = '[{"id": "f8d43c7e-a618-4c16-8829-3085f7202a67", "mapping": [{"id": "f384837e-ad8d-4dbe-b54e-87b57070bed1", "value": {"fileFieldName": "Name"}, "objectId": "germplasmName"}, {"id": "39628d14-458b-429b-8e66-bb48e0445a83", "value": {"fileFieldName": "Breeding Method"}, "objectId": "breedingMethod"}, {"id": "f1ba63e1-f5e4-433f-a53e-1c2f3e2fa71f", "value": {"fileFieldName": "Source"}, "objectId": "germplasmSource"}, {"id": "f5892565-f888-4596-be82-ab8eeabf37ce", "value": {"fileFieldName": "External UID"}, "objectId": "externalUID"}, {"id": "65507e5d-2d66-4595-8763-e772fe25c870", "value": {"fileFieldName": "Entry No"}, "objectId": "entryNo"}, {"id": "3eae24c1-ca4a-48a2-96d0-3cf4630acd3a", "value": {"fileFieldName": "Female Parent GID"}, "objectId": "femaleParentDBID"}, {"id": "2dbd7262-93a1-44b0-86b7-f5fca290b965", "value": {"fileFieldName": "Male Parent GID"}, "objectId": "maleParentDBID"}, {"id": "6f7f1539-6e8f-4ede-b7d3-3423cc63abec", "value": {"fileFieldName": "Female Parent Entry No"}, "objectId": "femaleParentEntryNo"}, {"id": "25fe9954-bca7-42f1-818a-5f71e242fa1f", "value": {"fileFieldName": "Male Parent Entry No"}, "objectId": "maleParentEntryNo"}, {"id": "15836d5f-8194-40a8-a771-114eaae31eb4", "objectId": "germplasmPUI"}, {"id": "675b6af8-5a17-4146-a503-2e4e1a65d5fa", "objectId": "acquisitionDate"}, {"id": "69a3bd3c-cebc-435c-acdd-0be62dda25ed", "objectId": "countryOfOrigin"}, {"id": "8ab25267-20f2-450e-89ca-21634ff8fadb", "objectId": "collection"}, {"id": "ce1701e2-2f61-4250-8595-9536e3f5ddcf", "objectId": "AdditionalInfo"}, {"id": "3470e9df-a028-45b7-943f-198bc62b6dbe", "objectId": "ExternalReference"}], "objectId": "Germplasm"}]', diff --git a/src/main/resources/db/migration/V0.5.33__alter-breeding-methods.sql b/src/main/resources/db/migration/V0.5.33__alter-breeding-methods.sql index d5ce8ccc1..162b98f3f 100644 --- a/src/main/resources/db/migration/V0.5.33__alter-breeding-methods.sql +++ b/src/main/resources/db/migration/V0.5.33__alter-breeding-methods.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- insert into breeding_method (name, abbreviation, description, category, genetic_diversity) values diff --git a/src/main/resources/db/migration/V0.5.34__created_shared_ontology_tables.sql b/src/main/resources/db/migration/V0.5.34__created_shared_ontology_tables.sql index abaa1b8fb..a25c1b5f0 100644 --- a/src/main/resources/db/migration/V0.5.34__created_shared_ontology_tables.sql +++ b/src/main/resources/db/migration/V0.5.34__created_shared_ontology_tables.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- CREATE TABLE program_shared_ontology ( like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, diff --git a/src/main/resources/db/migration/V0.5.35__germplasm_import_add_synonyms.sql b/src/main/resources/db/migration/V0.5.35__germplasm_import_add_synonyms.sql index f583f2e06..ac16cc670 100644 --- a/src/main/resources/db/migration/V0.5.35__germplasm_import_add_synonyms.sql +++ b/src/main/resources/db/migration/V0.5.35__germplasm_import_add_synonyms.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- update importer_mapping set mapping = '[{"id": "f8d43c7e-a618-4c16-8829-3085f7202a67", "mapping": [{"id": "f384837e-ad8d-4dbe-b54e-87b57070bed1", "value": {"fileFieldName": "Name"}, "objectId": "germplasmName"}, {"id": "39628d14-458b-429b-8e66-bb48e0445a83", "value": {"fileFieldName": "Breeding Method"}, "objectId": "breedingMethod"}, {"id": "f1ba63e1-f5e4-433f-a53e-1c2f3e2fa71f", "value": {"fileFieldName": "Source"}, "objectId": "germplasmSource"}, {"id": "f5892565-f888-4596-be82-ab8eeabf37ce", "value": {"fileFieldName": "External UID"}, "objectId": "externalUID"}, {"id": "65507e5d-2d66-4595-8763-e772fe25c870", "value": {"fileFieldName": "Entry No"}, "objectId": "entryNo"}, {"id": "3eae24c1-ca4a-48a2-96d0-3cf4630acd3a", "value": {"fileFieldName": "Female Parent GID"}, "objectId": "femaleParentDBID"}, {"id": "2dbd7262-93a1-44b0-86b7-f5fca290b965", "value": {"fileFieldName": "Male Parent GID"}, "objectId": "maleParentDBID"}, {"id": "6f7f1539-6e8f-4ede-b7d3-3423cc63abec", "value": {"fileFieldName": "Female Parent Entry No"}, "objectId": "femaleParentEntryNo"}, {"id": "25fe9954-bca7-42f1-818a-5f71e242fa1f", "value": {"fileFieldName": "Male Parent Entry No"}, "objectId": "maleParentEntryNo"}, {"id": "b910adfe-a474-47a0-8410-514578898436", "value": {"fileFieldName": "Synonyms"}, "objectId": "synonyms"}, {"id": "15836d5f-8194-40a8-a771-114eaae31eb4", "objectId": "germplasmPUI"}, {"id": "675b6af8-5a17-4146-a503-2e4e1a65d5fa", "objectId": "acquisitionDate"}, {"id": "69a3bd3c-cebc-435c-acdd-0be62dda25ed", "objectId": "countryOfOrigin"}, {"id": "8ab25267-20f2-450e-89ca-21634ff8fadb", "objectId": "collection"}, {"id": "ce1701e2-2f61-4250-8595-9536e3f5ddcf", "objectId": "AdditionalInfo"}, {"id": "3470e9df-a028-45b7-943f-198bc62b6dbe", "objectId": "ExternalReference"}], "objectId": "Germplasm"}]', diff --git a/src/main/resources/db/migration/V0.5.3__create-program-tables.sql b/src/main/resources/db/migration/V0.5.3__create-program-tables.sql index 8be53b717..9f628bb78 100644 --- a/src/main/resources/db/migration/V0.5.3__create-program-tables.sql +++ b/src/main/resources/db/migration/V0.5.3__create-program-tables.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- CREATE TABLE program ( like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, diff --git a/src/main/resources/db/migration/V0.5.4__create-program-data.sql b/src/main/resources/db/migration/V0.5.4__create-program-data.sql index 0ae9889b2..4ea9d64b9 100644 --- a/src/main/resources/db/migration/V0.5.4__create-program-data.sql +++ b/src/main/resources/db/migration/V0.5.4__create-program-data.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- DO $$ DECLARE diff --git a/src/main/resources/db/migration/V0.5.5__create-user-role-tables.sql b/src/main/resources/db/migration/V0.5.5__create-user-role-tables.sql index 625024bbd..591fff82f 100644 --- a/src/main/resources/db/migration/V0.5.5__create-user-role-tables.sql +++ b/src/main/resources/db/migration/V0.5.5__create-user-role-tables.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- CREATE TABLE system_user_role ( like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, diff --git a/src/main/resources/db/migration/V0.5.6__create-new-test-user.sql b/src/main/resources/db/migration/V0.5.6__create-new-test-user.sql index f6333bce4..3f25944cd 100644 --- a/src/main/resources/db/migration/V0.5.6__create-new-test-user.sql +++ b/src/main/resources/db/migration/V0.5.6__create-new-test-user.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- DO $$ DECLARE diff --git a/src/main/resources/db/migration/V0.5.7__add-user-active-flag.sql b/src/main/resources/db/migration/V0.5.7__add-user-active-flag.sql index d386977cf..229013458 100644 --- a/src/main/resources/db/migration/V0.5.7__add-user-active-flag.sql +++ b/src/main/resources/db/migration/V0.5.7__add-user-active-flag.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- ALTER TABLE bi_user ADD active bool NOT NULL DEFAULT true; diff --git a/src/main/resources/db/migration/V0.5.8__add-inactive-test-user.sql b/src/main/resources/db/migration/V0.5.8__add-inactive-test-user.sql index fc565e790..6b77a3365 100644 --- a/src/main/resources/db/migration/V0.5.8__add-inactive-test-user.sql +++ b/src/main/resources/db/migration/V0.5.8__add-inactive-test-user.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- DO $$ DECLARE diff --git a/src/main/resources/db/migration/V0.5.9__add-program-user-active-flag.sql b/src/main/resources/db/migration/V0.5.9__add-program-user-active-flag.sql index 2fbfdc931..2cfc72c12 100644 --- a/src/main/resources/db/migration/V0.5.9__add-program-user-active-flag.sql +++ b/src/main/resources/db/migration/V0.5.9__add-program-user-active-flag.sql @@ -1,18 +1,18 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- ALTER TABLE program_user_role ADD active bool NOT NULL DEFAULT true; diff --git a/src/main/resources/db/migration/V0.7.1__add_breeding_methods.sql b/src/main/resources/db/migration/V0.7.1__add_breeding_methods.sql index 4bf58b94d..b6973ad5a 100644 --- a/src/main/resources/db/migration/V0.7.1__add_breeding_methods.sql +++ b/src/main/resources/db/migration/V0.7.1__add_breeding_methods.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- insert into breeding_method (abbreviation, name, description, genetic_diversity, category) diff --git a/src/main/resources/db/migration/V0_5_29__add_program_keys.sql b/src/main/resources/db/migration/V0_5_29__add_program_keys.sql index 0d832bb9a..3e63e48db 100644 --- a/src/main/resources/db/migration/V0_5_29__add_program_keys.sql +++ b/src/main/resources/db/migration/V0_5_29__add_program_keys.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- UPDATE program SET key = 'STPALF' WHERE name = 'StPaul-Alfalfa'; UPDATE program SET key = 'PROALF' WHERE name = 'Prosser-Alfalfa'; diff --git a/src/main/resources/db/migration/V1.0.10__add_onto_term_type.sql b/src/main/resources/db/migration/V1.0.10__add_onto_term_type.sql index 872df67e7..97231fa53 100644 --- a/src/main/resources/db/migration/V1.0.10__add_onto_term_type.sql +++ b/src/main/resources/db/migration/V1.0.10__add_onto_term_type.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- CREATE TYPE "term_type" AS ENUM ( 'PHENOTYPE', diff --git a/src/main/resources/db/migration/V1.0.11__create_program_breeding_methods.sql b/src/main/resources/db/migration/V1.0.11__create_program_breeding_methods.sql index 1637517e5..9ead47f2c 100644 --- a/src/main/resources/db/migration/V1.0.11__create_program_breeding_methods.sql +++ b/src/main/resources/db/migration/V1.0.11__create_program_breeding_methods.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- create table program_breeding_method ( diff --git a/src/main/resources/db/migration/V1.0.12__create_genotypedataimport_mapping.sql b/src/main/resources/db/migration/V1.0.12__create_genotypedataimport_mapping.sql index c15ee8668..4181e330c 100644 --- a/src/main/resources/db/migration/V1.0.12__create_genotypedataimport_mapping.sql +++ b/src/main/resources/db/migration/V1.0.12__create_genotypedataimport_mapping.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- DO $$ DECLARE diff --git a/src/main/resources/db/migration/V1.0.14__germplasm_import_add_GID.sql b/src/main/resources/db/migration/V1.0.14__germplasm_import_add_GID.sql index 0739723ef..f635c876d 100644 --- a/src/main/resources/db/migration/V1.0.14__germplasm_import_add_GID.sql +++ b/src/main/resources/db/migration/V1.0.14__germplasm_import_add_GID.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- update importer_mapping set mapping = '[{"id": "f8d43c7e-a618-4c16-8829-3085f7202a67", "mapping": [{"id": "f384837e-ad8d-4dbe-b54e-87b57070bed1", "value": {"fileFieldName": "Name"}, "objectId": "germplasmName"}, {"id": "39628d14-458b-429b-8e66-bb48e0445a83", "value": {"fileFieldName": "Breeding Method"}, "objectId": "breedingMethod"}, {"id": "f1ba63e1-f5e4-433f-a53e-1c2f3e2fa71f", "value": {"fileFieldName": "Source"}, "objectId": "germplasmSource"}, {"id": "f5892565-f888-4596-be82-ab8eeabf37ce", "value": {"fileFieldName": "External UID"}, "objectId": "externalUID"}, {"id": "65507e5d-2d66-4595-8763-e772fe25c870", "value": {"fileFieldName": "Entry No"}, "objectId": "entryNo"}, {"id": "3eae24c1-ca4a-48a2-96d0-3cf4630acd3a", "value": {"fileFieldName": "Female Parent GID"}, "objectId": "femaleParentDBID"}, {"id": "2dbd7262-93a1-44b0-86b7-f5fca290b965", "value": {"fileFieldName": "Male Parent GID"}, "objectId": "maleParentDBID"}, {"id": "6f7f1539-6e8f-4ede-b7d3-3423cc63abec", "value": {"fileFieldName": "Female Parent Entry No"}, "objectId": "femaleParentEntryNo"}, {"id": "25fe9954-bca7-42f1-818a-5f71e242fa1f", "value": {"fileFieldName": "Male Parent Entry No"}, "objectId": "maleParentEntryNo"}, {"id": "b910adfe-a474-47a0-8410-514578898436", "value": {"fileFieldName": "Synonyms"}, "objectId": "synonyms"}, {"id": "15836d5f-8194-40a8-a771-114eaae31eb4", "objectId": "germplasmPUI"}, {"id": "675b6af8-5a17-4146-a503-2e4e1a65d5fa", "objectId": "acquisitionDate"}, {"id": "69a3bd3c-cebc-435c-acdd-0be62dda25ed", "objectId": "countryOfOrigin"}, {"id": "8ab25267-20f2-450e-89ca-21634ff8fadb", "objectId": "collection"}, {"id": "bc09c6e1-866f-45c3-a285-a25859e8c982", "value": {"fileFieldName": "GID"}, "objectId": "germplasmAccessionNumber"}, {"id": "ce1701e2-2f61-4250-8595-9536e3f5ddcf", "objectId": "AdditionalInfo"}, {"id": "3470e9df-a028-45b7-943f-198bc62b6dbe", "objectId": "ExternalReference"}], "objectId": "Germplasm"}]', diff --git a/src/main/resources/db/migration/V1.0.1__add_experiment_template_system_mapping.sql b/src/main/resources/db/migration/V1.0.1__add_experiment_template_system_mapping.sql index 8e9c85147..39655b2b2 100644 --- a/src/main/resources/db/migration/V1.0.1__add_experiment_template_system_mapping.sql +++ b/src/main/resources/db/migration/V1.0.1__add_experiment_template_system_mapping.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- DO $$ DECLARE diff --git a/src/main/resources/db/migration/V1.0.2__update_experiment_template_system_mapping.sql b/src/main/resources/db/migration/V1.0.2__update_experiment_template_system_mapping.sql index 9c3492ca4..d84339fa1 100644 --- a/src/main/resources/db/migration/V1.0.2__update_experiment_template_system_mapping.sql +++ b/src/main/resources/db/migration/V1.0.2__update_experiment_template_system_mapping.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- DO $$ diff --git a/src/main/resources/db/migration/V1.0.3__create_obs_unit_sequence.sql b/src/main/resources/db/migration/V1.0.3__create_obs_unit_sequence.sql index f0bed3308..8b7c779a2 100644 --- a/src/main/resources/db/migration/V1.0.3__create_obs_unit_sequence.sql +++ b/src/main/resources/db/migration/V1.0.3__create_obs_unit_sequence.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- alter table program add column obs_unit_sequence text; diff --git a/src/main/resources/db/migration/V1.0.4__create_experiment_sequence.sql b/src/main/resources/db/migration/V1.0.4__create_experiment_sequence.sql index c5ce9461d..cb6bdbf25 100644 --- a/src/main/resources/db/migration/V1.0.4__create_experiment_sequence.sql +++ b/src/main/resources/db/migration/V1.0.4__create_experiment_sequence.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- alter table program add column IF NOT EXISTS exp_sequence text; diff --git a/src/main/resources/db/migration/V1.0.5__experiment_template_obsUnitID.sql b/src/main/resources/db/migration/V1.0.5__experiment_template_obsUnitID.sql index d82a2edc2..d7896b6f6 100644 --- a/src/main/resources/db/migration/V1.0.5__experiment_template_obsUnitID.sql +++ b/src/main/resources/db/migration/V1.0.5__experiment_template_obsUnitID.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- DO $$ diff --git a/src/main/resources/db/migration/V1.0.6__create_environment_sequence.sql b/src/main/resources/db/migration/V1.0.6__create_environment_sequence.sql index 1bc60dfc7..dfa6c37d9 100644 --- a/src/main/resources/db/migration/V1.0.6__create_environment_sequence.sql +++ b/src/main/resources/db/migration/V1.0.6__create_environment_sequence.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- alter table program add column IF NOT EXISTS env_sequence text; diff --git a/src/main/resources/db/migration/V1.0.7__delete_obs_unit_sequence.sql b/src/main/resources/db/migration/V1.0.7__delete_obs_unit_sequence.sql index af85cee96..2ab13a52b 100644 --- a/src/main/resources/db/migration/V1.0.7__delete_obs_unit_sequence.sql +++ b/src/main/resources/db/migration/V1.0.7__delete_obs_unit_sequence.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- alter table program drop column obs_unit_sequence; diff --git a/src/main/resources/db/migration/V1.0.8__update_experiment_obs_id_template_system_mapping.sql b/src/main/resources/db/migration/V1.0.8__update_experiment_obs_id_template_system_mapping.sql index d1ac21bd2..5737ca907 100644 --- a/src/main/resources/db/migration/V1.0.8__update_experiment_obs_id_template_system_mapping.sql +++ b/src/main/resources/db/migration/V1.0.8__update_experiment_obs_id_template_system_mapping.sql @@ -1,19 +1,19 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- DO $$ diff --git a/src/main/resources/db/migration/V1.0.9__add_dynamic_column_storage.sql b/src/main/resources/db/migration/V1.0.9__add_dynamic_column_storage.sql index 5f65addf6..7d5c91b2d 100644 --- a/src/main/resources/db/migration/V1.0.9__add_dynamic_column_storage.sql +++ b/src/main/resources/db/migration/V1.0.9__add_dynamic_column_storage.sql @@ -1,18 +1,18 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- alter table importer_import add column dynamic_column_names text[]; \ No newline at end of file diff --git a/src/main/resources/db/migration/beforeClean.sql b/src/main/resources/db/migration/beforeClean.sql index 13e2df295..9f7ebc7a9 100644 --- a/src/main/resources/db/migration/beforeClean.sql +++ b/src/main/resources/db/migration/beforeClean.sql @@ -1,18 +1,18 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- drop extension if exists "uuid-ossp" cascade; \ No newline at end of file From 40b712b18ddafdffc0a23a86e4e620a1f3e0ae23 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 25 Sep 2023 12:39:39 -0400 Subject: [PATCH 003/220] [BI-1847] - removed space from "(C )" --- .../imports/experimentObservation/ExperimentObservation.java | 1 - src/test/resources/files/fileutil/file_all_null_columns.csv | 2 +- src/test/resources/sql/ImportControllerIntegrationTest.sql | 4 ++-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index d9a1a2fe7..f45cd83a3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -401,7 +401,6 @@ public static final class Columns { public static final String GERMPLASM_NAME = "Germplasm Name"; public static final String GERMPLASM_GID = "Germplasm GID"; public static final String TEST_CHECK = "Test (T) or Check (C)"; - public static final String OLD_TEST_CHECK = "Test (T) or Check (C )"; public static final String EXP_TITLE = "Exp Title"; public static final String EXP_DESCRIPTION = "Exp Description"; public static final String EXP_UNIT = "Exp Unit"; diff --git a/src/test/resources/files/fileutil/file_all_null_columns.csv b/src/test/resources/files/fileutil/file_all_null_columns.csv index ceb3dbef3..8bf92a120 100644 --- a/src/test/resources/files/fileutil/file_all_null_columns.csv +++ b/src/test/resources/files/fileutil/file_all_null_columns.csv @@ -1,4 +1,4 @@ -Germplasm Name,Germplasm GID,Test (T) or Check (C ),Exp Title,Exp Description,Exp Unit,Exp Type,Env,Env Location,Env Year,Exp Unit ID,Exp Replicate #,Exp Block #,Row,Column,Treatment Factors,ObsUnitID,Color,INCAUDPS,INCW10SUM,INCW6SUM,,, +Germplasm Name,Germplasm GID,Test (T) or Check (C),Exp Title,Exp Description,Exp Unit,Exp Type,Env,Env Location,Env Year,Exp Unit ID,Exp Replicate #,Exp Block #,Row,Column,Treatment Factors,ObsUnitID,Color,INCAUDPS,INCW10SUM,INCW6SUM,,, BRG,453,C,KRSP22-3,INSV phenotyping trial BRG x Eruption RIL population and parents,Plot,Disease resistance screening,"Salinas, CA 2022","Salinas, CA",2022,89,1,1,,,,,Green,120,6,0,,, Eruption,454,C,KRSP22-3,INSV phenotyping trial BRG x Eruption RIL population and parents,Plot,Disease resistance screening,"Salinas, CA 2022","Salinas, CA",2022,90,1,1,,,,,Medium red,110,4,0,,, BxE16-001,455,,KRSP22-3,INSV phenotyping trial BRG x Eruption RIL population and parents,Plot,Disease resistance screening,"Salinas, CA 2022","Salinas, CA",2022,91,1,1,,,,,Dark red,110,6,0,,, \ No newline at end of file diff --git a/src/test/resources/sql/ImportControllerIntegrationTest.sql b/src/test/resources/sql/ImportControllerIntegrationTest.sql index 8a93b472e..c036b466f 100644 --- a/src/test/resources/sql/ImportControllerIntegrationTest.sql +++ b/src/test/resources/sql/ImportControllerIntegrationTest.sql @@ -32,8 +32,8 @@ INSERT INTO public.importer_mapping (name,import_type_id,mapping,file,draft,crea INSERT INTO public.importer_mapping (name,import_type_id,mapping,file,draft,created_by,updated_by) values ('ExperimentsTemplateMap','ExperimentImport', - '[{"id": "726a9f10-4892-4204-9e52-bd2b1d735f65", "value": {"fileFieldName": "Germplasm Name"}, "objectId": "germplasmName"}, {"id": "98774e20-6f89-4d6a-a7c9-f88887228ed6", "value": {"fileFieldName": "Germplasm GID"}, "objectId": "gid"}, {"id": "880ef0c9-4e3e-42d4-9edc-667684a91889", "value": {"fileFieldName": "Test (T) or Check (C )"}, "objectId": "test_or_check"}, {"id": "b693eca7-efcd-4518-a9d3-db0b037a76ee", "value": {"fileFieldName": "Exp Title"}, "objectId": "exp_title"}, {"id": "df340215-db6e-4219-a3b7-119f297b81c3", "value": {"fileFieldName": "Exp Description"}, "objectId": "expDescription"}, {"id": "9ca7cc81-562c-43a7-989a-41da309f603d", "value": {"fileFieldName": "Exp Unit"}, "objectId": "expUnit"}, {"id": "27215777-c8f9-4fe7-a7ac-918d6168b0dd", "value": {"fileFieldName": "Exp Type"}, "objectId": "expType"}, {"id": "19d220e2-dff0-4a3a-ad6e-32f4d8602b5c", "value": {"fileFieldName": "Env"}, "objectId": "env"}, {"id": "861518b9-5c9e-4fe5-b31e-baf16e27155d", "value": {"fileFieldName": "Env Location"}, "objectId": "envLocation"}, {"id": "667355c3-dae1-4a64-94c8-ac2d543bd474", "value": {"fileFieldName": "Env Year"}, "objectId": "envYear"}, {"id": "ad11f2df-c5b4-4a05-8e52-c57625140061", "value": {"fileFieldName": "Exp Unit ID"}, "objectId": "expUnitId"}, {"id": "639b40ec-20f8-4659-8464-6a4be997ac7a", "value": {"fileFieldName": "Exp Replicate #"}, "objectId": "expReplicateNo"}, {"id": "2a62a80f-d8ba-42c4-9997-3b4ac8a965aa", "value": {"fileFieldName": "Exp Block #"}, "objectId": "expBlockNo"}, {"id": "f3e7de69-21ad-4cda-b1cc-a5e1987fb931", "value": {"fileFieldName": "Row"}, "objectId": "row"}, {"id": "251c5bbd-fc4d-4371-a4ce-4e2686f6837e", "value": {"fileFieldName": "Column"}, "objectId": "column"}, {"id": "ce5f61f2-f1de-45a4-8baf-e2471a5d863d", "value": {"fileFieldName": "Treatment Factors"}, "objectId": "treatmentFactors"}, {"id": "c5b8276f-e777-4385-a80f-5199abe63aac", "value": {"fileFieldName": "ObsUnitID"}, "objectId": "ObsUnitID"}]', - '[{"Env": "New Study", "Row": 4, "Column": 5, "Env Year": 2012, "Exp Type": "phenotyping", "Exp Unit": "plot", "Exp Title": "New Trial", "ObsUnitID": "", "Exp Block #": 2, "Exp Unit ID": 3, "Env Location": "New Location", "Germplasm GID": 1, "Germplasm Name": "Test", "Exp Description": "A new trial", "Exp Replicate #": 0, "Treatment Factors": "Jam application", "Test (T) or Check (C )": true}]', + '[{"id": "726a9f10-4892-4204-9e52-bd2b1d735f65", "value": {"fileFieldName": "Germplasm Name"}, "objectId": "germplasmName"}, {"id": "98774e20-6f89-4d6a-a7c9-f88887228ed6", "value": {"fileFieldName": "Germplasm GID"}, "objectId": "gid"}, {"id": "880ef0c9-4e3e-42d4-9edc-667684a91889", "value": {"fileFieldName": "Test (T) or Check (C)"}, "objectId": "test_or_check"}, {"id": "b693eca7-efcd-4518-a9d3-db0b037a76ee", "value": {"fileFieldName": "Exp Title"}, "objectId": "exp_title"}, {"id": "df340215-db6e-4219-a3b7-119f297b81c3", "value": {"fileFieldName": "Exp Description"}, "objectId": "expDescription"}, {"id": "9ca7cc81-562c-43a7-989a-41da309f603d", "value": {"fileFieldName": "Exp Unit"}, "objectId": "expUnit"}, {"id": "27215777-c8f9-4fe7-a7ac-918d6168b0dd", "value": {"fileFieldName": "Exp Type"}, "objectId": "expType"}, {"id": "19d220e2-dff0-4a3a-ad6e-32f4d8602b5c", "value": {"fileFieldName": "Env"}, "objectId": "env"}, {"id": "861518b9-5c9e-4fe5-b31e-baf16e27155d", "value": {"fileFieldName": "Env Location"}, "objectId": "envLocation"}, {"id": "667355c3-dae1-4a64-94c8-ac2d543bd474", "value": {"fileFieldName": "Env Year"}, "objectId": "envYear"}, {"id": "ad11f2df-c5b4-4a05-8e52-c57625140061", "value": {"fileFieldName": "Exp Unit ID"}, "objectId": "expUnitId"}, {"id": "639b40ec-20f8-4659-8464-6a4be997ac7a", "value": {"fileFieldName": "Exp Replicate #"}, "objectId": "expReplicateNo"}, {"id": "2a62a80f-d8ba-42c4-9997-3b4ac8a965aa", "value": {"fileFieldName": "Exp Block #"}, "objectId": "expBlockNo"}, {"id": "f3e7de69-21ad-4cda-b1cc-a5e1987fb931", "value": {"fileFieldName": "Row"}, "objectId": "row"}, {"id": "251c5bbd-fc4d-4371-a4ce-4e2686f6837e", "value": {"fileFieldName": "Column"}, "objectId": "column"}, {"id": "ce5f61f2-f1de-45a4-8baf-e2471a5d863d", "value": {"fileFieldName": "Treatment Factors"}, "objectId": "treatmentFactors"}, {"id": "c5b8276f-e777-4385-a80f-5199abe63aac", "value": {"fileFieldName": "ObsUnitID"}, "objectId": "ObsUnitID"}]', + '[{"Env": "New Study", "Row": 4, "Column": 5, "Env Year": 2012, "Exp Type": "phenotyping", "Exp Unit": "plot", "Exp Title": "New Trial", "ObsUnitID": "", "Exp Block #": 2, "Exp Unit ID": 3, "Env Location": "New Location", "Germplasm GID": 1, "Germplasm Name": "Test", "Exp Description": "A new trial", "Exp Replicate #": 0, "Treatment Factors": "Jam application", "Test (T) or Check (C)": true}]', false,user_id,user_id); END $$; From 123d6fb99fe585f4b7de29f0d0fed30e6519668b Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 25 Sep 2023 12:40:23 -0400 Subject: [PATCH 004/220] [BI-1847] - added migration to rename test or check removed space from "(C )" --- .../V1.15.0__update_test_or_check_name.sql | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 src/main/resources/db/migration/V1.15.0__update_test_or_check_name.sql diff --git a/src/main/resources/db/migration/V1.15.0__update_test_or_check_name.sql b/src/main/resources/db/migration/V1.15.0__update_test_or_check_name.sql new file mode 100644 index 000000000..08e28a6ba --- /dev/null +++ b/src/main/resources/db/migration/V1.15.0__update_test_or_check_name.sql @@ -0,0 +1,26 @@ +-- +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +DO $$ + +BEGIN +update importer_mapping +set + mapping = '[{"id": "726a9f10-4892-4204-9e52-bd2b1d735f65", "value": {"fileFieldName": "Germplasm Name"}, "objectId": "germplasmName"}, {"id": "98774e20-6f89-4d6a-a7c9-f88887228ed6", "value": {"fileFieldName": "Germplasm GID"}, "objectId": "gid"}, {"id": "880ef0c9-4e3e-42d4-9edc-667684a91889", "value": {"fileFieldName": "Test (T) or Check (C)"}, "objectId": "test_or_check"}, {"id": "b693eca7-efcd-4518-a9d3-db0b037a76ee", "value": {"fileFieldName": "Exp Title"}, "objectId": "exp_title"}, {"id": "df340215-db6e-4219-a3b7-119f297b81c3", "value": {"fileFieldName": "Exp Description"}, "objectId": "expDescription"}, {"id": "9ca7cc81-562c-43a7-989a-41da309f603d", "value": {"fileFieldName": "Exp Unit"}, "objectId": "expUnit"}, {"id": "27215777-c8f9-4fe7-a7ac-918d6168b0dd", "value": {"fileFieldName": "Exp Type"}, "objectId": "expType"}, {"id": "19d220e2-dff0-4a3a-ad6e-32f4d8602b5c", "value": {"fileFieldName": "Env"}, "objectId": "env"}, {"id": "861518b9-5c9e-4fe5-b31e-baf16e27155d", "value": {"fileFieldName": "Env Location"}, "objectId": "envLocation"}, {"id": "667355c3-dae1-4a64-94c8-ac2d543bd474", "value": {"fileFieldName": "Env Year"}, "objectId": "envYear"}, {"id": "ad11f2df-c5b4-4a05-8e52-c57625140061", "value": {"fileFieldName": "Exp Unit ID"}, "objectId": "expUnitId"}, {"id": "639b40ec-20f8-4659-8464-6a4be997ac7a", "value": {"fileFieldName": "Exp Replicate #"}, "objectId": "expReplicateNo"}, {"id": "2a62a80f-d8ba-42c4-9997-3b4ac8a965aa", "value": {"fileFieldName": "Exp Block #"}, "objectId": "expBlockNo"}, {"id": "f3e7de69-21ad-4cda-b1cc-a5e1987fb931", "value": {"fileFieldName": "Row"}, "objectId": "row"}, {"id": "251c5bbd-fc4d-4371-a4ce-4e2686f6837e", "value": {"fileFieldName": "Column"}, "objectId": "column"}, {"id": "ce5f61f2-f1de-45a4-8baf-e2471a5d863d", "value": {"fileFieldName": "Treatment Factors"}, "objectId": "treatmentFactors"}, {"id": "c5b8276f-e777-4385-a80f-5199abe63aac", "value": {"fileFieldName": "ObsUnitID"}, "objectId": "ObsUnitID"}]', + file = '[{"Env": "New Study", "Row": 4, "Column": 5, "Env Year": 2012, "Exp Type": "phenotyping", "Exp Unit": "plot", "Exp Title": "New Trial", "ObsUnitID": "", "Exp Block #": 2, "Exp Unit ID": 3, "Env Location": "New Location", "Germplasm GID": 1, "Germplasm Name": "Test", "Exp Description": "A new trial", "Exp Replicate #": 0, "Treatment Factors": "Jam application", "Test (T) or Check (C)": true}]' +where import_type_id = 'ExperimentImport'; +END $$; \ No newline at end of file From 7a28a6084584840bd919d75806a7ce2ad320904e Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 25 Sep 2023 16:51:31 -0400 Subject: [PATCH 005/220] [BI-1852] check for duplicate breeding methods --- .../controller/BreedingMethodController.java | 4 +- .../impl/BreedingMethodServiceImpl.java | 78 +++++++++++++++---- .../validators/TraitValidatorUnitTest.java | 2 +- 3 files changed, 68 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/breedinginsight/api/v1/controller/BreedingMethodController.java b/src/main/java/org/breedinginsight/api/v1/controller/BreedingMethodController.java index 6e89bd7cd..c9399dfef 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/BreedingMethodController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/BreedingMethodController.java @@ -64,7 +64,7 @@ public HttpResponse>> getSyst @Post("programs/{programId}/breeding-methods") @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roles = {ProgramSecuredRole.BREEDER}) - public HttpResponse> createProgramBreedingMethod(@PathVariable UUID programId, @Body ProgramBreedingMethodEntity breedingMethod) throws BadRequestException, ApiException { + public HttpResponse createProgramBreedingMethod(@PathVariable UUID programId, @Body ProgramBreedingMethodEntity breedingMethod) throws BadRequestException, ApiException { log.debug("Saving new program breeding method"); try { @@ -79,6 +79,8 @@ public HttpResponse> createProgramBreeding response.metadata = metadata; return HttpResponse.ok(response); + } catch (BadRequestException ex) { + return HttpResponse.badRequest(ex.getMessage()); } catch (Exception e) { log.error("Error creating program breeding method", e); throw e; diff --git a/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java b/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java index 9b5c9fab6..83958ed71 100644 --- a/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java +++ b/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java @@ -1,6 +1,5 @@ package org.breedinginsight.services.impl; -import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService; @@ -14,8 +13,9 @@ import java.util.*; import java.util.stream.Collectors; +import static java.lang.String.format; + @Singleton -@Slf4j public class BreedingMethodServiceImpl implements BreedingMethodService { private final BreedingMethodDAO breedingMethodDAO; @@ -60,19 +60,21 @@ public List fetchBreedingMethodsInUse(UUID programI } @Override - public ProgramBreedingMethodEntity createBreedingMethod(ProgramBreedingMethodEntity breedingMethod, UUID programId, UUID userId) throws BadRequestException, ApiException { - if(!validateBreedingMethod(breedingMethod)) { - throw new BadRequestException("Missing required data"); - } + public ProgramBreedingMethodEntity createBreedingMethod(ProgramBreedingMethodEntity breedingMethod, UUID programId, UUID userId) throws BadRequestException { + validate(breedingMethod, programId); return dsl.transactionResult(() -> breedingMethodDAO.createProgramMethod(breedingMethod, programId, userId)); } + private boolean methodAlreadyExist(ProgramBreedingMethodEntity breedingMethod, UUID programId) { + List programMethods = getBreedingMethods(programId); + List systemMethods = getSystemBreedingMethods(); + return isDuplicateMethodFoundAnywhere(breedingMethod, systemMethods, programMethods); + } + @Override public ProgramBreedingMethodEntity updateBreedingMethod(ProgramBreedingMethodEntity breedingMethod, UUID programId, UUID userId) throws BadRequestException, ApiException { - if(!validateBreedingMethod(breedingMethod)) { - throw new BadRequestException("Missing required data"); - } + validate(breedingMethod, programId); List inUseMethods = fetchBreedingMethodsInUse(programId); if(inUseMethods.stream().anyMatch(method -> method.getId().equals(breedingMethod.getId()))) { @@ -82,6 +84,16 @@ public ProgramBreedingMethodEntity updateBreedingMethod(ProgramBreedingMethodEnt return dsl.transactionResult(() -> breedingMethodDAO.updateProgramMethod(breedingMethod, programId, userId)); } + private void validate(ProgramBreedingMethodEntity breedingMethod, UUID programId) throws BadRequestException { + if (isMissingRequiredFields(breedingMethod)) { + throw new BadRequestException("Missing required data"); + } + if (methodAlreadyExist(breedingMethod, programId)) { + throw new BadRequestException(format("A method with name:'%s' or abbreviation:'%s already exist", breedingMethod.getName(), breedingMethod.getAbbreviation())); + } + } + + @Override public void enableSystemMethods(List systemBreedingMethods, UUID programId, UUID userId) throws ApiException, BadRequestException { List inUseMethods = fetchBreedingMethodsInUse(programId); @@ -111,10 +123,48 @@ public void deleteBreedingMethod(UUID programId, UUID breedingMethodId) throws A dsl.transaction(() -> breedingMethodDAO.deleteProgramMethod(programId, breedingMethodId)); } - private boolean validateBreedingMethod(ProgramBreedingMethodEntity method) { - return StringUtils.isNotBlank(method.getName()) - && StringUtils.isNotBlank(method.getAbbreviation()) - && StringUtils.isNotBlank(method.getCategory()) - && StringUtils.isNotBlank(method.getGeneticDiversity()); + public boolean isMissingRequiredFields(ProgramBreedingMethodEntity method) { + return StringUtils.isBlank(method.getName()) + || StringUtils.isBlank(method.getAbbreviation()) + || StringUtils.isBlank(method.getCategory()) + || StringUtils.isBlank(method.getGeneticDiversity()); } + + public boolean isDuplicateMethodFoundAnywhere(ProgramBreedingMethodEntity testMethod, List systemBreedingMethodEntityList, List programBreedingMethodEntityList) { + boolean foundDup = isDuplicateMethodFoundOnList(testMethod, systemBreedingMethodEntityList); + if (!foundDup && programBreedingMethodEntityList!=null){ + foundDup = isDuplicateMethodFoundOnList(testMethod, programBreedingMethodEntityList); + } + return foundDup; + } + + private boolean isDuplicateMethodFoundOnList(ProgramBreedingMethodEntity testMethod, List programBreedingMethodEntityList) { + boolean foundDup = false; + for (ProgramBreedingMethodEntity method: programBreedingMethodEntityList) { + if(areMethodsDuplicate(testMethod, method)){ + foundDup = true; + break; + } + } + return foundDup; + } + + + public boolean areMethodsDuplicate(ProgramBreedingMethodEntity testMethod, ProgramBreedingMethodEntity method) { + boolean isDup = false; + + if(testMethod.getName()!= null && testMethod.getName().equals(method.getName())){ + isDup = true; + } + else if(testMethod.getAbbreviation()!= null && testMethod.getAbbreviation().equals(method.getAbbreviation())){ + isDup = true; + } + else if(testMethod.getName()==null && method.getName()==null || + testMethod.getAbbreviation()==null && method.getAbbreviation()==null){ + isDup = true; + } + + return isDup; + } + } diff --git a/src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java b/src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java index f40151f19..cc563772c 100644 --- a/src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java +++ b/src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java @@ -40,6 +40,7 @@ import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class TraitValidatorUnitTest { @@ -86,7 +87,6 @@ public void missingMethod() { ValidationErrors validationErrors = traitValidatorService.checkRequiredTraitFields(List.of(trait), new TraitValidatorError()); - assertEquals(1, validationErrors.getRowErrors().size(), "Wrong number of row errors returned"); RowValidationErrors rowValidationErrors = validationErrors.getRowErrors().get(0); assertEquals(1, rowValidationErrors.getErrors().size(), "Wrong number of errors for row"); From f864ac14405586c695bde16876f1f50ec74d32d9 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 26 Sep 2023 09:37:21 -0400 Subject: [PATCH 006/220] [BI-1340] - updated endpoint authorization made getSubscribedOntology (GET) accessible to any authenticated user --- .../api/v1/controller/OntologyController.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/api/v1/controller/OntologyController.java b/src/main/java/org/breedinginsight/api/v1/controller/OntologyController.java index 093d9c0da..b4d675075 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/OntologyController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/OntologyController.java @@ -4,6 +4,8 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.MediaType; import io.micronaut.http.annotation.*; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRole; @@ -81,7 +83,7 @@ public HttpResponse>> getAvailablePrograms } /** - * Accepts a list of programs to shared the ontology with. + * Accepts a list of programs to share the ontology with. * @param programId * @return List of programs successfully shared to with acceptable status * { @@ -210,7 +212,7 @@ public HttpResponse unsubscribeOntology( @Get("/programs/{programId}/ontology/subscribe") @Produces(MediaType.APPLICATION_JSON) @AddMetadata - @ProgramSecured(roles = {ProgramSecuredRole.BREEDER}) + @Secured(SecurityRule.IS_AUTHENTICATED) public HttpResponse>> getSubscribedOntology( @PathVariable UUID programId) { try { From 24e99f042041c1a2d56c45b04df27ee3cef716ac Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Tue, 26 Sep 2023 12:32:27 -0400 Subject: [PATCH 007/220] [BI-1852] respond with 400 errer with exception's text Add unit test --- .../controller/BreedingMethodController.java | 4 +- .../impl/BreedingMethodServiceImpl.java | 31 ++-- .../impl/BreedingMethodServiceImplTest.java | 135 ++++++++++++++++++ 3 files changed, 151 insertions(+), 19 deletions(-) create mode 100644 src/test/java/org/breedinginsight/services/impl/BreedingMethodServiceImplTest.java diff --git a/src/main/java/org/breedinginsight/api/v1/controller/BreedingMethodController.java b/src/main/java/org/breedinginsight/api/v1/controller/BreedingMethodController.java index c9399dfef..eca384476 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/BreedingMethodController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/BreedingMethodController.java @@ -112,7 +112,7 @@ public HttpResponse>> getProg @Put("programs/{programId}/breeding-methods/{breedingMethodId}") @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roles = {ProgramSecuredRole.BREEDER}) - public HttpResponse> updateProgramBreedingMethod(@PathVariable UUID programId, @PathVariable UUID breedingMethodId, @Body ProgramBreedingMethodEntity breedingMethod) throws BadRequestException, ApiException { + public HttpResponse updateProgramBreedingMethod(@PathVariable UUID programId, @PathVariable UUID breedingMethodId, @Body ProgramBreedingMethodEntity breedingMethod) throws BadRequestException, ApiException { log.debug("Saving new program breeding method"); try { @@ -130,6 +130,8 @@ public HttpResponse> updateProgramBreeding response.metadata = metadata; return HttpResponse.ok(response); + } catch (BadRequestException ex) { + return HttpResponse.badRequest(ex.getMessage()); } catch (Exception e) { log.error("Error updating program breeding method", e); throw e; diff --git a/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java b/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java index 83958ed71..3c523682e 100644 --- a/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java +++ b/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java @@ -67,19 +67,17 @@ public ProgramBreedingMethodEntity createBreedingMethod(ProgramBreedingMethodEnt } private boolean methodAlreadyExist(ProgramBreedingMethodEntity breedingMethod, UUID programId) { - List programMethods = getBreedingMethods(programId); - List systemMethods = getSystemBreedingMethods(); - return isDuplicateMethodFoundAnywhere(breedingMethod, systemMethods, programMethods); + List programAndSystemMethods = getBreedingMethods(programId); + return isDuplicateMethodFoundOnList(breedingMethod, programAndSystemMethods); } @Override public ProgramBreedingMethodEntity updateBreedingMethod(ProgramBreedingMethodEntity breedingMethod, UUID programId, UUID userId) throws BadRequestException, ApiException { - validate(breedingMethod, programId); - List inUseMethods = fetchBreedingMethodsInUse(programId); if(inUseMethods.stream().anyMatch(method -> method.getId().equals(breedingMethod.getId()))) { throw new BadRequestException("Breeding method is not allowed to be edited"); } + validate(breedingMethod, programId); return dsl.transactionResult(() -> breedingMethodDAO.updateProgramMethod(breedingMethod, programId, userId)); } @@ -123,24 +121,16 @@ public void deleteBreedingMethod(UUID programId, UUID breedingMethodId) throws A dsl.transaction(() -> breedingMethodDAO.deleteProgramMethod(programId, breedingMethodId)); } - public boolean isMissingRequiredFields(ProgramBreedingMethodEntity method) { + boolean isMissingRequiredFields(ProgramBreedingMethodEntity method) { return StringUtils.isBlank(method.getName()) || StringUtils.isBlank(method.getAbbreviation()) || StringUtils.isBlank(method.getCategory()) || StringUtils.isBlank(method.getGeneticDiversity()); } - public boolean isDuplicateMethodFoundAnywhere(ProgramBreedingMethodEntity testMethod, List systemBreedingMethodEntityList, List programBreedingMethodEntityList) { - boolean foundDup = isDuplicateMethodFoundOnList(testMethod, systemBreedingMethodEntityList); - if (!foundDup && programBreedingMethodEntityList!=null){ - foundDup = isDuplicateMethodFoundOnList(testMethod, programBreedingMethodEntityList); - } - return foundDup; - } - - private boolean isDuplicateMethodFoundOnList(ProgramBreedingMethodEntity testMethod, List programBreedingMethodEntityList) { + boolean isDuplicateMethodFoundOnList(ProgramBreedingMethodEntity method, List programBreedingMethodEntityList) { boolean foundDup = false; - for (ProgramBreedingMethodEntity method: programBreedingMethodEntityList) { + for (ProgramBreedingMethodEntity testMethod: programBreedingMethodEntityList) { if(areMethodsDuplicate(testMethod, method)){ foundDup = true; break; @@ -150,9 +140,14 @@ private boolean isDuplicateMethodFoundOnList(ProgramBreedingMethodEntity testMet } - public boolean areMethodsDuplicate(ProgramBreedingMethodEntity testMethod, ProgramBreedingMethodEntity method) { - boolean isDup = false; + boolean areMethodsDuplicate(ProgramBreedingMethodEntity testMethod, ProgramBreedingMethodEntity method) { + // SPECIAL CASE: If the two methods are the same method, then they are not duplicates + if( (testMethod.getId()!=null) && testMethod.getId().equals(method.getId()) ){ + return false; + } + + boolean isDup = false; if(testMethod.getName()!= null && testMethod.getName().equals(method.getName())){ isDup = true; } diff --git a/src/test/java/org/breedinginsight/services/impl/BreedingMethodServiceImplTest.java b/src/test/java/org/breedinginsight/services/impl/BreedingMethodServiceImplTest.java new file mode 100644 index 000000000..9e7ec3c87 --- /dev/null +++ b/src/test/java/org/breedinginsight/services/impl/BreedingMethodServiceImplTest.java @@ -0,0 +1,135 @@ +package org.breedinginsight.services.impl; + +import lombok.SneakyThrows; +import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService; +import org.breedinginsight.dao.db.tables.pojos.ProgramBreedingMethodEntity; +import org.breedinginsight.daos.BreedingMethodDAO; +import org.jooq.DSLContext; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class BreedingMethodServiceImplTest { + BreedingMethodServiceImpl breedingMethodService; + BreedingMethodDAO breedingMethodDAO = null; + BrAPIGermplasmService germplasmService = null; + DSLContext dsl = null; + + @BeforeAll + public void setup() { + breedingMethodService = new BreedingMethodServiceImpl(breedingMethodDAO, germplasmService, dsl); + } + + @Test + public void hasRequiredFields(){ + ProgramBreedingMethodEntity method= new ProgramBreedingMethodEntity(); + assertTrue(breedingMethodService.isMissingRequiredFields(method),"the method has blanks"); + + method.setName("Dave"); + method.setAbbreviation("DRP"); + method.setCategory("human"); + method.setGeneticDiversity("none"); + assertFalse(breedingMethodService.isMissingRequiredFields(method),"the method has all required fields"); + } + + @Test + @SneakyThrows + public void isDupOfProgramMethod() { + List programBreedingMethodEntityList = new ArrayList<>(); + programBreedingMethodEntityList.add(makeMethod("1")); + programBreedingMethodEntityList.add(makeMethod("2")); + + boolean isDup = breedingMethodService.isDuplicateMethodFoundOnList(makeMethod("1"), programBreedingMethodEntityList); + assertTrue(isDup, "Duplicate found in list."); + } + + @Test + @SneakyThrows + public void isNotADupMethod() { + List programBreedingMethodEntityList = new ArrayList<>(); + programBreedingMethodEntityList.add(makeMethod("1")); + programBreedingMethodEntityList.add(makeMethod("2")); + + + ProgramBreedingMethodEntity methodUnique = makeMethod("unique");; + boolean isDup = breedingMethodService.isDuplicateMethodFoundOnList(methodUnique, programBreedingMethodEntityList); + assertFalse(isDup, "No duplicates method found."); + + } + @Test + @SneakyThrows + public void methodNameEqual() { + ProgramBreedingMethodEntity method = makeMethod("ABC"); + ProgramBreedingMethodEntity testMethod = makeMethod("ABC"); + testMethod.setAbbreviation("misatch Abbreviation"); + assertTrue( + breedingMethodService.areMethodsDuplicate(testMethod, method), + "Method Names are Equal" + ); + assertTrue( + breedingMethodService.areMethodsDuplicate(method, testMethod), + "Method Names are Equal (switched order)" + ); + + testMethod.setName("misatch Name"); + assertFalse( + breedingMethodService.areMethodsDuplicate(testMethod, method), + "Method Names are not Equal" + ); + testMethod.setName(null); + assertFalse( + breedingMethodService.areMethodsDuplicate(testMethod, method), + "Method Names are not Equal (one is null)" + ); + + testMethod.setName("name"); + method.setName(null); + assertFalse( + breedingMethodService.areMethodsDuplicate(testMethod, method), + "Method Names are not Equal (the other is null)" + ); + + } + + @Test + @SneakyThrows + public void methodAbbreviationEqual() { + ProgramBreedingMethodEntity method = makeMethod("ABC"); + ProgramBreedingMethodEntity testMethod = makeMethod("ABC"); + testMethod.setName("misatch Name"); + assertTrue( + breedingMethodService.areMethodsDuplicate(testMethod, method), + "Method Abbreviations are Equal" + ); + assertTrue( + breedingMethodService.areMethodsDuplicate(method, testMethod), + "Method Abbreviations are Equal (switched order)" + ); + + testMethod.setAbbreviation("misatch Abbreviation"); + assertFalse( + breedingMethodService.areMethodsDuplicate(method, testMethod), + "Method Abbreviations are not equal." + ); + testMethod.setAbbreviation(null); + assertFalse( + breedingMethodService.areMethodsDuplicate(method, testMethod), + "Method Abbreviations are not equal (one is null)." + ); + } + + // Helper Methods // + private ProgramBreedingMethodEntity makeMethod(String suffix){ + ProgramBreedingMethodEntity method = new ProgramBreedingMethodEntity(); + method.setName("Name" + suffix); + method.setAbbreviation("Abbreviation" + suffix); + return method; + } + +} From 3212e8af783a6c7f0dffb47faec510f7ee40b2aa Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Tue, 26 Sep 2023 12:45:10 -0400 Subject: [PATCH 008/220] [BI-1852]fixed some existing 'Problems' in BreedingMethodController.java --- .../api/v1/controller/BreedingMethodController.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/api/v1/controller/BreedingMethodController.java b/src/main/java/org/breedinginsight/api/v1/controller/BreedingMethodController.java index eca384476..7b3fc2bb5 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/BreedingMethodController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/BreedingMethodController.java @@ -7,7 +7,6 @@ import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.api.auth.*; import org.breedinginsight.api.model.v1.response.DataResponse; @@ -17,18 +16,14 @@ import org.breedinginsight.api.model.v1.response.metadata.Status; import org.breedinginsight.api.model.v1.response.metadata.StatusCode; import org.breedinginsight.api.v1.controller.metadata.AddMetadata; -import org.breedinginsight.dao.db.tables.pojos.BreedingMethodEntity; import org.breedinginsight.dao.db.tables.pojos.ProgramBreedingMethodEntity; -import org.breedinginsight.daos.BreedingMethodDAO; import org.breedinginsight.services.BreedingMethodService; import org.breedinginsight.services.exceptions.BadRequestException; import javax.inject.Inject; -import javax.validation.Valid; import java.util.ArrayList; import java.util.List; import java.util.UUID; -import java.util.stream.Collectors; @Slf4j @Controller("/${micronaut.bi.api.version}") @@ -64,7 +59,7 @@ public HttpResponse>> getSyst @Post("programs/{programId}/breeding-methods") @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roles = {ProgramSecuredRole.BREEDER}) - public HttpResponse createProgramBreedingMethod(@PathVariable UUID programId, @Body ProgramBreedingMethodEntity breedingMethod) throws BadRequestException, ApiException { + public HttpResponse createProgramBreedingMethod(@PathVariable UUID programId, @Body ProgramBreedingMethodEntity breedingMethod) throws ApiException { log.debug("Saving new program breeding method"); try { @@ -112,7 +107,7 @@ public HttpResponse>> getProg @Put("programs/{programId}/breeding-methods/{breedingMethodId}") @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roles = {ProgramSecuredRole.BREEDER}) - public HttpResponse updateProgramBreedingMethod(@PathVariable UUID programId, @PathVariable UUID breedingMethodId, @Body ProgramBreedingMethodEntity breedingMethod) throws BadRequestException, ApiException { + public HttpResponse updateProgramBreedingMethod(@PathVariable UUID programId, @PathVariable UUID breedingMethodId, @Body ProgramBreedingMethodEntity breedingMethod) throws ApiException { log.debug("Saving new program breeding method"); try { From 91ccde280b93ef7c6b131e008c98b0310fe8d4d6 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 26 Sep 2023 13:53:33 -0400 Subject: [PATCH 009/220] [BI-1847] - revert change to migrations keep checksums consistent --- .../V0.5.0__create-base-entities.sql | 32 +++++++++---------- .../V0.5.10__create_ontology_tables.sql | 32 +++++++++---------- .../V0.5.11__create-upload-table.sql | 32 +++++++++---------- .../V0.5.12__add-another-test-user.sql | 32 +++++++++---------- .../V0.5.13__add-trait-abbreviation.sql | 32 +++++++++---------- .../migration/V0.5.14__set-orcid-unique.sql | 32 +++++++++---------- .../V0.5.15__create-inventory-tables.sql | 32 +++++++++---------- .../V0.5.16__add-user-account-token.sql | 32 +++++++++---------- .../V0.5.17__remove_unused_trait_columns.sql | 32 +++++++++---------- .../V0.5.18__remove_code_data_type.sql | 32 +++++++++---------- .../migration/V0.5.19__set_trait_active.sql | 32 +++++++++---------- .../migration/V0.5.1__create-user-tables.sql | 32 +++++++++---------- .../V0.5.20__add-program-brapi-url.sql | 32 +++++++++---------- .../V0.5.21__create-brapi-import-tables.sql | 32 +++++++++---------- .../V0.5.22__rename-trait-name-column.sql | 32 +++++++++---------- ...3__create-system-level-import-mappings.sql | 32 +++++++++---------- .../db/migration/V0.5.24__add-species.sql | 32 +++++++++---------- .../V0.5.25__create-breeding-method-table.sql | 32 +++++++++---------- .../db/migration/V0.5.26__add-program-key.sql | 32 +++++++++---------- ..._add_germplasm_template_system_mapping.sql | 32 +++++++++---------- .../V0.5.28__remove_trait_abbreviations.sql | 32 +++++++++---------- .../migration/V0.5.2__populate-user-data.sql | 32 +++++++++---------- .../V0.5.30__create_germplasm_sequence.sql | 32 +++++++++---------- ...2__germplasm_import_change_dbid_to_gid.sql | 32 +++++++++---------- .../V0.5.33__alter-breeding-methods.sql | 32 +++++++++---------- ...0.5.34__created_shared_ontology_tables.sql | 32 +++++++++---------- ...V0.5.35__germplasm_import_add_synonyms.sql | 32 +++++++++---------- .../V0.5.3__create-program-tables.sql | 32 +++++++++---------- .../migration/V0.5.4__create-program-data.sql | 32 +++++++++---------- .../V0.5.5__create-user-role-tables.sql | 32 +++++++++---------- .../V0.5.6__create-new-test-user.sql | 32 +++++++++---------- .../V0.5.7__add-user-active-flag.sql | 32 +++++++++---------- .../V0.5.8__add-inactive-test-user.sql | 32 +++++++++---------- .../V0.5.9__add-program-user-active-flag.sql | 32 +++++++++---------- .../V0.7.1__add_breeding_methods.sql | 32 +++++++++---------- .../migration/V0_5_29__add_program_keys.sql | 32 +++++++++---------- .../migration/V1.0.10__add_onto_term_type.sql | 32 +++++++++---------- ....0.11__create_program_breeding_methods.sql | 32 +++++++++---------- ....12__create_genotypedataimport_mapping.sql | 32 +++++++++---------- .../V1.0.14__germplasm_import_add_GID.sql | 32 +++++++++---------- ...add_experiment_template_system_mapping.sql | 32 +++++++++---------- ...ate_experiment_template_system_mapping.sql | 32 +++++++++---------- .../V1.0.3__create_obs_unit_sequence.sql | 32 +++++++++---------- .../V1.0.4__create_experiment_sequence.sql | 32 +++++++++---------- .../V1.0.5__experiment_template_obsUnitID.sql | 32 +++++++++---------- .../V1.0.6__create_environment_sequence.sql | 32 +++++++++---------- .../V1.0.7__delete_obs_unit_sequence.sql | 32 +++++++++---------- ...eriment_obs_id_template_system_mapping.sql | 32 +++++++++---------- .../V1.0.9__add_dynamic_column_storage.sql | 32 +++++++++---------- .../V1.15.0__update_test_or_check_name.sql | 32 +++++++++---------- .../resources/db/migration/beforeClean.sql | 32 +++++++++---------- 51 files changed, 816 insertions(+), 816 deletions(-) diff --git a/src/main/resources/db/migration/V0.5.0__create-base-entities.sql b/src/main/resources/db/migration/V0.5.0__create-base-entities.sql index 28a68474a..5c28fc61e 100644 --- a/src/main/resources/db/migration/V0.5.0__create-base-entities.sql +++ b/src/main/resources/db/migration/V0.5.0__create-base-entities.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -- We need the public schema so we can create the uuid object in it create schema if not exists public; diff --git a/src/main/resources/db/migration/V0.5.10__create_ontology_tables.sql b/src/main/resources/db/migration/V0.5.10__create_ontology_tables.sql index 66c2e0765..2108c9975 100644 --- a/src/main/resources/db/migration/V0.5.10__create_ontology_tables.sql +++ b/src/main/resources/db/migration/V0.5.10__create_ontology_tables.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TYPE "data_type" AS ENUM ( 'CODE', diff --git a/src/main/resources/db/migration/V0.5.11__create-upload-table.sql b/src/main/resources/db/migration/V0.5.11__create-upload-table.sql index 0e3f6bb03..5f0f84d71 100644 --- a/src/main/resources/db/migration/V0.5.11__create-upload-table.sql +++ b/src/main/resources/db/migration/V0.5.11__create-upload-table.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TYPE "upload_type" AS ENUM ( 'TRAIT', diff --git a/src/main/resources/db/migration/V0.5.12__add-another-test-user.sql b/src/main/resources/db/migration/V0.5.12__add-another-test-user.sql index 6767ab0b7..8083ddcc1 100644 --- a/src/main/resources/db/migration/V0.5.12__add-another-test-user.sql +++ b/src/main/resources/db/migration/V0.5.12__add-another-test-user.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ DO $$ DECLARE diff --git a/src/main/resources/db/migration/V0.5.13__add-trait-abbreviation.sql b/src/main/resources/db/migration/V0.5.13__add-trait-abbreviation.sql index f2edeacb4..33c610135 100644 --- a/src/main/resources/db/migration/V0.5.13__add-trait-abbreviation.sql +++ b/src/main/resources/db/migration/V0.5.13__add-trait-abbreviation.sql @@ -1,18 +1,18 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ ALTER TABLE trait ADD COLUMN abbreviations text[] DEFAULT '{}'; \ No newline at end of file diff --git a/src/main/resources/db/migration/V0.5.14__set-orcid-unique.sql b/src/main/resources/db/migration/V0.5.14__set-orcid-unique.sql index 9a027a40b..ddbd23380 100644 --- a/src/main/resources/db/migration/V0.5.14__set-orcid-unique.sql +++ b/src/main/resources/db/migration/V0.5.14__set-orcid-unique.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ ALTER TABLE bi_user ADD CONSTRAINT orcid_unique UNIQUE (orcid); \ No newline at end of file diff --git a/src/main/resources/db/migration/V0.5.15__create-inventory-tables.sql b/src/main/resources/db/migration/V0.5.15__create-inventory-tables.sql index ed7824d41..36dabe7bf 100644 --- a/src/main/resources/db/migration/V0.5.15__create-inventory-tables.sql +++ b/src/main/resources/db/migration/V0.5.15__create-inventory-tables.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TYPE "entity_level" AS ENUM ( 'POPULATION', diff --git a/src/main/resources/db/migration/V0.5.16__add-user-account-token.sql b/src/main/resources/db/migration/V0.5.16__add-user-account-token.sql index 3afa712f3..f6d1f3e66 100644 --- a/src/main/resources/db/migration/V0.5.16__add-user-account-token.sql +++ b/src/main/resources/db/migration/V0.5.16__add-user-account-token.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ ALTER TABLE bi_user ADD COLUMN account_token text; diff --git a/src/main/resources/db/migration/V0.5.17__remove_unused_trait_columns.sql b/src/main/resources/db/migration/V0.5.17__remove_unused_trait_columns.sql index ceb7eacf5..e55e4dff5 100644 --- a/src/main/resources/db/migration/V0.5.17__remove_unused_trait_columns.sql +++ b/src/main/resources/db/migration/V0.5.17__remove_unused_trait_columns.sql @@ -1,18 +1,18 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ ALTER TABLE method DROP COLUMN method_name; \ No newline at end of file diff --git a/src/main/resources/db/migration/V0.5.18__remove_code_data_type.sql b/src/main/resources/db/migration/V0.5.18__remove_code_data_type.sql index c94558ce0..672d4a1ca 100644 --- a/src/main/resources/db/migration/V0.5.18__remove_code_data_type.sql +++ b/src/main/resources/db/migration/V0.5.18__remove_code_data_type.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ ALTER TYPE data_type RENAME TO data_type_old; diff --git a/src/main/resources/db/migration/V0.5.19__set_trait_active.sql b/src/main/resources/db/migration/V0.5.19__set_trait_active.sql index f60313540..d73c27432 100644 --- a/src/main/resources/db/migration/V0.5.19__set_trait_active.sql +++ b/src/main/resources/db/migration/V0.5.19__set_trait_active.sql @@ -1,18 +1,18 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ UPDATE trait set active = true where active is NULL; \ No newline at end of file diff --git a/src/main/resources/db/migration/V0.5.1__create-user-tables.sql b/src/main/resources/db/migration/V0.5.1__create-user-tables.sql index afbfb539f..38d14d49e 100644 --- a/src/main/resources/db/migration/V0.5.1__create-user-tables.sql +++ b/src/main/resources/db/migration/V0.5.1__create-user-tables.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TABLE bi_user ( like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, diff --git a/src/main/resources/db/migration/V0.5.20__add-program-brapi-url.sql b/src/main/resources/db/migration/V0.5.20__add-program-brapi-url.sql index 017d7a92b..9a6ffb28d 100644 --- a/src/main/resources/db/migration/V0.5.20__add-program-brapi-url.sql +++ b/src/main/resources/db/migration/V0.5.20__add-program-brapi-url.sql @@ -1,18 +1,18 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ ALTER TABLE program ADD COLUMN brapi_url text; diff --git a/src/main/resources/db/migration/V0.5.21__create-brapi-import-tables.sql b/src/main/resources/db/migration/V0.5.21__create-brapi-import-tables.sql index a88405a01..ac5726fe4 100644 --- a/src/main/resources/db/migration/V0.5.21__create-brapi-import-tables.sql +++ b/src/main/resources/db/migration/V0.5.21__create-brapi-import-tables.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ create table importer_mapping ( like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, diff --git a/src/main/resources/db/migration/V0.5.22__rename-trait-name-column.sql b/src/main/resources/db/migration/V0.5.22__rename-trait-name-column.sql index 5e9c6e3c3..a3979b5f7 100644 --- a/src/main/resources/db/migration/V0.5.22__rename-trait-name-column.sql +++ b/src/main/resources/db/migration/V0.5.22__rename-trait-name-column.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ DO $$ diff --git a/src/main/resources/db/migration/V0.5.23__create-system-level-import-mappings.sql b/src/main/resources/db/migration/V0.5.23__create-system-level-import-mappings.sql index c7402f408..b15accfdb 100644 --- a/src/main/resources/db/migration/V0.5.23__create-system-level-import-mappings.sql +++ b/src/main/resources/db/migration/V0.5.23__create-system-level-import-mappings.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -- Create new importer_mapping_program create table importer_mapping_program ( diff --git a/src/main/resources/db/migration/V0.5.24__add-species.sql b/src/main/resources/db/migration/V0.5.24__add-species.sql index d040e19ac..39f470a05 100644 --- a/src/main/resources/db/migration/V0.5.24__add-species.sql +++ b/src/main/resources/db/migration/V0.5.24__add-species.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ DO $$ DECLARE diff --git a/src/main/resources/db/migration/V0.5.25__create-breeding-method-table.sql b/src/main/resources/db/migration/V0.5.25__create-breeding-method-table.sql index 45716d4c3..535d2115c 100644 --- a/src/main/resources/db/migration/V0.5.25__create-breeding-method-table.sql +++ b/src/main/resources/db/migration/V0.5.25__create-breeding-method-table.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ create table breeding_method ( like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, diff --git a/src/main/resources/db/migration/V0.5.26__add-program-key.sql b/src/main/resources/db/migration/V0.5.26__add-program-key.sql index 4e2fb0d90..859ea8ad4 100644 --- a/src/main/resources/db/migration/V0.5.26__add-program-key.sql +++ b/src/main/resources/db/migration/V0.5.26__add-program-key.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ --Add key column to programs ALTER TABLE program ADD COLUMN key text; \ No newline at end of file diff --git a/src/main/resources/db/migration/V0.5.27__add_germplasm_template_system_mapping.sql b/src/main/resources/db/migration/V0.5.27__add_germplasm_template_system_mapping.sql index caa34b50a..c7ff87677 100644 --- a/src/main/resources/db/migration/V0.5.27__add_germplasm_template_system_mapping.sql +++ b/src/main/resources/db/migration/V0.5.27__add_germplasm_template_system_mapping.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ DO $$ DECLARE diff --git a/src/main/resources/db/migration/V0.5.28__remove_trait_abbreviations.sql b/src/main/resources/db/migration/V0.5.28__remove_trait_abbreviations.sql index a41b74bc1..1d3901baf 100644 --- a/src/main/resources/db/migration/V0.5.28__remove_trait_abbreviations.sql +++ b/src/main/resources/db/migration/V0.5.28__remove_trait_abbreviations.sql @@ -1,18 +1,18 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ ALTER TABLE trait DROP COLUMN abbreviations ; \ No newline at end of file diff --git a/src/main/resources/db/migration/V0.5.2__populate-user-data.sql b/src/main/resources/db/migration/V0.5.2__populate-user-data.sql index 7abc34a37..c7fe7e5aa 100644 --- a/src/main/resources/db/migration/V0.5.2__populate-user-data.sql +++ b/src/main/resources/db/migration/V0.5.2__populate-user-data.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ insert into bi_user (id, name, created_by, updated_by) values diff --git a/src/main/resources/db/migration/V0.5.30__create_germplasm_sequence.sql b/src/main/resources/db/migration/V0.5.30__create_germplasm_sequence.sql index d127b0fe3..60c022582 100644 --- a/src/main/resources/db/migration/V0.5.30__create_germplasm_sequence.sql +++ b/src/main/resources/db/migration/V0.5.30__create_germplasm_sequence.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ alter table program add column germplasm_sequence text; diff --git a/src/main/resources/db/migration/V0.5.32__germplasm_import_change_dbid_to_gid.sql b/src/main/resources/db/migration/V0.5.32__germplasm_import_change_dbid_to_gid.sql index d943bfc62..32cd0e5fc 100644 --- a/src/main/resources/db/migration/V0.5.32__germplasm_import_change_dbid_to_gid.sql +++ b/src/main/resources/db/migration/V0.5.32__germplasm_import_change_dbid_to_gid.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ update importer_mapping set mapping = '[{"id": "f8d43c7e-a618-4c16-8829-3085f7202a67", "mapping": [{"id": "f384837e-ad8d-4dbe-b54e-87b57070bed1", "value": {"fileFieldName": "Name"}, "objectId": "germplasmName"}, {"id": "39628d14-458b-429b-8e66-bb48e0445a83", "value": {"fileFieldName": "Breeding Method"}, "objectId": "breedingMethod"}, {"id": "f1ba63e1-f5e4-433f-a53e-1c2f3e2fa71f", "value": {"fileFieldName": "Source"}, "objectId": "germplasmSource"}, {"id": "f5892565-f888-4596-be82-ab8eeabf37ce", "value": {"fileFieldName": "External UID"}, "objectId": "externalUID"}, {"id": "65507e5d-2d66-4595-8763-e772fe25c870", "value": {"fileFieldName": "Entry No"}, "objectId": "entryNo"}, {"id": "3eae24c1-ca4a-48a2-96d0-3cf4630acd3a", "value": {"fileFieldName": "Female Parent GID"}, "objectId": "femaleParentDBID"}, {"id": "2dbd7262-93a1-44b0-86b7-f5fca290b965", "value": {"fileFieldName": "Male Parent GID"}, "objectId": "maleParentDBID"}, {"id": "6f7f1539-6e8f-4ede-b7d3-3423cc63abec", "value": {"fileFieldName": "Female Parent Entry No"}, "objectId": "femaleParentEntryNo"}, {"id": "25fe9954-bca7-42f1-818a-5f71e242fa1f", "value": {"fileFieldName": "Male Parent Entry No"}, "objectId": "maleParentEntryNo"}, {"id": "15836d5f-8194-40a8-a771-114eaae31eb4", "objectId": "germplasmPUI"}, {"id": "675b6af8-5a17-4146-a503-2e4e1a65d5fa", "objectId": "acquisitionDate"}, {"id": "69a3bd3c-cebc-435c-acdd-0be62dda25ed", "objectId": "countryOfOrigin"}, {"id": "8ab25267-20f2-450e-89ca-21634ff8fadb", "objectId": "collection"}, {"id": "ce1701e2-2f61-4250-8595-9536e3f5ddcf", "objectId": "AdditionalInfo"}, {"id": "3470e9df-a028-45b7-943f-198bc62b6dbe", "objectId": "ExternalReference"}], "objectId": "Germplasm"}]', diff --git a/src/main/resources/db/migration/V0.5.33__alter-breeding-methods.sql b/src/main/resources/db/migration/V0.5.33__alter-breeding-methods.sql index 162b98f3f..d5ce8ccc1 100644 --- a/src/main/resources/db/migration/V0.5.33__alter-breeding-methods.sql +++ b/src/main/resources/db/migration/V0.5.33__alter-breeding-methods.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ insert into breeding_method (name, abbreviation, description, category, genetic_diversity) values diff --git a/src/main/resources/db/migration/V0.5.34__created_shared_ontology_tables.sql b/src/main/resources/db/migration/V0.5.34__created_shared_ontology_tables.sql index a25c1b5f0..abaa1b8fb 100644 --- a/src/main/resources/db/migration/V0.5.34__created_shared_ontology_tables.sql +++ b/src/main/resources/db/migration/V0.5.34__created_shared_ontology_tables.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TABLE program_shared_ontology ( like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, diff --git a/src/main/resources/db/migration/V0.5.35__germplasm_import_add_synonyms.sql b/src/main/resources/db/migration/V0.5.35__germplasm_import_add_synonyms.sql index ac16cc670..f583f2e06 100644 --- a/src/main/resources/db/migration/V0.5.35__germplasm_import_add_synonyms.sql +++ b/src/main/resources/db/migration/V0.5.35__germplasm_import_add_synonyms.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ update importer_mapping set mapping = '[{"id": "f8d43c7e-a618-4c16-8829-3085f7202a67", "mapping": [{"id": "f384837e-ad8d-4dbe-b54e-87b57070bed1", "value": {"fileFieldName": "Name"}, "objectId": "germplasmName"}, {"id": "39628d14-458b-429b-8e66-bb48e0445a83", "value": {"fileFieldName": "Breeding Method"}, "objectId": "breedingMethod"}, {"id": "f1ba63e1-f5e4-433f-a53e-1c2f3e2fa71f", "value": {"fileFieldName": "Source"}, "objectId": "germplasmSource"}, {"id": "f5892565-f888-4596-be82-ab8eeabf37ce", "value": {"fileFieldName": "External UID"}, "objectId": "externalUID"}, {"id": "65507e5d-2d66-4595-8763-e772fe25c870", "value": {"fileFieldName": "Entry No"}, "objectId": "entryNo"}, {"id": "3eae24c1-ca4a-48a2-96d0-3cf4630acd3a", "value": {"fileFieldName": "Female Parent GID"}, "objectId": "femaleParentDBID"}, {"id": "2dbd7262-93a1-44b0-86b7-f5fca290b965", "value": {"fileFieldName": "Male Parent GID"}, "objectId": "maleParentDBID"}, {"id": "6f7f1539-6e8f-4ede-b7d3-3423cc63abec", "value": {"fileFieldName": "Female Parent Entry No"}, "objectId": "femaleParentEntryNo"}, {"id": "25fe9954-bca7-42f1-818a-5f71e242fa1f", "value": {"fileFieldName": "Male Parent Entry No"}, "objectId": "maleParentEntryNo"}, {"id": "b910adfe-a474-47a0-8410-514578898436", "value": {"fileFieldName": "Synonyms"}, "objectId": "synonyms"}, {"id": "15836d5f-8194-40a8-a771-114eaae31eb4", "objectId": "germplasmPUI"}, {"id": "675b6af8-5a17-4146-a503-2e4e1a65d5fa", "objectId": "acquisitionDate"}, {"id": "69a3bd3c-cebc-435c-acdd-0be62dda25ed", "objectId": "countryOfOrigin"}, {"id": "8ab25267-20f2-450e-89ca-21634ff8fadb", "objectId": "collection"}, {"id": "ce1701e2-2f61-4250-8595-9536e3f5ddcf", "objectId": "AdditionalInfo"}, {"id": "3470e9df-a028-45b7-943f-198bc62b6dbe", "objectId": "ExternalReference"}], "objectId": "Germplasm"}]', diff --git a/src/main/resources/db/migration/V0.5.3__create-program-tables.sql b/src/main/resources/db/migration/V0.5.3__create-program-tables.sql index 9f628bb78..8be53b717 100644 --- a/src/main/resources/db/migration/V0.5.3__create-program-tables.sql +++ b/src/main/resources/db/migration/V0.5.3__create-program-tables.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TABLE program ( like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, diff --git a/src/main/resources/db/migration/V0.5.4__create-program-data.sql b/src/main/resources/db/migration/V0.5.4__create-program-data.sql index 4ea9d64b9..0ae9889b2 100644 --- a/src/main/resources/db/migration/V0.5.4__create-program-data.sql +++ b/src/main/resources/db/migration/V0.5.4__create-program-data.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ DO $$ DECLARE diff --git a/src/main/resources/db/migration/V0.5.5__create-user-role-tables.sql b/src/main/resources/db/migration/V0.5.5__create-user-role-tables.sql index 591fff82f..625024bbd 100644 --- a/src/main/resources/db/migration/V0.5.5__create-user-role-tables.sql +++ b/src/main/resources/db/migration/V0.5.5__create-user-role-tables.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TABLE system_user_role ( like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, diff --git a/src/main/resources/db/migration/V0.5.6__create-new-test-user.sql b/src/main/resources/db/migration/V0.5.6__create-new-test-user.sql index 3f25944cd..f6333bce4 100644 --- a/src/main/resources/db/migration/V0.5.6__create-new-test-user.sql +++ b/src/main/resources/db/migration/V0.5.6__create-new-test-user.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ DO $$ DECLARE diff --git a/src/main/resources/db/migration/V0.5.7__add-user-active-flag.sql b/src/main/resources/db/migration/V0.5.7__add-user-active-flag.sql index 229013458..d386977cf 100644 --- a/src/main/resources/db/migration/V0.5.7__add-user-active-flag.sql +++ b/src/main/resources/db/migration/V0.5.7__add-user-active-flag.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ ALTER TABLE bi_user ADD active bool NOT NULL DEFAULT true; diff --git a/src/main/resources/db/migration/V0.5.8__add-inactive-test-user.sql b/src/main/resources/db/migration/V0.5.8__add-inactive-test-user.sql index 6b77a3365..fc565e790 100644 --- a/src/main/resources/db/migration/V0.5.8__add-inactive-test-user.sql +++ b/src/main/resources/db/migration/V0.5.8__add-inactive-test-user.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ DO $$ DECLARE diff --git a/src/main/resources/db/migration/V0.5.9__add-program-user-active-flag.sql b/src/main/resources/db/migration/V0.5.9__add-program-user-active-flag.sql index 2cfc72c12..2fbfdc931 100644 --- a/src/main/resources/db/migration/V0.5.9__add-program-user-active-flag.sql +++ b/src/main/resources/db/migration/V0.5.9__add-program-user-active-flag.sql @@ -1,18 +1,18 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ ALTER TABLE program_user_role ADD active bool NOT NULL DEFAULT true; diff --git a/src/main/resources/db/migration/V0.7.1__add_breeding_methods.sql b/src/main/resources/db/migration/V0.7.1__add_breeding_methods.sql index b6973ad5a..4bf58b94d 100644 --- a/src/main/resources/db/migration/V0.7.1__add_breeding_methods.sql +++ b/src/main/resources/db/migration/V0.7.1__add_breeding_methods.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ insert into breeding_method (abbreviation, name, description, genetic_diversity, category) diff --git a/src/main/resources/db/migration/V0_5_29__add_program_keys.sql b/src/main/resources/db/migration/V0_5_29__add_program_keys.sql index 3e63e48db..0d832bb9a 100644 --- a/src/main/resources/db/migration/V0_5_29__add_program_keys.sql +++ b/src/main/resources/db/migration/V0_5_29__add_program_keys.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ UPDATE program SET key = 'STPALF' WHERE name = 'StPaul-Alfalfa'; UPDATE program SET key = 'PROALF' WHERE name = 'Prosser-Alfalfa'; diff --git a/src/main/resources/db/migration/V1.0.10__add_onto_term_type.sql b/src/main/resources/db/migration/V1.0.10__add_onto_term_type.sql index 97231fa53..872df67e7 100644 --- a/src/main/resources/db/migration/V1.0.10__add_onto_term_type.sql +++ b/src/main/resources/db/migration/V1.0.10__add_onto_term_type.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ CREATE TYPE "term_type" AS ENUM ( 'PHENOTYPE', diff --git a/src/main/resources/db/migration/V1.0.11__create_program_breeding_methods.sql b/src/main/resources/db/migration/V1.0.11__create_program_breeding_methods.sql index 9ead47f2c..1637517e5 100644 --- a/src/main/resources/db/migration/V1.0.11__create_program_breeding_methods.sql +++ b/src/main/resources/db/migration/V1.0.11__create_program_breeding_methods.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ create table program_breeding_method ( diff --git a/src/main/resources/db/migration/V1.0.12__create_genotypedataimport_mapping.sql b/src/main/resources/db/migration/V1.0.12__create_genotypedataimport_mapping.sql index 4181e330c..c15ee8668 100644 --- a/src/main/resources/db/migration/V1.0.12__create_genotypedataimport_mapping.sql +++ b/src/main/resources/db/migration/V1.0.12__create_genotypedataimport_mapping.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ DO $$ DECLARE diff --git a/src/main/resources/db/migration/V1.0.14__germplasm_import_add_GID.sql b/src/main/resources/db/migration/V1.0.14__germplasm_import_add_GID.sql index f635c876d..0739723ef 100644 --- a/src/main/resources/db/migration/V1.0.14__germplasm_import_add_GID.sql +++ b/src/main/resources/db/migration/V1.0.14__germplasm_import_add_GID.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ update importer_mapping set mapping = '[{"id": "f8d43c7e-a618-4c16-8829-3085f7202a67", "mapping": [{"id": "f384837e-ad8d-4dbe-b54e-87b57070bed1", "value": {"fileFieldName": "Name"}, "objectId": "germplasmName"}, {"id": "39628d14-458b-429b-8e66-bb48e0445a83", "value": {"fileFieldName": "Breeding Method"}, "objectId": "breedingMethod"}, {"id": "f1ba63e1-f5e4-433f-a53e-1c2f3e2fa71f", "value": {"fileFieldName": "Source"}, "objectId": "germplasmSource"}, {"id": "f5892565-f888-4596-be82-ab8eeabf37ce", "value": {"fileFieldName": "External UID"}, "objectId": "externalUID"}, {"id": "65507e5d-2d66-4595-8763-e772fe25c870", "value": {"fileFieldName": "Entry No"}, "objectId": "entryNo"}, {"id": "3eae24c1-ca4a-48a2-96d0-3cf4630acd3a", "value": {"fileFieldName": "Female Parent GID"}, "objectId": "femaleParentDBID"}, {"id": "2dbd7262-93a1-44b0-86b7-f5fca290b965", "value": {"fileFieldName": "Male Parent GID"}, "objectId": "maleParentDBID"}, {"id": "6f7f1539-6e8f-4ede-b7d3-3423cc63abec", "value": {"fileFieldName": "Female Parent Entry No"}, "objectId": "femaleParentEntryNo"}, {"id": "25fe9954-bca7-42f1-818a-5f71e242fa1f", "value": {"fileFieldName": "Male Parent Entry No"}, "objectId": "maleParentEntryNo"}, {"id": "b910adfe-a474-47a0-8410-514578898436", "value": {"fileFieldName": "Synonyms"}, "objectId": "synonyms"}, {"id": "15836d5f-8194-40a8-a771-114eaae31eb4", "objectId": "germplasmPUI"}, {"id": "675b6af8-5a17-4146-a503-2e4e1a65d5fa", "objectId": "acquisitionDate"}, {"id": "69a3bd3c-cebc-435c-acdd-0be62dda25ed", "objectId": "countryOfOrigin"}, {"id": "8ab25267-20f2-450e-89ca-21634ff8fadb", "objectId": "collection"}, {"id": "bc09c6e1-866f-45c3-a285-a25859e8c982", "value": {"fileFieldName": "GID"}, "objectId": "germplasmAccessionNumber"}, {"id": "ce1701e2-2f61-4250-8595-9536e3f5ddcf", "objectId": "AdditionalInfo"}, {"id": "3470e9df-a028-45b7-943f-198bc62b6dbe", "objectId": "ExternalReference"}], "objectId": "Germplasm"}]', diff --git a/src/main/resources/db/migration/V1.0.1__add_experiment_template_system_mapping.sql b/src/main/resources/db/migration/V1.0.1__add_experiment_template_system_mapping.sql index 39655b2b2..8e9c85147 100644 --- a/src/main/resources/db/migration/V1.0.1__add_experiment_template_system_mapping.sql +++ b/src/main/resources/db/migration/V1.0.1__add_experiment_template_system_mapping.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ DO $$ DECLARE diff --git a/src/main/resources/db/migration/V1.0.2__update_experiment_template_system_mapping.sql b/src/main/resources/db/migration/V1.0.2__update_experiment_template_system_mapping.sql index d84339fa1..9c3492ca4 100644 --- a/src/main/resources/db/migration/V1.0.2__update_experiment_template_system_mapping.sql +++ b/src/main/resources/db/migration/V1.0.2__update_experiment_template_system_mapping.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ DO $$ diff --git a/src/main/resources/db/migration/V1.0.3__create_obs_unit_sequence.sql b/src/main/resources/db/migration/V1.0.3__create_obs_unit_sequence.sql index 8b7c779a2..f0bed3308 100644 --- a/src/main/resources/db/migration/V1.0.3__create_obs_unit_sequence.sql +++ b/src/main/resources/db/migration/V1.0.3__create_obs_unit_sequence.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ alter table program add column obs_unit_sequence text; diff --git a/src/main/resources/db/migration/V1.0.4__create_experiment_sequence.sql b/src/main/resources/db/migration/V1.0.4__create_experiment_sequence.sql index cb6bdbf25..c5ce9461d 100644 --- a/src/main/resources/db/migration/V1.0.4__create_experiment_sequence.sql +++ b/src/main/resources/db/migration/V1.0.4__create_experiment_sequence.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ alter table program add column IF NOT EXISTS exp_sequence text; diff --git a/src/main/resources/db/migration/V1.0.5__experiment_template_obsUnitID.sql b/src/main/resources/db/migration/V1.0.5__experiment_template_obsUnitID.sql index d7896b6f6..d82a2edc2 100644 --- a/src/main/resources/db/migration/V1.0.5__experiment_template_obsUnitID.sql +++ b/src/main/resources/db/migration/V1.0.5__experiment_template_obsUnitID.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ DO $$ diff --git a/src/main/resources/db/migration/V1.0.6__create_environment_sequence.sql b/src/main/resources/db/migration/V1.0.6__create_environment_sequence.sql index dfa6c37d9..1bc60dfc7 100644 --- a/src/main/resources/db/migration/V1.0.6__create_environment_sequence.sql +++ b/src/main/resources/db/migration/V1.0.6__create_environment_sequence.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ alter table program add column IF NOT EXISTS env_sequence text; diff --git a/src/main/resources/db/migration/V1.0.7__delete_obs_unit_sequence.sql b/src/main/resources/db/migration/V1.0.7__delete_obs_unit_sequence.sql index 2ab13a52b..af85cee96 100644 --- a/src/main/resources/db/migration/V1.0.7__delete_obs_unit_sequence.sql +++ b/src/main/resources/db/migration/V1.0.7__delete_obs_unit_sequence.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ alter table program drop column obs_unit_sequence; diff --git a/src/main/resources/db/migration/V1.0.8__update_experiment_obs_id_template_system_mapping.sql b/src/main/resources/db/migration/V1.0.8__update_experiment_obs_id_template_system_mapping.sql index 5737ca907..d1ac21bd2 100644 --- a/src/main/resources/db/migration/V1.0.8__update_experiment_obs_id_template_system_mapping.sql +++ b/src/main/resources/db/migration/V1.0.8__update_experiment_obs_id_template_system_mapping.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ DO $$ diff --git a/src/main/resources/db/migration/V1.0.9__add_dynamic_column_storage.sql b/src/main/resources/db/migration/V1.0.9__add_dynamic_column_storage.sql index 7d5c91b2d..5f65addf6 100644 --- a/src/main/resources/db/migration/V1.0.9__add_dynamic_column_storage.sql +++ b/src/main/resources/db/migration/V1.0.9__add_dynamic_column_storage.sql @@ -1,18 +1,18 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ alter table importer_import add column dynamic_column_names text[]; \ No newline at end of file diff --git a/src/main/resources/db/migration/V1.15.0__update_test_or_check_name.sql b/src/main/resources/db/migration/V1.15.0__update_test_or_check_name.sql index 08e28a6ba..09325d586 100644 --- a/src/main/resources/db/migration/V1.15.0__update_test_or_check_name.sql +++ b/src/main/resources/db/migration/V1.15.0__update_test_or_check_name.sql @@ -1,19 +1,19 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ DO $$ diff --git a/src/main/resources/db/migration/beforeClean.sql b/src/main/resources/db/migration/beforeClean.sql index 9f7ebc7a9..13e2df295 100644 --- a/src/main/resources/db/migration/beforeClean.sql +++ b/src/main/resources/db/migration/beforeClean.sql @@ -1,18 +1,18 @@ --- --- See the NOTICE file distributed with this work for additional information --- regarding copyright ownership. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ drop extension if exists "uuid-ossp" cascade; \ No newline at end of file From 177eeb30c098b05a266d2a2f842384debf83e99e Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Tue, 26 Sep 2023 15:29:12 -0400 Subject: [PATCH 010/220] [BI-1852] fixed error message --- .../services/impl/BreedingMethodServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java b/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java index 3c523682e..38c63b1ae 100644 --- a/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java +++ b/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java @@ -87,7 +87,7 @@ private void validate(ProgramBreedingMethodEntity breedingMethod, UUID programId throw new BadRequestException("Missing required data"); } if (methodAlreadyExist(breedingMethod, programId)) { - throw new BadRequestException(format("A method with name:'%s' or abbreviation:'%s already exist", breedingMethod.getName(), breedingMethod.getAbbreviation())); + throw new BadRequestException(format("A method with name:'%s' or abbreviation:'%s' already exist.", breedingMethod.getName(), breedingMethod.getAbbreviation())); } } From cc4ddf67a3e406e492ce46b202305da76d87c79c Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Wed, 27 Sep 2023 12:14:42 -0400 Subject: [PATCH 011/220] [BI-1852] Improved error messages. fixed bug. --- .../impl/BreedingMethodServiceImpl.java | 52 +++++++---- .../impl/BreedingMethodServiceImplTest.java | 91 ++++++++++++------- 2 files changed, 92 insertions(+), 51 deletions(-) diff --git a/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java b/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java index 38c63b1ae..239a3115d 100644 --- a/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java +++ b/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java @@ -66,10 +66,6 @@ public ProgramBreedingMethodEntity createBreedingMethod(ProgramBreedingMethodEnt return dsl.transactionResult(() -> breedingMethodDAO.createProgramMethod(breedingMethod, programId, userId)); } - private boolean methodAlreadyExist(ProgramBreedingMethodEntity breedingMethod, UUID programId) { - List programAndSystemMethods = getBreedingMethods(programId); - return isDuplicateMethodFoundOnList(breedingMethod, programAndSystemMethods); - } @Override public ProgramBreedingMethodEntity updateBreedingMethod(ProgramBreedingMethodEntity breedingMethod, UUID programId, UUID userId) throws BadRequestException, ApiException { @@ -86,8 +82,13 @@ private void validate(ProgramBreedingMethodEntity breedingMethod, UUID programId if (isMissingRequiredFields(breedingMethod)) { throw new BadRequestException("Missing required data"); } - if (methodAlreadyExist(breedingMethod, programId)) { - throw new BadRequestException(format("A method with name:'%s' or abbreviation:'%s' already exist.", breedingMethod.getName(), breedingMethod.getAbbreviation())); + + List programAndSystemMethods = getBreedingMethods(programId); + if( isDuplicateMethodNameFoundOnList(breedingMethod, programAndSystemMethods)){ + throw new BadRequestException(format("'%s' is already defined in the system.", breedingMethod.getName())); + } + if( isDuplicateMethodAbbreviationFoundOnList(breedingMethod, programAndSystemMethods)){ + throw new BadRequestException(format("'%s' is already defined in the system.", breedingMethod.getAbbreviation())); } } @@ -127,11 +128,10 @@ boolean isMissingRequiredFields(ProgramBreedingMethodEntity method) { || StringUtils.isBlank(method.getCategory()) || StringUtils.isBlank(method.getGeneticDiversity()); } - - boolean isDuplicateMethodFoundOnList(ProgramBreedingMethodEntity method, List programBreedingMethodEntityList) { + boolean isDuplicateMethodNameFoundOnList(ProgramBreedingMethodEntity method, List programBreedingMethodEntityList) { boolean foundDup = false; for (ProgramBreedingMethodEntity testMethod: programBreedingMethodEntityList) { - if(areMethodsDuplicate(testMethod, method)){ + if(isDuplicateName(testMethod, method)){ foundDup = true; break; } @@ -139,26 +139,46 @@ boolean isDuplicateMethodFoundOnList(ProgramBreedingMethodEntity method, List programBreedingMethodEntityList) { + boolean foundDup = false; + for (ProgramBreedingMethodEntity testMethod: programBreedingMethodEntityList) { + if(isDuplicateAbbreviation(testMethod, method)){ + foundDup = true; + break; + } + } + return foundDup; + } - boolean areMethodsDuplicate(ProgramBreedingMethodEntity testMethod, ProgramBreedingMethodEntity method) { - + boolean isDuplicateName(ProgramBreedingMethodEntity testMethod, ProgramBreedingMethodEntity method) { // SPECIAL CASE: If the two methods are the same method, then they are not duplicates if( (testMethod.getId()!=null) && testMethod.getId().equals(method.getId()) ){ return false; } boolean isDup = false; - if(testMethod.getName()!= null && testMethod.getName().equals(method.getName())){ + if(testMethod.getName()!= null && testMethod.getName().equalsIgnoreCase(method.getName())){ isDup = true; } - else if(testMethod.getAbbreviation()!= null && testMethod.getAbbreviation().equals(method.getAbbreviation())){ + else if(testMethod.getName()==null && method.getName()==null ){ isDup = true; } - else if(testMethod.getName()==null && method.getName()==null || - testMethod.getAbbreviation()==null && method.getAbbreviation()==null){ - isDup = true; + return isDup; + } + + boolean isDuplicateAbbreviation(ProgramBreedingMethodEntity testMethod, ProgramBreedingMethodEntity method) { + // SPECIAL CASE: If the two methods are the same method, then they are not duplicates + if( (testMethod.getId()!=null) && testMethod.getId().equals(method.getId()) ){ + return false; } + boolean isDup = false; + if(testMethod.getAbbreviation()!= null && testMethod.getAbbreviation().equalsIgnoreCase(method.getAbbreviation())){ + isDup = true; + } + else if(testMethod.getAbbreviation()==null && method.getAbbreviation()==null ){ + isDup = true; + } return isDup; } diff --git a/src/test/java/org/breedinginsight/services/impl/BreedingMethodServiceImplTest.java b/src/test/java/org/breedinginsight/services/impl/BreedingMethodServiceImplTest.java index 9e7ec3c87..428c8fe88 100644 --- a/src/test/java/org/breedinginsight/services/impl/BreedingMethodServiceImplTest.java +++ b/src/test/java/org/breedinginsight/services/impl/BreedingMethodServiceImplTest.java @@ -11,6 +11,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.UUID; import static org.junit.jupiter.api.Assertions.*; @@ -40,95 +41,115 @@ public void hasRequiredFields(){ @Test @SneakyThrows - public void isDupOfProgramMethod() { + public void isDuplicateMethodNameFoundOnList() { List programBreedingMethodEntityList = new ArrayList<>(); - programBreedingMethodEntityList.add(makeMethod("1")); - programBreedingMethodEntityList.add(makeMethod("2")); + programBreedingMethodEntityList.add(makeMethod("1", null)); + programBreedingMethodEntityList.add(makeMethod("2", null)); - boolean isDup = breedingMethodService.isDuplicateMethodFoundOnList(makeMethod("1"), programBreedingMethodEntityList); - assertTrue(isDup, "Duplicate found in list."); + boolean isDup = breedingMethodService.isDuplicateMethodNameFoundOnList(makeMethod("1", null), programBreedingMethodEntityList); + assertTrue(isDup, "Duplicate name found in list."); } @Test @SneakyThrows - public void isNotADupMethod() { + public void isDuplicateMethodAbbreviationFoundOnList() { List programBreedingMethodEntityList = new ArrayList<>(); - programBreedingMethodEntityList.add(makeMethod("1")); - programBreedingMethodEntityList.add(makeMethod("2")); + programBreedingMethodEntityList.add(makeMethod(null, "1")); + programBreedingMethodEntityList.add(makeMethod(null, "2")); + + boolean isDup = breedingMethodService.isDuplicateMethodAbbreviationFoundOnList(makeMethod(null, "1"), programBreedingMethodEntityList); + assertTrue(isDup, "Duplicate abbreviation found in list."); + isDup = breedingMethodService.isDuplicateMethodAbbreviationFoundOnList(makeMethod(null, "unique"), programBreedingMethodEntityList); + assertFalse(isDup, "Duplicate abbreviation NOT found in list."); + } - ProgramBreedingMethodEntity methodUnique = makeMethod("unique");; - boolean isDup = breedingMethodService.isDuplicateMethodFoundOnList(methodUnique, programBreedingMethodEntityList); - assertFalse(isDup, "No duplicates method found."); - } @Test @SneakyThrows - public void methodNameEqual() { - ProgramBreedingMethodEntity method = makeMethod("ABC"); - ProgramBreedingMethodEntity testMethod = makeMethod("ABC"); - testMethod.setAbbreviation("misatch Abbreviation"); + public void isDuplicateName() { + ProgramBreedingMethodEntity method = makeMethod("ABC", null); + ProgramBreedingMethodEntity testMethod = makeMethod("ABC", null); + ProgramBreedingMethodEntity testMethodLowerCase = makeMethod("abc", null); + + + assertTrue( - breedingMethodService.areMethodsDuplicate(testMethod, method), + breedingMethodService.isDuplicateName(testMethod, method), "Method Names are Equal" ); assertTrue( - breedingMethodService.areMethodsDuplicate(method, testMethod), + breedingMethodService.isDuplicateName(method, testMethod), "Method Names are Equal (switched order)" ); + assertTrue( + breedingMethodService.isDuplicateName(method, testMethodLowerCase), + "Method Names are Equal (one is LowerCase)" + ); + testMethod.setId(method.getId()); + assertFalse( + breedingMethodService.isDuplicateName(testMethod, method), + "The methods are the same method (not duplicate data)" + ); + testMethod.setName("misatch Name"); + testMethod.setId(UUID.randomUUID()); assertFalse( - breedingMethodService.areMethodsDuplicate(testMethod, method), + breedingMethodService.isDuplicateName(testMethod, method), "Method Names are not Equal" ); testMethod.setName(null); assertFalse( - breedingMethodService.areMethodsDuplicate(testMethod, method), + breedingMethodService.isDuplicateName(testMethod, method), "Method Names are not Equal (one is null)" ); testMethod.setName("name"); method.setName(null); assertFalse( - breedingMethodService.areMethodsDuplicate(testMethod, method), + breedingMethodService.isDuplicateName(testMethod, method), "Method Names are not Equal (the other is null)" ); - } @Test @SneakyThrows - public void methodAbbreviationEqual() { - ProgramBreedingMethodEntity method = makeMethod("ABC"); - ProgramBreedingMethodEntity testMethod = makeMethod("ABC"); - testMethod.setName("misatch Name"); + public void isDuplicateAbbreviation() { + ProgramBreedingMethodEntity method = makeMethod("ABC", "ABC"); + ProgramBreedingMethodEntity testMethod = makeMethod("mismatch_name", "ABC"); + ProgramBreedingMethodEntity testMethodLowerCase = makeMethod("mismatch_name", "abc"); assertTrue( - breedingMethodService.areMethodsDuplicate(testMethod, method), + breedingMethodService.isDuplicateAbbreviation(testMethod, method), "Method Abbreviations are Equal" ); assertTrue( - breedingMethodService.areMethodsDuplicate(method, testMethod), + breedingMethodService.isDuplicateAbbreviation(method, testMethod), "Method Abbreviations are Equal (switched order)" ); - - testMethod.setAbbreviation("misatch Abbreviation"); + assertTrue( + breedingMethodService.isDuplicateAbbreviation(testMethodLowerCase, method), + "Method Abbreviations are Equal (one is lower case)" + ); + testMethod.setAbbreviation("misatch_abbreviation"); assertFalse( - breedingMethodService.areMethodsDuplicate(method, testMethod), + breedingMethodService.isDuplicateAbbreviation(method, testMethod), "Method Abbreviations are not equal." ); testMethod.setAbbreviation(null); assertFalse( - breedingMethodService.areMethodsDuplicate(method, testMethod), + breedingMethodService.isDuplicateAbbreviation(method, testMethod), "Method Abbreviations are not equal (one is null)." ); } // Helper Methods // - private ProgramBreedingMethodEntity makeMethod(String suffix){ + private ProgramBreedingMethodEntity makeMethod(String nameSuffix, String abbrevSuffix){ ProgramBreedingMethodEntity method = new ProgramBreedingMethodEntity(); - method.setName("Name" + suffix); - method.setAbbreviation("Abbreviation" + suffix); + + method.setName("Name" + nameSuffix!=null? nameSuffix : "junk"); + method.setAbbreviation("Abbreviation" + abbrevSuffix!=null? abbrevSuffix : "junk"); + method.setId(UUID.randomUUID()); return method; } From 45d54d76021280d8e380c002968921507091b92c Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 29 Sep 2023 13:53:22 -0400 Subject: [PATCH 012/220] [BI-1852]changed returned type 'HttpResponse' to 'HttpResponse' to the linter happy --- .../api/v1/controller/BreedingMethodController.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/api/v1/controller/BreedingMethodController.java b/src/main/java/org/breedinginsight/api/v1/controller/BreedingMethodController.java index 7b3fc2bb5..9d80f9115 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/BreedingMethodController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/BreedingMethodController.java @@ -59,7 +59,7 @@ public HttpResponse>> getSyst @Post("programs/{programId}/breeding-methods") @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roles = {ProgramSecuredRole.BREEDER}) - public HttpResponse createProgramBreedingMethod(@PathVariable UUID programId, @Body ProgramBreedingMethodEntity breedingMethod) throws ApiException { + public HttpResponse createProgramBreedingMethod(@PathVariable UUID programId, @Body ProgramBreedingMethodEntity breedingMethod) throws ApiException{ log.debug("Saving new program breeding method"); try { @@ -107,7 +107,7 @@ public HttpResponse>> getProg @Put("programs/{programId}/breeding-methods/{breedingMethodId}") @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roles = {ProgramSecuredRole.BREEDER}) - public HttpResponse updateProgramBreedingMethod(@PathVariable UUID programId, @PathVariable UUID breedingMethodId, @Body ProgramBreedingMethodEntity breedingMethod) throws ApiException { + public HttpResponse updateProgramBreedingMethod(@PathVariable UUID programId, @PathVariable UUID breedingMethodId, @Body ProgramBreedingMethodEntity breedingMethod) throws ApiException { log.debug("Saving new program breeding method"); try { From 070e1822a6e33bc44559e6e7714410bd5162ffd0 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 29 Sep 2023 14:59:24 -0400 Subject: [PATCH 013/220] [BI-1852] explisitly declaired 'protected' methods fixed type-o in test titles Improved error messages --- .../impl/BreedingMethodServiceImpl.java | 20 ++++++++++++------- .../impl/BreedingMethodServiceImplTest.java | 10 +++++----- .../validators/TraitValidatorUnitTest.java | 1 - 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java b/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java index 239a3115d..19e880309 100644 --- a/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java +++ b/src/main/java/org/breedinginsight/services/impl/BreedingMethodServiceImpl.java @@ -85,10 +85,10 @@ private void validate(ProgramBreedingMethodEntity breedingMethod, UUID programId List programAndSystemMethods = getBreedingMethods(programId); if( isDuplicateMethodNameFoundOnList(breedingMethod, programAndSystemMethods)){ - throw new BadRequestException(format("'%s' is already defined in the system.", breedingMethod.getName())); + throw new BadRequestException(format("A breeding method with the name '%s' is already defined in the system.", breedingMethod.getName())); } if( isDuplicateMethodAbbreviationFoundOnList(breedingMethod, programAndSystemMethods)){ - throw new BadRequestException(format("'%s' is already defined in the system.", breedingMethod.getAbbreviation())); + throw new BadRequestException(format("A breeding method with the abbreviation '%s' is already defined in the system.", breedingMethod.getAbbreviation())); } } @@ -122,13 +122,16 @@ public void deleteBreedingMethod(UUID programId, UUID breedingMethodId) throws A dsl.transaction(() -> breedingMethodDAO.deleteProgramMethod(programId, breedingMethodId)); } - boolean isMissingRequiredFields(ProgramBreedingMethodEntity method) { + //'protected' instead of 'private' so it is accessible to unit test. + protected boolean isMissingRequiredFields(ProgramBreedingMethodEntity method) { return StringUtils.isBlank(method.getName()) || StringUtils.isBlank(method.getAbbreviation()) || StringUtils.isBlank(method.getCategory()) || StringUtils.isBlank(method.getGeneticDiversity()); } - boolean isDuplicateMethodNameFoundOnList(ProgramBreedingMethodEntity method, List programBreedingMethodEntityList) { + + //'protected' instead of 'private' so it is accessible to unit test. + protected boolean isDuplicateMethodNameFoundOnList(ProgramBreedingMethodEntity method, List programBreedingMethodEntityList) { boolean foundDup = false; for (ProgramBreedingMethodEntity testMethod: programBreedingMethodEntityList) { if(isDuplicateName(testMethod, method)){ @@ -139,7 +142,8 @@ boolean isDuplicateMethodNameFoundOnList(ProgramBreedingMethodEntity method, Lis return foundDup; } - boolean isDuplicateMethodAbbreviationFoundOnList(ProgramBreedingMethodEntity method, List programBreedingMethodEntityList) { + //'protected' instead of 'private' so it is accessible to unit test. + protected boolean isDuplicateMethodAbbreviationFoundOnList(ProgramBreedingMethodEntity method, List programBreedingMethodEntityList) { boolean foundDup = false; for (ProgramBreedingMethodEntity testMethod: programBreedingMethodEntityList) { if(isDuplicateAbbreviation(testMethod, method)){ @@ -150,7 +154,8 @@ boolean isDuplicateMethodAbbreviationFoundOnList(ProgramBreedingMethodEntity met return foundDup; } - boolean isDuplicateName(ProgramBreedingMethodEntity testMethod, ProgramBreedingMethodEntity method) { + //'protected' instead of 'private' so it is accessible to unit test. + protected boolean isDuplicateName(ProgramBreedingMethodEntity testMethod, ProgramBreedingMethodEntity method) { // SPECIAL CASE: If the two methods are the same method, then they are not duplicates if( (testMethod.getId()!=null) && testMethod.getId().equals(method.getId()) ){ return false; @@ -166,7 +171,8 @@ else if(testMethod.getName()==null && method.getName()==null ){ return isDup; } - boolean isDuplicateAbbreviation(ProgramBreedingMethodEntity testMethod, ProgramBreedingMethodEntity method) { + //'protected' instead of 'private' so it is accessible to unit test. + protected boolean isDuplicateAbbreviation(ProgramBreedingMethodEntity testMethod, ProgramBreedingMethodEntity method) { // SPECIAL CASE: If the two methods are the same method, then they are not duplicates if( (testMethod.getId()!=null) && testMethod.getId().equals(method.getId()) ){ return false; diff --git a/src/test/java/org/breedinginsight/services/impl/BreedingMethodServiceImplTest.java b/src/test/java/org/breedinginsight/services/impl/BreedingMethodServiceImplTest.java index 428c8fe88..cc59fd6ae 100644 --- a/src/test/java/org/breedinginsight/services/impl/BreedingMethodServiceImplTest.java +++ b/src/test/java/org/breedinginsight/services/impl/BreedingMethodServiceImplTest.java @@ -47,7 +47,7 @@ public void isDuplicateMethodNameFoundOnList() { programBreedingMethodEntityList.add(makeMethod("2", null)); boolean isDup = breedingMethodService.isDuplicateMethodNameFoundOnList(makeMethod("1", null), programBreedingMethodEntityList); - assertTrue(isDup, "Duplicate name found in list."); + assertTrue(isDup, "Duplicate name found in list."); } @Test @@ -58,9 +58,9 @@ public void isDuplicateMethodAbbreviationFoundOnList() { programBreedingMethodEntityList.add(makeMethod(null, "2")); boolean isDup = breedingMethodService.isDuplicateMethodAbbreviationFoundOnList(makeMethod(null, "1"), programBreedingMethodEntityList); - assertTrue(isDup, "Duplicate abbreviation found in list."); + assertTrue(isDup, "Duplicate abbreviation found in list."); isDup = breedingMethodService.isDuplicateMethodAbbreviationFoundOnList(makeMethod(null, "unique"), programBreedingMethodEntityList); - assertFalse(isDup, "Duplicate abbreviation NOT found in list."); + assertFalse(isDup, "Duplicate abbreviation NOT found in list."); } @@ -147,8 +147,8 @@ public void isDuplicateAbbreviation() { private ProgramBreedingMethodEntity makeMethod(String nameSuffix, String abbrevSuffix){ ProgramBreedingMethodEntity method = new ProgramBreedingMethodEntity(); - method.setName("Name" + nameSuffix!=null? nameSuffix : "junk"); - method.setAbbreviation("Abbreviation" + abbrevSuffix!=null? abbrevSuffix : "junk"); + method.setName("Name" + ( nameSuffix != null? nameSuffix : "junk" )); + method.setAbbreviation("Abbreviation" + ( abbrevSuffix != null? abbrevSuffix : "junk" )); method.setId(UUID.randomUUID()); return method; } diff --git a/src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java b/src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java index cc563772c..a7fa4507b 100644 --- a/src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java +++ b/src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java @@ -40,7 +40,6 @@ import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class TraitValidatorUnitTest { From dce81c47d1457af6bf8f1a54c9e4c65cf6089483 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 2 Oct 2023 13:48:43 +0000 Subject: [PATCH 014/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 31b76262a..e72b0c594 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+582 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/688b269b294fc5f968747b69dc80a9b93f9e2603 +version=v0.9.0+588 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/193c451d29df83b7cb1a98210941fc05e69d7880 From c75f2586fa9945cda9730b71a2cce82f8da630e6 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 3 Oct 2023 17:45:01 +0000 Subject: [PATCH 015/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index e72b0c594..65879b77d 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+588 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/193c451d29df83b7cb1a98210941fc05e69d7880 +version=v0.9.0+590 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/7f16bf306bf10aae92e270cf9f8202291d249c0c From 87f63d25179eb9f23baf582375b8c13f16e5bf5e Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 4 Oct 2023 18:05:12 +0000 Subject: [PATCH 016/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 65879b77d..ec223cd3c 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+590 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/7f16bf306bf10aae92e270cf9f8202291d249c0c +version=v0.9.0+592 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/ef45b94585a713b42c1f71b6cb6e733213c108c9 From 5007bf029db7b0fd01d20aef15b71af1c7702a7f Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 4 Oct 2023 17:03:05 -0400 Subject: [PATCH 017/220] [BI-1943] - avoid unintended value collision --- .../utilities/response/ResponseUtilsIntegrationTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java b/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java index 1b5543868..8411aa285 100644 --- a/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java +++ b/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java @@ -108,10 +108,11 @@ public void setup() throws MissingRequiredInfoException, UnprocessableEntityExce .slope(new BigDecimal("1.1")) .build()); for(int i = 2; i < 25; i++) { + // Ensure random slope values don't overlap with hardcoded value by adding 2 to the unsigned magnitude. newLocations.add(ProgramLocationRequest.builder() .name("place"+i) .abbreviation("abbrev"+i) - .slope(new BigDecimal(((Math.random()*10)%2 == 0 ? "" : "-")+Math.random()*20.0)) + .slope(new BigDecimal(((Math.random()*10)%2 == 0 ? "" : "-")+(Math.random()*20.0 + 2.0))) .build()); } for(int i = 25; i < 31; i++) { From 7e1f6a2094c4df8356ff188742648d647d4a2fb4 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 5 Oct 2023 10:56:57 -0400 Subject: [PATCH 018/220] [BI-1943] - simplified --- .../utilities/response/ResponseUtilsIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java b/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java index 8411aa285..5b4b608dc 100644 --- a/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java +++ b/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java @@ -112,7 +112,7 @@ public void setup() throws MissingRequiredInfoException, UnprocessableEntityExce newLocations.add(ProgramLocationRequest.builder() .name("place"+i) .abbreviation("abbrev"+i) - .slope(new BigDecimal(((Math.random()*10)%2 == 0 ? "" : "-")+(Math.random()*20.0 + 2.0))) + .slope(new BigDecimal((Math.random() > 0.5 ? "" : "-")+(Math.random() + 2.0))) .build()); } for(int i = 25; i < 31; i++) { From 08ab94c4daa58d992d8108738e7aa4e0494247f5 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 5 Oct 2023 11:14:19 -0400 Subject: [PATCH 019/220] [BI-1943] - simplified further --- .../utilities/response/ResponseUtilsIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java b/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java index 5b4b608dc..7cd16622a 100644 --- a/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java +++ b/src/test/java/org/breedinginsight/utilities/response/ResponseUtilsIntegrationTest.java @@ -112,7 +112,7 @@ public void setup() throws MissingRequiredInfoException, UnprocessableEntityExce newLocations.add(ProgramLocationRequest.builder() .name("place"+i) .abbreviation("abbrev"+i) - .slope(new BigDecimal((Math.random() > 0.5 ? "" : "-")+(Math.random() + 2.0))) + .slope(new BigDecimal((Math.random() > 0.5 ? "" : "-")+Math.random())) .build()); } for(int i = 25; i < 31; i++) { From 10f28ef2ecf12a66c8395cc603ac585f64704855 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Mar 2023 20:44:55 +0000 Subject: [PATCH 020/220] Bump postgresql from 42.3.2 to 42.3.8 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.3.2 to 42.3.8. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.3.2...REL42.3.8) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 53c76ed6e..b12c7b648 100644 --- a/pom.xml +++ b/pom.xml @@ -72,7 +72,7 @@ -Xmx1024m -XX:MaxPermSize=256m --enable-preview 3.16.3 - 42.3.2 + 42.3.8 1.2.10 1.18.22 2.8.9 From 153b7db202a39f56c9e8865dcf5d6961e1359b5d Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 6 Oct 2023 14:35:15 +0000 Subject: [PATCH 021/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index ec223cd3c..8a871b034 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+592 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/ef45b94585a713b42c1f71b6cb6e733213c108c9 +version=v0.9.0+596 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/6068610501eb72237ddff24b1c26eae5ad011116 From 429c7f1fef51225dbd7491e1a85e9bbef8a44cce Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 6 Oct 2023 11:07:18 -0400 Subject: [PATCH 022/220] [BI-1890] added validateTestOrCheck() method --- .../services/processors/ExperimentProcessor.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index c9e7f5aa0..fde0f4d72 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -557,6 +557,7 @@ private ValidationErrors validateFields(List importRows, Validation if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); } + validateTestOrCheck(importRow, validationErrors, rowNum); //Check if existing environment. If so, ObsUnitId must be assigned if ((mappedImportRow.getStudy().getState() == ImportObjectState.EXISTING) @@ -784,6 +785,16 @@ private void validateGermplasm(ExperimentObservation importRow, ValidationErrors } } + private void validateTestOrCheck(ExperimentObservation importRow, ValidationErrors validationErrors, int rowNum) { + String testOrCheck = importRow.getTestOrCheck(); + if ( ! ( testOrCheck==null || testOrCheck.isBlank() + || "C".equalsIgnoreCase(testOrCheck) || "CHECK".equalsIgnoreCase(testOrCheck) + || "T".equalsIgnoreCase(testOrCheck) || "TEST".equalsIgnoreCase(testOrCheck) ) + ){ + addRowError(Columns.TEST_CHECK, String.format("Invalid value (%s)", testOrCheck), validationErrors, rowNum) ; + } + } + private PendingImportObject getGidPOI(ExperimentObservation importRow) { if (this.existingGermplasmByGID.containsKey(importRow.getGid())) { return existingGermplasmByGID.get(importRow.getGid()); From 889a6bf4a5e38e1673f9a373ee0aa6707066407e Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 6 Oct 2023 11:22:10 -0400 Subject: [PATCH 023/220] [BI-1890] Fixed problems (not bugs) reported by InteliJ --- .../processors/ExperimentProcessor.java | 86 ++++++------------- 1 file changed, 25 insertions(+), 61 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index fde0f4d72..f5a7fe84e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -41,7 +41,6 @@ import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.base.AdditionalInfo; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; @@ -74,8 +73,6 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import static org.breedinginsight.brapps.importer.services.FileMappingUtil.EXPERIMENT_TEMPLATE_NAME; - @Slf4j @Prototype public class ExperimentProcessor implements Processor { @@ -113,8 +110,6 @@ public class ExperimentProcessor implements Processor { // used to make the yearsToSeasonDbId() function more efficient private final Map yearToSeasonDbIdCache = new HashMap<>(); - // used to make the seasonDbIdtoYear() function more efficient - private final Map seasonDbIdToYearCache = new HashMap<>(); //These BrapiData-objects are initially populated by the getExistingBrapiData() method, // then updated by the getNewBrapiData() method. @@ -126,13 +121,13 @@ public class ExperimentProcessor implements Processor { // initialized by getExistingBrapiData() ) private Map> observationUnitByNameNoScope = null; - private Map> observationByHash = new HashMap<>(); + private final Map> observationByHash = new HashMap<>(); // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the getNewBrapiData() method private Map> existingGermplasmByGID = null; // Associates timestamp columns to associated phenotype column name for ease of storage - private Map> timeStampColByPheno = new HashMap<>(); + private final Map> timeStampColByPheno = new HashMap<>(); @Inject public ExperimentProcessor(DSLContext dsl, @@ -220,7 +215,7 @@ public Map process( } } - List referencedTraits = verifyTraits(program.getId(), phenotypeCols, timestampCols, validationErrors); + List referencedTraits = verifyTraits(program.getId(), phenotypeCols, timestampCols); //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval for (Column tsColumn : timestampCols) { @@ -254,7 +249,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program List newTrials = ProcessorData.getNewObjects(this.trialByNameNoScope); Map mutatedTrialsById = ProcessorData - .getMutationsByObjectId(trialByNameNoScope, trial -> trial.getTrialDbId()); + .getMutationsByObjectId(trialByNameNoScope, BrAPITrial::getTrialDbId); List newLocations = ProcessorData.getNewObjects(this.locationByName) .stream() @@ -274,7 +269,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program return request; }).collect(Collectors.toList()); Map datasetNewDataById = ProcessorData - .getMutationsByObjectId(obsVarDatasetByName, listDetails -> listDetails.getListDbId()); + .getMutationsByObjectId(obsVarDatasetByName, BrAPIListSummary::getListDbId); List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); // filter out observations with no 'value' so they will not be saved @@ -287,9 +282,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program try { List createdDatasets = new ArrayList<>(brAPIListDAO.createBrAPILists(newDatasetRequests, program.getId(), upload)); - createdDatasets.forEach(summary -> { - obsVarDatasetByName.get(summary.getListName()).getBrAPIObject().setListDbId(summary.getListDbId()); - }); + createdDatasets.forEach(summary -> obsVarDatasetByName.get(summary.getListName()).getBrAPIObject().setListDbId(summary.getListDbId())); List createdTrials = new ArrayList<>(brapiTrialDAO.createBrAPITrials(newTrials, program.getId(), upload)); // set the DbId to the for each newly created trial @@ -403,7 +396,7 @@ private void prepareDataForValidation(List importRows, List verifyTraits(UUID programId, List> phenotypeCols, List> timestampCols, ValidationErrors validationErrors) { + private List verifyTraits(UUID programId, List> phenotypeCols, List> timestampCols) { Set varNames = phenotypeCols.stream() .map(Column::name) .collect(Collectors.toSet()); @@ -543,8 +536,8 @@ private String getObservationHash(String observationUnitName, String variableNam return DigestUtils.sha256Hex(concat); } - private ValidationErrors validateFields(List importRows, ValidationErrors validationErrors, Map mappedBrAPIImport, List referencedTraits, Program program, - List> phenotypeCols, boolean commit) throws MissingRequiredInfoException, ApiException { + private void validateFields(List importRows, ValidationErrors validationErrors, Map mappedBrAPIImport, List referencedTraits, Program program, + List> phenotypeCols, boolean commit) throws MissingRequiredInfoException, ApiException { //fetching any existing observations for any OUs in the import Map existingObsByObsHash = fetchExistingObservations(referencedTraits, program); Map colVarMap = referencedTraits.stream().collect(Collectors.toMap(Trait::getObservationVariableName, Function.identity())); @@ -569,7 +562,6 @@ private ValidationErrors validateFields(List importRows, Validation validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, existingObsByObsHash); } - return validationErrors; } private void validateObservationUnits(ValidationErrors validationErrors, Set uniqueStudyAndObsUnit, int rowNum, ExperimentObservation importRow) { @@ -834,20 +826,18 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra } - private PendingImportObject fetchOrCreateObservationPIO(Program program, - User user, - ExperimentObservation importRow, - String variableName, - String value, - String timeStampValue, - boolean commit, - String seasonDbId, - PendingImportObject obsUnitPIO) { + private void fetchOrCreateObservationPIO(Program program, + User user, + ExperimentObservation importRow, + String variableName, + String value, + String timeStampValue, + boolean commit, + String seasonDbId, + PendingImportObject obsUnitPIO) { PendingImportObject pio; String key = getImportObservationHash(importRow, variableName); - if (this.observationByHash.containsKey(key)) { - pio = observationByHash.get(key); - } else { + if (!this.observationByHash.containsKey(key)) { PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); @@ -865,7 +855,6 @@ private PendingImportObject fetchOrCreateObservationPIO(Progra pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); this.observationByHash.put(key, pio); } - return pio; } private void addObsVarsToDatasetDetails(PendingImportObject pio, List referencedTraits, Program program) { BrAPIListDetails details = pio.getBrAPIObject(); @@ -886,7 +875,7 @@ private void addObsVarsToDatasetDetails(PendingImportObject pi } }); } - private PendingImportObject fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) { + private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) { PendingImportObject pio; PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); String name = String.format("Observation Dataset [%s-%s]", @@ -913,7 +902,6 @@ private PendingImportObject fetchOrCreateDatasetPIO(Experiment obsVarDatasetByName.put(name, pio); } addObsVarsToDatasetDetails(pio, referencedTraits, program); - return pio; } private PendingImportObject fetchOrCreateStudyPIO(Program program, boolean commit, String expSequenceValue, ExperimentObservation importRow, Supplier envNextVal) { @@ -941,16 +929,13 @@ private PendingImportObject fetchOrCreateStudyPIO(Program program, b return pio; } - private PendingImportObject fetchOrCreateLocationPIO(ExperimentObservation importRow) { + private void fetchOrCreateLocationPIO(ExperimentObservation importRow) { PendingImportObject pio; - if (locationByName.containsKey((importRow.getEnvLocation()))) { - pio = locationByName.get(importRow.getEnvLocation()); - } else { + if (! locationByName.containsKey((importRow.getEnvLocation()))) { ProgramLocation newLocation = importRow.constructProgramLocation(); pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); this.locationByName.put(importRow.getEnvLocation(), pio); } - return pio; } private PendingImportObject fetchOrCreateTrialPIO(Program program, User user, boolean commit, ExperimentObservation importRow, Supplier expNextVal) { @@ -1171,7 +1156,6 @@ private Map> initializeObserva private Map> initializeTrialByNameNoScope(Program program, List experimentImportRows) { Map> trialByName = new HashMap<>(); - String programKey = program.getKey(); initializeTrialsForExistingObservationUnits(program, trialByName); @@ -1303,7 +1287,7 @@ private Map> initializeObsVarDatas BrAPIListDetails dataSetDetails = brAPIListDAO .getListById(existingDatasets.get(0).getListDbId(), program.getId()) .getResult(); - processAndCacheObsVarDataset(dataSetDetails, program, obsVarDatasetByName); + processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); } catch (ApiException e) { log.error(Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); @@ -1327,7 +1311,7 @@ private Optional> getTrialPIO(List> obsVarDatasetByName) { + private void processAndCacheObsVarDataset(BrAPIListDetails existingList, Map> obsVarDatasetByName) { BrAPIExternalReference xref = Utilities.getExternalReference(existingList.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) .orElseThrow(() -> new IllegalStateException("External references wasn't found for list (dbid): " + existingList.getListDbId())); @@ -1574,17 +1558,6 @@ private String yearToSeasonDbId(String year, UUID programId) { return dbID; } - private String seasonDbIdToYear(String seasonDbId, UUID programId) { - String year = null; - if (this.seasonDbIdToYearCache.containsKey(seasonDbId)) { // get it from cache if possible - year = this.seasonDbIdToYearCache.get(seasonDbId); - } else { - year = this.seasonDbIdToYearFromDatabase(seasonDbId, programId); - this.seasonDbIdToYearCache.put(seasonDbId, year); - } - return year; - } - private String yearToSeasonDbIdFromDatabase(String year, UUID programId) { BrAPISeason targetSeason = null; List seasons; @@ -1611,14 +1584,5 @@ private String yearToSeasonDbIdFromDatabase(String year, UUID programId) { return (targetSeason == null) ? null : targetSeason.getSeasonDbId(); } - private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { - BrAPISeason season = null; - try { - season = this.brAPISeasonDAO.getSeasonById(seasonDbId, programId); - } catch (ApiException e) { - log.error(Utilities.generateApiExceptionLogMessage(e), e); - } - Integer yearInt = (season == null) ? null : season.getYear(); - return (yearInt == null) ? "" : yearInt.toString(); - } + } From 0367c572c8a8947e035f2654e5bca56264220088 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 9 Oct 2023 18:51:27 +0000 Subject: [PATCH 024/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 8a871b034..6e36c8d8b 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+596 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/6068610501eb72237ddff24b1c26eae5ad011116 +version=v0.9.0+600 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/b4dca76d7739cf379ea35d858e57da9cf5a108ab From 84cce7dc49f686dec394601ab45755db35fd7397 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 9 Oct 2023 15:48:40 -0400 Subject: [PATCH 025/220] [BI-1930] made observationvariable validations with ontologies case-insensitive --- .../processors/ExperimentProcessor.java | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index c9e7f5aa0..66bc7db7e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -24,6 +24,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.*; @@ -41,7 +42,6 @@ import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.base.AdditionalInfo; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; @@ -70,12 +70,9 @@ import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.*; -import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; -import static org.breedinginsight.brapps.importer.services.FileMappingUtil.EXPERIMENT_TEMPLATE_NAME; - @Slf4j @Prototype public class ExperimentProcessor implements Processor { @@ -444,10 +441,11 @@ private List verifyTraits(UUID programId, List> phenotypeCols, private List fetchFileTraits(UUID programId, Collection varNames) { try { + Collection upperCaseVarNames = varNames.stream().map(String::toUpperCase).collect(Collectors.toList()); List traits = ontologyService.getTraitsByProgramId(programId, true); // filter out just traits specified in file return traits.stream() - .filter(e -> varNames.contains(e.getObservationVariableName())) + .filter(e -> upperCaseVarNames.contains(e.getObservationVariableName().toUpperCase())) .collect(Collectors.toList()); } catch (DoesNotExistException e) { log.error(e.getMessage(), e); @@ -547,8 +545,10 @@ private ValidationErrors validateFields(List importRows, Validation List> phenotypeCols, boolean commit) throws MissingRequiredInfoException, ApiException { //fetching any existing observations for any OUs in the import Map existingObsByObsHash = fetchExistingObservations(referencedTraits, program); - Map colVarMap = referencedTraits.stream().collect(Collectors.toMap(Trait::getObservationVariableName, Function.identity())); - + CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); + for ( Trait trait: referencedTraits) { + colVarMap.put(trait.getObservationVariableName(),trait); + } Set uniqueStudyAndObsUnit = new HashSet<>(); for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); @@ -619,7 +619,7 @@ private Map fetchExistingObservations(List refe .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } - private void validateObservations(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow, List> phenotypeCols, Map colVarMap, Map existingObservations) { + private void validateObservations(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow, List> phenotypeCols, CaseInsensitiveMap colVarMap, Map existingObservations) { phenotypeCols.forEach(phenoCol -> { var importHash = getImportObservationHash(importRow, phenoCol.name()); if(existingObservations.containsKey(importHash) && StringUtils.isNotBlank(phenoCol.getString(rowNum)) && !existingObservations.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { @@ -969,8 +969,10 @@ private void updateObservationDependencyValues(Program program) { // Update ObservationVariable DbIds List traits = getTraitList(program); - Map traitMap = traits.stream().collect(Collectors.toMap(TraitEntity::getObservationVariableName, Function.identity())); - + CaseInsensitiveMap traitMap = new CaseInsensitiveMap<>(); + for ( Trait trait: traits) { + traitMap.put(trait.getObservationVariableName(),trait); + } for (PendingImportObject observation : this.observationByHash.values()) { String observationVariableName = observation.getBrAPIObject().getObservationVariableName(); if (observationVariableName != null && traitMap.containsKey(observationVariableName)) { From 3ccee1e7b20ec131071c2cf4333a7bf52d73b7b1 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 12 Oct 2023 23:32:33 +0000 Subject: [PATCH 026/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 6e36c8d8b..689815dc5 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+600 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/b4dca76d7739cf379ea35d858e57da9cf5a108ab +version=v0.9.0+606 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/535940796e7d02c33d3e94a6afb4957333e458d2 From f318dbb1a96fe1ae51816af38cccac1e2b15d5ae Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 13 Oct 2023 09:55:58 -0400 Subject: [PATCH 027/220] [BI-1930] Fixed bug with sorting observation variables. --- .../brapps/importer/services/FileMappingUtil.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java index e26f375c5..e603a7d21 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java @@ -18,6 +18,7 @@ package org.breedinginsight.brapps.importer.services; import io.reactivex.functions.Function; +import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.tuple.MutablePair; import org.apache.commons.lang3.tuple.Pair; import org.breedinginsight.brapps.importer.model.config.MappedImportRelation; @@ -76,7 +77,7 @@ public List> findFileRelationships(Table data, List List sortByField(List sortedFields, List unsortedItems, Function fieldGetter) { - HashMap sortOrder = new HashMap<>(); + CaseInsensitiveMap sortOrder = new CaseInsensitiveMap<>(); for (int i = 0; i < sortedFields.size(); i++) { sortOrder.put(sortedFields.get(i), i); } From b1bfa80b6adaf7098de3c7dd383d43093d0dce6f Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 13 Oct 2023 16:37:55 +0000 Subject: [PATCH 028/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 689815dc5..40b9391ee 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+606 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/535940796e7d02c33d3e94a6afb4957333e458d2 +version=v0.9.0+608 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/ebb51275919060b57871b6d3f1ab1d07a31e29ac From dfe9802f230a456ee94cd2c906953bc976ca9f41 Mon Sep 17 00:00:00 2001 From: timparsons Date: Fri, 13 Oct 2023 15:46:41 -0400 Subject: [PATCH 029/220] [BI-1965] upgrading mongo version --- docker-compose.yml | 2 +- .../geno/impl/GigwaGenotypeServiceImplIntegrationTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 12cedd454..6c75b9736 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -144,7 +144,7 @@ services: source: gigwa_data target: /usr/local/tomcat/config mongo: - image: mongo:4.2.21 + image: mongo:4.2.24 container_name: ${GIGWA_CONTAINER_NAME:-gigwa}_db restart: always command: --profile 0 --slowms 60000 --storageEngine wiredTiger --wiredTigerCollectionBlockCompressor=zstd --directoryperdb --quiet diff --git a/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java b/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java index d16102713..afa593e7c 100644 --- a/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java +++ b/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java @@ -218,7 +218,7 @@ BrAPIEndpointProvider brAPIEndpointProvider() { public GigwaGenotypeServiceImplIntegrationTest() { super(); - mongo = new GenericContainer<>("mongo:4.2.21") + mongo = new GenericContainer<>("mongo:4.2.24") .withNetwork(super.getNetwork()) .withNetworkAliases("gigwa_db") .withImagePullPolicy(PullPolicy.defaultPolicy()) From 8e3d1d0e1303498c91c2595d3dfcce6e015635ac Mon Sep 17 00:00:00 2001 From: timparsons Date: Fri, 13 Oct 2023 15:47:24 -0400 Subject: [PATCH 030/220] Fixing header to match what Gigwa expects --- .../services/geno/impl/GigwaGenotypeServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java b/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java index e0c669e51..97c78cf05 100644 --- a/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java +++ b/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java @@ -82,7 +82,7 @@ @Slf4j public class GigwaGenotypeServiceImpl implements GenotypeService { private static final String AUTHORIZATION = "Authorization"; - private static final String X_FORWARDED_SERVER = "X-Forwarded-Server"; + private static final String X_FORWARDED_FOR = "X-Forwarded-For"; private static final String BEARER = "Bearer "; private static final String GIGWA_REST_BASE_PATH = "gigwa/rest"; private static final String GIGWA_BRAPI_BASE_PATH = GIGWA_REST_BASE_PATH + BrapiVersion.BRAPI_V2; @@ -613,7 +613,7 @@ private String submitRequestToGigwa(OkHttpClient client, Program program, UUID e .build()) .header(AUTHORIZATION, BEARER + gigwaAuthToken) - .header(X_FORWARDED_SERVER, referenceSource) + .header(X_FORWARDED_FOR, referenceSource) .post(RequestBody.create("", MediaType.parse("text/plain"))) .build(); From b3d7e571c982532ed44a9e04a8edcf5543203994 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 16 Oct 2023 14:30:47 -0400 Subject: [PATCH 031/220] [BI-1914]Created default sorts --- .../brapi/v2/services/BrAPITrialService.java | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index b90f7ef8e..2aac1bda7 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -11,6 +11,7 @@ import org.brapi.v2.model.core.*; import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; import org.brapi.v2.model.germ.BrAPIGermplasm; + import org.brapi.v2.model.pheno.*; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; @@ -18,6 +19,7 @@ import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.FileMappingUtil; import org.breedinginsight.model.BrAPIConstants; @@ -235,7 +237,11 @@ public DownloadFile exportObservations( List files = new ArrayList<>(); // Generate a file for each study. for (Map.Entry>> entry: rowsByStudyId.entrySet()) { - StreamedFile streamedFile = writeToStreamedFile(columns, entry.getValue(), fileType, "Experiment Data"); + + List> rows = entry.getValue(); + sortExportRows(rows); + //TODO the sheetName should be "Data". It should no longer be "Experiment Data" + StreamedFile streamedFile = writeToStreamedFile(columns, rows, fileType, "Experiment Data"); String name = makeFileName(experiment, program, studyByDbId.get(entry.getKey()).getStudyName()) + fileType.getExtension(); // Add to file list. files.add(new DownloadFile(name, streamedFile)); @@ -252,7 +258,9 @@ public DownloadFile exportObservations( } } else { List> exportRows = new ArrayList<>(rowByOUId.values()); + sortExportRows(exportRows); // write export data to requested file format + //TODO the sheetName should be "Data". It should no longer be "Experiment Data" StreamedFile streamedFile = writeToStreamedFile(columns, exportRows, fileType, "Experiment Data"); // Set filename. String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); @@ -308,6 +316,7 @@ public Dataset getDatasetData(Program program, UUID experimentId, UUID datsetId, log.debug("fetching observations for dataset: " + datsetId); List data = observationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, obsVarDbIds, program); log.debug("building dataset object for dataset: " + datsetId); + sortObservationUnit(datasetOUs); Dataset dataset = new Dataset(experimentId.toString(), data, datasetOUs, datasetObsVars); if (stats) { Integer ouCount = datasetOUs.size(); @@ -549,4 +558,16 @@ private List filterDatasetByEnvironment( .collect(Collectors.toList()); } + private void sortObservationUnit(List ous){ + ous.sort(Comparator.comparing(BrAPIObservationUnit::getStudyName) + .thenComparing(BrAPIObservationUnit::getObservationUnitName)); + } + + private void sortExportRows(List> exportRows){ + Comparator> envComparator = Comparator.comparing(row -> ( row.get(Columns.ENV).toString())); + Comparator> expUnitIdComparator = + Comparator.comparing(row -> (row.get(Columns.EXP_UNIT_ID).toString())); + + exportRows.sort(envComparator.thenComparing(expUnitIdComparator)); + } } \ No newline at end of file From 60e4ed3fe54bff49bf44f966335957c8f1032912 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 16 Oct 2023 14:34:44 -0400 Subject: [PATCH 032/220] [BI-1914] Spread sheet's sheet Name should be 'Data' not 'Experiment Data' --- .../brapi/v2/services/BrAPITrialService.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 2aac1bda7..020654985 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -229,7 +229,7 @@ public DownloadFile exportObservations( // Initialize key with empty list if it is not present. if (!rowsByStudyId.containsKey(studyId)) { - rowsByStudyId.put(studyId, new ArrayList>()); + rowsByStudyId.put(studyId, new ArrayList<>()); } // Add row to appropriate list in rowsByStudyId. rowsByStudyId.get(studyId).add(row); @@ -240,8 +240,7 @@ public DownloadFile exportObservations( List> rows = entry.getValue(); sortExportRows(rows); - //TODO the sheetName should be "Data". It should no longer be "Experiment Data" - StreamedFile streamedFile = writeToStreamedFile(columns, rows, fileType, "Experiment Data"); + StreamedFile streamedFile = writeToStreamedFile(columns, rows, fileType, "Data"); String name = makeFileName(experiment, program, studyByDbId.get(entry.getKey()).getStudyName()) + fileType.getExtension(); // Add to file list. files.add(new DownloadFile(name, streamedFile)); @@ -260,8 +259,7 @@ public DownloadFile exportObservations( List> exportRows = new ArrayList<>(rowByOUId.values()); sortExportRows(exportRows); // write export data to requested file format - //TODO the sheetName should be "Data". It should no longer be "Experiment Data" - StreamedFile streamedFile = writeToStreamedFile(columns, exportRows, fileType, "Experiment Data"); + StreamedFile streamedFile = writeToStreamedFile(columns, exportRows, fileType, "Data"); // Set filename. String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); String fileName = makeFileName(experiment, program, envFilenameFragment) + fileType.getExtension(); From d5c4ad7b81a86e93b7af803abfb62cd989d82ed9 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 16 Oct 2023 15:09:50 -0400 Subject: [PATCH 033/220] [BI-1914] tightened up code --- .../brapi/v2/services/BrAPITrialService.java | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 020654985..3f5036911 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -32,6 +32,7 @@ import org.breedinginsight.services.writers.CSVWriter; import org.breedinginsight.services.writers.ExcelWriter; import org.breedinginsight.utilities.Utilities; +import org.jetbrains.annotations.NotNull; import javax.inject.Inject; import javax.inject.Singleton; @@ -59,6 +60,7 @@ public class BrAPITrialService { private final BrAPIObservationUnitDAO ouDAO; private final BrAPIGermplasmDAO germplasmDAO; private final FileMappingUtil fileMappingUtil; + private static final String SHEET_NAME = "Data"; @Inject public BrAPITrialService(@Property(name = "brapi.server.reference-source") String referenceSource, @@ -237,10 +239,9 @@ public DownloadFile exportObservations( List files = new ArrayList<>(); // Generate a file for each study. for (Map.Entry>> entry: rowsByStudyId.entrySet()) { - List> rows = entry.getValue(); - sortExportRows(rows); - StreamedFile streamedFile = writeToStreamedFile(columns, rows, fileType, "Data"); + sortDefaultForExportRows(rows); + StreamedFile streamedFile = writeToStreamedFile(columns, rows, fileType, SHEET_NAME); String name = makeFileName(experiment, program, studyByDbId.get(entry.getKey()).getStudyName()) + fileType.getExtension(); // Add to file list. files.add(new DownloadFile(name, streamedFile)); @@ -257,9 +258,9 @@ public DownloadFile exportObservations( } } else { List> exportRows = new ArrayList<>(rowByOUId.values()); - sortExportRows(exportRows); + sortDefaultForExportRows(exportRows); // write export data to requested file format - StreamedFile streamedFile = writeToStreamedFile(columns, exportRows, fileType, "Data"); + StreamedFile streamedFile = writeToStreamedFile(columns, exportRows, fileType, SHEET_NAME); // Set filename. String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); String fileName = makeFileName(experiment, program, envFilenameFragment) + fileType.getExtension(); @@ -314,7 +315,7 @@ public Dataset getDatasetData(Program program, UUID experimentId, UUID datsetId, log.debug("fetching observations for dataset: " + datsetId); List data = observationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, obsVarDbIds, program); log.debug("building dataset object for dataset: " + datsetId); - sortObservationUnit(datasetOUs); + sortDefaultForObservationUnit(datasetOUs); Dataset dataset = new Dataset(experimentId.toString(), data, datasetOUs, datasetObsVars); if (stats) { Integer ouCount = datasetOUs.size(); @@ -556,13 +557,13 @@ private List filterDatasetByEnvironment( .collect(Collectors.toList()); } - private void sortObservationUnit(List ous){ + private void sortDefaultForObservationUnit(List ous) { ous.sort(Comparator.comparing(BrAPIObservationUnit::getStudyName) .thenComparing(BrAPIObservationUnit::getObservationUnitName)); } - private void sortExportRows(List> exportRows){ - Comparator> envComparator = Comparator.comparing(row -> ( row.get(Columns.ENV).toString())); + private void sortDefaultForExportRows(@NotNull List> exportRows) { + Comparator> envComparator = Comparator.comparing(row -> (row.get(Columns.ENV).toString())); Comparator> expUnitIdComparator = Comparator.comparing(row -> (row.get(Columns.EXP_UNIT_ID).toString())); From 0e2ffa6fa2c7c7e47d3dc1c892a48d48e0bcb25a Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 20 Oct 2023 16:33:47 +0000 Subject: [PATCH 034/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 40b9391ee..3f67822b2 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+608 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/ebb51275919060b57871b6d3f1ab1d07a31e29ac +version=v0.9.0+614 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/16402029bf9f16d6dbb0c001c30b07cc3fd681a8 From 38c6a6572f439a0b7351e7f5a4428a4dd715db0c Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 27 Oct 2023 14:33:32 -0400 Subject: [PATCH 035/220] [BI-1915] ontology export/download --- .../api/v1/controller/TraitController.java | 25 +++- .../model/imports/DataTypeTranslator.java | 35 ++++++ .../model/imports/TermTypeTranslator.java | 13 ++ .../services/OntologyService.java | 116 ++++++++++++++++++ .../parsers/trait/TraitFileColumns.java | 62 ++++++---- .../parsers/trait/TraitFileParser.java | 4 +- 6 files changed, 225 insertions(+), 30 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/DataTypeTranslator.java diff --git a/src/main/java/org/breedinginsight/api/v1/controller/TraitController.java b/src/main/java/org/breedinginsight/api/v1/controller/TraitController.java index 7f44398cd..46775d974 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/TraitController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/TraitController.java @@ -17,10 +17,12 @@ package org.breedinginsight.api.v1.controller; +import io.micronaut.http.HttpHeaders; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; import io.micronaut.http.MediaType; import io.micronaut.http.annotation.*; +import io.micronaut.http.server.types.files.StreamedFile; import lombok.extern.slf4j.Slf4j; import org.breedinginsight.api.auth.*; import org.breedinginsight.api.model.v1.request.query.QueryParams; @@ -35,6 +37,8 @@ import org.breedinginsight.api.model.v1.validators.QueryValid; import org.breedinginsight.api.model.v1.validators.SearchValid; import org.breedinginsight.api.v1.controller.metadata.AddMetadata; +import org.breedinginsight.brapps.importer.model.exports.FileType; +import org.breedinginsight.model.DownloadFile; import org.breedinginsight.model.Editable; import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; @@ -52,7 +56,6 @@ import java.util.List; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.atomic.AtomicReference; @Slf4j @Controller("/${micronaut.bi.api.version}") @@ -88,6 +91,26 @@ public HttpResponse>> getTraits( } } + @Get("/programs/{programId}/traits/export{?fileExtension,isActive}") + @Produces(value = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse getTraitsExport( + @PathVariable("programId") UUID programId, @QueryValue(defaultValue = "XLSX") String fileExtension, @QueryValue(defaultValue = "true") Boolean isActive) { + String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu."; + try { + FileType extension = Enum.valueOf(FileType.class, fileExtension); + DownloadFile ontologyFile = ontologyService.exportOntology(programId, extension, isActive); + HttpResponse ontologyExport = HttpResponse.ok(ontologyFile.getStreamedFile()).header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename="+ontologyFile.getFileName()+extension.getExtension()); + return ontologyExport; + } + catch (Exception e) { + log.info(e.getMessage(), e); + e.printStackTrace(); + HttpResponse response = HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, downloadErrorMessage).contentType(MediaType.TEXT_PLAIN).body(downloadErrorMessage); + return response; + } + } + @Post("/programs/{programId}/traits/search{?queryParams*}") @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DataTypeTranslator.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DataTypeTranslator.java new file mode 100644 index 000000000..d0861edb7 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DataTypeTranslator.java @@ -0,0 +1,35 @@ +package org.breedinginsight.brapps.importer.model.imports; + +import org.breedinginsight.dao.db.enums.DataType; +import org.breedinginsight.dao.db.enums.TermType; + +import java.util.Map; +import java.util.Optional; + +public class DataTypeTranslator { + + private static final Map displayToTypeMap = Map.of( + "Date YYYY-MM-DD", DataType.DATE, + "Numerical", DataType.NUMERICAL, + "Nominal", DataType.NOMINAL, + "Ordinal", DataType.ORDINAL, + "Text", DataType.TEXT); + + + public static Optional getTypeFromUserDisplayName(String displayName) { + return Optional.ofNullable(displayToTypeMap.get(displayName)); + } + + public static String getDisplayNameFromType(DataType type){ + String displayName = ""; + if(type!=null) { + for (String key : displayToTypeMap.keySet()) { + if (type == displayToTypeMap.get(key)) { + displayName = key; + break; + } + } + } + return displayName; + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/TermTypeTranslator.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/TermTypeTranslator.java index b53af17d7..225ba7ee6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/TermTypeTranslator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/TermTypeTranslator.java @@ -14,4 +14,17 @@ public class TermTypeTranslator { public static Optional getTermTypeFromUserDisplayName(String userDisplayName) { return Optional.ofNullable(userDisplayToTermTypeMap.get(userDisplayName)); } + + public static String getDisplayNameFromTermType(TermType termType){ + String displayName = ""; + if(termType!=null) { + for (String key : userDisplayToTermTypeMap.keySet()) { + if (termType == userDisplayToTermTypeMap.get(key)) { + displayName = key; + break; + } + } + } + return displayName; + } } diff --git a/src/main/java/org/breedinginsight/services/OntologyService.java b/src/main/java/org/breedinginsight/services/OntologyService.java index 702073a54..71508155a 100644 --- a/src/main/java/org/breedinginsight/services/OntologyService.java +++ b/src/main/java/org/breedinginsight/services/OntologyService.java @@ -4,21 +4,34 @@ import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.multipart.CompletedFileUpload; import io.micronaut.http.server.exceptions.InternalServerException; +import io.micronaut.http.server.types.files.StreamedFile; +import org.apache.commons.lang3.StringUtils; import org.brapi.v2.model.core.BrAPIProgram; +import org.brapi.v2.model.pheno.BrAPIScaleValidValuesCategories; import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.api.model.v1.request.SharedOntologyProgramRequest; import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapps.importer.model.exports.FileType; +import org.breedinginsight.brapps.importer.model.imports.DataTypeTranslator; +import org.breedinginsight.brapps.importer.model.imports.TermTypeTranslator; import org.breedinginsight.dao.db.tables.pojos.ProgramSharedOntologyEntity; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.ProgramOntologyDAO; import org.breedinginsight.daos.TraitDAO; import org.breedinginsight.model.*; import org.breedinginsight.services.exceptions.*; +import org.breedinginsight.services.parsers.trait.TraitFileColumns; +import org.breedinginsight.services.writers.CSVWriter; +import org.breedinginsight.services.writers.ExcelWriter; +import org.jooq.DataType; +import org.jooq.exception.IOException; import javax.inject.Inject; import javax.inject.Singleton; import javax.validation.constraints.NotNull; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; import java.util.*; import java.util.stream.Collectors; @@ -342,6 +355,43 @@ public List updateTraits(UUID programId, List traits, Authenticate return traitService.updateTraits(lookupId, traits, actingUser); } + + public DownloadFile exportOntology(UUID programId, FileType fileExtension, boolean isActive) throws IllegalArgumentException, IOException, java.io.IOException { + List columns = TraitFileColumns.getOrderedColumns(); + + //Retrieve trait list data + List traits = traitDAO.getTraitsFullByProgramId(programId); + //Filter traits for Active or Archived + traits = traits.stream().filter(trait -> trait.getActive()==isActive).collect(Collectors.toList()); + //Sort list in default (trait name) order. + traits.sort(Comparator.comparing(trait -> trait.getObservationVariableName().toLowerCase())); + + // make file Name + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + Program program = null; + try { + program = getProgram(programId); + } catch (DoesNotExistException e) { + e.printStackTrace(); + } + String activeOrArchive = isActive ? "Active" : "Archive"; + String programName = program.getName(); + String fileName = programName + "_" + activeOrArchive + "_Ontology_" + timestamp; + + StreamedFile downloadFile; + + //Convert traits list to List> data to pass into file writer + List> processedData = processData(traits); + + if (fileExtension == FileType.CSV){ + downloadFile = CSVWriter.writeToDownload(columns, processedData, fileExtension); + } else { + downloadFile = ExcelWriter.writeToDownload("Data", columns, processedData, fileExtension); + } + return new DownloadFile(fileName, downloadFile); + } + public List createTraits(UUID programId, List traits, AuthenticatedUser actingUser, Boolean throwDuplicateErrors) throws DoesNotExistException, ValidatorException, BadRequestException { UUID lookupId = getSubscribedOntologyProgramId(programId); @@ -408,5 +458,71 @@ public List getSubscribeOntologyOptions(UUID programId) thro ).collect(Collectors.toList()); return subscriptionOptions; } + public List> processData(List traits) { + List> processedData = new ArrayList<>(); + + for (Trait trait : traits) { + HashMap row = new HashMap<>(); + row.put(TraitFileColumns.NAME.toString(), trait.getObservationVariableName()); + row.put(TraitFileColumns.FULL_NAME.toString(), trait.getFullName()); + row.put(TraitFileColumns.TERM_TYPE.toString(), TermTypeTranslator.getDisplayNameFromTermType( trait.getTermType() )); + row.put(TraitFileColumns.DESCRIPTION.toString(), trait.getTraitDescription()); + //SYNONYMS + String synonymsAsStr = null; + if(trait.getSynonyms() != null) { + synonymsAsStr = String.join("; ", trait.getSynonyms()); + } + row.put(TraitFileColumns.SYNONYMS.toString(), synonymsAsStr); + //STATUS + if(trait.getActive()) { + row.put(TraitFileColumns.STATUS.toString(), "active"); + } else { + row.put(TraitFileColumns.STATUS.toString(), "archived"); + + } + //TAGS + String tagsAsStr = null; + if(trait.getTags() != null) { + tagsAsStr = String.join("; ", trait.getTags()); + } + row.put(TraitFileColumns.TAGS.toString(), tagsAsStr); + + row.put(TraitFileColumns.TRAIT_ENTITY.toString(), trait.getEntity()); + row.put(TraitFileColumns.TRAIT_ATTRIBUTE.toString(), trait.getAttribute()); + Method method = trait.getMethod(); + if(method!=null) { + row.put(TraitFileColumns.METHOD_DESCRIPTION.toString(), method.getDescription()); + row.put(TraitFileColumns.METHOD_CLASS.toString(), method.getMethodClass()); + row.put(TraitFileColumns.METHOD_FORMULA.toString(), method.getFormula()); + } + Scale scale = trait.getScale(); + if(scale!=null) { + + row.put(TraitFileColumns.SCALE_CLASS.toString(), DataTypeTranslator.getDisplayNameFromType(scale.getDataType())); + row.put(TraitFileColumns.SCALE_NAME.toString(), scale.getScaleName()); + row.put(TraitFileColumns.SCALE_DECIMAL_PLACES.toString(), scale.getDecimalPlaces()); + row.put(TraitFileColumns.SCALE_LOWER_LIMIT.toString(), scale.getValidValueMin()); + row.put(TraitFileColumns.SCALE_UPPER_LIMIT.toString(), scale.getValidValueMax()); + //SCALE_CATEGORIES + String categoriesAsStr = null; + if(scale.getCategories() != null) { + categoriesAsStr = scale.getCategories().stream() + .map(this::makeCategoryString) + .collect(Collectors.joining("; ")); + } + row.put(TraitFileColumns.SCALE_CATEGORIES.toString(), categoriesAsStr); + } + processedData.add(row); + } + return processedData; + } + private String makeCategoryString(BrAPIScaleValidValuesCategories category){ + StringBuilder stringBuilder = new StringBuilder( category.getValue() ); + if( StringUtils.isNotBlank( category.getLabel() ) ){ + stringBuilder.append("="); + stringBuilder.append( category.getLabel() ); + } + return stringBuilder.toString(); + } } diff --git a/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileColumns.java b/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileColumns.java index a9d7d1e45..f3782adf7 100644 --- a/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileColumns.java +++ b/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileColumns.java @@ -16,46 +16,54 @@ */ package org.breedinginsight.services.parsers.trait; +import org.breedinginsight.model.Column; + import java.util.Arrays; +import java.util.List; import java.util.Set; import java.util.stream.Collectors; public enum TraitFileColumns { - NAME("Name"), - SYNONYMS("Synonyms"), - TRAIT_ENTITY("Trait entity"), - TRAIT_ATTRIBUTE("Trait attribute"), - DESCRIPTION("Description"), - STATUS("Status"), - METHOD_DESCRIPTION("Method description"), - METHOD_CLASS("Method class"), - METHOD_FORMULA("Method formula"), - SCALE_NAME("Units"), - SCALE_CLASS("Scale class"), - SCALE_DECIMAL_PLACES("Scale decimal places"), - SCALE_LOWER_LIMIT("Scale lower limit"), - SCALE_UPPER_LIMIT("Scale upper limit"), - SCALE_CATEGORIES("Scale categories"), - TAGS("Tags"), - FULL_NAME("Full name"), - TERM_TYPE("Term Type"); - - - private String value; - - TraitFileColumns(String value) { - this.value = value; + NAME("Name", Column.ColumnDataType.STRING), + FULL_NAME("Full name", Column.ColumnDataType.STRING), + TERM_TYPE("Term Type", Column.ColumnDataType.STRING), + DESCRIPTION("Description", Column.ColumnDataType.STRING), + SYNONYMS("Synonyms", Column.ColumnDataType.STRING), + STATUS("Status",Column.ColumnDataType.STRING), + TAGS("Tags", Column.ColumnDataType.STRING), + TRAIT_ENTITY("Trait entity", Column.ColumnDataType.STRING), + TRAIT_ATTRIBUTE("Trait attribute", Column.ColumnDataType.STRING), + METHOD_DESCRIPTION("Method description", Column.ColumnDataType.STRING), + METHOD_CLASS("Method class", Column.ColumnDataType.STRING), + METHOD_FORMULA("Method formula", Column.ColumnDataType.STRING), + SCALE_CLASS("Scale class", Column.ColumnDataType.STRING), + SCALE_NAME("Units", Column.ColumnDataType.STRING), + SCALE_DECIMAL_PLACES("Scale decimal places", Column.ColumnDataType.INTEGER), + SCALE_LOWER_LIMIT("Scale lower limit", Column.ColumnDataType.INTEGER), + SCALE_UPPER_LIMIT("Scale upper limit", Column.ColumnDataType.INTEGER), + SCALE_CATEGORIES("Scale categories", Column.ColumnDataType.STRING); + + private final Column column; + + TraitFileColumns(String value, Column.ColumnDataType dataType) { + this.column = new Column(value, dataType); } @Override public String toString() { - return value; + return column.getValue(); } - public static Set getColumns() { + public static Set getColumnNames() { return Arrays.stream(TraitFileColumns.values()) - .map(value -> value.toString()) + .map(value -> value.column.getValue()) .collect(Collectors.toSet()); } + + public static List getOrderedColumns() { + return Arrays.stream(TraitFileColumns.values()) + .map(value -> value.column) + .collect(Collectors.toList()); + } } \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java b/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java index 3d81eac24..771518e71 100644 --- a/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java +++ b/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java @@ -95,7 +95,7 @@ public List parseExcel(@NonNull InputStream inputStream) throws ParsingEx throw new ParsingException(ParsingExceptionType.MISSING_SHEET); } - List records = ExcelParser.parse(sheet, TraitFileColumns.getColumns()); + List records = ExcelParser.parse(sheet, TraitFileColumns.getColumnNames()); return excelRecordsToTraits(records); } @@ -117,7 +117,7 @@ public List parseCsv(@NonNull InputStream inputStream) throws ParsingExce } Sheet excelSheet = convertCsvToExcel(records); - List excelRecords = ExcelParser.parse(excelSheet, TraitFileColumns.getColumns()); + List excelRecords = ExcelParser.parse(excelSheet, TraitFileColumns.getColumnNames()); return excelRecordsToTraits(excelRecords); } From fe5aed9f7c145115f625c58605403f824b5f0a70 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Wed, 1 Nov 2023 08:23:00 -0400 Subject: [PATCH 036/220] [BI-1915]fixed text for scale class DATE --- .../brapps/importer/model/imports/DataTypeTranslator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DataTypeTranslator.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DataTypeTranslator.java index d0861edb7..39a93639f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DataTypeTranslator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DataTypeTranslator.java @@ -9,7 +9,7 @@ public class DataTypeTranslator { private static final Map displayToTypeMap = Map.of( - "Date YYYY-MM-DD", DataType.DATE, + "Date", DataType.DATE, "Numerical", DataType.NUMERICAL, "Nominal", DataType.NOMINAL, "Ordinal", DataType.ORDINAL, From 92e1ad2e8b044ef6124049495ee667776c289fcf Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 2 Nov 2023 11:22:37 -0400 Subject: [PATCH 037/220] [BI-1914]WIP --- .../brapi/v2/services/BrAPITrialService.java | 50 +++++++++++--- .../utilities/IntOrderComparator.java | 67 +++++++++++++++++++ 2 files changed, 109 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/breedinginsight/utilities/IntOrderComparator.java diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 3f5036911..1bc1af3b6 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -31,6 +31,7 @@ import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; import org.breedinginsight.services.writers.CSVWriter; import org.breedinginsight.services.writers.ExcelWriter; +import org.breedinginsight.utilities.IntOrderComparator; import org.breedinginsight.utilities.Utilities; import org.jetbrains.annotations.NotNull; @@ -43,6 +44,8 @@ import java.time.format.DateTimeFormatter; import java.util.*; import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -557,16 +560,47 @@ private List filterDatasetByEnvironment( .collect(Collectors.toList()); } + static Comparator alphaNumComparator = (str1, str2) -> { + Pattern p = Pattern.compile("^\\d+"); + Matcher m = p.matcher(str1); + Integer number1 = null; + if (!m.find()) { + return str1.compareToIgnoreCase(str2); + } + else { + Integer number2 = null; + number1 = Integer.parseInt(m.group()); + m = p.matcher(str2); + if (!m.find()) { + return str1.compareToIgnoreCase(str2); + } + else { + number2 = Integer.parseInt(m.group()); + int comparison = number1.compareTo(number2); + if (comparison != 0) { + return comparison; + } + else { + return str1.compareToIgnoreCase(str2); + } + } + } + }; + private void sortDefaultForObservationUnit(List ous) { - ous.sort(Comparator.comparing(BrAPIObservationUnit::getStudyName) - .thenComparing(BrAPIObservationUnit::getObservationUnitName)); + Comparator studyNameComparator = Comparator.comparing(BrAPIObservationUnit::getStudyName, new IntOrderComparator()); + Comparator ouNameComparator = Comparator.comparing(BrAPIObservationUnit::getObservationUnitName, new IntOrderComparator()); + ous.sort( (studyNameComparator).thenComparing(ouNameComparator)); } private void sortDefaultForExportRows(@NotNull List> exportRows) { - Comparator> envComparator = Comparator.comparing(row -> (row.get(Columns.ENV).toString())); - Comparator> expUnitIdComparator = - Comparator.comparing(row -> (row.get(Columns.EXP_UNIT_ID).toString())); - - exportRows.sort(envComparator.thenComparing(expUnitIdComparator)); +// Comparator> envComparator = Comparator.comparing(row -> (row.get(Columns.ENV).toString())); +// Comparator> expUnitIdComparator = +// Comparator.comparing(row -> (row.get(Columns.EXP_UNIT_ID).toString())); +// +// exportRows.sort(envComparator.thenComparing(expUnitIdComparator)); +// + exportRows.sort( (row1, row2) -> alphaNumComparator.compare((row1.get(Columns.EXP_UNIT_ID).toString()), (row2.get(Columns.EXP_UNIT_ID).toString()))); + exportRows.sort( (row1, row2) -> alphaNumComparator.compare((row1.get(Columns.ENV).toString()), (row2.get(Columns.ENV).toString()))); } -} \ No newline at end of file +} diff --git a/src/main/java/org/breedinginsight/utilities/IntOrderComparator.java b/src/main/java/org/breedinginsight/utilities/IntOrderComparator.java new file mode 100644 index 000000000..5614c4c77 --- /dev/null +++ b/src/main/java/org/breedinginsight/utilities/IntOrderComparator.java @@ -0,0 +1,67 @@ +package org.breedinginsight.utilities; + +import java.util.Comparator; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +/* +If you have text(optional), followed by some digits, followed by more text(optional), this Comparator +will sort the digits in numeric order. The text (and any subsequent digits) will be in alpha-numeric order. +*/ +public class IntOrderComparator implements Comparator { + @Override + public int compare(String str1, String str2) { + //static finals to make the Matcher::group code more readable + final int PREFIX = 1; + final int NUMBERS = 2; + final int SUFFIX = 3; + + // convert null strings to blank + str1 = (str1 == null) ? "" : str1; + str2 = (str2 == null) ? "" : str2; + + //The real work begins + + // NOTE: The last group includes all remaining text and digits + Pattern p = Pattern.compile("^([^\\d]*)(\\d*)(.*)$"); + + Matcher m1 = p.matcher(str1); + Matcher m2 = p.matcher(str2); + m1.find(); // needed to let m1.group() work + m2.find(); // needed to let m2.group() work + + String prefix1 = m1.group(PREFIX); + String prefix2 = m2.group(PREFIX); + Integer number1 = m1.group(NUMBERS).length() > 0 ? Integer.parseInt(m1.group(NUMBERS)) : null; + Integer number2 = m2.group(NUMBERS).length() > 0 ? Integer.parseInt(m2.group(NUMBERS)) : null; + String suffix1 = m1.group(SUFFIX); + String suffix2 = m2.group(SUFFIX); + + if (!prefix1.equals(prefix2)) { + return prefix1.compareTo(prefix2); + } + /*if the prefixes are equal, sort by numbers and suffixes.*/ + + // an empty number goes before a number (EX. 'a' before 'a1') + if (number1 == null && number2 != null) { + return -1; + } + if (number1 != null && number2 == null) { + return 1; + } + if (number1 == null && number2 == null) { + return 0; + } + + if (!number1.equals(number2)) { + return number1.compareTo(number2); + } + /*if the prefixes and numbers are equal, sort by suffixes.*/ + if (!suffix1.equals(suffix2)) { + return suffix1.compareTo(suffix2); + } + + /* if all else fails, compare the strings + (I think this will always return 0)*/ + return str1.compareTo(str2); + } +} From d7cba83e9866ecf713f265472081fccc3380d091 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 2 Nov 2023 11:53:16 -0400 Subject: [PATCH 038/220] [BI-1914] Use IntOrderComparator --- .../brapi/v2/services/BrAPITrialService.java | 42 +++---------------- 1 file changed, 6 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 1bc1af3b6..7935f5be6 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -560,33 +560,6 @@ private List filterDatasetByEnvironment( .collect(Collectors.toList()); } - static Comparator alphaNumComparator = (str1, str2) -> { - Pattern p = Pattern.compile("^\\d+"); - Matcher m = p.matcher(str1); - Integer number1 = null; - if (!m.find()) { - return str1.compareToIgnoreCase(str2); - } - else { - Integer number2 = null; - number1 = Integer.parseInt(m.group()); - m = p.matcher(str2); - if (!m.find()) { - return str1.compareToIgnoreCase(str2); - } - else { - number2 = Integer.parseInt(m.group()); - int comparison = number1.compareTo(number2); - if (comparison != 0) { - return comparison; - } - else { - return str1.compareToIgnoreCase(str2); - } - } - } - }; - private void sortDefaultForObservationUnit(List ous) { Comparator studyNameComparator = Comparator.comparing(BrAPIObservationUnit::getStudyName, new IntOrderComparator()); Comparator ouNameComparator = Comparator.comparing(BrAPIObservationUnit::getObservationUnitName, new IntOrderComparator()); @@ -594,13 +567,10 @@ private void sortDefaultForObservationUnit(List ous) { } private void sortDefaultForExportRows(@NotNull List> exportRows) { -// Comparator> envComparator = Comparator.comparing(row -> (row.get(Columns.ENV).toString())); -// Comparator> expUnitIdComparator = -// Comparator.comparing(row -> (row.get(Columns.EXP_UNIT_ID).toString())); -// -// exportRows.sort(envComparator.thenComparing(expUnitIdComparator)); -// - exportRows.sort( (row1, row2) -> alphaNumComparator.compare((row1.get(Columns.EXP_UNIT_ID).toString()), (row2.get(Columns.EXP_UNIT_ID).toString()))); - exportRows.sort( (row1, row2) -> alphaNumComparator.compare((row1.get(Columns.ENV).toString()), (row2.get(Columns.ENV).toString()))); - } + Comparator> envComparator = Comparator.comparing(row -> (row.get(Columns.ENV).toString()), new IntOrderComparator()); + Comparator> expUnitIdComparator = + Comparator.comparing(row -> (row.get(Columns.EXP_UNIT_ID).toString()), new IntOrderComparator()); + + exportRows.sort(envComparator.thenComparing(expUnitIdComparator)); + } } From 5fd64cf2262b87a354a63f1f3d29cc022cb4f53d Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 2 Nov 2023 11:54:23 -0400 Subject: [PATCH 039/220] [BI-1914] removed unused import statement --- .../breedinginsight/brapi/v2/services/BrAPITrialService.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 7935f5be6..9847b762d 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -44,8 +44,6 @@ import java.time.format.DateTimeFormatter; import java.util.*; import java.util.function.Function; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; From 4b28150a843b98536d256c9887a6b1c810edd7d6 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 2 Nov 2023 16:01:58 -0400 Subject: [PATCH 040/220] [BI-1914] improved language in IntOrderComparator --- .../utilities/IntOrderComparator.java | 42 +++++++++++++------ 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/breedinginsight/utilities/IntOrderComparator.java b/src/main/java/org/breedinginsight/utilities/IntOrderComparator.java index 5614c4c77..cb01ff5d9 100644 --- a/src/main/java/org/breedinginsight/utilities/IntOrderComparator.java +++ b/src/main/java/org/breedinginsight/utilities/IntOrderComparator.java @@ -4,15 +4,28 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; /* -If you have text(optional), followed by some digits, followed by more text(optional), this Comparator -will sort the digits in numeric order. The text (and any subsequent digits) will be in alpha-numeric order. +If you have text(optional), followed by some digits(optional), followed by more text(optional), this Comparator +will sort the digits in numeric order. The trailing text (and any subsequent digits) will be in +alpha-numeric order. + +In other words; this comparator will assume that strings being compared are comprised of a text _prefix_, and an +_integer_, followed by a _suffix_. + +This comparator will fist compare the prefix as an alpha-numeric, the integer as numeric and +then the the suffix as an alpha-numeric. +For Example: +1) if a string is 'big4team', then prefix='big', integer=4, suffix='team' +2) if a string is '12monkeys', then prefix='', integer=12, suffix='monkeys' +3) if a string is 'abcd', then prefix='abcd', integer=null, suffix='' +4) if a string is 'libnum14.3', then prefix='libnum', integer=14, suffix='.4' + */ public class IntOrderComparator implements Comparator { @Override public int compare(String str1, String str2) { //static finals to make the Matcher::group code more readable final int PREFIX = 1; - final int NUMBERS = 2; + final int INTEGER = 2; final int SUFFIX = 3; // convert null strings to blank @@ -26,36 +39,37 @@ public int compare(String str1, String str2) { Matcher m1 = p.matcher(str1); Matcher m2 = p.matcher(str2); + m1.find(); // needed to let m1.group() work m2.find(); // needed to let m2.group() work String prefix1 = m1.group(PREFIX); String prefix2 = m2.group(PREFIX); - Integer number1 = m1.group(NUMBERS).length() > 0 ? Integer.parseInt(m1.group(NUMBERS)) : null; - Integer number2 = m2.group(NUMBERS).length() > 0 ? Integer.parseInt(m2.group(NUMBERS)) : null; + Integer integer1 = m1.group(INTEGER).length() > 0 ? Integer.parseInt(m1.group(INTEGER)) : null; + Integer integer2 = m2.group(INTEGER).length() > 0 ? Integer.parseInt(m2.group(INTEGER)) : null; String suffix1 = m1.group(SUFFIX); String suffix2 = m2.group(SUFFIX); if (!prefix1.equals(prefix2)) { return prefix1.compareTo(prefix2); } - /*if the prefixes are equal, sort by numbers and suffixes.*/ + /*if the prefixes are equal, sort by integers and suffixes.*/ - // an empty number goes before a number (EX. 'a' before 'a1') - if (number1 == null && number2 != null) { + // an empty integer goes before an integer (EX. 'a' before 'a1') + if (integer1 == null && integer2 != null) { return -1; } - if (number1 != null && number2 == null) { + if (integer1 != null && integer2 == null) { return 1; } - if (number1 == null && number2 == null) { + if (integer1 == null && integer2 == null) { return 0; } - if (!number1.equals(number2)) { - return number1.compareTo(number2); + if (!integer1.equals(integer2)) { + return integer1.compareTo(integer2); } - /*if the prefixes and numbers are equal, sort by suffixes.*/ + /*if the prefixes and integers are equal, sort by suffixes.*/ if (!suffix1.equals(suffix2)) { return suffix1.compareTo(suffix2); } @@ -65,3 +79,5 @@ public int compare(String str1, String str2) { return str1.compareTo(str2); } } + + From 35eeabd8854fbf6c3ee401c6c0b5b17d92eb8fab Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 7 Nov 2023 03:55:32 +0000 Subject: [PATCH 041/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 3f67822b2..c2e39fb54 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+614 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/16402029bf9f16d6dbb0c001c30b07cc3fd681a8 +version=v0.9.0+622 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/770333d4358a56a1e139a3ce864161ec6b500e83 From 921fd0856e1d627fe144c88d99816617337820ba Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Tue, 7 Nov 2023 12:18:06 -0500 Subject: [PATCH 042/220] [BI-1914] changed Integer to BigInteger Changed the wording from 'integers' to 'digits' --- .../utilities/IntOrderComparator.java | 51 ++++++++++--------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/src/main/java/org/breedinginsight/utilities/IntOrderComparator.java b/src/main/java/org/breedinginsight/utilities/IntOrderComparator.java index cb01ff5d9..bbd1ca813 100644 --- a/src/main/java/org/breedinginsight/utilities/IntOrderComparator.java +++ b/src/main/java/org/breedinginsight/utilities/IntOrderComparator.java @@ -1,5 +1,6 @@ package org.breedinginsight.utilities; +import java.math.BigInteger; import java.util.Comparator; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -8,24 +9,23 @@ If you have text(optional), followed by some digits(optional), followed by more will sort the digits in numeric order. The trailing text (and any subsequent digits) will be in alpha-numeric order. -In other words; this comparator will assume that strings being compared are comprised of a text _prefix_, and an -_integer_, followed by a _suffix_. +In other words; this comparator will assume that strings being compared are comprised of a text _prefix_, followed by +_digits_, followed by a _suffix_. -This comparator will fist compare the prefix as an alpha-numeric, the integer as numeric and +This comparator will fist compare the prefix as an alpha-numeric, the digits as a numeric and then the the suffix as an alpha-numeric. For Example: -1) if a string is 'big4team', then prefix='big', integer=4, suffix='team' -2) if a string is '12monkeys', then prefix='', integer=12, suffix='monkeys' -3) if a string is 'abcd', then prefix='abcd', integer=null, suffix='' -4) if a string is 'libnum14.3', then prefix='libnum', integer=14, suffix='.4' - +1) if a string is 'big4team', then prefix='big', digits=4, suffix='team' +2) if a string is '12monkeys', then prefix='', digits=12, suffix='monkeys' +3) if a string is 'abcd', then prefix='abcd', digits=null, suffix='' +4) if a string is 'libnum14.3', then prefix='libnum', digits=14, suffix='.4' */ public class IntOrderComparator implements Comparator { @Override public int compare(String str1, String str2) { //static finals to make the Matcher::group code more readable final int PREFIX = 1; - final int INTEGER = 2; + final int DIGITS = 2; final int SUFFIX = 3; // convert null strings to blank @@ -45,31 +45,32 @@ public int compare(String str1, String str2) { String prefix1 = m1.group(PREFIX); String prefix2 = m2.group(PREFIX); - Integer integer1 = m1.group(INTEGER).length() > 0 ? Integer.parseInt(m1.group(INTEGER)) : null; - Integer integer2 = m2.group(INTEGER).length() > 0 ? Integer.parseInt(m2.group(INTEGER)) : null; + BigInteger digits1 = m1.group(DIGITS).length() > 0 ? new BigInteger(m1.group(DIGITS)) : null; + BigInteger digits2 = m2.group(DIGITS).length() > 0 ? new BigInteger(m2.group(DIGITS)) : null; String suffix1 = m1.group(SUFFIX); String suffix2 = m2.group(SUFFIX); if (!prefix1.equals(prefix2)) { return prefix1.compareTo(prefix2); } - /*if the prefixes are equal, sort by integers and suffixes.*/ - - // an empty integer goes before an integer (EX. 'a' before 'a1') - if (integer1 == null && integer2 != null) { - return -1; - } - if (integer1 != null && integer2 == null) { - return 1; + /*if the prefixes are equal, sort by digits and suffixes.*/ + + // an empty digit is less than any digit (EX. 'a' is less than 'a1') + if( digits1 == null){ + if(digits2 == null){ + return 0; //if both digits are null (equal) + } + else { + return -1; //if there are no digits in str1 but digits in str2 (str1 < str2) + } } - if (integer1 == null && integer2 == null) { - return 0; + if( digits2 == null){ + return 1; //if there are digits in str1 but no digits in str2 (str1 > str2) } - - if (!integer1.equals(integer2)) { - return integer1.compareTo(integer2); + if (!digits1.equals(digits2)) { + return digits1.compareTo(digits2); } - /*if the prefixes and integers are equal, sort by suffixes.*/ + /*if the prefixes and digits are equal, sort by suffixes.*/ if (!suffix1.equals(suffix2)) { return suffix1.compareTo(suffix2); } From 0b42edb8735c4b5d45cb1bbdfb94582d0adb4a7f Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 2 Nov 2023 14:07:03 -0400 Subject: [PATCH 043/220] Added sorghum and hemp to brapi species --- src/main/resources/brapi/sql/species.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/resources/brapi/sql/species.sql b/src/main/resources/brapi/sql/species.sql index c57dd2ac9..9f8f6fe25 100644 --- a/src/main/resources/brapi/sql/species.sql +++ b/src/main/resources/brapi/sql/species.sql @@ -29,4 +29,6 @@ INSERT INTO crop (id, crop_name) VALUES ('13', 'Strawberry') ON CONFLICT DO NOTH INSERT INTO crop (id, crop_name) VALUES ('14', 'Honey') ON CONFLICT DO NOTHING; INSERT INTO crop (id, crop_name) VALUES ('15', 'Pecan') ON CONFLICT DO NOTHING; INSERT INTO crop (id, crop_name) VALUES ('16', 'Lettuce') ON CONFLICT DO NOTHING; -INSERT INTO crop (id, crop_name) VALUES ('17', 'Cotton') ON CONFLICT DO NOTHING; \ No newline at end of file +INSERT INTO crop (id, crop_name) VALUES ('17', 'Cotton') ON CONFLICT DO NOTHING; +INSERT INTO crop (id, crop_name) VALUES ('18', 'Sorghum') ON CONFLICT DO NOTHING; +INSERT INTO crop (id, crop_name) VALUES ('19', 'Hemp') ON CONFLICT DO NOTHING; \ No newline at end of file From e44706ea1b2a74fb11c776e0a88af4d61ce37e3c Mon Sep 17 00:00:00 2001 From: Nick <53413353+nickpalladino@users.noreply.github.com> Date: Thu, 2 Nov 2023 14:09:52 -0400 Subject: [PATCH 044/220] Added sorghum and hemp --- .../db/migration/V0.5.36__add-new-species.sql | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 src/main/resources/db/migration/V0.5.36__add-new-species.sql diff --git a/src/main/resources/db/migration/V0.5.36__add-new-species.sql b/src/main/resources/db/migration/V0.5.36__add-new-species.sql new file mode 100644 index 000000000..d6cac5494 --- /dev/null +++ b/src/main/resources/db/migration/V0.5.36__add-new-species.sql @@ -0,0 +1,30 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +DO $$ +DECLARE +user_id UUID; +BEGIN + +user_id := (SELECT id FROM bi_user WHERE name = 'system'); + +insert into species (common_name, description, created_by, updated_by) +values + ('Sorghum', 'Sorghum is a genus of about 25 species of flowering plants in the grass family (Poaceae).', user_id, user_id), + ('Hemp', 'Hemp is a class of Cannabis cultivars grown specifically for industrial use.', user_id, user_id); + +END $$; \ No newline at end of file From b0bf2ae6fdb6ee1e271f7f893e07d7ef25cf9b72 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 7 Nov 2023 20:21:35 +0000 Subject: [PATCH 045/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index c2e39fb54..931322be1 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+622 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/770333d4358a56a1e139a3ce864161ec6b500e83 +version=v0.9.0+624 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/ccddd50e1b3dd138af6321e38969339d612c6bd2 From fe6eb4737b2ffe01561d2d038f13c55f5d6f7d51 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Wed, 8 Nov 2023 10:24:47 -0500 Subject: [PATCH 046/220] [BI-1915] Fixed IntellaJ 'problems' ade code more readable --- .../api/v1/controller/TraitController.java | 14 ++--- .../model/imports/DataTypeTranslator.java | 1 - .../services/OntologyService.java | 57 ++++++++++--------- .../parsers/trait/TraitFileParser.java | 12 ++-- 4 files changed, 43 insertions(+), 41 deletions(-) diff --git a/src/main/java/org/breedinginsight/api/v1/controller/TraitController.java b/src/main/java/org/breedinginsight/api/v1/controller/TraitController.java index 46775d974..3e6e1869c 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/TraitController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/TraitController.java @@ -61,10 +61,10 @@ @Controller("/${micronaut.bi.api.version}") public class TraitController { - private TraitService traitService; - private SecurityService securityService; - private TraitQueryMapper traitQueryMapper; - private OntologyService ontologyService; + private final TraitService traitService; + private final SecurityService securityService; + private final TraitQueryMapper traitQueryMapper; + private final OntologyService ontologyService; @Inject public TraitController(TraitService traitService, SecurityService securityService, @@ -94,7 +94,7 @@ public HttpResponse>> getTraits( @Get("/programs/{programId}/traits/export{?fileExtension,isActive}") @Produces(value = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse getTraitsExport( + public HttpResponse getTraitsExport( @PathVariable("programId") UUID programId, @QueryValue(defaultValue = "XLSX") String fileExtension, @QueryValue(defaultValue = "true") Boolean isActive) { String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu."; try { @@ -163,7 +163,7 @@ public HttpResponse> getTraitEditable(@PathVariable UUID prog @Post("/programs/{programId}/traits") @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roles = {ProgramSecuredRole.BREEDER}) - public HttpResponse>> createTraits(@PathVariable UUID programId, @Body @Valid List traits) { + public HttpResponse createTraits(@PathVariable UUID programId, @Body @Valid List traits) { AuthenticatedUser actingUser = securityService.getUser(); try { List createdTraits = ontologyService.createTraits(programId, traits, actingUser, true); @@ -191,7 +191,7 @@ public HttpResponse>> createTraits(@PathVariable UU @Put("/programs/{programId}/traits") @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roles = {ProgramSecuredRole.BREEDER}) - public HttpResponse>> updateTraits(@PathVariable UUID programId, @Body @Valid List traits) { + public HttpResponse updateTraits(@PathVariable UUID programId, @Body @Valid List traits) { AuthenticatedUser actingUser = securityService.getUser(); try { List updatedTraits = ontologyService.updateTraits(programId, traits, actingUser); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DataTypeTranslator.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DataTypeTranslator.java index 39a93639f..17fb4a178 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/DataTypeTranslator.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/DataTypeTranslator.java @@ -1,7 +1,6 @@ package org.breedinginsight.brapps.importer.model.imports; import org.breedinginsight.dao.db.enums.DataType; -import org.breedinginsight.dao.db.enums.TermType; import java.util.Map; import java.util.Optional; diff --git a/src/main/java/org/breedinginsight/services/OntologyService.java b/src/main/java/org/breedinginsight/services/OntologyService.java index 71508155a..58cb5175f 100644 --- a/src/main/java/org/breedinginsight/services/OntologyService.java +++ b/src/main/java/org/breedinginsight/services/OntologyService.java @@ -16,6 +16,7 @@ import org.breedinginsight.brapps.importer.model.imports.DataTypeTranslator; import org.breedinginsight.brapps.importer.model.imports.TermTypeTranslator; import org.breedinginsight.dao.db.tables.pojos.ProgramSharedOntologyEntity; +import org.breedinginsight.dao.db.tables.pojos.TraitEntity; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.ProgramOntologyDAO; import org.breedinginsight.daos.TraitDAO; @@ -24,7 +25,7 @@ import org.breedinginsight.services.parsers.trait.TraitFileColumns; import org.breedinginsight.services.writers.CSVWriter; import org.breedinginsight.services.writers.ExcelWriter; -import org.jooq.DataType; + import org.jooq.exception.IOException; import javax.inject.Inject; @@ -38,11 +39,11 @@ @Singleton public class OntologyService { - private ProgramDAO programDAO; - private ProgramOntologyDAO programOntologyDAO; - private TraitDAO traitDAO; - private TraitService traitService; - private TraitUploadService traitUploadService; + private final ProgramDAO programDAO; + private final ProgramOntologyDAO programOntologyDAO; + private final TraitDAO traitDAO; + private final TraitService traitService; + private final TraitUploadService traitUploadService; @Inject public OntologyService(ProgramDAO programDAO, ProgramOntologyDAO programOntologyDAO, TraitDAO traitDAO, TraitService traitService, TraitUploadService traitUploadService) { @@ -59,7 +60,7 @@ public OntologyService(ProgramDAO programDAO, ProgramOntologyDAO programOntology * @param sharedOnly -- True = return only shared programs, False = get all shareable programs * @return List */ - public List getSharedOntology(@NotNull UUID programId, @NotNull Boolean sharedOnly) throws DoesNotExistException { + public List getSharedOntology(@NotNull UUID programId, @NotNull Boolean sharedOnly) throws DoesNotExistException { // Get program with that id Program program = getProgram(programId); @@ -100,7 +101,7 @@ public List getSharedOntology(@NotNull UUID programId, @NotNull Boolean sharedOn unsharableIds.addAll(shareTargetIds); formattedPrograms.addAll(matchingPrograms.stream() .filter(matchingProgram -> !unsharableIds.contains(matchingProgram.getId())) - .map(matchingProgram -> formatResponse(matchingProgram)) + .map(this::formatResponse) .collect(Collectors.toList())); } @@ -115,7 +116,7 @@ private List getSharedProgramsFormatted(Program program) { sharedOntologies.stream().map(ProgramSharedOntologyEntity::getSharedProgramId).collect(Collectors.toList())); // Get the programs in a lookup map Map sharedProgramsMap = new HashMap<>(); - sharedPrograms.stream().forEach(sharedProgram -> sharedProgramsMap.put(sharedProgram.getId(), sharedProgram)); + sharedPrograms.forEach(sharedProgram -> sharedProgramsMap.put(sharedProgram.getId(), sharedProgram)); // Format shared programs response return sharedOntologies.stream().map(sharedOntology -> @@ -170,7 +171,7 @@ private Boolean ontologyIsEditable(ProgramSharedOntologyEntity sharedOntologyEnt if (sharedOntologyEntity.getActive()) { // Get all trait ids for the program List traitIds = traitService.getSubscribedOntologyTraits(sharedOntologyEntity.getSharedProgramId()).stream() - .map(trait -> trait.getId()) + .map(TraitEntity::getId) .collect(Collectors.toList()); // Get the brapi program id @@ -216,7 +217,7 @@ public List shareOntology(@NotNull UUID programId, Authenticated // Check shareability, same brapi server, same species List matchingPrograms = getMatchingPrograms(program); Set matchingProgramsSet = new HashSet<>(); - matchingPrograms.stream().forEach(matchingProgram -> matchingProgramsSet.add(matchingProgram.getId())); + matchingPrograms.forEach(matchingProgram -> matchingProgramsSet.add(matchingProgram.getId())); Set shareProgramIdsSet = new HashSet<>(); ValidationErrors validationErrors = new ValidationErrors(); @@ -262,7 +263,7 @@ public List shareOntology(@NotNull UUID programId, Authenticated */ public void revokeOntology(@NotNull UUID programId, @NotNull UUID sharedProgramId) throws UnprocessableEntityException, DoesNotExistException { // Check that program exists - Program program = getProgram(programId); + getProgram(programId); // Check that shared program exists Optional optionalSharedOntology = programOntologyDAO.getSharedOntologyById(programId, sharedProgramId); @@ -324,7 +325,7 @@ public SubscribedOntology subscribeOntology(UUID programId, UUID sharingProgramI public void unsubscribeOntology(UUID programId, UUID sharingProgramId) throws DoesNotExistException, UnprocessableEntityException { // Check that program exists - Program program = getProgram(programId); + getProgram(programId); // Check that shared program exists Optional optionalSharedOntology = programOntologyDAO.getSharedOntologyById(sharingProgramId, programId); @@ -367,17 +368,7 @@ public DownloadFile exportOntology(UUID programId, FileType fileExtension, boole traits.sort(Comparator.comparing(trait -> trait.getObservationVariableName().toLowerCase())); // make file Name - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); - String timestamp = formatter.format(OffsetDateTime.now()); - Program program = null; - try { - program = getProgram(programId); - } catch (DoesNotExistException e) { - e.printStackTrace(); - } - String activeOrArchive = isActive ? "Active" : "Archive"; - String programName = program.getName(); - String fileName = programName + "_" + activeOrArchive + "_Ontology_" + timestamp; + String fileName = makeFileName(programId, isActive); StreamedFile downloadFile; @@ -392,6 +383,20 @@ public DownloadFile exportOntology(UUID programId, FileType fileExtension, boole return new DownloadFile(fileName, downloadFile); } + private String makeFileName(UUID programId, boolean isActive) { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + Program program = null; + try { + program = getProgram(programId); + } catch (DoesNotExistException e) { + e.printStackTrace(); + } + String activeOrArchive = isActive ? "Active" : "Archive"; + String programName = program==null? "program" : program.getName(); + return programName + "_" + activeOrArchive + "_Ontology_" + timestamp; + } + public List createTraits(UUID programId, List traits, AuthenticatedUser actingUser, Boolean throwDuplicateErrors) throws DoesNotExistException, ValidatorException, BadRequestException { UUID lookupId = getSubscribedOntologyProgramId(programId); @@ -440,8 +445,8 @@ public UUID getSubscribedOntologyProgramId(UUID programId) throws DoesNotExistEx } public List getSubscribeOntologyOptions(UUID programId) throws DoesNotExistException { - - Program program = getProgram(programId); + // Check that program exists + getProgram(programId); List sharedOntologies = programOntologyDAO.getSubscriptionOptions(programId); List programs = programDAO.get(sharedOntologies.stream().map(ProgramSharedOntologyEntity::getProgramId).collect(Collectors.toList())); diff --git a/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java b/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java index 771518e71..66c7134f6 100644 --- a/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java +++ b/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java @@ -65,10 +65,9 @@ public class TraitFileParser { private static final String TRAIT_STATUS_ACTIVE = "active"; private static final String TRAIT_STATUS_ARCHIVED = "archived"; - private final static Set TRAIT_STATUS_VALID_VALUES = Collections.unmodifiableSet( - Set.of(TRAIT_STATUS_ACTIVE, TRAIT_STATUS_ARCHIVED)); + private final static Set TRAIT_STATUS_VALID_VALUES = Set.of(TRAIT_STATUS_ACTIVE, TRAIT_STATUS_ARCHIVED); - private TraitFileValidatorError traitValidatorError; + private final TraitFileValidatorError traitValidatorError; @Inject public TraitFileParser(TraitFileValidatorError traitValidatorError){ @@ -103,7 +102,6 @@ public List parseExcel(@NonNull InputStream inputStream) throws ParsingEx // no sheets RFC4180 public List parseCsv(@NonNull InputStream inputStream) throws ParsingException, ValidatorException { - ArrayList traits = new ArrayList<>(); InputStreamReader in = new InputStreamReader(inputStream); Iterable records = null; @@ -135,7 +133,7 @@ private List excelRecordsToTraits(List records) throws Valid .build(); - Boolean active; + boolean active; String traitStatus = parseExcelValueAsString(record, TraitFileColumns.STATUS); if (traitStatus == null) { active = true; @@ -145,7 +143,7 @@ private List excelRecordsToTraits(List records) throws Valid ParsingExceptionType.INVALID_TRAIT_STATUS.toString(), HttpStatus.UNPROCESSABLE_ENTITY); validationErrors.addError(traitValidatorError.getRowNumber(i), error); } - active = !traitStatus.toLowerCase().equals(TRAIT_STATUS_ARCHIVED); + active = !traitStatus.equalsIgnoreCase(TRAIT_STATUS_ARCHIVED); } // Normalize and capitalize method class @@ -354,7 +352,7 @@ private BrAPIScaleValidValuesCategories parseCategory(String value) throws Unpro else if (labelMeaning.length == 1) { category.setValue(labelMeaning[0].trim()); } else if (labelMeaning.length > 2){ - // The case where there are multiple category delimiters in a value. Could be cause by bad list delimeter. + // The case where there are multiple category delimiters in a value. Could be cause by bad list delimiter. throw new UnprocessableEntityException("Unable to parse categories"); } From 1808030432be4b8302aeb37ef40338e7d3dd59ca Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 8 Nov 2023 17:03:52 +0000 Subject: [PATCH 047/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 931322be1..13d78346a 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+624 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/ccddd50e1b3dd138af6321e38969339d612c6bd2 +version=v0.9.0+628 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/2eb843e45e727d2e6f4dffd3cef4c9d99dd94c19 From 7582bd0ba3d070b55c40693a87444db4c0c68cf4 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Wed, 8 Nov 2023 12:14:53 -0500 Subject: [PATCH 048/220] [BI-1915] only include caigories in ontology export, if the scale class is ORDINA or NOMINAL --- .../services/OntologyService.java | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/services/OntologyService.java b/src/main/java/org/breedinginsight/services/OntologyService.java index 58cb5175f..90de17ec8 100644 --- a/src/main/java/org/breedinginsight/services/OntologyService.java +++ b/src/main/java/org/breedinginsight/services/OntologyService.java @@ -15,6 +15,7 @@ import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.brapps.importer.model.imports.DataTypeTranslator; import org.breedinginsight.brapps.importer.model.imports.TermTypeTranslator; +import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.dao.db.tables.pojos.ProgramSharedOntologyEntity; import org.breedinginsight.dao.db.tables.pojos.TraitEntity; import org.breedinginsight.daos.ProgramDAO; @@ -509,12 +510,7 @@ public List> processData(List traits) { row.put(TraitFileColumns.SCALE_LOWER_LIMIT.toString(), scale.getValidValueMin()); row.put(TraitFileColumns.SCALE_UPPER_LIMIT.toString(), scale.getValidValueMax()); //SCALE_CATEGORIES - String categoriesAsStr = null; - if(scale.getCategories() != null) { - categoriesAsStr = scale.getCategories().stream() - .map(this::makeCategoryString) - .collect(Collectors.joining("; ")); - } + String categoriesAsStr = makeCategoriesString(scale); row.put(TraitFileColumns.SCALE_CATEGORIES.toString(), categoriesAsStr); } @@ -522,9 +518,21 @@ public List> processData(List traits) { } return processedData; } - private String makeCategoryString(BrAPIScaleValidValuesCategories category){ + + private String makeCategoriesString(Scale scale) { + String categoriesAsStr = null; + if(scale.getCategories() != null && + (scale.getDataType()==DataType.ORDINAL || scale.getDataType() == DataType.NOMINAL) ) { + categoriesAsStr = scale.getCategories().stream() + .map(cat -> makeCategoryString(cat, scale.getDataType()) ) + .collect(Collectors.joining("; ")); + } + return categoriesAsStr; + } + + private String makeCategoryString(BrAPIScaleValidValuesCategories category, DataType scaleClass){ StringBuilder stringBuilder = new StringBuilder( category.getValue() ); - if( StringUtils.isNotBlank( category.getLabel() ) ){ + if( StringUtils.isNotBlank( category.getLabel() ) && scaleClass == DataType.ORDINAL ){ stringBuilder.append("="); stringBuilder.append( category.getLabel() ); } From 75729242a7977110411a5e62e0d49c39806adacf Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Wed, 25 Oct 2023 10:14:58 -0400 Subject: [PATCH 049/220] [BI-1922] Can download Germplasm --- .../brapi/v2/GermplasmController.java | 20 +++++++++++ .../v2/services/BrAPIGermplasmService.java | 33 ++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java b/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java index bbf17db26..408860c95 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java @@ -157,6 +157,26 @@ public HttpResponse germplasmListExport( } } + @Get("/programs/{programId}/germplasm/export{?fileExtension}") + @Produces(value = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse germplasmExport( + @PathVariable("programId") UUID programId, @QueryValue(defaultValue = "XLSX") String fileExtension) { + String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu."; + try { + FileType extension = Enum.valueOf(FileType.class, fileExtension); + DownloadFile germplasmListFile = germplasmService.exportGermplasm(programId, extension); + HttpResponse germplasmExport = HttpResponse.ok(germplasmListFile.getStreamedFile()).header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename="+germplasmListFile.getFileName()+extension.getExtension()); + return germplasmExport; + } + catch (Exception e) { + log.info(e.getMessage(), e); + e.printStackTrace(); + HttpResponse response = HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, downloadErrorMessage).contentType(MediaType.TEXT_PLAIN).body(downloadErrorMessage); + return response; + } + } + @Get("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/germplasm/{germplasmId}") @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java index 719b76b5e..54b466b55 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java @@ -191,6 +191,37 @@ public List getGermplasmByList(UUID programId, String listDbId) return germplasm; } else throw new ApiException(); } + public DownloadFile exportGermplasm(UUID programId, FileType fileExtension) throws IllegalArgumentException, ApiException, IOException { + List columns = GermplasmFileColumns.getOrderedColumns(); + + //Retrieve germplasm list data + List germplasm = germplasmDAO.getGermplasm(programId); + germplasm.sort(Comparator.comparingInt(germ -> Integer.parseInt(germ.getAccessionNumber()))); + + // make file Name + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + String programKey = "program"; + Optional optionalProgram = programService.getById(programId); + if (optionalProgram.isPresent()) { + Program program = optionalProgram. get(); + programKey = program.getKey(); + } + String fileName = "germplasm[" + programKey + "]_" + timestamp; + + StreamedFile downloadFile; + //Convert list data to List> data to pass into file writer + List> processedData = processListData(germplasm, new UUID(0,0)); + + if (fileExtension == FileType.CSV){ + downloadFile = CSVWriter.writeToDownload(columns, processedData, fileExtension); + } else { + downloadFile = ExcelWriter.writeToDownload("Data", columns, processedData, fileExtension); + } + + return new DownloadFile(fileName, downloadFile); + } + public DownloadFile exportGermplasmList(UUID programId, String listId, FileType fileExtension) throws IllegalArgumentException, ApiException, IOException { List columns = GermplasmFileColumns.getOrderedColumns(); @@ -219,7 +250,7 @@ public DownloadFile exportGermplasmList(UUID programId, String listId, FileType if (fileExtension == FileType.CSV){ downloadFile = CSVWriter.writeToDownload(columns, processedData, fileExtension); } else { - downloadFile = ExcelWriter.writeToDownload("Germplasm Import", columns, processedData, fileExtension); + downloadFile = ExcelWriter.writeToDownload("Data", columns, processedData, fileExtension); } return new DownloadFile(fileName, downloadFile); From 819a0f9ffe17fc64547128b956553d9e9c36e99e Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 30 Oct 2023 10:56:46 -0400 Subject: [PATCH 050/220] [BI-1922] addess 'problems' identified by InelliJ improved export file name --- .../v2/services/BrAPIGermplasmService.java | 60 +++---------------- 1 file changed, 9 insertions(+), 51 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java index 54b466b55..cfd44f64b 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java @@ -1,25 +1,21 @@ package org.breedinginsight.brapi.v2.services; import io.micronaut.context.annotation.Property; -import io.micronaut.http.HttpRequest; import io.micronaut.http.server.exceptions.InternalServerException; import io.micronaut.http.server.types.files.StreamedFile; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; import lombok.extern.slf4j.Slf4j; -import org.brapi.v2.model.core.BrAPIListSummary; -import org.brapi.v2.model.core.BrAPIListTypes; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.germ.BrAPIGermplasmSynonyms; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.daos.BrAPIListDAO; import org.breedinginsight.brapps.importer.model.exports.FileType; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.model.Column; import org.breedinginsight.model.DownloadFile; -import org.breedinginsight.model.Pedigree; import org.breedinginsight.model.Program; import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.exceptions.DoesNotExistException; @@ -28,7 +24,6 @@ import org.breedinginsight.services.writers.ExcelWriter; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.utilities.Utilities; import javax.inject.Inject; import javax.inject.Singleton; @@ -77,39 +72,6 @@ public Optional getGermplasmByDBID(UUID programId, String germpl return germplasmDAO.getGermplasmByDBID(germplasmId, programId); } - public List getGermplasmListsByProgramId(UUID programId, HttpRequest request) throws DoesNotExistException, ApiException { - - if (!programService.exists(programId)) { - throw new DoesNotExistException("Program does not exist"); - } - - Optional optionalProgram = programService.getById(programId); - if(optionalProgram.isPresent()) { - Program program = optionalProgram.get(); - - List germplasmLists = brAPIListDAO.getListByTypeAndExternalRef(BrAPIListTypes.GERMPLASM, programId, Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS), programId); - - for (BrAPIListSummary germplasmList: germplasmLists) { - String listName = germplasmList.getListName(); - String newListName = removeAppendedKey(listName, program.getKey()); - germplasmList.setListName(newListName); - - //Retrieve germplasm details to get list owner name - //Due to listOwnerName not being stored in breedbase - BrAPIListDetails listData = brAPIListDAO.getListById(germplasmList.getListDbId(), programId).getResult(); - List germplasmNames = listData.getData().subList(0,1); - List germplasm = germplasmDAO.getGermplasmByRawName(germplasmNames, programId); - String createdBy = germplasm.get(0).getAdditionalInfo().getAsJsonObject("createdBy").get("userName").getAsString(); - germplasmList.setListOwnerName(createdBy); - } - - return germplasmLists; - } - else { - throw new DoesNotExistException("Program does not exist"); - } - } - public List> processListData(List germplasm, UUID germplasmListId){ List> processedData = new ArrayList<>(); @@ -164,7 +126,7 @@ public List> processListData(List germplasm, // Synonyms if (germplasmEntry.getSynonyms() != null && !germplasmEntry.getSynonyms().isEmpty()) { String joinedSynonyms = germplasmEntry.getSynonyms().stream() - .map(synonym -> synonym.getSynonym()) + .map(BrAPIGermplasmSynonyms::getSynonym) .collect(Collectors.joining(";")); row.put("Synonyms", joinedSynonyms); } @@ -201,13 +163,14 @@ public DownloadFile exportGermplasm(UUID programId, FileType fileExtension) thro // make file Name DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); String timestamp = formatter.format(OffsetDateTime.now()); - String programKey = "program"; + StringBuilder fileNameSB = new StringBuilder(); Optional optionalProgram = programService.getById(programId); if (optionalProgram.isPresent()) { - Program program = optionalProgram. get(); - programKey = program.getKey(); + Program program = optionalProgram.get(); + fileNameSB.append( program.getName() ); + fileNameSB.append("_"); } - String fileName = "germplasm[" + programKey + "]_" + timestamp; + String fileName = fileNameSB.append("germplasm").append("_").append(timestamp).toString(); StreamedFile downloadFile; //Convert list data to List> data to pass into file writer @@ -260,7 +223,7 @@ public UUID getGermplasmListId(BrAPIListDetails listData) { if(Objects.nonNull(listData.getExternalReferences()) && hasListExternalReference(listData.getExternalReferences())) { return UUID.fromString(listData.getExternalReferences().stream() .filter(e -> referenceSource.concat("/lists").equals(e.getReferenceSource())) - .map(e -> e.getReferenceID()).findAny().orElse("00000000-0000-0000-000000000000")); + .map(BrAPIExternalReference::getReferenceID).findAny().orElse("00000000-0000-0000-000000000000")); } else { return new UUID(0,0); } @@ -270,7 +233,7 @@ public UUID getGermplasmListId(BrAPIListNewRequest importList) { if(Objects.nonNull(importList.getExternalReferences()) && hasListExternalReference(importList.getExternalReferences())) { return UUID.fromString(importList.getExternalReferences().stream() .filter(e -> referenceSource.concat("/lists").equals(e.getReferenceSource())) - .map(e -> e.getReferenceID()).findAny().orElse("00000000-0000-0000-000000000000")); + .map(BrAPIExternalReference::getReferenceID).findAny().orElse("00000000-0000-0000-000000000000")); } else { return new UUID(0,0); } @@ -337,11 +300,6 @@ public List updateBrAPIGermplasm(List putBrAPIGe return germplasmDAO.updateBrAPIGermplasm(putBrAPIGermplasmList, programId, upload); } - public List importBrAPIGermplasm(List postBrAPIGermplasmList, List putBrAPIGermplasmList, - UUID programId, ImportUpload upload) throws ApiException { - return germplasmDAO.createBrAPIGermplasm(postBrAPIGermplasmList, programId, upload); - } - public List getRawGermplasmByAccessionNumber(ArrayList germplasmAccessionNumbers, UUID programId) throws ApiException { List germplasmList = germplasmDAO.getRawGermplasm(programId); List resultGermplasm = new ArrayList<>(); From faa034ae359edf44ee380abfe395ebf4e453dfd1 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 13 Nov 2023 15:26:12 -0500 Subject: [PATCH 051/220] [BI-1945] - removed indexes on tables that were removed --- src/main/resources/brapi/sql/create_indexes.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/resources/brapi/sql/create_indexes.sql b/src/main/resources/brapi/sql/create_indexes.sql index 6c0772bfd..59f3c32af 100644 --- a/src/main/resources/brapi/sql/create_indexes.sql +++ b/src/main/resources/brapi/sql/create_indexes.sql @@ -19,10 +19,8 @@ CREATE INDEX CONCURRENTLY IF NOT EXISTS "pedigree_edge_connected_node_id" ON ped CREATE INDEX CONCURRENTLY IF NOT EXISTS "pedigree_edge_edge_type" ON pedigree_edge (edge_type); CREATE INDEX CONCURRENTLY IF NOT EXISTS "program_external_references_program_entity_id" ON program_external_references (program_entity_id); CREATE INDEX CONCURRENTLY IF NOT EXISTS "external_reference_composite" ON external_reference (external_reference_source, external_reference_id); -CREATE INDEX CONCURRENTLY IF NOT EXISTS "program_additional_info_composite" ON program_additional_info (additional_info_id, program_entity_id); CREATE INDEX CONCURRENTLY IF NOT EXISTS "list_list_name" ON list (list_name); CREATE INDEX CONCURRENTLY IF NOT EXISTS "pedigree_node_germplasm_id" ON pedigree_node (germplasm_id); -CREATE INDEX CONCURRENTLY IF NOT EXISTS "germplasm_additional_info_germplasm_entity_id" ON germplasm_additional_info (germplasm_entity_id); CREATE INDEX CONCURRENTLY IF NOT EXISTS "germplasm_external_references_germplasm_entity_id" ON germplasm_external_references (germplasm_entity_id); CREATE INDEX CONCURRENTLY IF NOT EXISTS "germplasm_synonym_germplasm_id" ON germplasm_synonym (germplasm_id); CREATE INDEX CONCURRENTLY IF NOT EXISTS "germplasm_taxon_germplasm_id" ON germplasm_taxon (germplasm_id); \ No newline at end of file From 91030fcf960121c7c732028fc24a8c0009f30bb4 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 13 Nov 2023 15:26:50 -0500 Subject: [PATCH 052/220] [BI-1945] - updated species seed data --- src/main/resources/brapi/sql/species.sql | 38 ++++++++++++------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/main/resources/brapi/sql/species.sql b/src/main/resources/brapi/sql/species.sql index 9f8f6fe25..fe0dff1d9 100644 --- a/src/main/resources/brapi/sql/species.sql +++ b/src/main/resources/brapi/sql/species.sql @@ -13,22 +13,22 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -INSERT INTO crop (id, crop_name) VALUES ('1', 'Blueberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (id, crop_name) VALUES ('2', 'Salmon') ON CONFLICT DO NOTHING; -INSERT INTO crop (id, crop_name) VALUES ('3', 'Grape') ON CONFLICT DO NOTHING; -INSERT INTO crop (id, crop_name) VALUES ('4', 'Alfalfa') ON CONFLICT DO NOTHING; -INSERT INTO crop (id, crop_name) VALUES ('5', 'Sweet Potato') ON CONFLICT DO NOTHING; -INSERT INTO crop (id, crop_name) VALUES ('6', 'Trout') ON CONFLICT DO NOTHING; -INSERT INTO crop (id, crop_name) VALUES ('7', 'Soybean') ON CONFLICT DO NOTHING; -INSERT INTO crop (id, crop_name) VALUES ('8', 'Cranberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (id, crop_name) VALUES ('9', 'Cucumber') ON CONFLICT DO NOTHING; -INSERT INTO crop (id, crop_name) VALUES ('10', 'Oat') ON CONFLICT DO NOTHING; -INSERT INTO crop (id, crop_name) VALUES ('11', 'Citrus') ON CONFLICT DO NOTHING; -INSERT INTO crop (id, crop_name) VALUES ('12', 'Sugar Cane') ON CONFLICT DO NOTHING; -INSERT INTO crop (id, crop_name) VALUES ('13', 'Strawberry') ON CONFLICT DO NOTHING; -INSERT INTO crop (id, crop_name) VALUES ('14', 'Honey') ON CONFLICT DO NOTHING; -INSERT INTO crop (id, crop_name) VALUES ('15', 'Pecan') ON CONFLICT DO NOTHING; -INSERT INTO crop (id, crop_name) VALUES ('16', 'Lettuce') ON CONFLICT DO NOTHING; -INSERT INTO crop (id, crop_name) VALUES ('17', 'Cotton') ON CONFLICT DO NOTHING; -INSERT INTO crop (id, crop_name) VALUES ('18', 'Sorghum') ON CONFLICT DO NOTHING; -INSERT INTO crop (id, crop_name) VALUES ('19', 'Hemp') ON CONFLICT DO NOTHING; \ No newline at end of file +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '4', 'Blueberry') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '5', 'Salmon') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '6', 'Grape') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '7', 'Alfalfa') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '8', 'Sweet Potato') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '9', 'Trout') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '10', 'Soybean') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '11', 'Cranberry') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '12', 'Cucumber') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '13', 'Oat') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '14', 'Citrus') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '15', 'Sugar Cane') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '16', 'Strawberry') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '17', 'Honey') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '18', 'Pecan') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '19', 'Lettuce') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '20', 'Cotton') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '21', 'Sorghum') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '22', 'Hemp') ON CONFLICT DO NOTHING; \ No newline at end of file From 24f535d7af9fb8c481f02108d6618ae9d7df53b5 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 13 Nov 2023 16:29:56 -0500 Subject: [PATCH 053/220] [BI-1945] - updated application.properties --- .../brapi/properties/application.properties | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/resources/brapi/properties/application.properties b/src/main/resources/brapi/properties/application.properties index 32fc7437f..568d63c7b 100644 --- a/src/main/resources/brapi/properties/application.properties +++ b/src/main/resources/brapi/properties/application.properties @@ -15,20 +15,24 @@ # limitations under the License. # -application.properties.templateserver.port = 8080 +server.port = 8080 server.servlet.context-path=/brapi/v2 spring.datasource.url=jdbc:postgresql://${BRAPI_DB_SERVER}/${BRAPI_DB} +spring.datasource.hikari.data-source-properties.stringtype=unspecified spring.datasource.username=${BRAPI_DB_USER} spring.datasource.password=${BRAPI_DB_PASSWORD} spring.datasource.driver-class-name=org.postgresql.Driver -spring.jpa.hibernate.ddl-auto=create-only +spring.flyway.locations=classpath:db/migration,classpath:org/brapi/test/BrAPITestServer/db/migration +spring.flyway.schemas=public +spring.flyway.baselineOnMigrate=true + +spring.jpa.hibernate.ddl-auto=validate spring.jpa.show-sql=false -spring.jpa.properties.hibernate.hbm2ddl.import_files=sql/species.sql, sql/create_indexes.sql spring.mvc.dispatch-options-request=true security.oidc_discovery_url=https://example.com/auth/.well-known/openid-configuration -security.enabled=false \ No newline at end of file +security.enabled=false From 92e53eca9f50a6b8973c83d5d6b0a9d6df5557ee Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 13 Nov 2023 17:02:53 -0500 Subject: [PATCH 054/220] [BI-1945] - updated bind mount and paths --- docker-compose.yml | 2 +- src/main/resources/brapi/properties/application.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 12cedd454..15826ae2d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -100,7 +100,7 @@ services: - ${BRAPI_SERVER_PORT}:8080 volumes: - ./src/main/resources/brapi/properties/application.properties:/home/brapi/properties/application.properties - - ./src/main/resources/brapi/sql/:/home/brapi/sql/ + - ./src/main/resources/brapi/sql/:/home/brapi/db/sql/ networks: backend: aliases: diff --git a/src/main/resources/brapi/properties/application.properties b/src/main/resources/brapi/properties/application.properties index 568d63c7b..28a097f3a 100644 --- a/src/main/resources/brapi/properties/application.properties +++ b/src/main/resources/brapi/properties/application.properties @@ -25,7 +25,7 @@ spring.datasource.password=${BRAPI_DB_PASSWORD} spring.datasource.driver-class-name=org.postgresql.Driver -spring.flyway.locations=classpath:db/migration,classpath:org/brapi/test/BrAPITestServer/db/migration +spring.flyway.locations=classpath:db/migration,classpath:db/sql,classpath:org/brapi/test/BrAPITestServer/db/migration spring.flyway.schemas=public spring.flyway.baselineOnMigrate=true From cae376acc79d3428ca22e96716ae1c4f5090373b Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 13 Nov 2023 17:03:17 -0500 Subject: [PATCH 055/220] [BI-1945] - renamed for flyway --- .../brapi/sql/{create_indexes.sql => R__create_indexes.sql} | 0 src/main/resources/brapi/sql/{species.sql => R__species.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/brapi/sql/{create_indexes.sql => R__create_indexes.sql} (100%) rename src/main/resources/brapi/sql/{species.sql => R__species.sql} (100%) diff --git a/src/main/resources/brapi/sql/create_indexes.sql b/src/main/resources/brapi/sql/R__create_indexes.sql similarity index 100% rename from src/main/resources/brapi/sql/create_indexes.sql rename to src/main/resources/brapi/sql/R__create_indexes.sql diff --git a/src/main/resources/brapi/sql/species.sql b/src/main/resources/brapi/sql/R__species.sql similarity index 100% rename from src/main/resources/brapi/sql/species.sql rename to src/main/resources/brapi/sql/R__species.sql From 17e6db3d2e686205bd6cc0012918b54c652e9027 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 14 Nov 2023 14:27:20 -0500 Subject: [PATCH 056/220] [BI-1945] - fixed tests --- .../java/org/breedinginsight/BrAPITest.java | 1 + .../brapi/properties/application.properties | 10 ++++--- .../sql/brapi/mount/R__create_indexes.sql | 26 +++++++++++++++++++ src/test/resources/sql/brapi/species.sql | 25 +++++++++++++----- 4 files changed, 53 insertions(+), 9 deletions(-) create mode 100644 src/test/resources/sql/brapi/mount/R__create_indexes.sql diff --git a/src/test/java/org/breedinginsight/BrAPITest.java b/src/test/java/org/breedinginsight/BrAPITest.java index 731ff3455..8c5c35bad 100644 --- a/src/test/java/org/breedinginsight/BrAPITest.java +++ b/src/test/java/org/breedinginsight/BrAPITest.java @@ -60,6 +60,7 @@ public BrAPITest() { .withEnv("BRAPI_DB", "postgres") .withEnv("BRAPI_DB_USER", "postgres") .withEnv("BRAPI_DB_PASSWORD", "postgres") + .withClasspathResourceMapping("sql/brapi/mount", "/home/brapi/db/sql", BindMode.READ_WRITE) // HACK - READ_WRITE forces testcontainers to use a bind mount (which overwrites) instead of copying files. .withClasspathResourceMapping("brapi/properties/application.properties", "/home/brapi/properties/application.properties", BindMode.READ_ONLY) .waitingFor(Wait.forLogMessage(".*Started BrapiTestServer in \\d*.\\d* seconds.*", 1).withStartupTimeout(Duration.ofMinutes(1))); diff --git a/src/test/resources/brapi/properties/application.properties b/src/test/resources/brapi/properties/application.properties index 7e0d5c19c..28a097f3a 100644 --- a/src/test/resources/brapi/properties/application.properties +++ b/src/test/resources/brapi/properties/application.properties @@ -15,18 +15,22 @@ # limitations under the License. # -application.properties.templateserver.port = 8080 +server.port = 8080 server.servlet.context-path=/brapi/v2 spring.datasource.url=jdbc:postgresql://${BRAPI_DB_SERVER}/${BRAPI_DB} +spring.datasource.hikari.data-source-properties.stringtype=unspecified spring.datasource.username=${BRAPI_DB_USER} spring.datasource.password=${BRAPI_DB_PASSWORD} spring.datasource.driver-class-name=org.postgresql.Driver -spring.jpa.hibernate.ddl-auto=create-drop +spring.flyway.locations=classpath:db/migration,classpath:db/sql,classpath:org/brapi/test/BrAPITestServer/db/migration +spring.flyway.schemas=public +spring.flyway.baselineOnMigrate=true + +spring.jpa.hibernate.ddl-auto=validate spring.jpa.show-sql=false -#spring.jpa.properties.hibernate.hbm2ddl.import_files=sql/species.sql spring.mvc.dispatch-options-request=true diff --git a/src/test/resources/sql/brapi/mount/R__create_indexes.sql b/src/test/resources/sql/brapi/mount/R__create_indexes.sql new file mode 100644 index 000000000..59f3c32af --- /dev/null +++ b/src/test/resources/sql/brapi/mount/R__create_indexes.sql @@ -0,0 +1,26 @@ +-- See the NOTICE file distributed with this work for additional information +-- regarding copyright ownership. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- Indexes to improve read performance of Germplasm operations. +CREATE INDEX CONCURRENTLY IF NOT EXISTS "pedigree_edge_this_node_id" ON pedigree_edge (this_node_id); +CREATE INDEX CONCURRENTLY IF NOT EXISTS "pedigree_edge_connected_node_id" ON pedigree_edge (connceted_node_id); +CREATE INDEX CONCURRENTLY IF NOT EXISTS "pedigree_edge_edge_type" ON pedigree_edge (edge_type); +CREATE INDEX CONCURRENTLY IF NOT EXISTS "program_external_references_program_entity_id" ON program_external_references (program_entity_id); +CREATE INDEX CONCURRENTLY IF NOT EXISTS "external_reference_composite" ON external_reference (external_reference_source, external_reference_id); +CREATE INDEX CONCURRENTLY IF NOT EXISTS "list_list_name" ON list (list_name); +CREATE INDEX CONCURRENTLY IF NOT EXISTS "pedigree_node_germplasm_id" ON pedigree_node (germplasm_id); +CREATE INDEX CONCURRENTLY IF NOT EXISTS "germplasm_external_references_germplasm_entity_id" ON germplasm_external_references (germplasm_entity_id); +CREATE INDEX CONCURRENTLY IF NOT EXISTS "germplasm_synonym_germplasm_id" ON germplasm_synonym (germplasm_id); +CREATE INDEX CONCURRENTLY IF NOT EXISTS "germplasm_taxon_germplasm_id" ON germplasm_taxon (germplasm_id); \ No newline at end of file diff --git a/src/test/resources/sql/brapi/species.sql b/src/test/resources/sql/brapi/species.sql index 1449a4203..817a1c726 100644 --- a/src/test/resources/sql/brapi/species.sql +++ b/src/test/resources/sql/brapi/species.sql @@ -17,9 +17,22 @@ */ -- name: InsertSpecies -INSERT INTO crop (id, crop_name) VALUES ('1', 'Blueberry'); -INSERT INTO crop (id, crop_name) VALUES ('2', 'Salmon'); -INSERT INTO crop (id, crop_name) VALUES ('3', 'Grape'); -INSERT INTO crop (id, crop_name) VALUES ('4', 'Alfalfa'); -INSERT INTO crop (id, crop_name) VALUES ('5', 'Sweet Potato'); -INSERT INTO crop (id, crop_name) VALUES ('6', 'Trout'); +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '4', 'Blueberry') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '5', 'Salmon') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '6', 'Grape') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '7', 'Alfalfa') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '8', 'Sweet Potato') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '9', 'Trout') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '10', 'Soybean') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '11', 'Cranberry') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '12', 'Cucumber') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '13', 'Oat') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '14', 'Citrus') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '15', 'Sugar Cane') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '16', 'Strawberry') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '17', 'Honey') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '18', 'Pecan') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '19', 'Lettuce') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '20', 'Cotton') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '21', 'Sorghum') ON CONFLICT DO NOTHING; +INSERT INTO crop (auth_user_id, id, crop_name) VALUES ('anonymousUser', '22', 'Hemp') ON CONFLICT DO NOTHING; From e0cf8867ee3ed376245bbe498d59d7b82dfa3273 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 17 Nov 2023 12:45:56 -0500 Subject: [PATCH 057/220] [BI-1922] better error handeling --- .../java/org/breedinginsight/brapi/v2/GermplasmController.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java b/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java index 408860c95..568e58d99 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java @@ -170,8 +170,7 @@ public HttpResponse germplasmExport( return germplasmExport; } catch (Exception e) { - log.info(e.getMessage(), e); - e.printStackTrace(); + log.error(e.getMessage(), e); HttpResponse response = HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, downloadErrorMessage).contentType(MediaType.TEXT_PLAIN).body(downloadErrorMessage); return response; } From 6301e2c48ac5b313d994c46e1e61696b940c441a Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 17 Nov 2023 20:42:20 +0000 Subject: [PATCH 058/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 13d78346a..9c23d8ce6 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+628 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/2eb843e45e727d2e6f4dffd3cef4c9d99dd94c19 +version=v0.9.0+630 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/45952b4e8d0a5015fdfb9cb647a8aed2f1bf6036 From e3d4c58da6f12e862a531a97a2cf327e66c3a0db Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Sat, 18 Nov 2023 22:07:22 +0000 Subject: [PATCH 059/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 9c23d8ce6..3fe0d14be 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+630 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/45952b4e8d0a5015fdfb9cb647a8aed2f1bf6036 +version=v0.9.0+632 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/5881a34cd3b31b8f61eacb8004664b460a262afb From 5b17e9535323671c563c46a8d194267a66d5d4d2 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 21 Nov 2023 20:59:55 +0000 Subject: [PATCH 060/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 3fe0d14be..d557e1cf9 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+632 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/5881a34cd3b31b8f61eacb8004664b460a262afb +version=v0.9.0+634 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/bff69347c16f64500e2e7460b80872c76077abaa From 44be42045f7fe94a6805a84a0756e67d2dde18bb Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 29 Nov 2023 10:26:38 -0500 Subject: [PATCH 061/220] [BI-1945-hotfix] - updated bind mount paths --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 15826ae2d..e27abda29 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -100,7 +100,7 @@ services: - ${BRAPI_SERVER_PORT}:8080 volumes: - ./src/main/resources/brapi/properties/application.properties:/home/brapi/properties/application.properties - - ./src/main/resources/brapi/sql/:/home/brapi/db/sql/ + - ./src/main/resources/brapi/sql:/home/brapi/db/sql networks: backend: aliases: From 5b48dd217404066eb6caf0d68b429f9abc4cf631 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 29 Nov 2023 15:43:37 +0000 Subject: [PATCH 062/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index d557e1cf9..3a4f44851 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+634 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/bff69347c16f64500e2e7460b80872c76077abaa +version=v0.9.0+636 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/ab28861b880edeaa1e9ee9f788a1c23b8e3bcf3d From a57cc2808b1230a6615a2d8deee401586840c41e Mon Sep 17 00:00:00 2001 From: timparsons Date: Mon, 23 Oct 2023 11:39:58 -0400 Subject: [PATCH 063/220] [BI-1910] Converting BrAPIImportService to an interface --- .../model/imports/BrAPIImportService.java | 20 +++++++++---------- .../ExperimentImportService.java | 2 +- .../germplasm/GermplasmImportService.java | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java index eb4832b67..2bf412955 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java @@ -30,24 +30,24 @@ import java.util.List; -public abstract class BrAPIImportService { - public String getImportTypeId() {return null;} - public BrAPIImport getImportClass() {return null;} - public String getInvalidIntegerMsg(String columnName) { +public interface BrAPIImportService { + String getImportTypeId(); + BrAPIImport getImportClass(); + default String getInvalidIntegerMsg(String columnName) { return String.format("Column name \"%s\" must be integer type, but non-integer type provided.", columnName); } - public String getBlankRequiredFieldMsg(String fieldName) { + default String getBlankRequiredFieldMsg(String fieldName) { return String.format("Required field \"%s\" cannot contain empty values", fieldName); } - public String getMissingColumnMsg(String columnName) { + default String getMissingColumnMsg(String columnName) { return String.format("Column name \"%s\" does not exist in file", columnName); } - public String getMissingUserInputMsg(String fieldName) { + default String getMissingUserInputMsg(String fieldName) { return String.format("User input, \"%s\" is required", fieldName); } - public String getWrongUserInputDataTypeMsg(String fieldName, String typeName) { + default String getWrongUserInputDataTypeMsg(String fieldName, String typeName) { return String.format("User input, \"%s\" must be an %s", fieldName, typeName); } - public ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) - throws UnprocessableEntityException, DoesNotExistException, ValidatorException, ApiException, MissingRequiredInfoException {return null;} + ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) + throws UnprocessableEntityException, DoesNotExistException, ValidatorException, ApiException, MissingRequiredInfoException; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index 4978ed9b4..aa425a2da 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -39,7 +39,7 @@ @Singleton @Slf4j -public class ExperimentImportService extends BrAPIImportService { +public class ExperimentImportService implements BrAPIImportService { private final String IMPORT_TYPE_ID = "ExperimentImport"; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java index cadd379a0..e4084eae4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java @@ -39,7 +39,7 @@ @Singleton @Slf4j -public class GermplasmImportService extends BrAPIImportService { +public class GermplasmImportService implements BrAPIImportService { private final String IMPORT_TYPE_ID = "GermplasmImport"; From 9286d6025d63c289550471234b8f521bd32f47ff Mon Sep 17 00:00:00 2001 From: timparsons Date: Mon, 23 Oct 2023 11:40:38 -0400 Subject: [PATCH 064/220] [BI-1910] Adding support for XLSB file type --- .../brapps/importer/services/FileImportService.java | 3 ++- .../java/org/breedinginsight/services/TraitUploadService.java | 3 ++- .../breedinginsight/services/constants/SupportedMediaType.java | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 74051b813..0d8f7d971 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -156,7 +156,8 @@ private Table parseUploadedFile(CompletedFileUpload file) throws UnsupportedType throw new HttpStatusException(HttpStatus.BAD_REQUEST, "Error parsing csv: " + e.getMessage()); } } else if (mediaType.toString().equals(SupportedMediaType.XLS) || - mediaType.toString().equals(SupportedMediaType.XLSX)) { + mediaType.toString().equals(SupportedMediaType.XLSX) || + mediaType.toString().equals(SupportedMediaType.XLSB)) { try { //TODO: Allow them to pass in header row index in the future diff --git a/src/main/java/org/breedinginsight/services/TraitUploadService.java b/src/main/java/org/breedinginsight/services/TraitUploadService.java index 4534c095f..2caf03bcc 100644 --- a/src/main/java/org/breedinginsight/services/TraitUploadService.java +++ b/src/main/java/org/breedinginsight/services/TraitUploadService.java @@ -111,7 +111,8 @@ public ProgramUpload updateTraitUpload(UUID programId, CompletedFileUpload file, throw new HttpStatusException(HttpStatus.BAD_REQUEST, "Error parsing csv: " + e.getMessage()); } } else if (mediaType.toString().equals(SupportedMediaType.XLS) || - mediaType.toString().equals(SupportedMediaType.XLSX)) { + mediaType.toString().equals(SupportedMediaType.XLSX) || + mediaType.toString().equals(SupportedMediaType.XLSB)) { try { traits = parser.parseExcel(new BOMInputStream(file.getInputStream(), false)); } catch(IOException | ParsingException e) { diff --git a/src/main/java/org/breedinginsight/services/constants/SupportedMediaType.java b/src/main/java/org/breedinginsight/services/constants/SupportedMediaType.java index 34de108c0..4f42bacd1 100644 --- a/src/main/java/org/breedinginsight/services/constants/SupportedMediaType.java +++ b/src/main/java/org/breedinginsight/services/constants/SupportedMediaType.java @@ -22,4 +22,6 @@ public class SupportedMediaType { public static final String XLS = "application/vnd.ms-excel"; public static final String XLSX = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + public static final String XLSB = "application/vnd.ms-excel.sheet.binary.macroenabled.12"; + } From 0518d43481d8ba913a4d0cc0e6ae919edc2a9040 Mon Sep 17 00:00:00 2001 From: timparsons Date: Mon, 23 Oct 2023 11:41:38 -0400 Subject: [PATCH 065/220] [BI-1910] Expanding return type of the post function to not have to match the object type being saved --- .../org/breedinginsight/utilities/BrAPIDAOUtil.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index e0c470c9b..e03a67017 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -281,12 +281,12 @@ public List put(List brapiObjects, throw new UnsupportedOperationException(); } - public List post(List brapiObjects, - ImportUpload upload, - Function, ApiResponse> postMethod, - Consumer progressUpdateMethod) throws ApiException { + public List post(List brapiObjects, + ImportUpload upload, + Function, ApiResponse> postMethod, + Consumer progressUpdateMethod) throws ApiException { - List listResult = new ArrayList<>(); + List listResult = new ArrayList<>(); try { // Make the POST calls in chunks so we don't overload the brapi server Integer currentRightBorder = 0; @@ -314,7 +314,7 @@ public List post(List brapiObjects, if (result.getData() == null) { throw new ApiException("Response result is missing data", response.getStatusCode(), response.getHeaders(), response.getBody().toString()); } - List data = result.getData(); + List data = result.getData(); // TODO: Maybe move this outside of the loop if (data.size() != postChunk.size()) { throw new ApiException("Number of brapi objects returned does not equal number sent"); From 3863dcce8bca4795fdc69e794f912bdc2a9d863e Mon Sep 17 00:00:00 2001 From: timparsons Date: Mon, 23 Oct 2023 11:42:17 -0400 Subject: [PATCH 066/220] [BI-1910] upgrading to the latest brapi client --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b12c7b648..7cc635a9f 100644 --- a/pom.xml +++ b/pom.xml @@ -85,7 +85,7 @@ 31.0.1-jre 4.9.3 4.3.1 - 2.1-SNAPSHOT + 2.1.0 2.11.0 2.2.1 From 7fbbacd2cfd3d174f44396d22f7b73ee1741c7c6 Mon Sep 17 00:00:00 2001 From: timparsons Date: Mon, 23 Oct 2023 11:43:10 -0400 Subject: [PATCH 067/220] [BI-1910] Implementing support for importing sample submissions --- localstack/startup_info.json | 1 + .../api/model/PendingBrAPIImport.java | 4 + .../constants/BrAPIAdditionalInfoFields.java | 6 + .../brapps/importer/daos/BrAPIPlateDAO.java | 98 ++++++ .../brapps/importer/daos/BrAPISampleDAO.java | 105 +++++++ .../importer/model/imports/PendingImport.java | 5 +- .../sample/SampleSubmissionImport.java | 194 ++++++++++++ .../sample/SampleSubmissionImportService.java | 76 +++++ .../services/ExternalReferenceSource.java | 6 +- .../processors/SampleSubmissionProcessor.java | 281 ++++++++++++++++++ .../org/breedinginsight/daos/ProgramDAO.java | 2 + .../daos/SampleSubmissionDAO.java | 58 ++++ .../daos/impl/ProgramDAOImpl.java | 9 +- .../model/SampleSubmission.java | 60 ++++ .../services/SampleSubmissionService.java | 115 +++++++ .../resources/application-dev.yml.template | 10 + .../V1.16.0__create_sampleimport_mapping.sql | 108 +++++++ ...1.17.0__create_sample_submission_table.sql | 36 +++ 18 files changed, 1171 insertions(+), 3 deletions(-) create mode 100755 localstack/startup_info.json create mode 100644 src/main/java/org/breedinginsight/api/model/PendingBrAPIImport.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIPlateDAO.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISampleDAO.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java create mode 100644 src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java create mode 100644 src/main/java/org/breedinginsight/daos/SampleSubmissionDAO.java create mode 100644 src/main/java/org/breedinginsight/model/SampleSubmission.java create mode 100644 src/main/java/org/breedinginsight/services/SampleSubmissionService.java create mode 100644 src/main/resources/application-dev.yml.template create mode 100644 src/main/resources/db/migration/V1.16.0__create_sampleimport_mapping.sql create mode 100644 src/main/resources/db/migration/V1.17.0__create_sample_submission_table.sql diff --git a/localstack/startup_info.json b/localstack/startup_info.json new file mode 100755 index 000000000..a38d9d96e --- /dev/null +++ b/localstack/startup_info.json @@ -0,0 +1 @@ +[{"timestamp": "2022-11-17T17:01:36.401503", "localstack_version": "1.2.0", "localstack_ext_version": "1.2.0", "pro_activated": false}] \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/api/model/PendingBrAPIImport.java b/src/main/java/org/breedinginsight/api/model/PendingBrAPIImport.java new file mode 100644 index 000000000..6fc8bd194 --- /dev/null +++ b/src/main/java/org/breedinginsight/api/model/PendingBrAPIImport.java @@ -0,0 +1,4 @@ +package org.breedinginsight.api.model; + +public class PendingBrAPIImport { +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index d9daab6d0..821c39a98 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -48,4 +48,10 @@ public final class BrAPIAdditionalInfoFields { public static final String TREATMENTS = "treatments"; public static final String GID = "gid"; public static final String ENV_YEAR = "envYear"; + public static final String GERMPLASM_UUID = "germplasmId"; + public static final String SAMPLE_ORGANISM = "organism"; + public static final String SAMPLE_SPECIES = "species"; + public static final String OBS_UNIT_ID = "obsUnitID"; + public static final String GERMPLASM_NAME = "germplasmName"; + public static final String SUBMISSION_NAME = "submissionName"; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIPlateDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIPlateDAO.java new file mode 100644 index 000000000..d4bd47490 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIPlateDAO.java @@ -0,0 +1,98 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.daos; + +import io.micronaut.context.annotation.Property; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.modules.genotype.PlatesApi; +import org.brapi.v2.model.geno.BrAPIPlate; +import org.brapi.v2.model.geno.request.BrAPIPlateNewRequest; +import org.brapi.v2.model.geno.request.BrAPIPlateSearchRequest; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.List; +import java.util.stream.Collectors; + +@Slf4j +@Singleton +public class BrAPIPlateDAO { + + private final String referenceSource; + + private final ProgramDAO programDAO; + private final ImportDAO importDAO; + private final BrAPIDAOUtil brAPIDAOUtil; + private final BrAPIEndpointProvider brAPIEndpointProvider; + + @Inject + public BrAPIPlateDAO(ProgramDAO programDAO, + ImportDAO importDAO, + BrAPIDAOUtil brAPIDAOUtil, + BrAPIEndpointProvider brAPIEndpointProvider, + @Property(name = "brapi.server.reference-source") String referenceSource) { + this.referenceSource = referenceSource; + this.programDAO = programDAO; + this.importDAO = importDAO; + this.brAPIDAOUtil = brAPIDAOUtil; + this.brAPIEndpointProvider = brAPIEndpointProvider; + } + + public List createPlates(Program program, List platesToSave, ImportUpload upload) throws ApiException { + PlatesApi platesApi = brAPIEndpointProvider.get(programDAO.getSampleClient(program.getId()), PlatesApi.class); + List newPlatesRequests = platesToSave.stream() + .map(plate -> new BrAPIPlateNewRequest().additionalInfo(plate.getAdditionalInfo()) + .externalReferences(plate.getExternalReferences()) + .plateBarcode(plate.getPlateBarcode()) + .plateFormat(plate.getPlateFormat()) + .plateName(plate.getPlateName()) + .programDbId(plate.getProgramDbId()) + .sampleType(plate.getSampleType()) + .studyDbId(plate.getStudyDbId()) + .trialDbId(plate.getTrialDbId()) + ) + .collect(Collectors.toList()); + return brAPIDAOUtil.post(newPlatesRequests, upload, platesApi::platesPost, importDAO::update); + } + + public List readPlatesByIds(Program program, List plateExternalIds) throws ApiException { + PlatesApi platesApi = brAPIEndpointProvider.get(programDAO.getSampleClient(program.getId()), PlatesApi.class); + + BrAPIPlateSearchRequest request = new BrAPIPlateSearchRequest().externalReferenceIDs(plateExternalIds) + .externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PLATES))); + + return brAPIDAOUtil.search(platesApi::searchPlatesPost, platesApi::searchPlatesSearchResultsDbIdGet, request); + } + + public List readPlatesBySubmissionIds(Program program, List submissionExternalIds) throws ApiException { + PlatesApi platesApi = brAPIEndpointProvider.get(programDAO.getSampleClient(program.getId()), PlatesApi.class); + + BrAPIPlateSearchRequest request = new BrAPIPlateSearchRequest().externalReferenceIDs(submissionExternalIds) + .externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PLATE_SUBMISSIONS))); + + return brAPIDAOUtil.search(platesApi::searchPlatesPost, platesApi::searchPlatesSearchResultsDbIdGet, request); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISampleDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISampleDAO.java new file mode 100644 index 000000000..1cb9a531a --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISampleDAO.java @@ -0,0 +1,105 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.daos; + +import io.micronaut.context.annotation.Property; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.modules.genotype.SamplesApi; +import org.brapi.v2.model.geno.BrAPISample; +import org.brapi.v2.model.geno.request.BrAPISampleSearchRequest; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.Collections; +import java.util.List; +import java.util.UUID; + +@Slf4j +@Singleton +public class BrAPISampleDAO { + + private final String referenceSource; + + private final ProgramDAO programDAO; + private final ImportDAO importDAO; + private final BrAPIDAOUtil brAPIDAOUtil; + private final BrAPIEndpointProvider brAPIEndpointProvider; + + @Inject + public BrAPISampleDAO(ProgramDAO programDAO, + ImportDAO importDAO, + BrAPIDAOUtil brAPIDAOUtil, + BrAPIEndpointProvider brAPIEndpointProvider, + @Property(name = "brapi.server.reference-source") String referenceSource) { + this.referenceSource = referenceSource; + this.programDAO = programDAO; + this.importDAO = importDAO; + this.brAPIDAOUtil = brAPIDAOUtil; + this.brAPIEndpointProvider = brAPIEndpointProvider; + } + + public List createSamples(Program program, List samplesToSave, ImportUpload upload) throws ApiException { + SamplesApi samplesApi = brAPIEndpointProvider.get(programDAO.getSampleClient(program.getId()), SamplesApi.class); + + return brAPIDAOUtil.post(samplesToSave, upload, samplesApi::samplesPost, importDAO::update); + } + + public List readSamplesByIds(Program program, List sampleExternalIds) throws ApiException { + if(sampleExternalIds.isEmpty()) { + return Collections.emptyList(); + } + + BrAPISampleSearchRequest searchRequest = new BrAPISampleSearchRequest().externalReferenceIDs(sampleExternalIds) + .externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.SAMPLES))); + + SamplesApi samplesApi = brAPIEndpointProvider.get(programDAO.getSampleClient(program.getId()), SamplesApi.class); + return brAPIDAOUtil.search(samplesApi::searchSamplesPost, samplesApi::searchSamplesSearchResultsDbIdGet, searchRequest); + } + + public List readSamplesByPlateIds(Program program, List plateExternalIds) throws ApiException { + if(plateExternalIds.isEmpty()) { + return Collections.emptyList(); + } + + BrAPISampleSearchRequest searchRequest = new BrAPISampleSearchRequest().externalReferenceIDs(plateExternalIds) + .externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PLATES))); + + SamplesApi samplesApi = brAPIEndpointProvider.get(programDAO.getSampleClient(program.getId()), SamplesApi.class); + return brAPIDAOUtil.search(samplesApi::searchSamplesPost, samplesApi::searchSamplesSearchResultsDbIdGet, searchRequest); + } + + public List readSamplesBySubmissionIds(Program program, List submissionExternalIds) throws ApiException { + if(submissionExternalIds.isEmpty()) { + return Collections.emptyList(); + } + + BrAPISampleSearchRequest searchRequest = new BrAPISampleSearchRequest().externalReferenceIDs(submissionExternalIds) + .externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PLATE_SUBMISSIONS))); + + SamplesApi samplesApi = brAPIEndpointProvider.get(programDAO.getSampleClient(program.getId()), SamplesApi.class); + return brAPIDAOUtil.search(samplesApi::searchSamplesPost, samplesApi::searchSamplesSearchResultsDbIdGet, searchRequest); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java index f3d4f3180..e478fc385 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/PendingImport.java @@ -21,9 +21,10 @@ import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; -import org.brapi.v2.model.core.BrAPILocation; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.geno.BrAPIPlate; +import org.brapi.v2.model.geno.BrAPISample; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; @@ -44,6 +45,8 @@ public class PendingImport { private PendingImportObject study; private PendingImportObject observationUnit; private List> observations = new ArrayList<>(); + private PendingImportObject plate; + private PendingImportObject sample; @JsonIgnore public PendingImportObject getObservation() { diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java new file mode 100644 index 000000000..bf251f5b7 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java @@ -0,0 +1,194 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.imports.sample; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.geno.BrAPIPlate; +import org.brapi.v2.model.geno.BrAPIPlateFormat; +import org.brapi.v2.model.geno.BrAPISample; +import org.brapi.v2.model.geno.BrAPISampleTypeEnum; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.model.config.*; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import org.breedinginsight.utilities.Utilities; + +import java.util.*; + +@Getter +@Setter +@NoArgsConstructor +@ImportConfigMetadata(id = "SampleImport", name = "Sample Import", + description = "This import is used to create Genotype Samples") +public class SampleSubmissionImport implements BrAPIImport { + + private static final String SAMPLE_NAME_FORMAT = "%s__%s_%s%s"; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "plateId", name = Columns.PLATE_ID, description = "The ID which uniquely identifies this plate to the client making the request") + private String plateId; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "row", name = Columns.ROW, description = "The Row identifier for this samples location in the plate, ex: B") + private String row; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "column", name = Columns.COLUMN, description = "The Column identifier for this samples location in the plate, ex: 6") + private String column; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "organism", name = Columns.ORGANISM, description = "Scientific organism name") + private String organism; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "species", name = Columns.SPECIES, description = "Scientific species name") + private String species; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "germplasmName", name = Columns.GERMPLASM_NAME, description = "Name of germplasm") + private String germplasmName; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "gid", name = Columns.GERMPLASM_GID, description = "Unique germplasm identifier") + private String gid; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "obsUnitId", name = Columns.OBS_UNIT_ID, description = "The Observation Unit that this sample was collected from") + private String obsUnitId; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "tissue", name = Columns.TISSUE, description = "The type of tissue in this sample. List of accepted tissue types can be found in the Vendor Specs.") + private String tissue; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT) + @ImportFieldMetadata(id = "comment", name = Columns.COMMENT, description = "Generic comments about this sample for the vendor") + private String comment; + + @ImportFieldType(type= ImportFieldTypeEnum.TEXT, collectTime = ImportCollectTimeEnum.UPLOAD) + @ImportMappingRequired + @ImportFieldMetadata(id="submissionName", name="Submission Name", description = "Name of the submission imported.") + private String submissionName; + + public static final class Columns { + public static final String PLATE_ID = "PlateID"; + public static final String ROW = "Row"; + public static final String COLUMN = "Column"; + public static final String ORGANISM = "Organism"; + public static final String SPECIES = "Species"; + public static final String GERMPLASM_NAME = "Germplasm Name"; + public static final String GERMPLASM_GID = "Germplasm GID"; + public static final String TISSUE = "Tissue"; + public static final String COMMENT = "Comment"; + public static final String OBS_UNIT_ID = "ObsUnitID"; + } + + public BrAPISample constructBrAPISample(boolean commit, Program program, User user, BrAPIPlate plate, String referenceSource, String submissionId, BrAPIGermplasm germplasm, BrAPIObservationUnit ou) { + List xrefs = new ArrayList<>(); + xrefs.add(new BrAPIExternalReference().referenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS)) + .referenceId(program.getId() + .toString())); + String germXrefSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.GERMPLASM); + xrefs.add(germplasm.getExternalReferences() + .stream() + .filter(xref -> xref.getReferenceSource().equals(referenceSource) || xref.getReferenceSource().equals(germXrefSource)) + .findFirst() + .orElseThrow(() -> new IllegalStateException(String.format("Germplasm %s doesn't have an xref! -> %s", germplasm.getAccessionNumber(), germplasm.toString()))).referenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.GERMPLASM))); + if(commit) { + xrefs.add(new BrAPIExternalReference().referenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.SAMPLES)) + .referenceId(UUID.randomUUID().toString())); + xrefs.add(Utilities.getExternalReference(plate.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PLATES)) + .get()); + } + if (submissionId != null) { + xrefs.add(new BrAPIExternalReference().referenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PLATE_SUBMISSIONS)) + .referenceId(submissionId)); + } + + Map createdBy = new HashMap<>(); + createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_ID, + user.getId().toString()); + createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_NAME, user.getName()); + + BrAPISample brAPISample = new BrAPISample() + .putAdditionalInfoItem(BrAPIAdditionalInfoFields.CREATED_BY, createdBy) + .putAdditionalInfoItem(BrAPIAdditionalInfoFields.SAMPLE_SPECIES, species) + .putAdditionalInfoItem(BrAPIAdditionalInfoFields.SAMPLE_ORGANISM, organism) + .putAdditionalInfoItem(BrAPIAdditionalInfoFields.GID, germplasm.getAccessionNumber()) + .putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_NAME, germplasm.getDefaultDisplayName()) + .externalReferences(xrefs) + .plateName(plateId) + .plateDbId(plate.getPlateDbId()) + .sampleName(String.format(SAMPLE_NAME_FORMAT, germplasm.getDefaultDisplayName(), plate.getPlateName(), row, column)) + .germplasmDbId(germplasm.getGermplasmDbId()) + .row(row) + .column(Integer.valueOf(column)) + .tissueType(tissue) + .sampleDescription(comment); + + if (ou != null) { + brAPISample + .putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBS_UNIT_ID, + Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS)) + .get() + .getReferenceId()) + .observationUnitDbId(ou.getObservationUnitDbId()); + } + + return brAPISample; + } + + public BrAPIPlate constructBrAPIPlate(boolean commit, Program program, User user, String referenceSource, String submissionId) { + List xrefs = new ArrayList<>(); + xrefs.add(new BrAPIExternalReference().referenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS)) + .referenceId(program.getId() + .toString())); + + BrAPIPlate brAPIPlate = new BrAPIPlate(); + if(commit) { + xrefs.add(new BrAPIExternalReference().referenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PLATES)) + .referenceId(UUID.randomUUID().toString())); + } + if (submissionId != null) { + xrefs.add(new BrAPIExternalReference().referenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PLATE_SUBMISSIONS)) + .referenceId(submissionId)); + brAPIPlate.putAdditionalInfoItem(BrAPIAdditionalInfoFields.SUBMISSION_NAME, submissionName); + } + Map createdBy = new HashMap<>(); + createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_ID, + user.getId() + .toString()); + createdBy.put(BrAPIAdditionalInfoFields.CREATED_BY_USER_NAME, user.getName()); + + + + return brAPIPlate + .externalReferences(xrefs) + .putAdditionalInfoItem(BrAPIAdditionalInfoFields.CREATED_BY, createdBy) + .plateName(plateId) + .sampleType(BrAPISampleTypeEnum.fromValue(tissue.toUpperCase())) + .plateFormat(BrAPIPlateFormat.PLATE_96); + + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java new file mode 100644 index 000000000..637fa7ce3 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java @@ -0,0 +1,76 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.imports.sample; + +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; +import org.breedinginsight.brapps.importer.services.processors.Processor; +import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; +import org.breedinginsight.brapps.importer.services.processors.SampleSubmissionProcessor; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.User; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.services.exceptions.ValidatorException; +import tech.tablesaw.api.Table; + +import javax.inject.Inject; +import javax.inject.Provider; +import javax.inject.Singleton; +import java.util.List; + +@Singleton +@Slf4j +public class SampleSubmissionImportService implements BrAPIImportService { + private final String IMPORT_TYPE_ID = "SampleImport"; + private final Provider sampleProcessorProvider; + private final Provider processorManagerProvider; + + @Inject + public SampleSubmissionImportService(Provider sampleProcessorProvider, Provider processorManagerProvider) + { + this.sampleProcessorProvider = sampleProcessorProvider; + this.processorManagerProvider = processorManagerProvider; + } + + @Override + public String getImportTypeId() { + return IMPORT_TYPE_ID; + } + + @Override + public BrAPIImport getImportClass() { + return new SampleSubmissionImport(); + } + + @Override + public ImportPreviewResponse process(List brAPIImports, + Table data, + Program program, + ImportUpload upload, + User user, + Boolean commit) throws UnprocessableEntityException, DoesNotExistException, ValidatorException, ApiException, MissingRequiredInfoException { + List processors = List.of(sampleProcessorProvider.get()); + return processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); + } +} diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/ExternalReferenceSource.java b/src/main/java/org/breedinginsight/brapps/importer/services/ExternalReferenceSource.java index 78e051e08..45553e189 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/ExternalReferenceSource.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/ExternalReferenceSource.java @@ -10,7 +10,11 @@ public enum ExternalReferenceSource { OBSERVATION_UNITS("observationunits"), DATASET("dataset"), LISTS("lists"), - OBSERVATIONS("observations"); + OBSERVATIONS("observations"), + GERMPLASM("germplasm"), + PLATES("plates"), + SAMPLES("samples"), + PLATE_SUBMISSIONS("plates/submissions"); private String name; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java new file mode 100644 index 000000000..d3d4c1b0e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java @@ -0,0 +1,281 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.services.processors; + +import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Prototype; +import io.micronaut.http.HttpStatus; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.geno.BrAPIPlate; +import org.brapi.v2.model.geno.BrAPISample; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.model.v1.response.ValidationError; +import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.daos.BrAPIPlateDAO; +import org.breedinginsight.brapps.importer.daos.BrAPISampleDAO; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.PendingImport; +import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport; +import org.breedinginsight.brapps.importer.model.response.ImportObjectState; +import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; +import org.breedinginsight.brapps.importer.model.response.PendingImportObject; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.dao.db.tables.daos.SampleSubmissionDao; +import org.breedinginsight.dao.db.tables.pojos.SampleSubmissionEntity; +import org.breedinginsight.daos.SampleSubmissionDAO; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.SampleSubmission; +import org.breedinginsight.model.User; +import org.breedinginsight.services.SampleSubmissionService; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.utilities.Utilities; +import org.jooq.DSLContext; +import tech.tablesaw.api.Table; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +@Prototype +public class SampleSubmissionProcessor implements Processor { + + private static final String MISSING_REQUIRED_DATA = "Missing required data"; + private final String referenceSource; + private final BrAPIGermplasmDAO germplasmDAO; + private final BrAPIObservationUnitDAO observationUnitDAO; + private final SampleSubmissionService sampleSubmissionService; + private final DSLContext dsl; + private final BrAPIPlateDAO plateDAO; + private final BrAPISampleDAO sampleDAO; + + private SampleSubmission submission; + private Map germplasmByGID = new HashMap<>(); + private Map germplasmByDbId = new HashMap<>(); + private Map observationUnitsById = new HashMap<>(); + private Map> plateById = new HashMap<>(); + private Map plateLayouts = new HashMap<>(); + + @Inject + public SampleSubmissionProcessor(@Property(name = "brapi.server.reference-source") String referenceSource, + BrAPIGermplasmDAO germplasmDAO, + BrAPIObservationUnitDAO observationUnitDAO, + SampleSubmissionService sampleSubmissionService, + DSLContext dsl, + BrAPIPlateDAO plateDAO, + BrAPISampleDAO sampleDAO) { + this.referenceSource = referenceSource; + this.germplasmDAO = germplasmDAO; + this.observationUnitDAO = observationUnitDAO; + this.sampleSubmissionService = sampleSubmissionService; + this.dsl = dsl; + this.plateDAO = plateDAO; + this.sampleDAO = sampleDAO; + } + + @Override + public void getExistingBrapiData(List importRows, Program program) throws ValidatorException, ApiException { + Set gids = importRows.stream() + .filter((row -> StringUtils.isNotBlank(((SampleSubmissionImport) row).getGid()))) + .map(row -> ((SampleSubmissionImport) row).getGid()) + .collect(Collectors.toSet()); + + List germplasm = germplasmDAO.getGermplasm(program.getId()); + + List obsUnitIds = importRows.stream() + .filter((row -> StringUtils.isNotBlank(((SampleSubmissionImport) row).getObsUnitId()))) + .map(row -> ((SampleSubmissionImport) row).getObsUnitId()) + .distinct() + .collect(Collectors.toList()); + + List observationUnits = observationUnitDAO.getObservationUnitsById(obsUnitIds, program); + Set germDbIds = new HashSet<>(); + String ouRefSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS); + observationUnits.forEach(ou -> { + observationUnitsById.put(Utilities.getExternalReference(ou.getExternalReferences(), ouRefSource) + .get() + .getReferenceId(), ou); + germDbIds.add(ou.getGermplasmDbId()); + }); + germplasm.stream() + .filter(germ -> gids.contains(germ.getAccessionNumber()) || germDbIds.contains(germ.getGermplasmDbId())) + .forEach(germ -> { + germplasmByGID.put(germ.getAccessionNumber(), germ); + germplasmByDbId.put(germ.getGermplasmDbId(), germ); + }); + } + + @Override + public Map process(ImportUpload upload, + List importRows, + Map mappedBrAPIImport, + Table data, + Program program, + User user, + boolean commit) throws ValidatorException, MissingRequiredInfoException, ApiException { + ValidationErrors validationErrors = new ValidationErrors(); + + String submissionId = null; + if (commit) { + submissionId = UUID.randomUUID() + .toString(); + SampleSubmissionImport row = (SampleSubmissionImport) importRows.get(0); + submission = SampleSubmission.builder() + .id(UUID.fromString(submissionId)) + .name(row.getSubmissionName()) + .createdBy(user.getId()) + .build(); + } + + for (int i = 0; i < importRows.size(); i++) { + int rowNum = i + 2; //accounts for column header and 0-index of i + SampleSubmissionImport row = (SampleSubmissionImport) importRows.get(i); + + if (validRow(row, rowNum, validationErrors)) { + PendingImport pendingImport = new PendingImport(); + PendingImportObject plate = plateById.getOrDefault(row.getPlateId(), new PendingImportObject<>(ImportObjectState.NEW, row.constructBrAPIPlate(commit, program, user, referenceSource, submissionId))); + pendingImport.setPlate(plate); + plateById.putIfAbsent(plate.getBrAPIObject() + .getPlateName(), plate); + + BrAPIGermplasm germ; + if (StringUtils.isNotBlank(row.getObsUnitId())) { + germ = germplasmByDbId.get(observationUnitsById.get(row.getObsUnitId()) + .getGermplasmDbId()); + } else { + germ = germplasmByGID.get(row.getGid()); + } + pendingImport.setSample(new PendingImportObject<>(ImportObjectState.NEW, + row.constructBrAPISample(commit, program, user, plate.getBrAPIObject(), referenceSource, submissionId, germ, observationUnitsById.get(row.getObsUnitId())))); + mappedBrAPIImport.put(rowNum, pendingImport); + } + } + + if (validationErrors.hasErrors()) { + throw new ValidatorException(validationErrors); + } + + return Map.of("Plates", + ImportPreviewStatistics.builder() + .newObjectCount(plateById.size()) + .build(), + "Samples", + ImportPreviewStatistics.builder() + .newObjectCount(importRows.size()) + .build()); + } + + private boolean validRow(SampleSubmissionImport row, int rowNum, ValidationErrors validationErrors) { + int numErrorsBefore = validationErrors.getRowErrors() + .size(); + if (StringUtils.isBlank(row.getPlateId())) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.PLATE_ID, MISSING_REQUIRED_DATA, HttpStatus.UNPROCESSABLE_ENTITY)); + } + int plateRow = -1; + if (StringUtils.isBlank(row.getRow())) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.ROW, MISSING_REQUIRED_DATA, HttpStatus.UNPROCESSABLE_ENTITY)); + } else if (row.getRow() + .length() > 1 || row.getRow() + .toUpperCase() + .charAt(0) - 'A' < 0 || row.getRow() + .toUpperCase() + .charAt(0) - 'H' > 0) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.ROW, "Row must be a a letter between A and H", HttpStatus.UNPROCESSABLE_ENTITY)); + } else { + plateRow = row.getRow() + .toUpperCase() + .charAt(0) - 'A'; + } + int plateCol = -1; + if (StringUtils.isBlank(row.getColumn())) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.COLUMN, MISSING_REQUIRED_DATA, HttpStatus.UNPROCESSABLE_ENTITY)); + } else { + try { + plateCol = Integer.parseInt(row.getColumn()); + if (plateCol < 1 || plateCol > 12) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.COLUMN, "Column must be a number between 1 and 12", HttpStatus.UNPROCESSABLE_ENTITY)); + plateCol = -1; + } + } catch (NumberFormatException e) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.COLUMN, "Column must be a number between 1 and 12", HttpStatus.UNPROCESSABLE_ENTITY)); + } + } + if (StringUtils.isBlank(row.getOrganism())) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.ORGANISM, MISSING_REQUIRED_DATA, HttpStatus.UNPROCESSABLE_ENTITY)); + } + if (StringUtils.isBlank(row.getSpecies())) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.SPECIES, MISSING_REQUIRED_DATA, HttpStatus.UNPROCESSABLE_ENTITY)); + } + if (StringUtils.isBlank(row.getGid()) && StringUtils.isBlank(row.getObsUnitId())) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.GERMPLASM_GID, "One of GID or ObsUnitID is required", HttpStatus.UNPROCESSABLE_ENTITY)); + } else if (StringUtils.isNotBlank(row.getObsUnitId()) && !observationUnitsById.containsKey(row.getObsUnitId())) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.OBS_UNIT_ID, "Unknown ObsUnitID", HttpStatus.UNPROCESSABLE_ENTITY)); + } else if (StringUtils.isNotBlank(row.getGid()) && !germplasmByGID.containsKey(row.getGid())) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.GERMPLASM_GID, "Unknown germplasm GID", HttpStatus.UNPROCESSABLE_ENTITY)); + } + if (StringUtils.isBlank(row.getTissue())) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.TISSUE, MISSING_REQUIRED_DATA, HttpStatus.UNPROCESSABLE_ENTITY)); + } + + if (plateRow > -1 && plateCol > -1) { + int[][] plateLayout = plateLayouts.getOrDefault(row.getPlateId(), new int[9][13]); + if (plateLayout[plateRow][plateCol] > 0) { + validationErrors.addError(rowNum, + new ValidationError(SampleSubmissionImport.Columns.ROW + "/" + SampleSubmissionImport.Columns.COLUMN, + String.format("The sample in row %d is already in row: %s, column: %d", plateLayout[plateRow][plateCol], Character.toString('A' + plateRow), plateCol), + HttpStatus.UNPROCESSABLE_ENTITY)); + } else { + plateLayout[plateRow][plateCol] = rowNum; + plateLayouts.put(row.getPlateId(), plateLayout); + } + } + + return numErrorsBefore == validationErrors.getRowErrors() + .size(); + } + + @Override + public void validateDependencies(Map mappedBrAPIImport) throws ValidatorException { + + } + + @Override + public void postBrapiData(Map mappedBrAPIImport, Program program, ImportUpload upload) throws ValidatorException { + dsl.transaction(() -> { + List platesToSave = plateById.values().stream().map(PendingImportObject::getBrAPIObject).collect(Collectors.toList()); + List samplesToSave = mappedBrAPIImport.values().stream().map(row -> row.getSample().getBrAPIObject()).collect(Collectors.toList()); + + submission.setPlates(platesToSave); + submission.setSamples(samplesToSave); + + sampleSubmissionService.createSubmission(submission, program, upload); + }); + } + + @Override + public String getName() { + return "SampleSubmission"; + } +} diff --git a/src/main/java/org/breedinginsight/daos/ProgramDAO.java b/src/main/java/org/breedinginsight/daos/ProgramDAO.java index 3d3600da7..e60481f35 100644 --- a/src/main/java/org/breedinginsight/daos/ProgramDAO.java +++ b/src/main/java/org/breedinginsight/daos/ProgramDAO.java @@ -62,4 +62,6 @@ public interface ProgramDAO extends DAO { BrAPIClient getCoreClient(UUID programId); BrAPIClient getPhenoClient(UUID programId); + + BrAPIClient getSampleClient(UUID programId); } diff --git a/src/main/java/org/breedinginsight/daos/SampleSubmissionDAO.java b/src/main/java/org/breedinginsight/daos/SampleSubmissionDAO.java new file mode 100644 index 000000000..c8449ff5d --- /dev/null +++ b/src/main/java/org/breedinginsight/daos/SampleSubmissionDAO.java @@ -0,0 +1,58 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.daos; + +import org.breedinginsight.dao.db.Tables; +import org.breedinginsight.dao.db.tables.daos.SampleSubmissionDao; +import org.breedinginsight.dao.db.tables.pojos.SampleSubmissionEntity; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.SampleSubmission; +import org.jooq.Configuration; +import org.jooq.DSLContext; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.breedinginsight.dao.db.Tables.SAMPLE_SUBMISSION; + +@Singleton +public class SampleSubmissionDAO extends SampleSubmissionDao { + + private DSLContext dsl; + + @Inject + public SampleSubmissionDAO(Configuration config, DSLContext dsl) { + super(config); + this.dsl = dsl; + } + + public List getBySubmissionId(Program program, UUID submissionId) { + return dsl.select() + .from(SAMPLE_SUBMISSION) + .where(SAMPLE_SUBMISSION.PROGRAM_ID.eq(program.getId())) + .and(SAMPLE_SUBMISSION.ID.eq(submissionId)) + .fetchInto(SampleSubmissionEntity.class) + .stream() + .map(SampleSubmission::new) + .collect(Collectors.toList()); + } + +} diff --git a/src/main/java/org/breedinginsight/daos/impl/ProgramDAOImpl.java b/src/main/java/org/breedinginsight/daos/impl/ProgramDAOImpl.java index 5818ab281..21231a05d 100644 --- a/src/main/java/org/breedinginsight/daos/impl/ProgramDAOImpl.java +++ b/src/main/java/org/breedinginsight/daos/impl/ProgramDAOImpl.java @@ -36,7 +36,6 @@ import org.breedinginsight.dao.db.tables.BiUserTable; import org.breedinginsight.dao.db.tables.daos.ProgramDao; import org.breedinginsight.dao.db.tables.pojos.ProgramEntity; -import org.breedinginsight.dao.db.tables.records.ProgramRecord; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.model.User; import org.breedinginsight.model.*; @@ -394,6 +393,14 @@ public BrAPIClient getPhenoClient(UUID programId) { return client; } + @Override + public BrAPIClient getSampleClient(UUID programId) { + String brapiUrl = defaultBrAPIPhenoUrl; + BrAPIClient client = new BrAPIClient(brapiUrl); + initializeHttpClient(client); + return client; + } + private void initializeHttpClient(BrAPIClient brapiClient) { brapiClient.setHttpClient(brapiClient.getHttpClient() .newBuilder() diff --git a/src/main/java/org/breedinginsight/model/SampleSubmission.java b/src/main/java/org/breedinginsight/model/SampleSubmission.java new file mode 100644 index 000000000..99d2030cc --- /dev/null +++ b/src/main/java/org/breedinginsight/model/SampleSubmission.java @@ -0,0 +1,60 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import io.micronaut.core.annotation.Introspected; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.experimental.Accessors; +import lombok.experimental.SuperBuilder; +import lombok.extern.jackson.Jacksonized; +import org.brapi.v2.model.geno.BrAPIPlate; +import org.brapi.v2.model.geno.BrAPISample; +import org.breedinginsight.dao.db.tables.pojos.SampleSubmissionEntity; + +import java.util.List; + +@Getter +@Setter +@Accessors(chain=true) +@ToString +@SuperBuilder +@NoArgsConstructor +@Introspected +@Jacksonized +//@JsonIgnoreProperties(value = {"createdBy", "updatedBy"}) +public class SampleSubmission extends SampleSubmissionEntity { + private List plates; + private List samples; + + public SampleSubmission(SampleSubmissionEntity entity) { + this.setId(entity.getId()); + this.setName(entity.getName()); + this.setSubmitted(entity.getSubmitted()); + this.setVendorOrderId(entity.getVendorOrderId()); + this.setShipmentforms(entity.getShipmentforms()); + this.setProgramId(entity.getProgramId()); + this.setCreatedAt(entity.getCreatedAt()); + this.setCreatedBy(entity.getCreatedBy()); + this.setUpdatedAt(entity.getUpdatedAt()); + this.setUpdatedBy(entity.getUpdatedBy()); + } +} diff --git a/src/main/java/org/breedinginsight/services/SampleSubmissionService.java b/src/main/java/org/breedinginsight/services/SampleSubmissionService.java new file mode 100644 index 000000000..3509e210c --- /dev/null +++ b/src/main/java/org/breedinginsight/services/SampleSubmissionService.java @@ -0,0 +1,115 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.services; + +import io.micronaut.context.annotation.Property; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.geno.BrAPIPlate; +import org.brapi.v2.model.geno.BrAPISample; +import org.breedinginsight.brapps.importer.daos.BrAPIPlateDAO; +import org.breedinginsight.brapps.importer.daos.BrAPISampleDAO; +import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.daos.SampleSubmissionDAO; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.SampleSubmission; +import org.breedinginsight.utilities.Utilities; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +@Singleton +public class SampleSubmissionService { + + private final String referenceSource; + + private final SampleSubmissionDAO submissionDAO; + private final BrAPIPlateDAO plateDAO; + private final BrAPISampleDAO sampleDAO; + + @Inject + public SampleSubmissionService(@Property(name = "brapi.server.reference-source") String referenceSource, SampleSubmissionDAO submissionDAO, BrAPIPlateDAO plateDAO, BrAPISampleDAO sampleDAO) { + this.referenceSource = referenceSource; + this.submissionDAO = submissionDAO; + this.plateDAO = plateDAO; + this.sampleDAO = sampleDAO; + } + + public SampleSubmission createSubmission(SampleSubmission submission, Program program, ImportUpload upload) throws ApiException { + submission.setProgramId(program.getId()); + submission.setCreatedBy(upload.getCreatedBy()); + submission.setUpdatedBy(upload.getUpdatedBy()); + submissionDAO.insert(submission); + + List savedPlates = plateDAO.createPlates(program, submission.getPlates(), upload); + submission.setPlates(savedPlates); + Map plateNameToDbId = savedPlates.stream().collect(Collectors.toMap(BrAPIPlate::getPlateName, BrAPIPlate::getPlateDbId)); + + List samplesToSave = submission.getSamples().stream().map(sample -> sample.plateDbId(plateNameToDbId.get(sample.getPlateName()))).collect(Collectors.toList()); + List savedSamples = sampleDAO.createSamples(program, samplesToSave, upload); + submission.setSamples(savedSamples); + + return submission; + } + + public Optional getSampleSubmission(Program program, UUID submissionId) throws ApiException { + return populateSubmissions(program, submissionDAO.getBySubmissionId(program, submissionId)).stream().findFirst(); + } + + public List getProgramSubmissions(Program program) { + return submissionDAO.fetchByProgramId(program.getId()) + .stream() + .map(SampleSubmission::new) + .collect(Collectors.toList()); + } + + private List populateSubmissions(Program program, List submissions) throws ApiException { + List submissionIds = submissions.stream().map(s -> s.getId().toString()).collect(Collectors.toList()); + List plates = plateDAO.readPlatesBySubmissionIds(program, submissionIds); + List samples = sampleDAO.readSamplesBySubmissionIds(program, submissionIds); + + Map> platesBySubmissionId = new HashMap<>(); + String submissionXrefSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PLATE_SUBMISSIONS); + plates.forEach(plate -> { + BrAPIExternalReference brAPIExternalReference = Utilities.getExternalReference(plate.getExternalReferences(), submissionXrefSource).orElseThrow(() -> new IllegalStateException(String.format("Plate %s does not have a submission ID", plate.getPlateName()))); + List submissionPlates = platesBySubmissionId.getOrDefault(brAPIExternalReference.getReferenceId(), new ArrayList<>()); + submissionPlates.add(plate); + platesBySubmissionId.putIfAbsent(brAPIExternalReference.getReferenceId(), submissionPlates); + }); + + Map> samplesBySubmissionId = new HashMap<>(); + samples.forEach(sample -> { + BrAPIExternalReference brAPIExternalReference = Utilities.getExternalReference(sample.getExternalReferences(), submissionXrefSource).orElseThrow(() -> new IllegalStateException(String.format("Plate %s does not have a submission ID", sample.getPlateName()))); + List submissionSamples = samplesBySubmissionId.getOrDefault(brAPIExternalReference.getReferenceId(), new ArrayList<>()); + submissionSamples.add(sample); + samplesBySubmissionId.putIfAbsent(brAPIExternalReference.getReferenceId(), submissionSamples); + }); + + submissions.forEach(submission -> { + submission.setPlates(platesBySubmissionId.get(submission.getId().toString())); + submission.setSamples(samplesBySubmissionId.get(submission.getId().toString())); + }); + + return submissions; + } +} diff --git a/src/main/resources/application-dev.yml.template b/src/main/resources/application-dev.yml.template new file mode 100644 index 000000000..81d9add14 --- /dev/null +++ b/src/main/resources/application-dev.yml.template @@ -0,0 +1,10 @@ +##For local development: +## 1. spin up a LocalStack container +## 2. copy this file and rename it to `application-dev.yml` +## 3. uncomment these lines, and update the `endpoint` value +## 4. set the environment variable MICRONAUT_ENVIRONMENTS=dev +#aws: +# s3: +# buckets: +# geno: +# endpoint: \ No newline at end of file diff --git a/src/main/resources/db/migration/V1.16.0__create_sampleimport_mapping.sql b/src/main/resources/db/migration/V1.16.0__create_sampleimport_mapping.sql new file mode 100644 index 000000000..eb88ec05d --- /dev/null +++ b/src/main/resources/db/migration/V1.16.0__create_sampleimport_mapping.sql @@ -0,0 +1,108 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +DO +$$ + DECLARE + user_id UUID; + BEGIN + + user_id := (SELECT id FROM bi_user WHERE name = 'system'); + + INSERT INTO public.importer_mapping (id, name, import_type_id, mapping, file, draft, created_at, updated_at, created_by, updated_by) + VALUES (uuid_generate_v4(), + 'SampleImport', + 'SampleImport', '[ + { + "id": "e4038afd-02b9-475a-88b2-4692e331c399", + "value": { + "fileFieldName": "PlateID" + }, + "objectId": "plateId" + }, + { + "id": "9488023c-ed28-4357-811e-7c517cbd9f68", + "value": { + "fileFieldName": "Row" + }, + "objectId": "row" + }, + { + "id": "54b47821-c868-4fbd-acd6-8b8a64a02230", + "value": { + "fileFieldName": "Column" + }, + "objectId": "column" + }, + { + "id": "f9e39005-e8be-4e83-92b8-a459fe296d2f", + "value": { + "fileFieldName": "Organism" + }, + "objectId": "organism" + }, + { + "id": "fe37a2a5-00e0-4076-b130-846f19b3defd", + "value": { + "fileFieldName": "Species" + }, + "objectId": "species" + }, + { + "id": "e5074e78-b8ba-474e-87df-716a759f0517", + "value": { + "fileFieldName": "Germplasm Name" + }, + "objectId": "germplasmName" + }, + { + "id": "15a20991-ee85-49c6-b5c3-5db4e3dca372", + "value": { + "fileFieldName": "GID" + }, + "objectId": "gid" + }, + { + "id": "11e2af68-ca45-4164-9bbb-326c68894fec", + "value": { + "fileFieldName": "ObsUnitID" + }, + "objectId": "obsUnitId" + }, + { + "id": "e7731381-7b92-42c3-97f0-38fb37208e8d", + "value": { + "fileFieldName": "Tissue" + }, + "objectId": "tissue" + }, + { + "id": "de7fed3a-ec44-4139-83cb-c773e09237bd", + "value": { + "fileFieldName": "Comment" + }, + "objectId": "comment" + } + ]', '[]', + false, + '2023-10-18 02:03:17 +00:00', + '2023-10-18 02:03:17 +00:00', + user_id, + user_id); + + END +$$; \ No newline at end of file diff --git a/src/main/resources/db/migration/V1.17.0__create_sample_submission_table.sql b/src/main/resources/db/migration/V1.17.0__create_sample_submission_table.sql new file mode 100644 index 000000000..ac4d8fd43 --- /dev/null +++ b/src/main/resources/db/migration/V1.17.0__create_sample_submission_table.sql @@ -0,0 +1,36 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +create table sample_submission +( + like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, + name TEXT, + submitted bool default false, + submittedDate timestamp(0) with time zone, + vendor_order_id TEXT, + vendor_status TEXT, + shipmentForms jsonb, + program_id UUID NOT NULL, + like base_edit_track_entity INCLUDING ALL +); + +ALTER TABLE sample_submission + ADD FOREIGN KEY (created_by) REFERENCES bi_user (id); +ALTER TABLE sample_submission + ADD FOREIGN KEY (updated_by) REFERENCES bi_user (id); +ALTER TABLE sample_submission + ADD FOREIGN KEY (program_id) REFERENCES program (id); \ No newline at end of file From f1088055de773c3b039c04f49602d4fc7e740de9 Mon Sep 17 00:00:00 2001 From: timparsons Date: Mon, 23 Oct 2023 11:45:53 -0400 Subject: [PATCH 068/220] [BI-1910] remove unintended class commit --- .../org/breedinginsight/api/model/PendingBrAPIImport.java | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 src/main/java/org/breedinginsight/api/model/PendingBrAPIImport.java diff --git a/src/main/java/org/breedinginsight/api/model/PendingBrAPIImport.java b/src/main/java/org/breedinginsight/api/model/PendingBrAPIImport.java deleted file mode 100644 index 6fc8bd194..000000000 --- a/src/main/java/org/breedinginsight/api/model/PendingBrAPIImport.java +++ /dev/null @@ -1,4 +0,0 @@ -package org.breedinginsight.api.model; - -public class PendingBrAPIImport { -} From e84c46dbc76f52a4cd86f954e81ba98fbe092b25 Mon Sep 17 00:00:00 2001 From: timparsons Date: Thu, 26 Oct 2023 15:32:24 -0400 Subject: [PATCH 069/220] [BI-1910] Implementing functionality to fetch and update sample submissions Able to fetch submissions. Able to generate a DArT submission file. Able to submit them via BrAPI to a vendor. Able to update the status. Able to automatically check the status of a submission. --- .../geno/SampleSubmissionController.java | 272 +++++++++++++++ .../brapi/v2/ExperimentController.java | 12 +- .../brapi/v2/services/BrAPITrialService.java | 15 +- .../sample/SampleSubmissionImport.java | 21 +- .../processors/SampleSubmissionProcessor.java | 39 +-- .../daos/SampleSubmissionDAO.java | 87 ++++- .../breedinginsight/model/GenotypeVendor.java | 37 ++ .../model/SampleSubmission.java | 48 ++- .../java/org/breedinginsight/model/User.java | 3 +- .../services/SampleSubmissionService.java | 329 +++++++++++++++++- .../breedinginsight/utilities/FileUtil.java | 12 + src/main/resources/application.yml | 7 +- ...1.17.0__create_sample_submission_table.sql | 10 +- 13 files changed, 822 insertions(+), 70 deletions(-) create mode 100644 src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java create mode 100644 src/main/java/org/breedinginsight/model/GenotypeVendor.java diff --git a/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java b/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java new file mode 100644 index 000000000..5385ea498 --- /dev/null +++ b/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java @@ -0,0 +1,272 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.api.v1.controller.geno; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import io.micronaut.http.*; +import io.micronaut.http.annotation.*; +import io.micronaut.http.server.types.files.StreamedFile; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.geno.BrAPIVendorOrderSubmission; +import org.breedinginsight.api.auth.*; +import org.breedinginsight.api.model.v1.response.DataResponse; +import org.breedinginsight.api.model.v1.response.Response; +import org.breedinginsight.api.model.v1.response.metadata.Metadata; +import org.breedinginsight.api.model.v1.response.metadata.Pagination; +import org.breedinginsight.api.model.v1.response.metadata.Status; +import org.breedinginsight.api.model.v1.response.metadata.StatusCode; +import org.breedinginsight.model.*; +import org.breedinginsight.services.ProgramService; +import org.breedinginsight.services.SampleSubmissionService; +import org.breedinginsight.services.UserService; +import org.breedinginsight.utilities.Utilities; + +import javax.annotation.Nullable; +import javax.inject.Inject; +import javax.validation.constraints.NotBlank; +import java.io.IOException; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@Slf4j +@Controller("/${micronaut.bi.api.version}") +@Secured(SecurityRule.IS_AUTHENTICATED) +public class SampleSubmissionController { + private final SampleSubmissionService sampleSubmissionService; + private final ProgramService programService; + private final SecurityService securityService; + private final UserService userService; + + private final Gson gson; + + @Inject + public SampleSubmissionController(SampleSubmissionService sampleSubmissionService, ProgramService programService, SecurityService securityService, UserService userService) { + this.sampleSubmissionService = sampleSubmissionService; + this.programService = programService; + this.securityService = securityService; + this.userService = userService; + this.gson = new Gson(); + } + + @Get("programs/{programId}/submissions") + @Produces(MediaType.APPLICATION_JSON) + @ProgramSecured(roleGroups = ProgramSecuredRoleGroup.ALL) + public HttpResponse>> getProgramSampleSubmissions(@PathVariable UUID programId) { + Optional program = programService.getById(programId); + if(program.isEmpty()) { + log.info(String.format("programId not found: %s", programId.toString())); + return HttpResponse.notFound(); + } + + List submissions = sampleSubmissionService.getProgramSubmissions(program.get()); + Metadata metadata = new Metadata(new Pagination(submissions.size(), submissions.size(), 1, 0), + List.of(new Status(StatusCode.INFO, "Successful Query"))); + Response> response = new Response<>(metadata, new DataResponse<>(submissions)); + return HttpResponse.ok(response); + } + + @Get("programs/{programId}/submissions/{submissionId}") + @Produces(MediaType.APPLICATION_JSON) + @ProgramSecured(roleGroups = ProgramSecuredRoleGroup.ALL) + public HttpResponse> getSubmissionById(@PathVariable UUID programId, @PathVariable UUID submissionId, @QueryValue(value = "details", defaultValue = "false") @Nullable Boolean fetchDetails) { + Optional program = programService.getById(programId); + if(program.isEmpty()) { + log.info(String.format("programId not found: %s", programId.toString())); + return HttpResponse.notFound(); + } + + try { + Optional submission = sampleSubmissionService.getSampleSubmission(program.get(), submissionId, fetchDetails); + + if(submission.isEmpty()) { + return HttpResponse.notFound(); + } + + Metadata metadata = new Metadata(new Pagination(1, 1, 1, 0), + List.of(new Status(StatusCode.INFO, "Successful Query"))); + Response response = new Response<>(metadata, submission.get()); + return HttpResponse.ok(response); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(); + } + } + + @Put("programs/{programId}/submissions/{submissionId}/status") + @Produces(MediaType.APPLICATION_JSON) + @ProgramSecured(roles = {ProgramSecuredRole.SYSTEM_ADMIN}) + public HttpResponse> updateSubmissionStatus(@PathVariable UUID programId, @PathVariable UUID submissionId, @Body String body) { + Optional program = programService.getById(programId); + if(program.isEmpty()) { + log.info(String.format("programId not found: %s", programId.toString())); + return HttpResponse.notFound(); + } + + AuthenticatedUser actingUser = securityService.getUser(); + Optional user = userService.getById(actingUser.getId()); + if (user.isEmpty()) { + return HttpResponse.unauthorized(); + } + + SampleSubmission.Status status = SampleSubmission.Status.fromValue(gson.fromJson(body, JsonObject.class).get("status").getAsString()); + if(status == null) { + HttpResponse response = HttpResponse.badRequest("Unrecognized status"); + return response; + } + + try { + Optional submission = sampleSubmissionService.updateSubmissionStatus(program.get(), submissionId, status, user.get()); + + if(submission.isEmpty()) { + return HttpResponse.notFound(); + } + + Metadata metadata = new Metadata(new Pagination(1, 1, 1, 0), + List.of(new Status(StatusCode.INFO, "Successful Update"))); + Response response = new Response<>(metadata, submission.get()); + return HttpResponse.ok(response); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(); + } + } + + + @Get("/programs/{programId}/submissions/{submissionId}/dart") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + @Produces(value={"text/csv", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/octet-stream"}) + public HttpResponse generateDArTFile(@PathVariable UUID programId, @PathVariable UUID submissionId) { + try { + Optional program = programService.getById(programId); + if(program.isEmpty()) { + return HttpResponse.notFound(); + } + Optional downloadFile = sampleSubmissionService.generateDArTFile(program.get(), submissionId); + if(downloadFile.isEmpty()) { + return HttpResponse.notFound(); + } + HttpResponse response = HttpResponse + .ok(downloadFile.get().getStreamedFile()) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + downloadFile.get().getFileName()); + return response; + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(); + } catch (IOException e) { + log.error("Error generating DArT file", e); + HttpResponse response = HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error generating DArT file").contentType(MediaType.TEXT_PLAIN).body("Error generating DArT file"); + return response; + } + } + + @Get("/programs/{programId}/submissions/{submissionId}/lookup") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + @Produces(value={"text/csv", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/octet-stream"}) + public HttpResponse generateLookupFile(@PathVariable UUID programId, @PathVariable UUID submissionId) { + try { + Optional program = programService.getById(programId); + if(program.isEmpty()) { + return HttpResponse.notFound(); + } + Optional downloadFile = sampleSubmissionService.generateLookupFile(program.get(), submissionId); + if(downloadFile.isEmpty()) { + return HttpResponse.notFound(); + } + HttpResponse response = HttpResponse + .ok(downloadFile.get().getStreamedFile()) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + downloadFile.get().getFileName()); + return response; + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(); + } catch (IOException e) { + log.error("Error generating lookup file", e); + HttpResponse response = HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error generating lookup file").contentType(MediaType.TEXT_PLAIN).body("Error generating lookup file"); + return response; + } + } + + @Post("programs/{programId}/submissions/{submissionId}/submit") + @Produces(MediaType.APPLICATION_JSON) + @ProgramSecured(roles = {ProgramSecuredRole.SYSTEM_ADMIN}) + public HttpResponse> submitOrder(@PathVariable UUID programId, @PathVariable UUID submissionId, @QueryValue(value = "vendor") @NotBlank String vendorName) { + try { + GenotypeVendor vendor = GenotypeVendor.fromName(vendorName); + if(vendor != null) { + Optional program = programService.getById(programId); + if (program.isEmpty()) { + return HttpResponse.notFound(); + } + + AuthenticatedUser actingUser = securityService.getUser(); + Optional user = userService.getById(actingUser.getId()); + if (user.isEmpty()) { + return HttpResponse.unauthorized(); + } + + Optional order = sampleSubmissionService.submitOrder(program.get(), submissionId, user.get(), vendor); + if (order.isEmpty()) { + return HttpResponse.notFound(); + } + + Metadata metadata = new Metadata(new Pagination(1, 1, 1, 0), + List.of(new Status(StatusCode.INFO, "Successful submission"))); + Response response = new Response<>(metadata, order.get()); + return HttpResponse.ok(response); + } else { + HttpResponse response = HttpResponse.badRequest("Unrecognized vendor"); + return response; + } + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(); + } + } + + @Get("programs/{programId}/submissions/{submissionId}/status") + @Produces(MediaType.APPLICATION_JSON) + @ProgramSecured(roles = {ProgramSecuredRole.SYSTEM_ADMIN}) + public HttpResponse> checkVendorStatus(@PathVariable UUID programId, @PathVariable UUID submissionId) { + Optional program = programService.getById(programId); + if(program.isEmpty()) { + log.info(String.format("programId not found: %s", programId.toString())); + return HttpResponse.notFound(); + } + + try { + Optional submission = sampleSubmissionService.checkVendorStatus(program.get(), submissionId); + + if(submission.isEmpty()) { + return HttpResponse.notFound(); + } + + Metadata metadata = new Metadata(new Pagination(1, 1, 1, 0), + List.of(new Status(StatusCode.INFO, "Successful Query"))); + Response response = new Response<>(metadata, submission.get()); + return HttpResponse.ok(response); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java index 443b81b4c..c819cc10c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java @@ -97,19 +97,21 @@ public HttpResponse datasetExport( @QueryValue @Valid ExperimentExportQuery queryParams) { String downloadErrorMessage = "An error occurred while generating the download file. Contact the development team at bidevteam@cornell.edu."; try { - Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program does not exist")); + Optional program = programService.getById(programId); + if(program.isEmpty()) { + return HttpResponse.notFound(); + } // if a list of environmentIds are sent, return multiple files (zipped), // else if a single environmentId is sent, return single file (CSV/Excel), // else (if no environmentIds are sent), return a single file (CSV/Excel) including all Environments. - DownloadFile downloadFile = experimentService.exportObservations(program, experimentId, queryParams); + DownloadFile downloadFile = experimentService.exportObservations(program.get(), experimentId, queryParams); - HttpResponse response = HttpResponse + return HttpResponse .ok(downloadFile.getStreamedFile()) .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + downloadFile.getFileName()); - return response; } catch (Exception e) { - log.info(e.getMessage(), e); + log.info(downloadErrorMessage, e); HttpResponse response = HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, downloadErrorMessage).contentType(MediaType.TEXT_PLAIN).body(downloadErrorMessage); return response; } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index b902132ad..95d34cd32 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -29,9 +29,8 @@ import org.breedinginsight.model.*; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; -import org.breedinginsight.services.writers.CSVWriter; -import org.breedinginsight.services.writers.ExcelWriter; import org.breedinginsight.utilities.IntOrderComparator; +import org.breedinginsight.utilities.FileUtil; import org.breedinginsight.utilities.Utilities; import org.jetbrains.annotations.NotNull; @@ -242,7 +241,7 @@ public DownloadFile exportObservations( for (Map.Entry>> entry: rowsByStudyId.entrySet()) { List> rows = entry.getValue(); sortDefaultForExportRows(rows); - StreamedFile streamedFile = writeToStreamedFile(columns, rows, fileType, SHEET_NAME); + StreamedFile streamedFile = FileUtil.writeToStreamedFile(columns, rows, fileType, SHEET_NAME); String name = makeFileName(experiment, program, studyByDbId.get(entry.getKey()).getStudyName()) + fileType.getExtension(); // Add to file list. files.add(new DownloadFile(name, streamedFile)); @@ -261,7 +260,7 @@ public DownloadFile exportObservations( List> exportRows = new ArrayList<>(rowByOUId.values()); sortDefaultForExportRows(exportRows); // write export data to requested file format - StreamedFile streamedFile = writeToStreamedFile(columns, exportRows, fileType, SHEET_NAME); + StreamedFile streamedFile = FileUtil.writeToStreamedFile(columns, exportRows, fileType, SHEET_NAME); // Set filename. String envFilenameFragment = params.getEnvironments() == null ? "All Environments" : params.getEnvironments(); String fileName = makeFileName(experiment, program, envFilenameFragment) + fileType.getExtension(); @@ -271,14 +270,6 @@ public DownloadFile exportObservations( return downloadFile; } - private StreamedFile writeToStreamedFile(List columns, List> data, FileType extension, String sheetName) throws IOException { - if (extension.equals(FileType.CSV)){ - return CSVWriter.writeToDownload(columns, data, extension); - } else { - return ExcelWriter.writeToDownload(sheetName, columns, data, extension); - } - } - private StreamedFile zipFiles(List files) throws IOException { PipedInputStream in = new PipedInputStream(); final PipedOutputStream out = new PipedOutputStream(in); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java index bf251f5b7..242b5e4a7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java @@ -44,7 +44,8 @@ description = "This import is used to create Genotype Samples") public class SampleSubmissionImport implements BrAPIImport { - private static final String SAMPLE_NAME_FORMAT = "%s__%s_%s%s"; + private static final String SAMPLE_NAME_FORMAT = "S_%s__%s_%s%02d"; + private static final String SAMPLE_NAME_ILLEGAL_CHARS_REGEX = "[^a-zA-Z0-9_]"; @ImportFieldType(type = ImportFieldTypeEnum.TEXT) @ImportFieldMetadata(id = "plateId", name = Columns.PLATE_ID, description = "The ID which uniquely identifies this plate to the client making the request") @@ -83,7 +84,7 @@ public class SampleSubmissionImport implements BrAPIImport { private String tissue; @ImportFieldType(type = ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id = "comment", name = Columns.COMMENT, description = "Generic comments about this sample for the vendor") + @ImportFieldMetadata(id = "comment", name = Columns.COMMENTS, description = "Generic comments about this sample for the vendor") private String comment; @ImportFieldType(type= ImportFieldTypeEnum.TEXT, collectTime = ImportCollectTimeEnum.UPLOAD) @@ -100,7 +101,7 @@ public static final class Columns { public static final String GERMPLASM_NAME = "Germplasm Name"; public static final String GERMPLASM_GID = "Germplasm GID"; public static final String TISSUE = "Tissue"; - public static final String COMMENT = "Comment"; + public static final String COMMENTS = "Comments"; public static final String OBS_UNIT_ID = "ObsUnitID"; } @@ -140,7 +141,7 @@ public BrAPISample constructBrAPISample(boolean commit, Program program, User us .externalReferences(xrefs) .plateName(plateId) .plateDbId(plate.getPlateDbId()) - .sampleName(String.format(SAMPLE_NAME_FORMAT, germplasm.getDefaultDisplayName(), plate.getPlateName(), row, column)) + .sampleName(generateSampleName(germplasm, plate)) .germplasmDbId(germplasm.getGermplasmDbId()) .row(row) .column(Integer.valueOf(column)) @@ -159,6 +160,18 @@ public BrAPISample constructBrAPISample(boolean commit, Program program, User us return brAPISample; } + private String generateSampleName(BrAPIGermplasm germplasm, BrAPIPlate plate) { + var legalGermplasmName = germplasm.getDefaultDisplayName().replaceAll(SAMPLE_NAME_ILLEGAL_CHARS_REGEX, "_"); + String name = String.format(SAMPLE_NAME_FORMAT, legalGermplasmName, plate.getPlateName(), row, Integer.parseInt(column)); + + if(name.length() > 100) { + //TODO what to do in this case?? + throw new RuntimeException("Generated sample name is too long"); + } + + return name; + } + public BrAPIPlate constructBrAPIPlate(boolean commit, Program program, User user, String referenceSource, String submissionId) { List xrefs = new ArrayList<>(); xrefs.add(new BrAPIExternalReference().referenceSource(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS)) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java index d3d4c1b0e..531df2a09 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java @@ -63,6 +63,12 @@ public class SampleSubmissionProcessor implements Processor { private static final String MISSING_REQUIRED_DATA = "Missing required data"; + private static final String MISSING_GERM_ASSOCIATION = "One of GID or ObsUnitID is required"; + private static final String UNKNOWN_OBS_UNIT_ID = "Unknown ObsUnitID"; + private static final String UNKNOWN_GID = "Unknown germplasm GID"; + private static final String INVALID_COLUMN = "Column must be a number between 1 and 12"; + private static final String INVALID_ROW = "Row must be a letter between A and H"; + private static final String MULTIPLE_SAMPLES_SINGLE_WELL = "The sample in row %d is already in row: %s, column: %d"; private final String referenceSource; private final BrAPIGermplasmDAO germplasmDAO; private final BrAPIObservationUnitDAO observationUnitDAO; @@ -188,25 +194,19 @@ public Map process(ImportUpload upload, } private boolean validRow(SampleSubmissionImport row, int rowNum, ValidationErrors validationErrors) { - int numErrorsBefore = validationErrors.getRowErrors() - .size(); + int numErrorsBefore = validationErrors.getRowErrors().size(); if (StringUtils.isBlank(row.getPlateId())) { validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.PLATE_ID, MISSING_REQUIRED_DATA, HttpStatus.UNPROCESSABLE_ENTITY)); } int plateRow = -1; if (StringUtils.isBlank(row.getRow())) { validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.ROW, MISSING_REQUIRED_DATA, HttpStatus.UNPROCESSABLE_ENTITY)); - } else if (row.getRow() - .length() > 1 || row.getRow() - .toUpperCase() - .charAt(0) - 'A' < 0 || row.getRow() - .toUpperCase() - .charAt(0) - 'H' > 0) { - validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.ROW, "Row must be a a letter between A and H", HttpStatus.UNPROCESSABLE_ENTITY)); + } else if (row.getRow().length() > 1 + || row.getRow().toUpperCase().charAt(0) - 'A' < 0 + || row.getRow().toUpperCase().charAt(0) - 'H' > 0) { + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.ROW, INVALID_ROW, HttpStatus.UNPROCESSABLE_ENTITY)); } else { - plateRow = row.getRow() - .toUpperCase() - .charAt(0) - 'A'; + plateRow = row.getRow().toUpperCase().charAt(0) - 'A'; } int plateCol = -1; if (StringUtils.isBlank(row.getColumn())) { @@ -215,25 +215,22 @@ private boolean validRow(SampleSubmissionImport row, int rowNum, ValidationError try { plateCol = Integer.parseInt(row.getColumn()); if (plateCol < 1 || plateCol > 12) { - validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.COLUMN, "Column must be a number between 1 and 12", HttpStatus.UNPROCESSABLE_ENTITY)); + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.COLUMN, INVALID_COLUMN, HttpStatus.UNPROCESSABLE_ENTITY)); plateCol = -1; } } catch (NumberFormatException e) { - validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.COLUMN, "Column must be a number between 1 and 12", HttpStatus.UNPROCESSABLE_ENTITY)); + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.COLUMN, INVALID_COLUMN, HttpStatus.UNPROCESSABLE_ENTITY)); } } if (StringUtils.isBlank(row.getOrganism())) { validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.ORGANISM, MISSING_REQUIRED_DATA, HttpStatus.UNPROCESSABLE_ENTITY)); } - if (StringUtils.isBlank(row.getSpecies())) { - validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.SPECIES, MISSING_REQUIRED_DATA, HttpStatus.UNPROCESSABLE_ENTITY)); - } if (StringUtils.isBlank(row.getGid()) && StringUtils.isBlank(row.getObsUnitId())) { - validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.GERMPLASM_GID, "One of GID or ObsUnitID is required", HttpStatus.UNPROCESSABLE_ENTITY)); + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.GERMPLASM_GID, MISSING_GERM_ASSOCIATION, HttpStatus.UNPROCESSABLE_ENTITY)); } else if (StringUtils.isNotBlank(row.getObsUnitId()) && !observationUnitsById.containsKey(row.getObsUnitId())) { - validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.OBS_UNIT_ID, "Unknown ObsUnitID", HttpStatus.UNPROCESSABLE_ENTITY)); + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.OBS_UNIT_ID, UNKNOWN_OBS_UNIT_ID, HttpStatus.UNPROCESSABLE_ENTITY)); } else if (StringUtils.isNotBlank(row.getGid()) && !germplasmByGID.containsKey(row.getGid())) { - validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.GERMPLASM_GID, "Unknown germplasm GID", HttpStatus.UNPROCESSABLE_ENTITY)); + validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.GERMPLASM_GID, UNKNOWN_GID, HttpStatus.UNPROCESSABLE_ENTITY)); } if (StringUtils.isBlank(row.getTissue())) { validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.TISSUE, MISSING_REQUIRED_DATA, HttpStatus.UNPROCESSABLE_ENTITY)); @@ -244,7 +241,7 @@ private boolean validRow(SampleSubmissionImport row, int rowNum, ValidationError if (plateLayout[plateRow][plateCol] > 0) { validationErrors.addError(rowNum, new ValidationError(SampleSubmissionImport.Columns.ROW + "/" + SampleSubmissionImport.Columns.COLUMN, - String.format("The sample in row %d is already in row: %s, column: %d", plateLayout[plateRow][plateCol], Character.toString('A' + plateRow), plateCol), + String.format(MULTIPLE_SAMPLES_SINGLE_WELL, plateLayout[plateRow][plateCol], Character.toString('A' + plateRow), plateCol), HttpStatus.UNPROCESSABLE_ENTITY)); } else { plateLayout[plateRow][plateCol] = rowNum; diff --git a/src/main/java/org/breedinginsight/daos/SampleSubmissionDAO.java b/src/main/java/org/breedinginsight/daos/SampleSubmissionDAO.java index c8449ff5d..a6f84596c 100644 --- a/src/main/java/org/breedinginsight/daos/SampleSubmissionDAO.java +++ b/src/main/java/org/breedinginsight/daos/SampleSubmissionDAO.java @@ -17,21 +17,28 @@ package org.breedinginsight.daos; +import org.brapi.v2.model.geno.response.BrAPIVendorOrderStatusResponseResult; import org.breedinginsight.dao.db.Tables; +import org.breedinginsight.dao.db.tables.BiUserTable; import org.breedinginsight.dao.db.tables.daos.SampleSubmissionDao; import org.breedinginsight.dao.db.tables.pojos.SampleSubmissionEntity; +import org.breedinginsight.dao.db.tables.records.SampleSubmissionRecord; import org.breedinginsight.model.Program; import org.breedinginsight.model.SampleSubmission; -import org.jooq.Configuration; -import org.jooq.DSLContext; +import org.breedinginsight.model.User; +import org.jooq.*; import javax.inject.Inject; import javax.inject.Singleton; +import java.time.OffsetDateTime; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.stream.Collectors; -import static org.breedinginsight.dao.db.Tables.SAMPLE_SUBMISSION; +import static org.breedinginsight.dao.db.Tables.*; +import static org.breedinginsight.dao.db.Tables.BI_USER; @Singleton public class SampleSubmissionDAO extends SampleSubmissionDao { @@ -45,14 +52,72 @@ public SampleSubmissionDAO(Configuration config, DSLContext dsl) { } public List getBySubmissionId(Program program, UUID submissionId) { - return dsl.select() - .from(SAMPLE_SUBMISSION) - .where(SAMPLE_SUBMISSION.PROGRAM_ID.eq(program.getId())) - .and(SAMPLE_SUBMISSION.ID.eq(submissionId)) - .fetchInto(SampleSubmissionEntity.class) - .stream() - .map(SampleSubmission::new) - .collect(Collectors.toList()); + return getRecords(List.of(SAMPLE_SUBMISSION.PROGRAM_ID.eq(program.getId()), SAMPLE_SUBMISSION.ID.eq(submissionId))); } + public List getByProgramId(UUID programId) { + return getRecords(List.of(SAMPLE_SUBMISSION.PROGRAM_ID.eq(programId))); + } + + public List getSubmittedAndNotComplete() { + return getRecords(List.of(SAMPLE_SUBMISSION.VENDOR_ORDER_ID.isNotNull(), + SAMPLE_SUBMISSION.VENDOR_STATUS.isNull().or(SAMPLE_SUBMISSION.VENDOR_STATUS.ne(BrAPIVendorOrderStatusResponseResult.StatusEnum.COMPLETED.name())))); + } + + public SampleSubmission update(SampleSubmission submission, User updatedBy) { + submission.setUpdatedAt(OffsetDateTime.now()); + submission.setUpdatedBy(updatedBy.getId()); + if(submission.getSubmittedByUser() != null) { + submission.setSubmittedBy(submission.getSubmittedByUser().getId()); + } else { + submission.setSubmittedBy(null); + } + + super.update(submission); + return submission; + } + + private List getRecords(List andConditions) { + BiUserTable createdByUser = BI_USER.as("createdByUser"); + BiUserTable updatedByUser = BI_USER.as("updatedByUser"); + BiUserTable submittedByUser = BI_USER.as("submittedByUser"); + try(SelectSelectStep select = dsl.select()) { + SelectConditionStep query = select + .from(SAMPLE_SUBMISSION) + .join(createdByUser).on(SAMPLE_SUBMISSION.CREATED_BY.eq(createdByUser.ID)) + .join(updatedByUser).on(SAMPLE_SUBMISSION.UPDATED_BY.eq(updatedByUser.ID)) + .leftJoin(submittedByUser).on(SAMPLE_SUBMISSION.SUBMITTED_BY.eq(submittedByUser.ID)) + .where("1=1"); + + for (Condition condition : andConditions) { + query = query.and(condition); + } + + return query.fetch() + .stream() + .map(record -> parseRecord(record, createdByUser, updatedByUser, submittedByUser)) + .collect(Collectors.toList()); + } + + } + + + private SampleSubmission parseRecord(Record record, BiUserTable createdByUser, BiUserTable updatedByUser, BiUserTable submittedByUser) { + SampleSubmission submission = new SampleSubmission(record.into(SampleSubmissionEntity.class)); + submission.setCreatedByUser(User.parseSQLRecord(record, createdByUser)); + submission.setUpdatedByUser(User.parseSQLRecord(record, updatedByUser)); + submission.setSubmittedByUser(User.parseSQLRecord(record, submittedByUser)); + //these explicit setters are needed because of overlapping column names with the bi_user table joined to the query + submission.setId(record.get(SAMPLE_SUBMISSION.ID)); + submission.setName(record.get(SAMPLE_SUBMISSION.NAME)); + submission.setProgramId(record.get(SAMPLE_SUBMISSION.PROGRAM_ID)); + submission.setCreatedAt(record.get(SAMPLE_SUBMISSION.CREATED_AT)); + submission.setUpdatedAt(record.get(SAMPLE_SUBMISSION.UPDATED_AT)); + submission.setCreatedBy(submission.getCreatedByUser().getId()); + submission.setUpdatedBy(submission.getUpdatedByUser().getId()); + if(submission.getSubmittedByUser() != null) { + submission.setSubmittedBy(submission.getSubmittedByUser().getId()); + } + return submission; + } } diff --git a/src/main/java/org/breedinginsight/model/GenotypeVendor.java b/src/main/java/org/breedinginsight/model/GenotypeVendor.java new file mode 100644 index 000000000..e5302d32c --- /dev/null +++ b/src/main/java/org/breedinginsight/model/GenotypeVendor.java @@ -0,0 +1,37 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.model; + +public enum GenotypeVendor { + DART("DArT"); + + private final String name; + + GenotypeVendor(String name) { + this.name = name; + } + + public static GenotypeVendor fromName(String name) { + for(var vendor : GenotypeVendor.values()) { + if(vendor.name.equalsIgnoreCase(name)) { + return vendor; + } + } + return null; + } +} diff --git a/src/main/java/org/breedinginsight/model/SampleSubmission.java b/src/main/java/org/breedinginsight/model/SampleSubmission.java index 99d2030cc..6202a5a6f 100644 --- a/src/main/java/org/breedinginsight/model/SampleSubmission.java +++ b/src/main/java/org/breedinginsight/model/SampleSubmission.java @@ -18,6 +18,8 @@ package org.breedinginsight.model; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; import io.micronaut.core.annotation.Introspected; import lombok.Getter; import lombok.NoArgsConstructor; @@ -28,7 +30,9 @@ import lombok.extern.jackson.Jacksonized; import org.brapi.v2.model.geno.BrAPIPlate; import org.brapi.v2.model.geno.BrAPISample; +import org.brapi.v2.model.geno.BrAPIShipmentForm; import org.breedinginsight.dao.db.tables.pojos.SampleSubmissionEntity; +import org.jooq.JSONB; import java.util.List; @@ -40,8 +44,12 @@ @NoArgsConstructor @Introspected @Jacksonized -//@JsonIgnoreProperties(value = {"createdBy", "updatedBy"}) +@JsonIgnoreProperties(value = {"createdBy", "updatedBy", "submittedBy", "shipmentforms"}) public class SampleSubmission extends SampleSubmissionEntity { + private User createdByUser; + private User updatedByUser; + private User submittedByUser; + private List shipmentForms; private List plates; private List samples; @@ -49,12 +57,50 @@ public SampleSubmission(SampleSubmissionEntity entity) { this.setId(entity.getId()); this.setName(entity.getName()); this.setSubmitted(entity.getSubmitted()); + this.setSubmittedDate(entity.getSubmittedDate()); + this.setSubmittedBy(entity.getSubmittedBy()); this.setVendorOrderId(entity.getVendorOrderId()); + this.setVendorStatus(entity.getVendorStatus()); + this.setVendorStatusLastCheck(entity.getVendorStatusLastCheck()); this.setShipmentforms(entity.getShipmentforms()); this.setProgramId(entity.getProgramId()); this.setCreatedAt(entity.getCreatedAt()); this.setCreatedBy(entity.getCreatedBy()); this.setUpdatedAt(entity.getUpdatedAt()); this.setUpdatedBy(entity.getUpdatedBy()); + + parseShipmentForms(super.getShipmentforms()); + } + + private void parseShipmentForms(JSONB shipmentforms) { + if(shipmentforms != null) { + Gson gson = new Gson(); + this.shipmentForms = gson.fromJson(shipmentforms.data(), new TypeToken>() {}.getType()); + } + } + + public enum Status { + NOT_SUBMITTED("NOT SUBMITTED"), + SUBMITTED("SUBMITTED"), + COMPLETED("COMPLETED"); + + private final String value; + + Status(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + public static Status fromValue(String value) { + for(Status status : Status.values()) { + if(status.value.equals(value)) { + return status; + } + } + return null; + } } } diff --git a/src/main/java/org/breedinginsight/model/User.java b/src/main/java/org/breedinginsight/model/User.java index 15b38c1be..1aaf2283e 100644 --- a/src/main/java/org/breedinginsight/model/User.java +++ b/src/main/java/org/breedinginsight/model/User.java @@ -32,6 +32,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.UUID; import static org.breedinginsight.dao.db.Tables.BI_USER; @@ -43,7 +44,7 @@ @Introspected @Jacksonized @JsonIgnoreProperties(value = {"accountToken"}) -public class User extends BiUserEntity{ +public class User extends BiUserEntity { @JsonInclude() @NotNull diff --git a/src/main/java/org/breedinginsight/services/SampleSubmissionService.java b/src/main/java/org/breedinginsight/services/SampleSubmissionService.java index 3509e210c..47a295c3a 100644 --- a/src/main/java/org/breedinginsight/services/SampleSubmissionService.java +++ b/src/main/java/org/breedinginsight/services/SampleSubmissionService.java @@ -18,22 +18,44 @@ package org.breedinginsight.services; import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Value; +import io.micronaut.scheduling.annotation.Scheduled; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.brapi.client.v2.ApiResponse; +import org.brapi.client.v2.BrAPIClient; +import org.brapi.client.v2.auth.Authentication; +import org.brapi.client.v2.auth.OAuth; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.modules.genotype.VendorApi; import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.geno.BrAPIPlate; -import org.brapi.v2.model.geno.BrAPISample; +import org.brapi.v2.model.geno.*; +import org.brapi.v2.model.geno.request.BrAPIVendorOrderSubmissionRequest; +import org.brapi.v2.model.geno.response.BrAPIVendorOrderStatusResponse; +import org.brapi.v2.model.geno.response.BrAPIVendorOrderStatusResponseResult; +import org.brapi.v2.model.geno.response.BrAPIVendorOrderSubmissionSingleResponse; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.daos.BrAPIPlateDAO; import org.breedinginsight.brapps.importer.daos.BrAPISampleDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.exports.FileType; +import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.daos.SampleSubmissionDAO; -import org.breedinginsight.model.Program; -import org.breedinginsight.model.SampleSubmission; +import org.breedinginsight.model.*; +import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.utilities.FileUtil; import org.breedinginsight.utilities.Utilities; +import org.jooq.DSLContext; +import org.jooq.JSONB; import javax.inject.Inject; import javax.inject.Singleton; +import java.io.IOException; +import java.time.Duration; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; import java.util.*; import java.util.stream.Collectors; @@ -41,24 +63,70 @@ @Singleton public class SampleSubmissionService { + private static final String COLUMN_GENOTYPE = "Genotype"; + private static final String VENDOR_NOT_SUBMITTED_STATUS = "NOT SUBMITTED"; + private static final String VENDOR_SUBMITTED_STATUS = "SUBMITTED"; private final String referenceSource; + private String dartBrapiUrl; + private String dartClientId; + private String dartToken; + private Duration requestTimeout; private final SampleSubmissionDAO submissionDAO; private final BrAPIPlateDAO plateDAO; private final BrAPISampleDAO sampleDAO; + private final BrAPIEndpointProvider brAPIEndpointProvider; + private final ProgramDAO programDAO; + private final DSLContext dsl; @Inject - public SampleSubmissionService(@Property(name = "brapi.server.reference-source") String referenceSource, SampleSubmissionDAO submissionDAO, BrAPIPlateDAO plateDAO, BrAPISampleDAO sampleDAO) { + public SampleSubmissionService(@Property(name = "brapi.server.reference-source") String referenceSource, + @Property(name = "brapi.server.vendor.dart-url") String dartBrapiUrl, + @Property(name = "brapi.server.vendor.dart-client-id") String dartClientId, + @Property(name = "brapi.server.vendor.dart-token") String dartToken, + @Value(value = "${brapi.read-timeout:5m}") Duration requestTimeout, + SampleSubmissionDAO submissionDAO, + BrAPIPlateDAO plateDAO, + BrAPISampleDAO sampleDAO, + BrAPIEndpointProvider brAPIEndpointProvider, + ProgramDAO programDAO, + DSLContext dsl) { this.referenceSource = referenceSource; + this.dartBrapiUrl = dartBrapiUrl; + this.dartClientId = dartClientId; + this.dartToken = dartToken; + this.requestTimeout = requestTimeout; this.submissionDAO = submissionDAO; this.plateDAO = plateDAO; this.sampleDAO = sampleDAO; + this.brAPIEndpointProvider = brAPIEndpointProvider; + this.programDAO = programDAO; + this.dsl = dsl; + } + + @Scheduled(fixedDelay = "${brapi.vendor-check-frequency}", initialDelay = "10s") + void checkSubmissionStatuses() { + log.trace("checking vendor order statuses"); + List submittedAndNotCompleted = submissionDAO.getSubmittedAndNotComplete(); + log.trace(submittedAndNotCompleted.size() + " orders to check"); + for(var order : submittedAndNotCompleted) { + try { + this.checkVendorStatus(order); + } catch (ApiException e) { + log.error("Error checking vendor order status: \n\n" + Utilities.generateApiExceptionLogMessage(e), e); + throw new RuntimeException(e); + } + } + log.trace("vendor order status checks complete, sleeping"); } public SampleSubmission createSubmission(SampleSubmission submission, Program program, ImportUpload upload) throws ApiException { submission.setProgramId(program.getId()); + submission.setCreatedByUser(upload.getCreatedByUser()); submission.setCreatedBy(upload.getCreatedBy()); + submission.setUpdatedByUser(upload.getUpdatedByUser()); submission.setUpdatedBy(upload.getUpdatedBy()); + submissionDAO.insert(submission); List savedPlates = plateDAO.createPlates(program, submission.getPlates(), upload); @@ -72,15 +140,16 @@ public SampleSubmission createSubmission(SampleSubmission submission, Program pr return submission; } - public Optional getSampleSubmission(Program program, UUID submissionId) throws ApiException { - return populateSubmissions(program, submissionDAO.getBySubmissionId(program, submissionId)).stream().findFirst(); + public Optional getSampleSubmission(Program program, UUID submissionId, boolean fetchDetails) throws ApiException { + if (fetchDetails) { + return populateSubmissions(program, submissionDAO.getBySubmissionId(program, submissionId)).stream().findFirst(); + } else { + return submissionDAO.getBySubmissionId(program, submissionId).stream().findFirst(); + } } public List getProgramSubmissions(Program program) { - return submissionDAO.fetchByProgramId(program.getId()) - .stream() - .map(SampleSubmission::new) - .collect(Collectors.toList()); + return submissionDAO.getByProgramId(program.getId()); } private List populateSubmissions(Program program, List submissions) throws ApiException { @@ -112,4 +181,242 @@ private List populateSubmissions(Program program, List generateDArTFile(Program program, UUID submissionId) throws ApiException, IOException { + Optional submission = getSampleSubmission(program, submissionId, true); + if (submission.isEmpty()) { + return Optional.empty(); + } + + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + String filename = Utilities.makePortableFilename(String.format("%s_DArT_Submission_%s.csv", submission.get().getName(), timestamp)); + + List columns = new ArrayList<>(); + columns.add(Column.builder().value(SampleSubmissionImport.Columns.PLATE_ID).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(SampleSubmissionImport.Columns.ROW).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(SampleSubmissionImport.Columns.COLUMN).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(SampleSubmissionImport.Columns.ORGANISM).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(SampleSubmissionImport.Columns.SPECIES).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(COLUMN_GENOTYPE).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(SampleSubmissionImport.Columns.TISSUE).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(SampleSubmissionImport.Columns.COMMENTS).dataType(Column.ColumnDataType.STRING).build()); + + //TODO sort the samples first + List> rows = new ArrayList<>(); + submission.get().getSamples().forEach(sample -> { + Map row = new HashMap<>(); + row.put(SampleSubmissionImport.Columns.PLATE_ID, sample.getPlateName()); + row.put(SampleSubmissionImport.Columns.ROW, sample.getRow()); + row.put(SampleSubmissionImport.Columns.COLUMN, sample.getColumn()); + row.put(SampleSubmissionImport.Columns.ORGANISM, sample.getAdditionalInfo().get(BrAPIAdditionalInfoFields.SAMPLE_ORGANISM).getAsString()); + row.put(SampleSubmissionImport.Columns.SPECIES, sample.getAdditionalInfo().has(BrAPIAdditionalInfoFields.SAMPLE_SPECIES) ? sample.getAdditionalInfo().get(BrAPIAdditionalInfoFields.SAMPLE_SPECIES).getAsString() : ""); + row.put(COLUMN_GENOTYPE, sample.getSampleName()); + row.put(SampleSubmissionImport.Columns.TISSUE, sample.getTissueType()); + row.put(SampleSubmissionImport.Columns.COMMENTS, sample.getSampleDescription()); + + rows.add(row); + }); + + + return Optional.of(new DownloadFile(filename, FileUtil.writeToStreamedFile(columns, rows, FileType.CSV, "Data"))); + } + + public Optional generateLookupFile(Program program, UUID submissionId) throws ApiException, IOException { + Optional submission = getSampleSubmission(program, submissionId, true); + if (submission.isEmpty()) { + return Optional.empty(); + } + + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + String filename = Utilities.makePortableFilename(String.format("%s_Lookup_File_%s.csv", submission.get().getName(), timestamp)); + + List columns = new ArrayList<>(); + columns.add(Column.builder().value(COLUMN_GENOTYPE).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(SampleSubmissionImport.Columns.GERMPLASM_NAME).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(SampleSubmissionImport.Columns.GERMPLASM_GID).dataType(Column.ColumnDataType.STRING).build()); + + //TODO sort the samples first + List> rows = new ArrayList<>(); + submission.get().getSamples().forEach(sample -> { + Map row = new HashMap<>(); + row.put(COLUMN_GENOTYPE, sample.getSampleName()); + row.put(SampleSubmissionImport.Columns.GERMPLASM_NAME, sample.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_NAME).getAsString()); + row.put(SampleSubmissionImport.Columns.GERMPLASM_GID, sample.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GID).getAsString()); + + rows.add(row); + }); + + + return Optional.of(new DownloadFile(filename, FileUtil.writeToStreamedFile(columns, rows, FileType.CSV, "Data"))); + } + + public Optional submitOrder(Program program, UUID submissionId, User submittingUser, GenotypeVendor vendor) throws ApiException, IllegalStateException { + Optional submissionOptional = getSampleSubmission(program, submissionId, true); + if (submissionOptional.isEmpty()) { + return Optional.empty(); + } + SampleSubmission submission = submissionOptional.get(); + + //prevent re-submission + if(Boolean.TRUE.equals(submission.getSubmitted())) { + if(StringUtils.isNotBlank(submission.getVendorOrderId())) { + return Optional.of(new BrAPIVendorOrderSubmission().orderId(submission.getVendorOrderId())); + } else { + throw new IllegalStateException("Submission has been manually submitted, cannot automatically submit to vendor"); + } + } + + Map platesForOrder = new HashMap<>(); + submission.getPlates().forEach(plate -> { + platesForOrder.put(plate.getPlateDbId(), new BrAPIVendorPlate() + .clientPlateBarcode(plate.getPlateBarcode()) + .clientPlateId(plate.getPlateName()) + .sampleSubmissionFormat(BrAPIPlateFormat.PLATE_96) + ); + }); + + submission.getSamples().forEach(sample -> { + BrAPIVendorPlate vendorPlate = platesForOrder.get(sample.getPlateDbId()); + vendorPlate.addSamplesItem(new BrAPIVendorSample() + .clientSampleId(sample.getSampleName()) + .clientSampleBarCode(sample.getSampleBarcode()) + .row(sample.getRow()) + .column(sample.getColumn()) + .comments(sample.getSampleDescription()) + .organismName(sample.getAdditionalInfo().get(BrAPIAdditionalInfoFields.SAMPLE_ORGANISM).getAsString()) + .speciesName(sample.getAdditionalInfo().has(BrAPIAdditionalInfoFields.SAMPLE_SPECIES) ? sample.getAdditionalInfo().get(BrAPIAdditionalInfoFields.SAMPLE_SPECIES).getAsString() : null) + .tissueType(sample.getTissueType()) + .well(String.format("%s%d", sample.getRow(), sample.getColumn())) + ); + }); + + //TODO get info for the specific vendor, and verify program has an account + BrAPIVendorOrderSubmissionRequest order = new BrAPIVendorOrderSubmissionRequest(); + VendorApi vendorApi; + if(GenotypeVendor.DART.equals(vendor)) { + order.setClientId(dartClientId); + vendorApi = getVendorApi(dartBrapiUrl, dartToken); + } else { + throw new IllegalStateException("Unrecognized vendor"); + } + + order.setNumberOfSamples(submission.getSamples().size()); + order.setPlates(new ArrayList<>(platesForOrder.values())); + + ApiResponse response = vendorApi.vendorOrdersPost(order); + + if (response.getBody() == null) { + throw new ApiException("Response is missing body", response.getStatusCode(), response.getHeaders(), null); + } + BrAPIVendorOrderSubmissionSingleResponse body = response.getBody(); + if (body.getResult() == null) { + throw new ApiException("Response body is missing result", response.getStatusCode(), response.getHeaders(), response.getBody().toString()); + } + BrAPIVendorOrderSubmission submittedOrder = body.getResult(); + + submission.setVendorOrderId(submittedOrder.getOrderId()); + submission.setVendorStatus(VENDOR_SUBMITTED_STATUS); + submission.setSubmitted(true); + submission.setSubmittedDate(OffsetDateTime.now()); + submission.setSubmittedByUser(submittingUser); + //TODO: TEMPORARY + submittedOrder.addShipmentFormsItem(new BrAPIShipmentForm().fileDescription("This is a shipment manifest form").fileName("Shipment Manifest").fileURL("https://vendor.org/forms/manifest.pdf")); + if(submittedOrder.getShipmentForms() != null) { + + submission.setShipmentforms(JSONB.valueOf(vendorApi.getApiClient().getJSON().serialize(submittedOrder.getShipmentForms()))); + } + + dsl.transaction(() -> { + submissionDAO.update(submission, submittingUser); + }); + + return Optional.of(submittedOrder); + } + + public Optional checkVendorStatus(Program program, UUID submissionId) throws ApiException { + Optional submissionOptional = getSampleSubmission(program, submissionId, true); + if (submissionOptional.isEmpty()) { + return Optional.empty(); + } + SampleSubmission submission = submissionOptional.get(); + + return Optional.of(checkVendorStatus(submission)); + } + + private SampleSubmission checkVendorStatus(SampleSubmission submission) throws ApiException { + if(submission.getVendorOrderId() == null || BrAPIVendorOrderStatusResponseResult.StatusEnum.COMPLETED.name().equalsIgnoreCase(submission.getVendorStatus())) { + return submission; + } + + VendorApi vendorApi = getVendorApi(dartBrapiUrl, dartToken); + ApiResponse response = vendorApi.vendorOrdersOrderIdStatusGet(submission.getVendorOrderId()); + if (response.getBody() == null) { + throw new ApiException("Response is missing body", response.getStatusCode(), response.getHeaders(), null); + } + BrAPIVendorOrderStatusResponse body = response.getBody(); + if (body.getResult() == null) { + throw new ApiException("Response body is missing result", response.getStatusCode(), response.getHeaders(), response.getBody().toString()); + } + BrAPIVendorOrderStatusResponseResult result = body.getResult(); + + if(result.getStatus() != null) { + submission.setVendorStatus(result.getStatus().name()); + } + submission.setVendorStatusLastCheck(OffsetDateTime.now()); + + dsl.transaction(() -> { + submissionDAO.update(submission, submission.getUpdatedByUser()); + }); + + return submission; + } + + private VendorApi getVendorApi(String url, String authToken) { + BrAPIClient client = new BrAPIClient(url); + client.setHttpClient(client.getHttpClient() + .newBuilder() + .readTimeout(requestTimeout) + .build()); + + Authentication authorizationToken = client.getAuthentication("AuthorizationToken"); + if(authorizationToken instanceof OAuth) { + ((OAuth)authorizationToken).setAccessToken(authToken); + } + + return brAPIEndpointProvider.get(client, VendorApi.class); + } + + public Optional updateSubmissionStatus(Program program, UUID submissionId, SampleSubmission.Status status, User user) throws ApiException { + Optional submissionOptional = this.getSampleSubmission(program, submissionId, false); + if(submissionOptional.isEmpty()) { + return Optional.empty(); + } + + SampleSubmission submission = submissionOptional.get(); + if(StringUtils.isBlank(submission.getVendorOrderId())) { + SampleSubmission.Status currentStatus = SampleSubmission.Status.fromValue(submission.getVendorStatus()); + if(currentStatus != status) { + if((currentStatus == null || currentStatus == SampleSubmission.Status.NOT_SUBMITTED) && status != SampleSubmission.Status.NOT_SUBMITTED) { + submission.setSubmitted(true); + submission.setSubmittedByUser(user); + submission.setSubmittedDate(OffsetDateTime.now()); + } + submission.setVendorStatus(status.getValue()); + + if(status == SampleSubmission.Status.NOT_SUBMITTED){ + submission.setSubmitted(false); + submission.setSubmittedBy(null); + submission.setSubmittedByUser(null); + submission.setSubmittedDate(null); + submission.setVendorStatus(null); + } + + return Optional.ofNullable(submissionDAO.update(submission, user)); + } + } + + return submissionOptional; + } } diff --git a/src/main/java/org/breedinginsight/utilities/FileUtil.java b/src/main/java/org/breedinginsight/utilities/FileUtil.java index 09a2962fa..985103bc1 100644 --- a/src/main/java/org/breedinginsight/utilities/FileUtil.java +++ b/src/main/java/org/breedinginsight/utilities/FileUtil.java @@ -17,14 +17,18 @@ package org.breedinginsight.utilities; +import io.micronaut.http.server.types.files.StreamedFile; import lombok.extern.slf4j.Slf4j; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; import org.apache.commons.io.input.BOMInputStream; import org.apache.poi.EncryptedDocumentException; import org.apache.poi.ss.usermodel.*; +import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.services.parsers.ParsingException; import org.breedinginsight.services.parsers.ParsingExceptionType; +import org.breedinginsight.services.writers.CSVWriter; +import org.breedinginsight.services.writers.ExcelWriter; import tech.tablesaw.api.ColumnType; import tech.tablesaw.api.StringColumn; import tech.tablesaw.api.Table; @@ -246,4 +250,12 @@ public static Table removeNullColumns(Table table) throws ParsingException { return table; } + + public static StreamedFile writeToStreamedFile(List columns, List> data, FileType extension, String sheetName) throws IOException { + if (extension.equals(FileType.CSV)){ + return CSVWriter.writeToDownload(columns, data, extension); + } else { + return ExcelWriter.writeToDownload(sheetName, columns, data, extension); + } + } } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 5b3e50999..929742eca 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -154,17 +154,22 @@ web: brapi: server: - default-url: ${BRAPI_DEFAULT_URL:`https://test-server.brapi.org`} + default-url: ${BRAPI_DEFAULT_URL} # leave these for future but all point to default for now core-url: ${brapi.server.default-url} pheno-url: ${brapi.server.default-url} geno-url: ${brapi.server.default-url} + vendor: + dart-url: ${DART_VENDOR_URL} + dart-client-id: ${DART_CLIENT_ID:potato-salad} + dart-token: ${DART_TOKEN:YYYY} reference-source: ${BRAPI_REFERENCE_SOURCE:breedinginsight.org} read-timeout: ${BRAPI_READ_TIMEOUT:10m} page-size: 1000 search: wait-time: 1000 post-group-size: ${POST_CHUNK_SIZE:1000} + vendor-check-frequency: ${VENDOR_CHECK_FREQUENCY:30s} email: relay-server: diff --git a/src/main/resources/db/migration/V1.17.0__create_sample_submission_table.sql b/src/main/resources/db/migration/V1.17.0__create_sample_submission_table.sql index ac4d8fd43..71c9561b2 100644 --- a/src/main/resources/db/migration/V1.17.0__create_sample_submission_table.sql +++ b/src/main/resources/db/migration/V1.17.0__create_sample_submission_table.sql @@ -20,10 +20,12 @@ create table sample_submission like base_entity INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES, name TEXT, submitted bool default false, - submittedDate timestamp(0) with time zone, + submitted_date timestamp(0) with time zone, + submitted_by UUID, vendor_order_id TEXT, vendor_status TEXT, - shipmentForms jsonb, + vendor_status_last_check timestamp(0) with time zone, + shipmentforms jsonb, program_id UUID NOT NULL, like base_edit_track_entity INCLUDING ALL ); @@ -33,4 +35,6 @@ ALTER TABLE sample_submission ALTER TABLE sample_submission ADD FOREIGN KEY (updated_by) REFERENCES bi_user (id); ALTER TABLE sample_submission - ADD FOREIGN KEY (program_id) REFERENCES program (id); \ No newline at end of file + ADD FOREIGN KEY (program_id) REFERENCES program (id); +ALTER TABLE sample_submission + ADD FOREIGN KEY (submitted_by) REFERENCES bi_user (id); \ No newline at end of file From 8682e0d6ba3c3c3d53473e27741ee1d16ac55a15 Mon Sep 17 00:00:00 2001 From: timparsons Date: Mon, 30 Oct 2023 11:06:32 -0400 Subject: [PATCH 070/220] [BI-1910] Updating brapi-java-client dependency to point to Central Repository --- pom.xml | 15 +++++++++++++-- settings.xml | 12 ------------ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/pom.xml b/pom.xml index 7cc635a9f..acf4e3071 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ 1.0-SNAPSHOT jar - Breeding Insight API + DeltaBreed API https://breedinginsight.org @@ -57,6 +57,10 @@ David Phillips drp227@cornell.edu + + Matthew Mandych + mlm483@cornell.edu + @@ -85,7 +89,7 @@ 31.0.1-jre 4.9.3 4.3.1 - 2.1.0 + 2.1-SNAPSHOT 2.11.0 2.2.1 @@ -149,6 +153,13 @@ false + + ossrh + https://s01.oss.sonatype.org/content/repositories/snapshots/ + + false + + diff --git a/settings.xml b/settings.xml index 4905dc995..c3867921d 100644 --- a/settings.xml +++ b/settings.xml @@ -49,13 +49,6 @@ true true - - github-brapi - GitHub Breeding Insight Apache Maven Packages - https://maven.pkg.github.com/Breeding-Insight/brapi - true - true - github-fannypack FannyPack github repository @@ -73,11 +66,6 @@ ${GITHUB_ACTOR} ${GITHUB_TOKEN} - - github-brapi - ${GITHUB_ACTOR} - ${GITHUB_TOKEN} - github-fannypack ${GITHUB_ACTOR} From 4d568192fc3e74454528dcf3151f0464522588df Mon Sep 17 00:00:00 2001 From: timparsons Date: Mon, 30 Oct 2023 11:07:03 -0400 Subject: [PATCH 071/220] [BI-1910] Adding new config for allowing vendor submission --- .../geno/SampleSubmissionController.java | 14 +++++++++++++- .../services/SampleSubmissionService.java | 6 +++--- src/main/resources/application.yml | 10 ++++++---- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java b/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java index 5385ea498..60d39d7ec 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java @@ -19,6 +19,7 @@ import com.google.gson.Gson; import com.google.gson.JsonObject; +import io.micronaut.context.annotation.Property; import io.micronaut.http.*; import io.micronaut.http.annotation.*; import io.micronaut.http.server.types.files.StreamedFile; @@ -52,6 +53,8 @@ @Controller("/${micronaut.bi.api.version}") @Secured(SecurityRule.IS_AUTHENTICATED) public class SampleSubmissionController { + + private final boolean brapiSubmissionEnabled; private final SampleSubmissionService sampleSubmissionService; private final ProgramService programService; private final SecurityService securityService; @@ -60,7 +63,8 @@ public class SampleSubmissionController { private final Gson gson; @Inject - public SampleSubmissionController(SampleSubmissionService sampleSubmissionService, ProgramService programService, SecurityService securityService, UserService userService) { + public SampleSubmissionController(@Property(name = "brapi.vendors.submission-enabled") boolean brapiSubmissionEnabled, SampleSubmissionService sampleSubmissionService, ProgramService programService, SecurityService securityService, UserService userService) { + this.brapiSubmissionEnabled = brapiSubmissionEnabled; this.sampleSubmissionService = sampleSubmissionService; this.programService = programService; this.securityService = securityService; @@ -210,6 +214,10 @@ public HttpResponse generateLookupFile(@PathVariable UUID programI @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roles = {ProgramSecuredRole.SYSTEM_ADMIN}) public HttpResponse> submitOrder(@PathVariable UUID programId, @PathVariable UUID submissionId, @QueryValue(value = "vendor") @NotBlank String vendorName) { + if(!brapiSubmissionEnabled) { + return HttpResponse.notFound(); + } + try { GenotypeVendor vendor = GenotypeVendor.fromName(vendorName); if(vendor != null) { @@ -247,6 +255,10 @@ public HttpResponse> submitOrder(@PathVaria @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roles = {ProgramSecuredRole.SYSTEM_ADMIN}) public HttpResponse> checkVendorStatus(@PathVariable UUID programId, @PathVariable UUID submissionId) { + if(!brapiSubmissionEnabled) { + return HttpResponse.notFound(); + } + Optional program = programService.getById(programId); if(program.isEmpty()) { log.info(String.format("programId not found: %s", programId.toString())); diff --git a/src/main/java/org/breedinginsight/services/SampleSubmissionService.java b/src/main/java/org/breedinginsight/services/SampleSubmissionService.java index 47a295c3a..7c117e34f 100644 --- a/src/main/java/org/breedinginsight/services/SampleSubmissionService.java +++ b/src/main/java/org/breedinginsight/services/SampleSubmissionService.java @@ -81,9 +81,9 @@ public class SampleSubmissionService { @Inject public SampleSubmissionService(@Property(name = "brapi.server.reference-source") String referenceSource, - @Property(name = "brapi.server.vendor.dart-url") String dartBrapiUrl, - @Property(name = "brapi.server.vendor.dart-client-id") String dartClientId, - @Property(name = "brapi.server.vendor.dart-token") String dartToken, + @Property(name = "brapi.vendors.dart.url") String dartBrapiUrl, + @Property(name = "brapi.vendors.dart.client-id") String dartClientId, + @Property(name = "brapi.vendors.dart.token") String dartToken, @Value(value = "${brapi.read-timeout:5m}") Duration requestTimeout, SampleSubmissionDAO submissionDAO, BrAPIPlateDAO plateDAO, diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 929742eca..13e35457b 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -159,11 +159,13 @@ brapi: core-url: ${brapi.server.default-url} pheno-url: ${brapi.server.default-url} geno-url: ${brapi.server.default-url} - vendor: - dart-url: ${DART_VENDOR_URL} - dart-client-id: ${DART_CLIENT_ID:potato-salad} - dart-token: ${DART_TOKEN:YYYY} reference-source: ${BRAPI_REFERENCE_SOURCE:breedinginsight.org} + vendors: + submission-enabled: ${BRAPI_VENDOR_SUBMISSION_ENABLED:false} + dart: + url: ${DART_VENDOR_URL:`https://test-server.brapi.org`} + client-id: ${DART_CLIENT_ID:potato-salad} + token: ${DART_TOKEN:YYYY} read-timeout: ${BRAPI_READ_TIMEOUT:10m} page-size: 1000 search: From 77c8a4c6909e8d3d0b423658b561709ddaff0382 Mon Sep 17 00:00:00 2001 From: timparsons Date: Thu, 2 Nov 2023 22:58:33 -0400 Subject: [PATCH 072/220] [BI-1910] Adding unit tests --- .env.template | 5 +- .../geno/SampleSubmissionController.java | 2 +- .../sample/SampleSubmissionImport.java | 8 +- .../importer/services/FileImportService.java | 2 +- .../processors/SampleSubmissionProcessor.java | 31 +- .../services/SampleSubmissionService.java | 30 +- src/main/resources/application.yml | 4 +- .../V1.16.0__create_sampleimport_mapping.sql | 4 +- ...leSubmissionControllerIntegrationTest.java | 502 +++++++++++++++ .../v2/ListControllerIntegrationTest.java | 2 +- .../importer/ExperimentFileImportTest.java | 71 +-- .../brapps/importer/ImportTestUtils.java | 2 +- .../SampleSubmissionFileImportTest.java | 592 ++++++++++++++++++ 13 files changed, 1176 insertions(+), 79 deletions(-) create mode 100644 src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java create mode 100644 src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java diff --git a/.env.template b/.env.template index 283b97128..0f100a7db 100644 --- a/.env.template +++ b/.env.template @@ -54,4 +54,7 @@ AWS_REGION= AWS_ACCESS_KEY_ID= AWS_SECRET_KEY= AWS_GENO_BUCKET= -AWS_S3_ENDPOINT= \ No newline at end of file +AWS_S3_ENDPOINT= + +BRAPI_VENDOR_SUBMISSION_ENABLED=false #can a submission be sent to a vendor via BrAPI +BRAPI_VENDOR_CHECK_FREQUENCY=1d #how often to check for vendor updates for sample submissions \ No newline at end of file diff --git a/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java b/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java index 60d39d7ec..43923c71a 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/geno/SampleSubmissionController.java @@ -63,7 +63,7 @@ public class SampleSubmissionController { private final Gson gson; @Inject - public SampleSubmissionController(@Property(name = "brapi.vendors.submission-enabled") boolean brapiSubmissionEnabled, SampleSubmissionService sampleSubmissionService, ProgramService programService, SecurityService securityService, UserService userService) { + public SampleSubmissionController(@Property(name = "brapi.vendor-submission-enabled") boolean brapiSubmissionEnabled, SampleSubmissionService sampleSubmissionService, ProgramService programService, SecurityService securityService, UserService userService) { this.brapiSubmissionEnabled = brapiSubmissionEnabled; this.sampleSubmissionService = sampleSubmissionService; this.programService = programService; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java index 242b5e4a7..65a6be3bc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java @@ -84,8 +84,8 @@ public class SampleSubmissionImport implements BrAPIImport { private String tissue; @ImportFieldType(type = ImportFieldTypeEnum.TEXT) - @ImportFieldMetadata(id = "comment", name = Columns.COMMENTS, description = "Generic comments about this sample for the vendor") - private String comment; + @ImportFieldMetadata(id = "comments", name = Columns.COMMENTS, description = "Generic comments about this sample for the vendor") + private String comments; @ImportFieldType(type= ImportFieldTypeEnum.TEXT, collectTime = ImportCollectTimeEnum.UPLOAD) @ImportMappingRequired @@ -99,7 +99,7 @@ public static final class Columns { public static final String ORGANISM = "Organism"; public static final String SPECIES = "Species"; public static final String GERMPLASM_NAME = "Germplasm Name"; - public static final String GERMPLASM_GID = "Germplasm GID"; + public static final String GERMPLASM_GID = "GID"; public static final String TISSUE = "Tissue"; public static final String COMMENTS = "Comments"; public static final String OBS_UNIT_ID = "ObsUnitID"; @@ -146,7 +146,7 @@ public BrAPISample constructBrAPISample(boolean commit, Program program, User us .row(row) .column(Integer.valueOf(column)) .tissueType(tissue) - .sampleDescription(comment); + .sampleDescription(comments); if (ou != null) { brAPISample diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java index 0d8f7d971..05c48601d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileImportService.java @@ -454,7 +454,7 @@ private void processFile(List finalBrAPIImportList, Table data, Pro progress.setUpdatedBy(actingUser.getId()); importDAO.update(upload); }catch (ValidatorException e) { - log.info("Validation errors", e); + log.info("Validation errors: \n" + e); ImportProgress progress = upload.getProgress(); progress.setStatuscode((short) HttpStatus.UNPROCESSABLE_ENTITY.getCode()); progress.setMessage("Multiple Errors"); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java index 531df2a09..d92ebb300 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java @@ -20,6 +20,7 @@ import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.HttpStatus; +import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; @@ -73,10 +74,6 @@ public class SampleSubmissionProcessor implements Processor { private final BrAPIGermplasmDAO germplasmDAO; private final BrAPIObservationUnitDAO observationUnitDAO; private final SampleSubmissionService sampleSubmissionService; - private final DSLContext dsl; - private final BrAPIPlateDAO plateDAO; - private final BrAPISampleDAO sampleDAO; - private SampleSubmission submission; private Map germplasmByGID = new HashMap<>(); private Map germplasmByDbId = new HashMap<>(); @@ -88,17 +85,11 @@ public class SampleSubmissionProcessor implements Processor { public SampleSubmissionProcessor(@Property(name = "brapi.server.reference-source") String referenceSource, BrAPIGermplasmDAO germplasmDAO, BrAPIObservationUnitDAO observationUnitDAO, - SampleSubmissionService sampleSubmissionService, - DSLContext dsl, - BrAPIPlateDAO plateDAO, - BrAPISampleDAO sampleDAO) { + SampleSubmissionService sampleSubmissionService) { this.referenceSource = referenceSource; this.germplasmDAO = germplasmDAO; this.observationUnitDAO = observationUnitDAO; this.sampleSubmissionService = sampleSubmissionService; - this.dsl = dsl; - this.plateDAO = plateDAO; - this.sampleDAO = sampleDAO; } @Override @@ -260,15 +251,21 @@ public void validateDependencies(Map mappedBrAPIImport) @Override public void postBrapiData(Map mappedBrAPIImport, Program program, ImportUpload upload) throws ValidatorException { - dsl.transaction(() -> { - List platesToSave = plateById.values().stream().map(PendingImportObject::getBrAPIObject).collect(Collectors.toList()); - List samplesToSave = mappedBrAPIImport.values().stream().map(row -> row.getSample().getBrAPIObject()).collect(Collectors.toList()); + List platesToSave = plateById.values().stream().map(PendingImportObject::getBrAPIObject).collect(Collectors.toList()); + List samplesToSave = mappedBrAPIImport.values().stream().map(row -> row.getSample().getBrAPIObject()).collect(Collectors.toList()); - submission.setPlates(platesToSave); - submission.setSamples(samplesToSave); + submission.setPlates(platesToSave); + submission.setSamples(samplesToSave); + try { sampleSubmissionService.createSubmission(submission, program, upload); - }); + } catch (ApiException e) { + log.error("Error saving sample submission import: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving sample submission import", e); + } catch (Exception e) { + log.error("Error saving sample submission import", e); + throw new InternalServerException(e.getMessage(), e); + } } @Override diff --git a/src/main/java/org/breedinginsight/services/SampleSubmissionService.java b/src/main/java/org/breedinginsight/services/SampleSubmissionService.java index 7c117e34f..2c629da55 100644 --- a/src/main/java/org/breedinginsight/services/SampleSubmissionService.java +++ b/src/main/java/org/breedinginsight/services/SampleSubmissionService.java @@ -67,10 +67,11 @@ public class SampleSubmissionService { private static final String VENDOR_NOT_SUBMITTED_STATUS = "NOT SUBMITTED"; private static final String VENDOR_SUBMITTED_STATUS = "SUBMITTED"; private final String referenceSource; - private String dartBrapiUrl; - private String dartClientId; - private String dartToken; - private Duration requestTimeout; + private final String dartBrapiUrl; + private final String dartClientId; + private final String dartToken; + private final Duration requestTimeout; + private final boolean brapiSubmissionEnabled; private final SampleSubmissionDAO submissionDAO; private final BrAPIPlateDAO plateDAO; @@ -85,6 +86,7 @@ public SampleSubmissionService(@Property(name = "brapi.server.reference-source") @Property(name = "brapi.vendors.dart.client-id") String dartClientId, @Property(name = "brapi.vendors.dart.token") String dartToken, @Value(value = "${brapi.read-timeout:5m}") Duration requestTimeout, + @Property(name = "brapi.vendor-submission-enabled") boolean brapiSubmissionEnabled, SampleSubmissionDAO submissionDAO, BrAPIPlateDAO plateDAO, BrAPISampleDAO sampleDAO, @@ -96,6 +98,7 @@ public SampleSubmissionService(@Property(name = "brapi.server.reference-source") this.dartClientId = dartClientId; this.dartToken = dartToken; this.requestTimeout = requestTimeout; + this.brapiSubmissionEnabled = brapiSubmissionEnabled; this.submissionDAO = submissionDAO; this.plateDAO = plateDAO; this.sampleDAO = sampleDAO; @@ -106,6 +109,9 @@ public SampleSubmissionService(@Property(name = "brapi.server.reference-source") @Scheduled(fixedDelay = "${brapi.vendor-check-frequency}", initialDelay = "10s") void checkSubmissionStatuses() { + if(!brapiSubmissionEnabled) { + return; + } log.trace("checking vendor order statuses"); List submittedAndNotCompleted = submissionDAO.getSubmittedAndNotComplete(); log.trace(submittedAndNotCompleted.size() + " orders to check"); @@ -127,15 +133,17 @@ public SampleSubmission createSubmission(SampleSubmission submission, Program pr submission.setUpdatedByUser(upload.getUpdatedByUser()); submission.setUpdatedBy(upload.getUpdatedBy()); - submissionDAO.insert(submission); + dsl.transaction(() -> { + submissionDAO.insert(submission); - List savedPlates = plateDAO.createPlates(program, submission.getPlates(), upload); - submission.setPlates(savedPlates); - Map plateNameToDbId = savedPlates.stream().collect(Collectors.toMap(BrAPIPlate::getPlateName, BrAPIPlate::getPlateDbId)); + List savedPlates = plateDAO.createPlates(program, submission.getPlates(), upload); + submission.setPlates(savedPlates); + Map plateNameToDbId = savedPlates.stream().collect(Collectors.toMap(BrAPIPlate::getPlateName, BrAPIPlate::getPlateDbId)); - List samplesToSave = submission.getSamples().stream().map(sample -> sample.plateDbId(plateNameToDbId.get(sample.getPlateName()))).collect(Collectors.toList()); - List savedSamples = sampleDAO.createSamples(program, samplesToSave, upload); - submission.setSamples(savedSamples); + List samplesToSave = submission.getSamples().stream().map(sample -> sample.plateDbId(plateNameToDbId.get(sample.getPlateName()))).collect(Collectors.toList()); + List savedSamples = sampleDAO.createSamples(program, samplesToSave, upload); + submission.setSamples(savedSamples); + }); return submission; } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 13e35457b..aec211b82 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -160,8 +160,9 @@ brapi: pheno-url: ${brapi.server.default-url} geno-url: ${brapi.server.default-url} reference-source: ${BRAPI_REFERENCE_SOURCE:breedinginsight.org} + vendor-submission-enabled: ${BRAPI_VENDOR_SUBMISSION_ENABLED:false} + vendor-check-frequency: ${BRAPI_VENDOR_CHECK_FREQUENCY:1d} vendors: - submission-enabled: ${BRAPI_VENDOR_SUBMISSION_ENABLED:false} dart: url: ${DART_VENDOR_URL:`https://test-server.brapi.org`} client-id: ${DART_CLIENT_ID:potato-salad} @@ -171,7 +172,6 @@ brapi: search: wait-time: 1000 post-group-size: ${POST_CHUNK_SIZE:1000} - vendor-check-frequency: ${VENDOR_CHECK_FREQUENCY:30s} email: relay-server: diff --git a/src/main/resources/db/migration/V1.16.0__create_sampleimport_mapping.sql b/src/main/resources/db/migration/V1.16.0__create_sampleimport_mapping.sql index eb88ec05d..50bac1903 100644 --- a/src/main/resources/db/migration/V1.16.0__create_sampleimport_mapping.sql +++ b/src/main/resources/db/migration/V1.16.0__create_sampleimport_mapping.sql @@ -93,9 +93,9 @@ $$ { "id": "de7fed3a-ec44-4139-83cb-c773e09237bd", "value": { - "fileFieldName": "Comment" + "fileFieldName": "Comments" }, - "objectId": "comment" + "objectId": "comments" } ]', '[]', false, diff --git a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java new file mode 100644 index 000000000..d41874783 --- /dev/null +++ b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java @@ -0,0 +1,502 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.api.v1.controller; + +import com.eclipsesource.json.Json; +import com.google.gson.*; +import com.google.gson.reflect.TypeToken; +import io.kowalski.fannypack.FannyPack; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.netty.cookies.NettyCookie; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import io.reactivex.Flowable; +import org.apache.commons.lang3.tuple.Pair; +import org.brapi.client.v2.BrAPIClient; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.geno.BrAPISample; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.breedinginsight.BrAPITest; +import org.breedinginsight.TestUtils; +import org.breedinginsight.api.model.v1.request.ProgramRequest; +import org.breedinginsight.api.model.v1.request.SpeciesRequest; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.ImportTestUtils; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport; +import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport.Columns; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.dao.db.tables.pojos.ProgramBreedingMethodEntity; +import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.SpeciesDAO; +import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.model.*; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.parsers.ParsingException; +import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; +import org.breedinginsight.services.writers.CSVWriter; +import org.breedinginsight.utilities.FileUtil; +import org.breedinginsight.utilities.Utilities; +import org.jetbrains.annotations.NotNull; +import org.jooq.DSLContext; +import org.junit.jupiter.api.*; +import org.testcontainers.containers.localstack.LocalStackContainer; +import tech.tablesaw.api.Table; + +import javax.inject.Inject; +import java.io.*; +import java.time.OffsetDateTime; +import java.util.*; + +import static io.micronaut.http.HttpRequest.*; +import static org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields.SUBMISSION_NAME; +import static org.junit.jupiter.api.Assertions.*; + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class SampleSubmissionControllerIntegrationTest extends BrAPITest { + + private Program program; + + private ImportTestUtils importTestUtils; + + private String experimentId; + private List envIds = new ArrayList<>(); + private final List> rows = new ArrayList<>(); + private final List columns = ExperimentFileColumns.getOrderedColumns(); + private List traits; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + @Inject + private DSLContext dsl; + @Inject + private UserDAO userDAO; + @Inject + private SpeciesDAO speciesDAO; + @Inject + private OntologyService ontologyService; + @Inject + private BrAPIGermplasmDAO germplasmDAO; + + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + +// private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) +// (json, type, context) -> OffsetDateTime.parse(json.getAsString())) +// .create(); + + private final Gson gson = new BrAPIClient().getJSON().getGson(); + + @BeforeAll + void setup() throws Exception { + importTestUtils = new ImportTestUtils(); + FannyPack fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); + FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); + + // Test User + User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + // Species + super.getBrapiDsl().execute(brapiFp.get("InsertSpecies")); + SpeciesEntity validSpecies = speciesDAO.findAll().get(0); + SpeciesRequest speciesRequest = SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); + + // Test Program + ProgramRequest programRequest = ProgramRequest.builder() + .name("Test Program") + .abbreviation("Test") + .documentationUrl("localhost:8080") + .objective("To test things") + .species(speciesRequest) + .key("TEST") + .build(); + program = TestUtils.insertAndFetchTestProgram(gson, client, programRequest); + + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId()); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + List germplasm = createGermplasm(96); + BrAPIExternalReference newReference = new BrAPIExternalReference(); + newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); + newReference.setReferenceID(program.getId().toString()); + + germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); + + germplasmDAO.createBrAPIGermplasm(germplasm, program.getId(), null); + } + + @NotNull + @Override + public Map getProperties() { + Map properties = super.getProperties(); + + properties.put("brapi.vendor-submission-enabled", "true"); + + Integer containerPort = getBrapiContainer().getMappedPort(8080); + String containerIp = getBrapiContainer().getContainerIpAddress(); + properties.put("brapi.vendors.dart.url", String.format("http://%s:%s/", containerIp, containerPort)); + + return properties; + } + + /* + Tests + - fetch all sample submissions + - fetch individual sample submission + - manual update submission status + - brapi submit + - check vendor status + + */ + + @Test + public void testFetchProgramSubmissions() throws IOException, InterruptedException { + List submissionIds = new ArrayList<>(); + submissionIds.add(createSubmission(program).getLeft().getId()); + submissionIds.add(createSubmission(program).getLeft().getId()); + + Flowable> call = client.exchange( + GET(String.format("/programs/%s/submissions", program.getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + List submissions = gson.fromJson(JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").getAsJsonArray("data"), new TypeToken>(){}.getType()); + assertTrue(submissions.size() >= 2); + + + List returnedSubmissionIds = new ArrayList<>(); + for(var submission : submissions) { + if(submissionIds.contains(submission.getId())) { + returnedSubmissionIds.add(submission.getId()); + } + } + assertEquals(submissionIds.size(), returnedSubmissionIds.size()); + } + + @Test + public void testFetchIndividualSubmissions() throws IOException, InterruptedException { + Pair>> uploadedSubmission = createSubmission(program); + + Flowable> call = client.exchange( + GET(String.format("/programs/%s/submissions/%s?details=true", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + SampleSubmission retrievedSubmission = gson.fromJson(JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result"), SampleSubmission.class); + + assertNotNull(retrievedSubmission); + assertEquals(uploadedSubmission.getLeft().getId(), retrievedSubmission.getId()); + assertEquals(uploadedSubmission.getLeft().getName(), retrievedSubmission.getName()); + assertEquals(1, retrievedSubmission.getPlates().size()); + assertEquals(96, retrievedSubmission.getSamples().size()); + } + + @Test + public void testManualUpdateSubmissions() throws IOException, InterruptedException { + Pair>> uploadedSubmission = createSubmission(program); + + Flowable> putCall = client.exchange( + PUT(String.format("/programs/%s/submissions/%s/status", program.getId(), uploadedSubmission.getLeft().getId()), "{status:\"SUBMITTED\"}").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse putResponse = putCall.blockingFirst(); + assertNotNull(putResponse.body()); + + Flowable> fetchCall = client.exchange( + GET(String.format("/programs/%s/submissions/%s?details=true", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse fetchResponse = fetchCall.blockingFirst(); + SampleSubmission retrievedSubmission = gson.fromJson(JsonParser.parseString(fetchResponse.body()).getAsJsonObject().getAsJsonObject("result"), SampleSubmission.class); + + assertNotNull(retrievedSubmission); + assertEquals("SUBMITTED", retrievedSubmission.getVendorStatus()); + assertNull(retrievedSubmission.getVendorOrderId()); + assertNull(retrievedSubmission.getVendorStatusLastCheck()); + } + + @Test + public void testSubmitViaBrAPI() throws IOException, InterruptedException { + Pair>> uploadedSubmission = createSubmission(program); + + Flowable> postCall = client.exchange( + POST(String.format("/programs/%s/submissions/%s/submit?vendor=dart", program.getId(), uploadedSubmission.getLeft().getId()), null).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse postResponse = postCall.blockingFirst(); + assertNotNull(postResponse.body()); + + Flowable> fetchCall = client.exchange( + GET(String.format("/programs/%s/submissions/%s?details=true", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse fetchResponse = fetchCall.blockingFirst(); + SampleSubmission retrievedSubmission = gson.fromJson(JsonParser.parseString(fetchResponse.body()).getAsJsonObject().getAsJsonObject("result"), SampleSubmission.class); + + assertNotNull(retrievedSubmission); + assertEquals("SUBMITTED", retrievedSubmission.getVendorStatus()); + assertNotNull(retrievedSubmission.getVendorOrderId()); + assertNull(retrievedSubmission.getVendorStatusLastCheck()); + } + + @Test + public void testCheckVendorStatus() throws IOException, InterruptedException { + Pair>> uploadedSubmission = createSubmission(program); + + Flowable> postCall = client.exchange( + POST(String.format("/programs/%s/submissions/%s/submit?vendor=dart", program.getId(), uploadedSubmission.getLeft().getId()), null).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse postResponse = postCall.blockingFirst(); + assertNotNull(postResponse.body()); + + Flowable> fetchCall = client.exchange( + GET(String.format("/programs/%s/submissions/%s?details=false", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse fetchResponse = fetchCall.blockingFirst(); + SampleSubmission retrievedSubmission = gson.fromJson(JsonParser.parseString(fetchResponse.body()).getAsJsonObject().getAsJsonObject("result"), SampleSubmission.class); + + assertNotNull(retrievedSubmission); + assertNotNull(retrievedSubmission.getVendorOrderId()); + assertNull(retrievedSubmission.getVendorStatusLastCheck()); + + Flowable> fetchStatus = client.exchange( + GET(String.format("/programs/%s/submissions/%s/status", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse fetchStatusResponse = fetchStatus.blockingFirst(); + assertNotNull(fetchStatusResponse.body()); + + Flowable> fetchSubmissionAfterStatus = client.exchange( + GET(String.format("/programs/%s/submissions/%s?details=false", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse fetchSubmissionAfterStatusResponse = fetchSubmissionAfterStatus.blockingFirst(); + SampleSubmission updatedStatusResponse = gson.fromJson(JsonParser.parseString(fetchSubmissionAfterStatusResponse.body()).getAsJsonObject().getAsJsonObject("result"), SampleSubmission.class); + assertNotNull(updatedStatusResponse.getVendorStatusLastCheck()); + + Thread.sleep(1000); + + fetchStatus = client.exchange( + GET(String.format("/programs/%s/submissions/%s/status", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + fetchStatusResponse = fetchStatus.blockingFirst(); + assertNotNull(fetchStatusResponse.body()); + + fetchSubmissionAfterStatus = client.exchange( + GET(String.format("/programs/%s/submissions/%s?details=false", program.getId(), uploadedSubmission.getLeft().getId())).cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + fetchSubmissionAfterStatusResponse = fetchSubmissionAfterStatus.blockingFirst(); + SampleSubmission updatedStatusResponse2 = gson.fromJson(JsonParser.parseString(fetchSubmissionAfterStatusResponse.body()).getAsJsonObject().getAsJsonObject("result"), SampleSubmission.class); + assertTrue(updatedStatusResponse2.getVendorStatusLastCheck().isAfter(updatedStatusResponse.getVendorStatusLastCheck())); + } + + @Test + public void testGenerateDArTFile() throws IOException, InterruptedException, ParsingException { + Pair>> uploadedSubmission = createSubmission(program); + + Flowable> call = client.exchange( + GET(String.format("/programs/%s/submissions/%s/dart", + program.getId().toString(), uploadedSubmission.getLeft().getId())) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + HttpResponse response = call.blockingFirst(); + + assertEquals(HttpStatus.OK, response.getStatus()); + + ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(response.body())); + Table lookupTable = FileUtil.parseTableFromCsv(bodyStream); + assertEquals(8, lookupTable.columnCount()); + assertEquals(Columns.PLATE_ID, lookupTable.column(0).name()); + assertEquals(Columns.ROW, lookupTable.column(1).name()); + assertEquals(Columns.COLUMN, lookupTable.column(2).name()); + assertEquals(Columns.ORGANISM, lookupTable.column(3).name()); + assertEquals(Columns.SPECIES, lookupTable.column(4).name()); + assertEquals("Genotype", lookupTable.column(5).name()); + assertEquals(Columns.TISSUE, lookupTable.column(6).name()); + assertEquals(Columns.COMMENTS, lookupTable.column(7).name()); + } + + @Test + public void testGenerateLookupFile() throws IOException, InterruptedException, ParsingException { + Pair>> uploadedSubmission = createSubmission(program); + + Flowable> call = client.exchange( + GET(String.format("/programs/%s/submissions/%s/lookup", + program.getId().toString(), uploadedSubmission.getLeft().getId())) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), byte[].class + ); + HttpResponse response = call.blockingFirst(); + + assertEquals(HttpStatus.OK, response.getStatus()); + + ByteArrayInputStream bodyStream = new ByteArrayInputStream(Objects.requireNonNull(response.body())); + Table lookupTable = FileUtil.parseTableFromCsv(bodyStream); + assertEquals(3, lookupTable.columnCount()); + assertEquals("Genotype", lookupTable.column(0).name()); + assertEquals("Germplasm Name", lookupTable.column(1).name()); + assertEquals("GID", lookupTable.column(2).name()); + } + + + private Pair>> createSubmission(Program program) throws IOException, InterruptedException { + Flowable> call = client.exchange( + GET("/import/mappings?importName=SampleImport").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + String sampleMappingId = JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + var submissionData = makeSubmission(); + var submission = new SampleSubmission(); + submission.setName("test-"+UUID.randomUUID()); + + JsonObject importResult = importTestUtils.uploadAndFetch( + writeSubmissionToFile(submissionData), + Map.of(SUBMISSION_NAME, submission.getName()), + true, + client, + program, + sampleMappingId); + JsonArray previewRows = importResult.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(96, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + BrAPISample sample = new Gson().fromJson(row.getAsJsonObject("sample").getAsJsonObject("brAPIObject"), BrAPISample.class); + BrAPIExternalReference xref = Utilities.getExternalReference(sample.getExternalReferences(), Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.PLATE_SUBMISSIONS)).get(); + submission.setId(UUID.fromString(xref.getReferenceId())); + + return Pair.of(submission, submissionData); + } + + private List> makeSubmission() { + List> validFile = new ArrayList<>(); + int germGidCounter = 1; + for(int i = 0; i < 8; i++) { + for(int j = 0; j < 12; j++) { + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A' + i)); + validRow.put(Columns.COLUMN, j+1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, germGidCounter++); + validRow.put(Columns.OBS_UNIT_ID, ""); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + } + } + + return validFile; + } + + private String createExperiment(Program program) throws IOException, InterruptedException { + Flowable> call = client.exchange( + GET("/import/mappings?importName=ExperimentsTemplateMap").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + String expMappingId = JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + JsonObject importResult = importTestUtils.uploadAndFetch( + importTestUtils.writeExperimentDataToFile(List.of(makeExpImportRow("Env1")), null), + null, + true, + client, + program, + expMappingId); + return importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + } + + private Map makeExpImportRow(String environment) { + Map row = new HashMap<>(); + row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); + row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); + row.put(ExperimentObservation.Columns.EXP_TITLE, "Test Exp"); + row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); + row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); + row.put(ExperimentObservation.Columns.ENV, environment); + row.put(ExperimentObservation.Columns.ENV_LOCATION, "Location A"); + row.put(ExperimentObservation.Columns.ENV_YEAR, "2023"); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, "a-1"); + row.put(ExperimentObservation.Columns.REP_NUM, "1"); + row.put(ExperimentObservation.Columns.BLOCK_NUM, "1"); + row.put(ExperimentObservation.Columns.ROW, "1"); + row.put(ExperimentObservation.Columns.COLUMN, "1"); + return row; + } + + public File writeSubmissionToFile(List> data) throws IOException { + File file = File.createTempFile("test", ".csv"); + + List columns = new ArrayList<>(); + columns.add(Column.builder().value(Columns.PLATE_ID).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.ROW).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.COLUMN).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.ORGANISM).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.SPECIES).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.GERMPLASM_NAME).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.GERMPLASM_GID).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.OBS_UNIT_ID).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.TISSUE).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.COMMENTS).dataType(Column.ColumnDataType.STRING).build()); + + ByteArrayOutputStream byteArrayOutputStream = CSVWriter.writeToCSV(columns, data); + FileOutputStream fos = new FileOutputStream(file); + fos.write(byteArrayOutputStream.toByteArray()); + + return file; + } + + private List createGermplasm(int numToCreate) { + List germplasm = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String gid = ""+(i+1); + BrAPIGermplasm testGermplasm = new BrAPIGermplasm(); + testGermplasm.setGermplasmName(String.format("Germplasm %s [TEST-%s]", gid, gid)); + testGermplasm.setSeedSource("Wild"); + testGermplasm.setAccessionNumber(gid); + testGermplasm.setDefaultDisplayName(String.format("Germplasm %s", gid)); + JsonObject additionalInfo = new JsonObject(); + additionalInfo.addProperty("importEntryNumber", gid); + additionalInfo.addProperty("breedingMethod", "Allopolyploid"); + testGermplasm.setAdditionalInfo(additionalInfo); + List externalRef = new ArrayList<>(); + BrAPIExternalReference testReference = new BrAPIExternalReference(); + testReference.setReferenceSource(BRAPI_REFERENCE_SOURCE); + testReference.setReferenceID(UUID.randomUUID().toString()); + externalRef.add(testReference); + testGermplasm.setExternalReferences(externalRef); + germplasm.add(testGermplasm); + } + + return germplasm; + } +} diff --git a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java index ccdf04e19..fdabb53db 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ListControllerIntegrationTest.java @@ -142,7 +142,7 @@ public void setup() { newExp.put(traits.get(0).getObservationVariableName(), "1"); JsonObject result = importTestUtils.uploadAndFetch( - importTestUtils.writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId + importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId ); } diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index ac8520a68..748299daf 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -18,7 +18,6 @@ package org.breedinginsight.brapps.importer; import com.google.gson.*; -import com.google.gson.reflect.TypeToken; import io.kowalski.fannypack.FannyPack; import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpResponse; @@ -50,7 +49,6 @@ import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.dao.db.tables.pojos.BiUserEntity; import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; import org.breedinginsight.daos.ProgramDAO; @@ -64,7 +62,6 @@ import org.breedinginsight.services.exceptions.BadRequestException; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.ValidatorException; -import org.breedinginsight.services.writers.CSVWriter; import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; import org.junit.jupiter.api.*; @@ -74,9 +71,7 @@ import org.opentest4j.AssertionFailedError; import javax.inject.Inject; -import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; import java.time.OffsetDateTime; import java.util.*; @@ -199,7 +194,7 @@ public void importNewExpNewLocNoObsSuccess() { validRow.put(Columns.COLUMN, "1"); validRow.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeDataToFile(List.of(validRow), null), null, true, client, program, mappingId); + Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(validRow), null), null, true, client, program, mappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); @@ -257,7 +252,7 @@ public void importNewExpMultiNewEnvNoObsSuccess() { secondEnv.put(Columns.COLUMN, "1"); secondEnv.put(Columns.TREATMENT_FACTORS, "Test treatment factors"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); + Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(firstEnv, secondEnv), null), null, true, client, program, mappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); @@ -304,7 +299,7 @@ public void importNewEnvExistingExpNoObsSuccess() { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); Map newEnv = new HashMap<>(); newEnv.put(Columns.GERMPLASM_GID, "1"); @@ -321,7 +316,7 @@ public void importNewEnvExistingExpNoObsSuccess() { newEnv.put(Columns.ROW, "1"); newEnv.put(Columns.COLUMN, "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -355,43 +350,43 @@ public void verifyMissingDataThrowsError(boolean commit) { Map noGID = new HashMap<>(base); noGID.remove(Columns.GERMPLASM_GID); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noGID), null), Columns.GERMPLASM_GID, commit); Map noExpTitle = new HashMap<>(base); noExpTitle.remove(Columns.EXP_TITLE); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpTitle), null), Columns.EXP_TITLE, commit); Map noExpUnit = new HashMap<>(base); noExpUnit.remove(Columns.EXP_UNIT); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnit), null), Columns.EXP_UNIT, commit); Map noExpType = new HashMap<>(base); noExpType.remove(Columns.EXP_TYPE); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpType), null), Columns.EXP_TYPE, commit); Map noEnv = new HashMap<>(base); noEnv.remove(Columns.ENV); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noEnv), null), Columns.ENV, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnv), null), Columns.ENV, commit); Map noEnvLoc = new HashMap<>(base); noEnvLoc.remove(Columns.ENV_LOCATION); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvLoc), null), Columns.ENV_LOCATION, commit); Map noExpUnitId = new HashMap<>(base); noExpUnitId.remove(Columns.EXP_UNIT_ID); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpUnitId), null), Columns.EXP_UNIT_ID, commit); Map noExpRep = new HashMap<>(base); noExpRep.remove(Columns.REP_NUM); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpRep), null), Columns.REP_NUM, commit); Map noExpBlock = new HashMap<>(base); noExpBlock.remove(Columns.BLOCK_NUM); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noExpBlock), null), Columns.BLOCK_NUM, commit); Map noEnvYear = new HashMap<>(base); noEnvYear.remove(Columns.ENV_YEAR); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(noEnvYear), null), Columns.ENV_YEAR, commit); } @Test @@ -415,7 +410,7 @@ public void importNewExpWithObsVar() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -465,7 +460,7 @@ public void verifyDiffYearSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(rows, null), Columns.ENV_YEAR, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_YEAR, commit); } @ParameterizedTest @@ -503,7 +498,7 @@ public void verifyDiffLocSameEnvThrowsError(boolean commit) { row.put(Columns.BLOCK_NUM, "2"); rows.add(row); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(rows, null), Columns.ENV_LOCATION, commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(rows, null), Columns.ENV_LOCATION, commit); } @ParameterizedTest @@ -528,7 +523,7 @@ public void importNewExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, commit, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -568,7 +563,7 @@ public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "Red"); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); } @ParameterizedTest @@ -591,14 +586,14 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); Map newOU = new HashMap<>(newExp); newOU.put(Columns.EXP_UNIT_ID, "a-2"); newOU.put(Columns.ROW, "1"); newOU.put(Columns.COLUMN, "2"); - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); + Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); @@ -632,7 +627,7 @@ public void importNewObsVarExisingOu() { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), null); - importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -660,7 +655,7 @@ public void importNewObsVarExisingOu() { newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); newObsVar.put(traits.get(1).getObservationVariableName(), null); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -697,7 +692,7 @@ public void importNewObsExisingOu(boolean commit) { newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -725,7 +720,7 @@ public void importNewObsExisingOu(boolean commit) { newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -764,7 +759,7 @@ public void verifyFailureImportNewObsExisingOuWithExistingObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -792,7 +787,7 @@ public void verifyFailureImportNewObsExisingOuWithExistingObs(boolean commit) { newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); newObservation.put(traits.get(0).getObservationVariableName(), "2"); - uploadAndVerifyFailure(program, importTestUtils.writeDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName(), commit); + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName(), commit); } /* @@ -822,7 +817,7 @@ public void importSecondExpAfterFirstExpWithObs() { newExpA.put(Columns.COLUMN, "1"); newExpA.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); + JsonObject resultA = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpA), traits), null, true, client, program, mappingId); JsonArray previewRowsA = resultA.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsA.size()); @@ -850,7 +845,7 @@ public void importSecondExpAfterFirstExpWithObs() { newExpB.put(Columns.COLUMN, "1"); newExpB.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); + JsonObject resultB = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExpB), traits), null, true, client, program, mappingId); JsonArray previewRowsB = resultB.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRowsB.size()); @@ -891,7 +886,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -920,7 +915,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "2"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -965,7 +960,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newExp.put(Columns.COLUMN, "1"); newExp.put(traits.get(0).getObservationVariableName(), "1"); - importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), null, true, client, program, mappingId); BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); @@ -996,7 +991,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newObservation.put(traits.get(0).getObservationVariableName(), ""); newObservation.put(traits.get(1).getObservationVariableName(), "2"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index b8b00374d..3a521aa55 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -196,7 +196,7 @@ public List createTraits(int numToCreate) { return traits; } - public File writeDataToFile(List> data, List traits) throws IOException { + public File writeExperimentDataToFile(List> data, List traits) throws IOException { File file = File.createTempFile("test", ".csv"); List columns = new ArrayList<>(); diff --git a/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java new file mode 100644 index 000000000..aa446cc55 --- /dev/null +++ b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java @@ -0,0 +1,592 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer; + +import com.google.gson.*; +import io.kowalski.fannypack.FannyPack; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.netty.cookies.NettyCookie; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import io.reactivex.Flowable; +import lombok.SneakyThrows; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.typeAdapters.PaginationTypeAdapter; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.BrAPIPagination; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.geno.BrAPIPlate; +import org.brapi.v2.model.geno.BrAPISample; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.BrAPITest; +import org.breedinginsight.TestUtils; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramRequest; +import org.breedinginsight.api.model.v1.request.SpeciesRequest; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.daos.*; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport.Columns; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.dao.db.tables.pojos.BiUserEntity; +import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.SpeciesDAO; +import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.model.Column; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.SampleSubmission; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.*; +import org.breedinginsight.services.exceptions.BadRequestException; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.services.writers.CSVWriter; +import org.breedinginsight.utilities.Utilities; +import org.jooq.DSLContext; +import org.junit.jupiter.api.*; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.junit.platform.commons.util.StringUtils; + +import javax.inject.Inject; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.*; + +import static io.micronaut.http.HttpRequest.GET; +import static org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields.SUBMISSION_NAME; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class SampleSubmissionFileImportTest extends BrAPITest { + + private FannyPack securityFp; + private String mappingId; + private BiUserEntity testUser; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + @Property(name = "brapi.server.core-url") + private String BRAPI_URL; + + @Inject + private SpeciesService speciesService; + @Inject + private UserDAO userDAO; + @Inject + private DSLContext dsl; + + @Inject + private SpeciesDAO speciesDAO; + + @Inject + private ProgramService programService; + + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + + private ImportTestUtils importTestUtils; + + @Inject + private OntologyService ontologyService; + + @Inject + private BrAPITrialDAO brAPITrialDAO; + + @Inject + private BrAPIStudyDAO brAPIStudyDAO; + + @Inject + private BrAPIObservationUnitDAO ouDAO; + + @Inject + private ProgramLocationService locationService; + + @Inject + private BrAPIGermplasmDAO germplasmDAO; + + @Inject + private BrAPIObservationDAO observationDAO; + + @Inject + private BrAPISeasonDAO seasonDAO; + + @Inject + private SampleSubmissionService sampleSubmissionService; + + private Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) + (json, type, context) -> OffsetDateTime.parse(json.getAsString())) + .registerTypeAdapter(BrAPIPagination.class, new PaginationTypeAdapter()) + .create(); + + @BeforeAll + public void setup() { + importTestUtils = new ImportTestUtils(); + Map setupObjects = importTestUtils.setup(client, gson, dsl, speciesService, userDAO, super.getBrapiDsl(), "SampleImport"); + mappingId = (String) setupObjects.get("mappingId"); + testUser = (BiUserEntity) setupObjects.get("testUser"); + securityFp = (FannyPack) setupObjects.get("securityFp"); + + } + + /* + Tests + - valid submission GID + - valid submission ObsUnitID + - missing columns error + - conflicting wells error + - bad GID error + - bad ObsUnitID error + */ + + @Test + @SneakyThrows + public void importGIDSuccess() { + Program program = createProgram("Import GID Success", "GIDS", "GIDS", BRAPI_REFERENCE_SOURCE, createGermplasm(96), null); + List> validFile = new ArrayList<>(); + + int germGidCounter = 1; + for(int i = 0; i < 8; i++) { + for(int j = 0; j < 12; j++) { + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A' + i)); + validRow.put(Columns.COLUMN, j+1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, germGidCounter++); + validRow.put(Columns.OBS_UNIT_ID, ""); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + } + } + + Flowable> call = importTestUtils.uploadDataFile(writeDataToFile(validFile), Map.of(SUBMISSION_NAME, "test-"+UUID.randomUUID()), true, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(96, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("NEW", row.getAsJsonObject("plate").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("sample").get("state").getAsString()); + + BrAPISample sample = new Gson().fromJson(row.getAsJsonObject("sample").getAsJsonObject("brAPIObject"), BrAPISample.class); + BrAPIExternalReference xref = Utilities.getExternalReference(sample.getExternalReferences(), Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.PLATE_SUBMISSIONS)).get(); + + assertFileSaved(validFile, program, UUID.fromString(xref.getReferenceId())); + } + + @Test + @SneakyThrows + public void importObsUnitIdSuccess() { + Program program = createProgram("Import ObsUnitID success", "OBSID", "OBSID", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + + var experimentId = createExperiment(program); + + BrAPITrial trial = brAPITrialDAO.getTrialById(program.getId(), UUID.fromString(experimentId)).get(); + + List ous = ouDAO.getObservationUnitsForTrialDbId(program.getId(), trial.getTrialDbId()); + BrAPIExternalReference obsUnitId = Utilities.getExternalReference(ous.get(0).getExternalReferences(), Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS)).get(); + + List> validFile = new ArrayList<>(); + + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, "A"); + validRow.put(Columns.COLUMN, 1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, ""); + validRow.put(Columns.OBS_UNIT_ID, obsUnitId.getReferenceId()); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + + Flowable> call = importTestUtils.uploadDataFile(writeDataToFile(validFile), Map.of(SUBMISSION_NAME, "test-"+UUID.randomUUID()), true, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(200, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("NEW", row.getAsJsonObject("plate").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("sample").get("state").getAsString()); + + BrAPISample sample = new Gson().fromJson(row.getAsJsonObject("sample").getAsJsonObject("brAPIObject"), BrAPISample.class); + BrAPIExternalReference xref = Utilities.getExternalReference(sample.getExternalReferences(), Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.PLATE_SUBMISSIONS)).get(); + + assertFileSaved(validFile, program, UUID.fromString(xref.getReferenceId())); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void importMissingGIDAndObsUnitIdFailure(boolean commit) { + Program program = createProgram("Missing GID/ObsUnit ID " + (commit ? "C" : "P"), "MGIOB"+ (commit ? "C" : "P"), "MGIOB"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, null, null); + List> validFile = new ArrayList<>(); + + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A')); + validRow.put(Columns.COLUMN, 1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, ""); + validRow.put(Columns.OBS_UNIT_ID, ""); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + + uploadAndVerifyFailure(program, writeDataToFile(validFile), Columns.GERMPLASM_GID, commit); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void verifyMissingDataThrowsError(boolean commit) { + Program program = createProgram("Missing Req Cols "+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), "MISS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(96), null); + Map base = new HashMap<>(); + base.put(Columns.PLATE_ID, "valid_1"); + base.put(Columns.ROW, "A"); + base.put(Columns.COLUMN, 1); + base.put(Columns.ORGANISM, "TEST"); + base.put(Columns.SPECIES, "TEST"); + base.put(Columns.GERMPLASM_NAME, ""); + base.put(Columns.GERMPLASM_GID, "1"); + base.put(Columns.OBS_UNIT_ID, ""); + base.put(Columns.TISSUE, "TEST"); + base.put(Columns.COMMENTS, "Test sample"); + + createUploadAndVerifyFailure(program, base, Columns.PLATE_ID, commit); + createUploadAndVerifyFailure(program, base, Columns.ROW, commit); + createUploadAndVerifyFailure(program, base, Columns.COLUMN, commit); + createUploadAndVerifyFailure(program, base, Columns.ORGANISM, commit); + createUploadAndVerifyFailure(program, base, Columns.TISSUE, commit); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void importInvalidGIDFailure(boolean commit) { + Program program = createProgram("Invalid GID " + (commit ? "C" : "P"), "INGID"+ (commit ? "C" : "P"), "INGID"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, null, null); + List> validFile = new ArrayList<>(); + + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A')); + validRow.put(Columns.COLUMN, 1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, ""); + validRow.put(Columns.OBS_UNIT_ID, ""); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + + uploadAndVerifyFailure(program, writeDataToFile(validFile), Columns.GERMPLASM_GID, commit); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void importInvalidObsUnitIdFailure(boolean commit) { + Program program = createProgram("Invalid ObsUnit ID " + (commit ? "C" : "P"), "INOBS"+ (commit ? "C" : "P"), "INOBS"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, null, null); + List> validFile = new ArrayList<>(); + + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A')); + validRow.put(Columns.COLUMN, 1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, ""); + validRow.put(Columns.OBS_UNIT_ID, "hgfhgfhg"); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + + uploadAndVerifyFailure(program, writeDataToFile(validFile), Columns.OBS_UNIT_ID, commit); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void importConflictingWellsFailure(boolean commit) { + Program program = createProgram("Conflicting Wells " + (commit ? "C" : "P"), "WELL"+ (commit ? "C" : "P"), "WELL"+ (commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(2), null); + List> validFile = new ArrayList<>(); + + Map validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A')); + validRow.put(Columns.COLUMN, 1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, 1); + validRow.put(Columns.OBS_UNIT_ID, ""); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + validRow = new HashMap<>(); + validRow.put(Columns.PLATE_ID, "valid_1"); + validRow.put(Columns.ROW, Character.toString('A')); + validRow.put(Columns.COLUMN, 1); + validRow.put(Columns.ORGANISM, "TEST"); + validRow.put(Columns.SPECIES, "TEST"); + validRow.put(Columns.GERMPLASM_NAME, ""); + validRow.put(Columns.GERMPLASM_GID, 2); + validRow.put(Columns.OBS_UNIT_ID, ""); + validRow.put(Columns.TISSUE, "TEST"); + validRow.put(Columns.COMMENTS, "Test sample"); + validFile.add(validRow); + + uploadAndVerifyFailure(program, writeDataToFile(validFile), Columns.ROW + "/" + Columns.COLUMN, commit); + } + + private void assertFileSaved(List> validFile, Program program, UUID submissionId) throws ApiException { + Optional submission = sampleSubmissionService.getSampleSubmission(program, submissionId, true); + assertTrue(submission.isPresent(), "Could not find sample submission by ID: " + submissionId); + for(var row : validFile) { + assertRowSaved(row, program, submission.get()); + } + } + + private Map assertRowSaved(Map expected, Program program, SampleSubmission submission) { + Map ret = new HashMap<>(); + + Optional plate = submission.getPlates().stream().filter(p -> p.getPlateName().equals(expected.get(Columns.PLATE_ID))).findFirst(); + assertTrue(plate.isPresent(), "plate not found"); + + Optional sample = submission.getSamples().stream().filter(s -> s.getPlateName().equals(expected.get(Columns.PLATE_ID)) + && s.getRow().equals(expected.get(Columns.ROW)) + && s.getColumn().equals(expected.get(Columns.COLUMN))) + .findFirst(); + assertTrue(sample.isPresent(), String.format("sample %s%s not found", expected.get(Columns.ROW), expected.get(Columns.COLUMN))); + + assertEquals(expected.get(Columns.ORGANISM), sample.get().getAdditionalInfo().get(BrAPIAdditionalInfoFields.SAMPLE_ORGANISM).getAsString()); + assertEquals(expected.get(Columns.SPECIES), sample.get().getAdditionalInfo().get(BrAPIAdditionalInfoFields.SAMPLE_SPECIES).getAsString()); + assertTrue(sample.get().getAdditionalInfo().has(BrAPIAdditionalInfoFields.GERMPLASM_NAME)); + assertEquals(expected.get(Columns.TISSUE), sample.get().getTissueType()); + assertEquals(expected.get(Columns.COMMENTS), sample.get().getSampleDescription()); + + if(StringUtils.isNotBlank((String)expected.get(Columns.OBS_UNIT_ID))) { + assertTrue(sample.get().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBS_UNIT_ID)); + assertEquals(expected.get(Columns.OBS_UNIT_ID), sample.get().getAdditionalInfo().get(BrAPIAdditionalInfoFields.OBS_UNIT_ID).getAsString()); + } else { + assertEquals(String.valueOf(expected.get(Columns.GERMPLASM_GID)), sample.get().getAdditionalInfo().get(BrAPIAdditionalInfoFields.GID).getAsString()); + } + + return ret; + } + + private Program createProgram(String name, String abbv, String key, String referenceSource, List germplasm, List traits) throws ApiException, DoesNotExistException, ValidatorException, BadRequestException { + SpeciesEntity validSpecies = speciesDAO.findAll().get(0); + SpeciesRequest speciesRequest = SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); + ProgramRequest programRequest1 = ProgramRequest.builder() + .name(name) + .abbreviation(abbv) + .documentationUrl("localhost:8080") + .objective("To test things") + .species(speciesRequest) + .key(key) + .build(); + + + TestUtils.insertAndFetchTestProgram(gson, client, programRequest1); + + // Get main program + Program program = programService.getByKey(key).get(); + + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId().toString()); + + if(germplasm != null && !germplasm.isEmpty()) { + BrAPIExternalReference newReference = new BrAPIExternalReference(); + newReference.setReferenceSource(String.format("%s/programs", referenceSource)); + newReference.setReferenceID(program.getId().toString()); + + germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); + + germplasmDAO.createBrAPIGermplasm(germplasm, program.getId(), null); + } + + if(traits != null && !traits.isEmpty()) { + AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); + try { + ontologyService.createTraits(program.getId(), traits, user, false); + } catch (ValidatorException e) { + System.err.println(e.getErrors()); + throw e; + } + } + + return program; + } + + private List createGermplasm(int numToCreate) { + List germplasm = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String gid = ""+(i+1); + BrAPIGermplasm testGermplasm = new BrAPIGermplasm(); + testGermplasm.setGermplasmName(String.format("Germplasm %s [TEST-%s]", gid, gid)); + testGermplasm.setSeedSource("Wild"); + testGermplasm.setAccessionNumber(gid); + testGermplasm.setDefaultDisplayName(String.format("Germplasm %s", gid)); + JsonObject additionalInfo = new JsonObject(); + additionalInfo.addProperty("importEntryNumber", gid); + additionalInfo.addProperty("breedingMethod", "Allopolyploid"); + testGermplasm.setAdditionalInfo(additionalInfo); + List externalRef = new ArrayList<>(); + BrAPIExternalReference testReference = new BrAPIExternalReference(); + testReference.setReferenceSource(BRAPI_REFERENCE_SOURCE); + testReference.setReferenceID(UUID.randomUUID().toString()); + externalRef.add(testReference); + testGermplasm.setExternalReferences(externalRef); + germplasm.add(testGermplasm); + } + + return germplasm; + } + + private void createUploadAndVerifyFailure(Program program, Map base, String columnToRemove, boolean commit) throws IOException, InterruptedException { + Map invalidRow = new HashMap<>(base); + invalidRow.remove(columnToRemove); + uploadAndVerifyFailure(program, writeDataToFile(List.of(invalidRow)), columnToRemove, commit); + } + + private JsonObject uploadAndVerifyFailure(Program program, File file, String expectedColumnError, boolean commit) throws InterruptedException, IOException { + Flowable> call = importTestUtils.uploadDataFile(file, Map.of(SUBMISSION_NAME, "test-"+UUID.randomUUID()), true, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + JsonArray rowErrors = result.getAsJsonObject("progress").getAsJsonArray("rowErrors"); + assertEquals(1, rowErrors.size()); + JsonArray fieldErrors = rowErrors.get(0).getAsJsonObject().getAsJsonArray("errors"); + assertEquals(1, fieldErrors.size()); + JsonObject error = fieldErrors.get(0).getAsJsonObject(); + assertEquals(expectedColumnError, error.get("field").getAsString()); + assertEquals(422, error.get("httpStatusCode").getAsInt()); + + return result; + } + + public File writeDataToFile(List> data) throws IOException { + File file = File.createTempFile("test", ".csv"); + + List columns = new ArrayList<>(); + columns.add(Column.builder().value(Columns.PLATE_ID).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.ROW).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.COLUMN).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.ORGANISM).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.SPECIES).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.GERMPLASM_NAME).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.GERMPLASM_GID).dataType(Column.ColumnDataType.INTEGER).build()); + columns.add(Column.builder().value(Columns.OBS_UNIT_ID).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.TISSUE).dataType(Column.ColumnDataType.STRING).build()); + columns.add(Column.builder().value(Columns.COMMENTS).dataType(Column.ColumnDataType.STRING).build()); + + ByteArrayOutputStream byteArrayOutputStream = CSVWriter.writeToCSV(columns, data); + FileOutputStream fos = new FileOutputStream(file); + fos.write(byteArrayOutputStream.toByteArray()); + + return file; + } + + private String createExperiment(Program program) throws IOException, InterruptedException { + Flowable> call = client.exchange( + GET("/import/mappings?importName=ExperimentsTemplateMap").cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + String expMappingId = JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + JsonObject importResult = importTestUtils.uploadAndFetch( + importTestUtils.writeExperimentDataToFile(List.of(makeExpImportRow("Env1")), null), + null, + true, + client, + program, + expMappingId); + return importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + } + + private Map makeExpImportRow(String environment) { + Map row = new HashMap<>(); + row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); + row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); + row.put(ExperimentObservation.Columns.EXP_TITLE, "Test Exp"); + row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); + row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); + row.put(ExperimentObservation.Columns.ENV, environment); + row.put(ExperimentObservation.Columns.ENV_LOCATION, "Location A"); + row.put(ExperimentObservation.Columns.ENV_YEAR, "2023"); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, "a-1"); + row.put(ExperimentObservation.Columns.REP_NUM, "1"); + row.put(ExperimentObservation.Columns.BLOCK_NUM, "1"); + row.put(ExperimentObservation.Columns.ROW, "1"); + row.put(ExperimentObservation.Columns.COLUMN, "1"); + return row; + } + +} From e35be0c990e10ccd152c7e756928ea33705cd45c Mon Sep 17 00:00:00 2001 From: timparsons Date: Sun, 5 Nov 2023 21:43:04 -0500 Subject: [PATCH 073/220] [BI-1910] fixing failing test --- .../brapi/v2/BrAPIV2ControllerIntegrationTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java index d9c05ed21..ebb3979d1 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java @@ -387,6 +387,7 @@ private BrAPIObservationVariable generateVariable() { return new BrAPIObservationVariable().observationVariableName("test" + random) .commonCropName("Grape") .externalReferences(Collections.singletonList(new BrAPIExternalReference().referenceID("abc123") + .referenceId("abc123") .referenceSource("breedinginsight.org"))) .trait(new BrAPITrait().traitClass("Agronomic") .traitName("test trait" + random)) From 09fc55e33bfd8c5048ba34a4b5aba17a2ff3b7b0 Mon Sep 17 00:00:00 2001 From: timparsons Date: Wed, 22 Nov 2023 11:50:31 -0500 Subject: [PATCH 074/220] [BI-1910] Forcing row to be uppercase when saving --- .../importer/model/imports/sample/SampleSubmissionImport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java index 65a6be3bc..6ae9fc2ff 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java @@ -143,7 +143,7 @@ public BrAPISample constructBrAPISample(boolean commit, Program program, User us .plateDbId(plate.getPlateDbId()) .sampleName(generateSampleName(germplasm, plate)) .germplasmDbId(germplasm.getGermplasmDbId()) - .row(row) + .row(row.toUpperCase()) .column(Integer.valueOf(column)) .tissueType(tissue) .sampleDescription(comments); From 7ae9224d7abc01bda966b7e94500d5e7ba520b9a Mon Sep 17 00:00:00 2001 From: timparsons Date: Wed, 22 Nov 2023 11:55:34 -0500 Subject: [PATCH 075/220] [BI-1910] removing commented out code --- .../controller/SampleSubmissionControllerIntegrationTest.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java index d41874783..a73cc1ac8 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/SampleSubmissionControllerIntegrationTest.java @@ -102,10 +102,6 @@ public class SampleSubmissionControllerIntegrationTest extends BrAPITest { @Client("/${micronaut.bi.api.version}") private RxHttpClient client; -// private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) -// (json, type, context) -> OffsetDateTime.parse(json.getAsString())) -// .create(); - private final Gson gson = new BrAPIClient().getJSON().getGson(); @BeforeAll From 3da6372900f1bb4afc237d290651ea7a5ded3816 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 29 Nov 2023 19:50:04 +0000 Subject: [PATCH 076/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 3a4f44851..f33a96478 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+636 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/ab28861b880edeaa1e9ee9f788a1c23b8e3bcf3d +version=v0.9.0+638 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/aa58fd3eebb248e1653890572577b85123db7620 From 0c3494282975946eeef1127b7281fbcfeb5484d3 Mon Sep 17 00:00:00 2001 From: HMS17 Date: Fri, 1 Dec 2023 10:38:21 -0500 Subject: [PATCH 077/220] [BI-1867] - Ontology: Increase name length to 16 characters --- .../services/validators/TraitFileValidatorError.java | 2 +- .../services/validators/TraitValidatorService.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java b/src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java index 25f1b3540..0eab0ad37 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java @@ -147,7 +147,7 @@ public ValidationError getInsufficientOrdinalValError() { @Override public ValidationError getCharLimitObsVarNameMsg() { - return new ValidationError("Name", "Name exceeds 12 character limit", HttpStatus.UNPROCESSABLE_ENTITY); + return new ValidationError("Name", "Name exceeds 16 character limit", HttpStatus.UNPROCESSABLE_ENTITY); } @Override diff --git a/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java b/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java index 3943ba77f..1b893c8bc 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java @@ -184,7 +184,7 @@ public ValidationErrors checkTraitFieldsLength(List traits, TraitValidato Trait trait = traits.get(i); Method method = trait.getMethod(); - int shortCharLimit = 12; + int shortCharLimit = 16; int longCharLimit = 30; if ((trait.getObservationVariableName() != null) && (trait.getObservationVariableName().length() > shortCharLimit)) { From ddc28e81af4479b05f295f130ca4995c22e16367 Mon Sep 17 00:00:00 2001 From: HMS17 Date: Fri, 1 Dec 2023 14:33:58 -0500 Subject: [PATCH 078/220] [BI-1867] - Ontology: Increase name length to 16 characters --- .../services/validators/TraitValidatorUnitTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java b/src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java index a7fa4507b..184e7308e 100644 --- a/src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java +++ b/src/test/java/org/breedinginsight/services/validators/TraitValidatorUnitTest.java @@ -298,7 +298,7 @@ public void duplicateTraitsInFile() { public void charLimitExceeded() { Trait trait = new Trait(); - trait.setObservationVariableName("OverTwelveChar"); + trait.setObservationVariableName("IsOverSixteenChar"); trait.setProgramObservationLevel(ProgramObservationLevel.builder().name("Plant").build()); Scale scale = new Scale(); scale.setScaleName("Test Scale"); From ff84d150e07b9550fcda60fdd9b3789537c4db1e Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 6 Dec 2023 14:33:55 +0000 Subject: [PATCH 079/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index f33a96478..cd684ebb2 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+638 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/aa58fd3eebb248e1653890572577b85123db7620 +version=v0.9.0+640 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/4430904e4487fb18a67ff358b7a091702d72d459 From ac8d84497eb8dde5eee7c504684346973cfd9953 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 28 Nov 2023 14:28:27 -0500 Subject: [PATCH 080/220] add check for multiple exp titles in import --- .../importer/services/processors/ExperimentProcessor.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 426c78293..a19cae999 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -85,6 +85,7 @@ public class ExperimentProcessor implements Processor { "If you’re trying to add these units to the experiment, please create a new environment" + " with all appropriate experiment units (NOTE: this will generate new Observation Unit Ids " + "for each experiment unit)."; + private static final String MULTIPLE_EXP_TITLES = "File contains more than one Experiment Title"; private static final String MIDNIGHT = "T00:00:00-00:00"; private static final String TIMESTAMP_PREFIX = "TS:"; private static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; @@ -552,7 +553,6 @@ private ValidationErrors validateFields(List importRows, Validation for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); - if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); } @@ -981,10 +981,12 @@ private PendingImportObject fetchOrCreateLocationPIO(Experiment return pio; } - private PendingImportObject fetchOrCreateTrialPIO(Program program, User user, boolean commit, ExperimentObservation importRow, Supplier expNextVal) { + private PendingImportObject fetchOrCreateTrialPIO(Program program, User user, boolean commit, ExperimentObservation importRow, Supplier expNextVal) throws UnprocessableEntityException { PendingImportObject pio; if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { pio = trialByNameNoScope.get(importRow.getExpTitle()); + } else if (!trialByNameNoScope.isEmpty()) { + throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); } else { UUID id = UUID.randomUUID(); String expSeqValue = null; From a433556d37f6eb4ad6875f31a5a3e8c8d585d7f8 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 1 Dec 2023 10:36:51 -0500 Subject: [PATCH 081/220] handle exception in exp processor public method --- .../importer/services/processors/ExperimentProcessor.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index a19cae999..e5ec4a5c5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -59,6 +59,7 @@ import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.services.exceptions.ValidatorException; import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; @@ -477,7 +478,12 @@ private void initNewBrapiData(List importRows, List> phen for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - PendingImportObject trialPIO = fetchOrCreateTrialPIO(program, user, commit, importRow, expNextVal); + PendingImportObject trialPIO = null; + try { + trialPIO = fetchOrCreateTrialPIO(program, user, commit, importRow, expNextVal); + } catch (UnprocessableEntityException e) { + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, e.getMessage()); + } String expSeqValue = null; if (commit) { From 2787525899e0d358ca41cfb3811b5e45773642d8 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 6 Dec 2023 15:13:55 +0000 Subject: [PATCH 082/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index cd684ebb2..d4fec2383 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+640 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/4430904e4487fb18a67ff358b7a091702d72d459 +version=v0.9.0+642 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/228d6dc9abc766a7de64138d6289026f5528308f From 2faaf9271c18b0455a056eaceb6fccc9853bf50b Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 21 Sep 2023 13:22:41 -0400 Subject: [PATCH 083/220] update ExperimentObservation.java to have fields overwrite and overwriteReason --- .../brapps/importer/model/config/ImportFieldTypeEnum.java | 1 + .../experimentObservation/ExperimentObservation.java | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/config/ImportFieldTypeEnum.java b/src/main/java/org/breedinginsight/brapps/importer/model/config/ImportFieldTypeEnum.java index bb176bd0d..d54b509ef 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/config/ImportFieldTypeEnum.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/config/ImportFieldTypeEnum.java @@ -21,6 +21,7 @@ @Getter public enum ImportFieldTypeEnum { + BOOLEAN("BOOLEAN"), TEXT("TEXT"), NUMERICAL("NUMERICAL"), INTEGER("INTEGER"), diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index f45cd83a3..20eb95745 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -47,6 +47,13 @@ @ImportConfigMetadata(id = "ExperimentImport", name = "Experiment Import", description = "This import is used to create Observation Unit and Experiment data") public class ExperimentObservation implements BrAPIImport { + @ImportFieldType(type = ImportFieldTypeEnum.BOOLEAN) + @ImportFieldMetadata(id = "overwrite", name = "Overwrite", description = "Boolean flag to overwrite existing observation") + private boolean overwrite; + + @ImportFieldType(type = ImportFieldTypeEnum.TEXT, collectTime = ImportCollectTimeEnum.UPLOAD) + @ImportFieldMetadata(id="overwriteReason", name="Overwrite Reason", description="Description of the reason for overwriting existing observations") + private String overwriteReason; @ImportFieldType(type = ImportFieldTypeEnum.TEXT) @ImportFieldMetadata(id = "germplasmName", name = Columns.GERMPLASM_NAME, description = "Name of germplasm") From 38231a9264680731ba2992ddb2441461ca0aacde Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 2 Oct 2023 15:42:23 -0400 Subject: [PATCH 084/220] add count of existing observations to preview --- .../processors/ExperimentProcessor.java | 61 +++++++++++++++---- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index e5ec4a5c5..4fa572cec 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -44,6 +44,7 @@ import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.base.AdditionalInfo; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; @@ -77,6 +78,8 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.breedinginsight.brapps.importer.services.FileMappingUtil.EXPERIMENT_TEMPLATE_NAME; + @Slf4j @Prototype public class ExperimentProcessor implements Processor { @@ -127,7 +130,7 @@ public class ExperimentProcessor implements Processor { private Map> observationUnitByNameNoScope = null; private final Map> observationByHash = new HashMap<>(); - + private Map existingObsByObsHash = new HashMap<>(); // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the getNewBrapiData() method private Map> existingGermplasmByGID = null; @@ -240,7 +243,7 @@ public Map process( log.debug("done processing experiment import"); // Construct our response object - return generateStatisticsMap(importRows); + return generateStatisticsMap(importRows, phenotypeCols); } @Override @@ -550,7 +553,7 @@ private String getObservationHash(String observationUnitName, String variableNam private ValidationErrors validateFields(List importRows, ValidationErrors validationErrors, Map mappedBrAPIImport, List referencedTraits, Program program, List> phenotypeCols, boolean commit) throws MissingRequiredInfoException, ApiException { //fetching any existing observations for any OUs in the import - Map existingObsByObsHash = fetchExistingObservations(referencedTraits, program); + this.existingObsByObsHash = fetchExistingObservations(referencedTraits, program); CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); for ( Trait trait: referencedTraits) { colVarMap.put(trait.getObservationVariableName(),trait); @@ -572,7 +575,7 @@ private ValidationErrors validateFields(List importRows, Validation validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); - validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, existingObsByObsHash); + validateObservations(validationErrors, rowNum, importRows.size(), importRow, phenotypeCols, colVarMap, commit); } return validationErrors; } @@ -625,22 +628,45 @@ private Map fetchExistingObservations(List refe .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } - private void validateObservations(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow, List> phenotypeCols, CaseInsensitiveMap colVarMap, Map existingObservations) { + private void validateObservations(ValidationErrors validationErrors, + int rowNum, + int numRows, + ExperimentObservation importRow, + List> phenotypeCols, + Map colVarMap, + boolean commit) { phenotypeCols.forEach(phenoCol -> { var importHash = getImportObservationHash(importRow, phenoCol.name()); - if(existingObservations.containsKey(importHash) && StringUtils.isNotBlank(phenoCol.getString(rowNum)) && !existingObservations.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { + + // error if import observation data already exists and user has not selected to overwrite + if(commit && !importRow.isOverwrite() && + this.existingObsByObsHash.containsKey(importHash) && + StringUtils.isNotBlank(phenoCol.getString(numRows - rowNum - 1)) && + !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(numRows - rowNum - 1))) { addRowError( phenoCol.name(), String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), validationErrors, rowNum ); - } else if(existingObservations.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)) || existingObservations.get(importHash).getValue().equals(phenoCol.getString(rowNum)))) { - BrAPIObservation existingObs = existingObservations.get(importHash); + + // preview case where observation has already been committed and the import row ObsVar data differs from what + // had been saved prior to import + } else if (this.existingObsByObsHash.containsKey(importHash) && + StringUtils.isNotBlank(phenoCol.getString(numRows - rowNum -1)) && + !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(numRows - rowNum -1))) { + observationByHash.get(importHash).setState(ImportObjectState.EXISTING); + + // preview case where observation has already been committed and import ObsVar data is either empty or the + // same as has been committed prior to import + } else if(this.existingObsByObsHash.containsKey(importHash) && + (StringUtils.isBlank(phenoCol.getString(numRows - rowNum -1)) || + this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(numRows - rowNum -1)))) { + BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); existingObs.setObservationVariableName(phenoCol.name()); observationByHash.get(importHash).setState(ImportObjectState.EXISTING); observationByHash.get(importHash).setBrAPIObject(existingObs); } else { - validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); + validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(numRows - rowNum -1), phenoCol.name(), validationErrors, rowNum); //Timestamp validation if(timeStampColByPheno.containsKey(phenoCol.name())) { @@ -734,7 +760,7 @@ private void addIfNotNull(HashSet set, String setValue) { } } - private Map generateStatisticsMap(List importRows) { + private Map generateStatisticsMap(List importRows, List> phenotypeCols) { // Data for stats. HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's @@ -757,6 +783,15 @@ private Map generateStatisticsMap(List preview != null && preview.getState() == ImportObjectState.EXISTING && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() .newObjectCount(environmentNameCounter.size()) .build(); @@ -769,12 +804,16 @@ private Map generateStatisticsMap(List Date: Mon, 2 Oct 2023 19:07:10 -0400 Subject: [PATCH 085/220] create BrAPIObservationDAO#updateBrAPIObservation --- .../constants/BrAPIAdditionalInfoFields.java | 1 + .../importer/daos/BrAPIObservationDAO.java | 25 +++++++++++++++++++ .../processors/ExperimentProcessor.java | 6 +++-- 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java index 821c39a98..27723e9d5 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java +++ b/src/main/java/org/breedinginsight/brapi/v2/constants/BrAPIAdditionalInfoFields.java @@ -47,6 +47,7 @@ public final class BrAPIAdditionalInfoFields { public static final String MALE_PARENT_UNKNOWN = "maleParentUnknown"; public static final String TREATMENTS = "treatments"; public static final String GID = "gid"; + public static final String CHANGELOG = "changeLog"; public static final String ENV_YEAR = "envYear"; public static final String GERMPLASM_UUID = "germplasmId"; public static final String SAMPLE_ORGANISM = "organism"; diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java index 7083cae0e..9c9ed184b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java @@ -16,6 +16,7 @@ */ package org.breedinginsight.brapps.importer.daos; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.brapi.client.v2.ApiResponse; import org.brapi.client.v2.model.exceptions.ApiException; @@ -24,11 +25,13 @@ import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.request.BrAPIObservationSearchRequest; import org.brapi.v2.model.pheno.response.BrAPIObservationListResponse; +import org.brapi.v2.model.pheno.response.BrAPIObservationSingleResponse; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.model.Program; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.breedinginsight.utilities.Utilities; import org.jetbrains.annotations.NotNull; import javax.inject.Inject; @@ -38,6 +41,7 @@ import static org.brapi.v2.model.BrAPIWSMIMEDataTypes.APPLICATION_JSON; @Singleton +@Slf4j public class BrAPIObservationDAO { private ProgramDAO programDAO; @@ -113,4 +117,25 @@ public List createBrAPIObservations(List brA return brAPIDAOUtil.post(brAPIObservationList, upload, api::observationsPost, importDAO::update); } + public BrAPIObservation updateBrAPIObservation(String dbId, BrAPIObservation observation, UUID programId) throws ApiException { + ObservationsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationsApi.class); + ApiResponse response; + BrAPIObservation updatedObservation = null; + try { + response = api.observationsObservationDbIdPut(dbId, observation); + if (response != null) { + BrAPIObservationSingleResponse body = response.getBody(); + if (body == null) { + throw new ApiException("Response is missing body", 0, response.getHeaders(), null); + } + updatedObservation = body.getResult(); + if (updatedObservation == null) { + throw new ApiException("Response body is missing result", 0, response.getHeaders(), response.getBody().toString()); + } + } + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e)); + } + return updatedObservation; + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 4fa572cec..24a70f82b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -280,6 +280,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program .getMutationsByObjectId(obsVarDatasetByName, BrAPIListSummary::getListDbId); List newObservationUnits = ProcessorData.getNewObjects(this.observationUnitByNameNoScope); + // filter out observations with no 'value' so they will not be saved List newObservations = ProcessorData.getNewObjects(this.observationByHash) .stream() @@ -654,7 +655,8 @@ private void validateObservations(ValidationErrors validationErrors, } else if (this.existingObsByObsHash.containsKey(importHash) && StringUtils.isNotBlank(phenoCol.getString(numRows - rowNum -1)) && !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(numRows - rowNum -1))) { - observationByHash.get(importHash).setState(ImportObjectState.EXISTING); + observationByHash.get(importHash).getBrAPIObject().setValue(phenoCol.getString(numRows - rowNum -1)); + observationByHash.get(importHash).setState(ImportObjectState.MUTATED); // preview case where observation has already been committed and import ObsVar data is either empty or the // same as has been committed prior to import @@ -786,7 +788,7 @@ private Map generateStatisticsMap(List preview != null && preview.getState() == ImportObjectState.EXISTING && + .filter(preview -> preview != null && (preview.getState() == ImportObjectState.EXISTING || preview.getState() == ImportObjectState.MUTATED) && !StringUtils.isBlank(preview.getBrAPIObject() .getValue())) .count() From 099023291915b5eb7c8010a2db22373fdb32e576 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 2 Oct 2023 21:58:18 -0400 Subject: [PATCH 086/220] add change log to additional info of mutated observation --- .../processors/ExperimentProcessor.java | 94 +++++++++++++------ 1 file changed, 65 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 24a70f82b..c659e88e3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -16,6 +16,9 @@ */ package org.breedinginsight.brapps.importer.services.processors; +import com.google.gson.JsonArray; +import com.google.gson.Gson; +import com.google.gson.JsonObject; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import io.micronaut.context.annotation.Property; @@ -27,6 +30,7 @@ import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.brapi.client.v2.JSON; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.*; @@ -46,6 +50,7 @@ import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.base.AdditionalInfo; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; +import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.imports.PendingImport; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; @@ -95,6 +100,9 @@ public class ExperimentProcessor implements Processor { private static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; private static final String COMMA_DELIMITER = ","; private static final String BLANK_FIELD = "Required field is blank"; + private static final String BLANK_FIELD_EXPERIMENT = "Field is blank when creating a new experiment"; + private static final String BLANK_FIELD_ENV = "Field is blank when creating a new environment"; + private static final String BLANK_FIELD_OBS = "Field is blank when creating new observations"; private static final String ENV_LOCATION_MISMATCH = "All locations must be the same for a given environment"; private static final String ENV_YEAR_MISMATCH = "All years must be the same for a given environment"; @@ -136,6 +144,8 @@ public class ExperimentProcessor implements Processor { // Associates timestamp columns to associated phenotype column name for ease of storage private final Map> timeStampColByPheno = new HashMap<>(); + private final Gson gson; + @Inject public ExperimentProcessor(DSLContext dsl, @@ -159,6 +169,7 @@ public ExperimentProcessor(DSLContext dsl, this.brAPIListDAO = brAPIListDAO; this.ontologyService = ontologyService; this.fileMappingUtil = fileMappingUtil; + this.gson = new JSON().getGson(); } @Override @@ -235,7 +246,7 @@ public Map process( prepareDataForValidation(importRows, phenotypeCols, mappedBrAPIImport); - validateFields(importRows, validationErrors, mappedBrAPIImport, referencedTraits, program, phenotypeCols, commit); + validateFields(importRows, validationErrors, mappedBrAPIImport, referencedTraits, program, phenotypeCols, commit, user); if (validationErrors.hasErrors()) { throw new ValidatorException(validationErrors); @@ -552,7 +563,7 @@ private String getObservationHash(String observationUnitName, String variableNam } private ValidationErrors validateFields(List importRows, ValidationErrors validationErrors, Map mappedBrAPIImport, List referencedTraits, Program program, - List> phenotypeCols, boolean commit) throws MissingRequiredInfoException, ApiException { + List> phenotypeCols, boolean commit, User user) throws MissingRequiredInfoException, ApiException { //fetching any existing observations for any OUs in the import this.existingObsByObsHash = fetchExistingObservations(referencedTraits, program); CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); @@ -576,7 +587,7 @@ private ValidationErrors validateFields(List importRows, Validation validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); - validateObservations(validationErrors, rowNum, importRows.size(), importRow, phenotypeCols, colVarMap, commit); + validateObservations(validationErrors, rowNum, importRows.size(), importRow, phenotypeCols, colVarMap, commit, user); } return validationErrors; } @@ -605,8 +616,8 @@ private Map fetchExistingObservations(List refe observationUnitByNameNoScope.values().forEach(ou -> { if(StringUtils.isNotBlank(ou.getBrAPIObject().getObservationUnitDbId())) { ouDbIds.add(ou.getBrAPIObject().getObservationUnitDbId()); - ouNameByDbId.put(ou.getBrAPIObject().getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getBrAPIObject().getObservationUnitName(), program.getKey())); } + ouNameByDbId.put(ou.getBrAPIObject().getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getBrAPIObject().getObservationUnitName(), program.getKey())); }); for (Trait referencedTrait : referencedTraits) { @@ -635,7 +646,8 @@ private void validateObservations(ValidationErrors validationErrors, ExperimentObservation importRow, List> phenotypeCols, Map colVarMap, - boolean commit) { + boolean commit, + User user) { phenotypeCols.forEach(phenoCol -> { var importHash = getImportObservationHash(importRow, phenoCol.name()); @@ -658,6 +670,27 @@ private void validateObservations(ValidationErrors validationErrors, observationByHash.get(importHash).getBrAPIObject().setValue(phenoCol.getString(numRows - rowNum -1)); observationByHash.get(importHash).setState(ImportObjectState.MUTATED); + // add a change log entry when updating the value of an observation + if (commit) { + BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); + String timestamp = formatter.format(OffsetDateTime.now()); + ChangeLogEntry change = new ChangeLogEntry(existingObsByObsHash.get(importHash).getValue(), + importRow.getOverwriteReason(), + user.getId(), + timestamp + ); + if (pendingObservation.getAdditionalInfo().isJsonNull()) { + pendingObservation.setAdditionalInfo(new JsonObject()); + pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + + if (pendingObservation.getAdditionalInfo() != null && !pendingObservation.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { + pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); + } + pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJson(change)); + } + // preview case where observation has already been committed and import ObsVar data is either empty or the // same as has been committed prior to import } else if(this.existingObsByObsHash.containsKey(importHash) && @@ -708,7 +741,12 @@ private void validateConditionallyRequired(ValidationErrors validationErrors, in .getState(); ImportObjectState envState = this.studyByNameNoScope.get(importRow.getEnv()).getState(); - String errorMessage = BLANK_FIELD; + String errorMessage = BLANK_FIELD_EXPERIMENT; + if (expState == ImportObjectState.EXISTING && envState == ImportObjectState.NEW) { + errorMessage = BLANK_FIELD_ENV; + } else if(expState == ImportObjectState.EXISTING && envState == ImportObjectState.EXISTING) { + errorMessage = BLANK_FIELD_OBS; + } if(expState == ImportObjectState.NEW || envState == ImportObjectState.NEW) { validateRequiredCell(importRow.getGid(), Columns.GERMPLASM_GID, errorMessage, validationErrors, rowNum); @@ -875,20 +913,18 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra } - private PendingImportObject fetchOrCreateObservationPIO(Program program, - User user, - ExperimentObservation importRow, - String variableName, - String value, - String timeStampValue, - boolean commit, - String seasonDbId, - PendingImportObject obsUnitPIO) { + private void fetchOrCreateObservationPIO(Program program, + User user, + ExperimentObservation importRow, + String variableName, + String value, + String timeStampValue, + boolean commit, + String seasonDbId, + PendingImportObject obsUnitPIO) { PendingImportObject pio; String key = getImportObservationHash(importRow, variableName); - if (this.observationByHash.containsKey(key)) { - pio = observationByHash.get(key); - } else { + if (!this.observationByHash.containsKey(key)) { PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); @@ -906,13 +942,17 @@ private PendingImportObject fetchOrCreateObservationPIO(Progra pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); this.observationByHash.put(key, pio); } - return pio; } private void addObsVarsToDatasetDetails(PendingImportObject pio, List referencedTraits, Program program) { BrAPIListDetails details = pio.getBrAPIObject(); referencedTraits.forEach(trait -> { - String id = trait.getRawObservationVariableName(); + String id = Utilities.appendProgramKey(trait.getObservationVariableName(), program.getKey()); + // Don't append the key if connected to a brapi service operating with legacy data(no appended program key) + if (trait.getFullName() == null) { + id = trait.getObservationVariableName(); + } + if (!details.getData().contains(id) && ImportObjectState.EXISTING != pio.getState()) { details.getData().add(id); } @@ -968,7 +1008,7 @@ private PendingImportObject fetchOrCreateStudyPIO(Program program, b // It is assumed that the study has only one season, And that the Years and not // the dbId's are stored in getSeason() list. - String year = newStudy.getSeasons().get(0); + String year = newStudy.getSeasons().get(0); // It is assumed that the study has only one season if (commit) { if(StringUtils.isNotBlank(year)) { String seasonID = this.yearToSeasonDbId(year, program.getId()); @@ -978,7 +1018,6 @@ private PendingImportObject fetchOrCreateStudyPIO(Program program, b addYearToStudyAdditionalInfo(program, newStudy, year); } - pio = new PendingImportObject<>(ImportObjectState.NEW, newStudy, id); this.studyByNameNoScope.put(importRow.getEnv(), pio); } @@ -1016,16 +1055,13 @@ private void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study, Str } } - private PendingImportObject fetchOrCreateLocationPIO(ExperimentObservation importRow) { + private void fetchOrCreateLocationPIO(ExperimentObservation importRow) { PendingImportObject pio; - if (locationByName.containsKey((importRow.getEnvLocation()))) { - pio = locationByName.get(importRow.getEnvLocation()); - } else { + if (! locationByName.containsKey((importRow.getEnvLocation()))) { ProgramLocation newLocation = importRow.constructProgramLocation(); pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); this.locationByName.put(importRow.getEnvLocation(), pio); } - return pio; } private PendingImportObject fetchOrCreateTrialPIO(Program program, User user, boolean commit, ExperimentObservation importRow, Supplier expNextVal) throws UnprocessableEntityException { @@ -1381,7 +1417,7 @@ private Map> initializeObsVarDatas BrAPIListDetails dataSetDetails = brAPIListDAO .getListById(existingDatasets.get(0).getListDbId(), program.getId()) .getResult(); - processAndCacheObsVarDataset(dataSetDetails, program, obsVarDatasetByName); + processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); } catch (ApiException e) { log.error(Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); @@ -1405,7 +1441,7 @@ private Optional> getTrialPIO(List> obsVarDatasetByName) { + private void processAndCacheObsVarDataset(BrAPIListDetails existingList, Map> obsVarDatasetByName) { BrAPIExternalReference xref = Utilities.getExternalReference(existingList.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) .orElseThrow(() -> new IllegalStateException("External references wasn't found for list (dbid): " + existingList.getListDbId())); From ad3d566848dc32b1296c67e21aac2d1c571a188c Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 2 Oct 2023 22:20:00 -0400 Subject: [PATCH 087/220] update mutated observations in postBrapiData --- .../services/processors/ExperimentProcessor.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index c659e88e3..ea9258f55 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -270,6 +270,9 @@ public void postBrapiData(Map mappedBrAPIImport, Program Map mutatedTrialsById = ProcessorData .getMutationsByObjectId(trialByNameNoScope, BrAPITrial::getTrialDbId); + Map mutatedObservationByDbId = ProcessorData + .getMutationsByObjectId(observationByHash, observation -> observation.getObservationDbId()); + List newLocations = ProcessorData.getNewObjects(this.locationByName) .stream() .map(location -> ProgramLocationRequest.builder() @@ -388,6 +391,19 @@ public void postBrapiData(Map mappedBrAPIImport, Program throw new InternalServerException(e.getMessage(), e); } }); + + mutatedObservationByDbId.forEach((id, observation) -> { + try { + brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); + } catch (ApiException e) { + log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException("Error saving experiment import", e); + } catch (Exception e) { + log.error("Error updating observation: ", e); + throw new InternalServerException(e.getMessage(), e); + } + }); + log.debug("experiment import complete"); } From d1e8fff33a76837e5918ff9c848fa90967336ec3 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 3 Oct 2023 13:19:04 -0400 Subject: [PATCH 088/220] add check for NPE --- .../importer/services/processors/ExperimentProcessor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index ea9258f55..87733e5a9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -691,8 +691,9 @@ private void validateObservations(ValidationErrors validationErrors, BrAPIObservation pendingObservation = observationByHash.get(importHash).getBrAPIObject(); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); String timestamp = formatter.format(OffsetDateTime.now()); + String reason = importRow.getOverwriteReason() != null ? importRow.getOverwriteReason() : ""; ChangeLogEntry change = new ChangeLogEntry(existingObsByObsHash.get(importHash).getValue(), - importRow.getOverwriteReason(), + reason, user.getId(), timestamp ); From 72dc5f79c53c50410724615e401cbb9f37862e97 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 4 Oct 2023 16:06:17 -0400 Subject: [PATCH 089/220] count mutated observations --- .../processors/ExperimentProcessor.java | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 87733e5a9..aba667ba3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -683,7 +683,7 @@ private void validateObservations(ValidationErrors validationErrors, } else if (this.existingObsByObsHash.containsKey(importHash) && StringUtils.isNotBlank(phenoCol.getString(numRows - rowNum -1)) && !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(numRows - rowNum -1))) { - observationByHash.get(importHash).getBrAPIObject().setValue(phenoCol.getString(numRows - rowNum -1)); + //observationByHash.get(importHash).getBrAPIObject().setValue(phenoCol.getString(numRows - rowNum -1)); observationByHash.get(importHash).setState(ImportObjectState.MUTATED); // add a change log entry when updating the value of an observation @@ -843,12 +843,22 @@ private Map generateStatisticsMap(List preview != null && (preview.getState() == ImportObjectState.EXISTING || preview.getState() == ImportObjectState.MUTATED) && + .filter(preview -> preview != null && preview.getState() == ImportObjectState.EXISTING && !StringUtils.isBlank(preview.getBrAPIObject() .getValue())) .count() ); + int numMutatedObservations = Math.toIntExact( + this.observationByHash.values() + .stream() + .filter(preview -> preview != null && preview.getState() == ImportObjectState.MUTATED && + !StringUtils.isBlank(preview.getBrAPIObject() + .getValue())) + .count() + ); + + ImportPreviewStatistics environmentStats = ImportPreviewStatistics.builder() .newObjectCount(environmentNameCounter.size()) .build(); @@ -864,13 +874,17 @@ private Map generateStatisticsMap(List Date: Sat, 7 Oct 2023 12:19:58 -0400 Subject: [PATCH 090/220] change overwrite field type to String --- .../ExperimentObservation.java | 4 ++-- .../importer/services/MappingManager.java | 19 ++++++++++++++----- .../processors/ExperimentProcessor.java | 3 ++- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 20eb95745..290c046ad 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -47,9 +47,9 @@ @ImportConfigMetadata(id = "ExperimentImport", name = "Experiment Import", description = "This import is used to create Observation Unit and Experiment data") public class ExperimentObservation implements BrAPIImport { - @ImportFieldType(type = ImportFieldTypeEnum.BOOLEAN) + @ImportFieldType(type = ImportFieldTypeEnum.BOOLEAN, collectTime = ImportCollectTimeEnum.UPLOAD) @ImportFieldMetadata(id = "overwrite", name = "Overwrite", description = "Boolean flag to overwrite existing observation") - private boolean overwrite; + private String overwrite; @ImportFieldType(type = ImportFieldTypeEnum.TEXT, collectTime = ImportCollectTimeEnum.UPLOAD) @ImportFieldMetadata(id="overwriteReason", name="Overwrite Reason", description="Description of the reason for overwriting existing observations") diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java b/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java index 5a3a2289f..a64705452 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java @@ -53,6 +53,12 @@ public class MappingManager { private ImportConfigManager configManager; + public static String wrongDataTypeMsg = "Column name \"%s\" must be integer type, but non-integer type provided."; + public static String blankRequiredField = "Required field \"%s\" cannot contain empty values"; + public static String missingColumn = "Column name \"%s\" does not exist in file"; + public static String missingUserInput = "User input, \"%s\" is required"; + public static String wrongUserInputDataType = "User input, \"%s\" must be an %s"; + @Inject MappingManager(ImportConfigManager configManager) { this.configManager = configManager; @@ -339,7 +345,7 @@ private void mapUserInputField(Object parent, Field field, Map u // Only supports user input at the top level of an object at the moment. No nested objects. Map String fieldId = metadata.id(); - if (!userInput.containsKey(fieldId) && required != null) { + if ((userInput == null || !userInput.containsKey(fieldId)) && required != null) { throw new UnprocessableEntityException(importService.getMissingUserInputMsg(metadata.name())); } else if (required != null && userInput.containsKey(fieldId) && userInput.get(fieldId).toString().isBlank()) { @@ -369,12 +375,15 @@ private void checkFieldType(ImportFieldTypeEnum expectedType, String column, Str private Boolean isCorrectType(ImportFieldTypeEnum expectedType, String value) { if (!value.isBlank()) { - if (expectedType == ImportFieldTypeEnum.INTEGER) { - try { + try { + if (expectedType == ImportFieldTypeEnum.INTEGER) { Integer d = Integer.parseInt(value); - } catch (NumberFormatException nfe) { - return false; } + if (expectedType == ImportFieldTypeEnum.BOOLEAN) { + Boolean b = Boolean.parseBoolean(value); + } + } catch (NumberFormatException nfe) { + return false; } } return true; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index aba667ba3..d4f496762 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -668,7 +668,7 @@ private void validateObservations(ValidationErrors validationErrors, var importHash = getImportObservationHash(importRow, phenoCol.name()); // error if import observation data already exists and user has not selected to overwrite - if(commit && !importRow.isOverwrite() && + if(commit && importRow.getOverwrite().equals("false") && this.existingObsByObsHash.containsKey(importHash) && StringUtils.isNotBlank(phenoCol.getString(numRows - rowNum - 1)) && !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(numRows - rowNum - 1))) { @@ -684,6 +684,7 @@ private void validateObservations(ValidationErrors validationErrors, StringUtils.isNotBlank(phenoCol.getString(numRows - rowNum -1)) && !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(numRows - rowNum -1))) { //observationByHash.get(importHash).getBrAPIObject().setValue(phenoCol.getString(numRows - rowNum -1)); + //this.existingObsByObsHash.get(importHash).setObservationVariableName(phenoCol.name()); observationByHash.get(importHash).setState(ImportObjectState.MUTATED); // add a change log entry when updating the value of an observation From cc944731a6aef40ef84a40bcce31e4067d4c82a4 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:11:55 -0400 Subject: [PATCH 091/220] [BI-1830] init observation pio for updated value --- .../processors/ExperimentProcessor.java | 98 +++++++++++++------ 1 file changed, 68 insertions(+), 30 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index d4f496762..498b6c8ee 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -16,8 +16,8 @@ */ package org.breedinginsight.brapps.importer.services.processors; -import com.google.gson.JsonArray; import com.google.gson.Gson; +import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -68,6 +68,7 @@ import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.services.exceptions.ValidatorException; import org.breedinginsight.utilities.Utilities; +import org.checkerframework.checker.nullness.Opt; import org.jooq.DSLContext; import tech.tablesaw.api.Table; import tech.tablesaw.columns.Column; @@ -146,7 +147,6 @@ public class ExperimentProcessor implements Processor { private final Map> timeStampColByPheno = new HashMap<>(); private final Gson gson; - @Inject public ExperimentProcessor(DSLContext dsl, BrAPITrialDAO brapiTrialDAO, @@ -490,7 +490,7 @@ private String getVariableNameFromColumn(Column column) { return column.name(); } - private void initNewBrapiData(List importRows, List> phenotypeCols, Program program, User user, List referencedTraits, boolean commit) { + private void initNewBrapiData(List importRows, List> phenotypeCols, Program program, User user, List referencedTraits, boolean commit) throws ApiException { String expSequenceName = program.getExpSequence(); if (expSequenceName == null) { @@ -554,7 +554,7 @@ private void initNewBrapiData(List importRows, List> phen } //column.name() gets phenotype name String seasonDbId = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); - fetchOrCreateObservationPIO(program, user, importRow, column.name(), column.getString(rowNum), dateTimeValue, commit, seasonDbId, obsUnitPIO); + fetchOrCreateObservationPIO(program, user, importRow, column.name(), column.getString(rowNum), dateTimeValue, commit, seasonDbId, obsUnitPIO, referencedTraits); } } } @@ -581,7 +581,6 @@ private String getObservationHash(String observationUnitName, String variableNam private ValidationErrors validateFields(List importRows, ValidationErrors validationErrors, Map mappedBrAPIImport, List referencedTraits, Program program, List> phenotypeCols, boolean commit, User user) throws MissingRequiredInfoException, ApiException { //fetching any existing observations for any OUs in the import - this.existingObsByObsHash = fetchExistingObservations(referencedTraits, program); CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); for ( Trait trait: referencedTraits) { colVarMap.put(trait.getObservationVariableName(),trait); @@ -596,10 +595,10 @@ private ValidationErrors validateFields(List importRows, Validation validateTestOrCheck(importRow, validationErrors, rowNum); //Check if existing environment. If so, ObsUnitId must be assigned - if ((mappedImportRow.getStudy().getState() == ImportObjectState.EXISTING) - && (StringUtils.isBlank(importRow.getObsUnitID()))) { - throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); - } +// if ((mappedImportRow.getStudy().getState() == ImportObjectState.EXISTING) +// && (StringUtils.isBlank(importRow.getObsUnitID()))) { +// throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); +// } validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); @@ -629,12 +628,24 @@ private Map fetchExistingObservations(List refe .map(PendingImportObject::getBrAPIObject) .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); - observationUnitByNameNoScope.values().forEach(ou -> { - if(StringUtils.isNotBlank(ou.getBrAPIObject().getObservationUnitDbId())) { - ouDbIds.add(ou.getBrAPIObject().getObservationUnitDbId()); + studyNameByDbId.keySet().stream().forEach(studyDbId -> { + try { + brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { + if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { + ouDbIds.add(ou.getObservationUnitDbId()); + } + ouNameByDbId.put(ou.getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + }); + } catch (ApiException e) { + throw new RuntimeException(e); } - ouNameByDbId.put(ou.getBrAPIObject().getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getBrAPIObject().getObservationUnitName(), program.getKey())); }); +// observationUnitByNameNoScope.values().forEach(ou -> { +// if(StringUtils.isNotBlank(ou.getBrAPIObject().getObservationUnitDbId())) { +// ouDbIds.add(ou.getBrAPIObject().getObservationUnitDbId()); +// } +// ouNameByDbId.put(ou.getBrAPIObject().getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getBrAPIObject().getObservationUnitName(), program.getKey())); +// }); for (Trait referencedTrait : referencedTraits) { variableDbIds.add(referencedTrait.getObservationVariableDbId()); @@ -685,7 +696,7 @@ private void validateObservations(ValidationErrors validationErrors, !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(numRows - rowNum -1))) { //observationByHash.get(importHash).getBrAPIObject().setValue(phenoCol.getString(numRows - rowNum -1)); //this.existingObsByObsHash.get(importHash).setObservationVariableName(phenoCol.name()); - observationByHash.get(importHash).setState(ImportObjectState.MUTATED); + //observationByHash.get(importHash).setState(ImportObjectState.MUTATED); // add a change log entry when updating the value of an observation if (commit) { @@ -795,7 +806,7 @@ private void validateConditionallyRequired(ValidationErrors validationErrors, in addRowError(Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); } } else { - validateRequiredCell(importRow.getObsUnitID(), Columns.OBS_UNIT_ID, errorMessage, validationErrors, rowNum); + //validateRequiredCell(importRow.getObsUnitID(), Columns.OBS_UNIT_ID, errorMessage, validationErrors, rowNum); } } @@ -914,7 +925,7 @@ private PendingImportObject getGidPOI(ExperimentObservation impo return null; } - private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) { + private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException { PendingImportObject pio; String key = createObservationUnitKey(importRow); if (this.observationUnitByNameNoScope.containsKey(key)) { @@ -935,10 +946,17 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); } PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); - UUID studyID = studyPIO.getId(); - UUID id = UUID.randomUUID(); - BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); - pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); + + List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); + List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); + if (!matchingOU.isEmpty()) { + pio = new PendingImportObject<>(ImportObjectState.EXISTING, matchingOU.get(0)); + } else { + UUID studyID = studyPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); + pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); + } this.observationUnitByNameNoScope.put(key, pio); } return pio; @@ -946,23 +964,43 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra private void fetchOrCreateObservationPIO(Program program, - User user, - ExperimentObservation importRow, - String variableName, - String value, - String timeStampValue, - boolean commit, - String seasonDbId, - PendingImportObject obsUnitPIO) { + User user, + ExperimentObservation importRow, + String variableName, + String value, + String timeStampValue, + boolean commit, + String seasonDbId, + PendingImportObject obsUnitPIO, + List referencedTraits) throws ApiException { PendingImportObject pio; + BrAPIObservation newObservation; String key = getImportObservationHash(importRow, variableName); - if (!this.observationByHash.containsKey(key)) { + existingObsByObsHash = fetchExistingObservations(referencedTraits, program); + if (existingObsByObsHash.containsKey(key)) { + if (StringUtils.isNotBlank(value) && + !existingObsByObsHash.get(key).getValue().equals(value)) { + + // prior observation with updated value + newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); + newObservation.setValue(value); + pio = new PendingImportObject<>(ImportObjectState.MUTATED, newObservation); + } else { + + // prior observation + pio = new PendingImportObject<>(ImportObjectState.EXISTING, existingObsByObsHash.get(key)); + } + + observationByHash.put(key, pio); + } else if (!this.observationByHash.containsKey(key)){ + + // new observation PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); UUID studyID = studyPIO.getId(); UUID id = UUID.randomUUID(); - BrAPIObservation newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject(), commit, program, user, BRAPI_REFERENCE_SOURCE, trialID, studyID, obsUnitPIO.getId(), id); + newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject(), commit, program, user, BRAPI_REFERENCE_SOURCE, trialID, studyID, obsUnitPIO.getId(), id); //NOTE: Can't parse invalid timestamp value, so have to skip if invalid. // Validation error should be thrown for offending value, but that doesn't happen until later downstream if (timeStampValue != null && !timeStampValue.isBlank() && (validDateValue(timeStampValue) || validDateTimeValue(timeStampValue))) { From 7e4f17ab86673dc873a44b903fac5377ed7a1648 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 18 Oct 2023 13:38:44 -0400 Subject: [PATCH 092/220] [BI-1830] refactor --- .../services/processors/ExperimentProcessor.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 498b6c8ee..89d42a263 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -946,15 +946,16 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString()); } PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); + UUID studyID = studyPIO.getId(); + UUID id = UUID.randomUUID(); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); - List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); - List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); - if (!matchingOU.isEmpty()) { - pio = new PendingImportObject<>(ImportObjectState.EXISTING, matchingOU.get(0)); + // check for existing units if this is an existing study + if (studyPIO.getBrAPIObject().getStudyDbId() != null) { + List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); + List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); + pio = !matchingOU.isEmpty() ? new PendingImportObject<>(ImportObjectState.EXISTING, matchingOU.get(0)) : new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); } else { - UUID studyID = studyPIO.getId(); - UUID id = UUID.randomUUID(); - BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); } this.observationUnitByNameNoScope.put(key, pio); From c9d5241d6a9b36ee2540f7074052a97989e81b87 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 18 Oct 2023 19:34:11 -0400 Subject: [PATCH 093/220] remove comments --- .../importer/services/processors/ExperimentProcessor.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 89d42a263..322f5241d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -694,9 +694,6 @@ private void validateObservations(ValidationErrors validationErrors, } else if (this.existingObsByObsHash.containsKey(importHash) && StringUtils.isNotBlank(phenoCol.getString(numRows - rowNum -1)) && !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(numRows - rowNum -1))) { - //observationByHash.get(importHash).getBrAPIObject().setValue(phenoCol.getString(numRows - rowNum -1)); - //this.existingObsByObsHash.get(importHash).setObservationVariableName(phenoCol.name()); - //observationByHash.get(importHash).setState(ImportObjectState.MUTATED); // add a change log entry when updating the value of an observation if (commit) { @@ -717,7 +714,7 @@ private void validateObservations(ValidationErrors validationErrors, if (pendingObservation.getAdditionalInfo() != null && !pendingObservation.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); } - pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJson(change)); + pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); } // preview case where observation has already been committed and import ObsVar data is either empty or the From d8b8c741822464a3eb79a6622eb5c7ad5bcf8694 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Sun, 22 Oct 2023 22:56:07 -0400 Subject: [PATCH 094/220] code cleanup --- .../processors/ExperimentProcessor.java | 37 +++++++------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 322f5241d..906f73db7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -234,7 +234,7 @@ public Map process( } } - List referencedTraits = verifyTraits(program.getId(), phenotypeCols, timestampCols, validationErrors); + List referencedTraits = verifyTraits(program.getId(), phenotypeCols, timestampCols); //Now know timestamps all valid phenotypes, can associate with phenotype column name for easy retrieval for (Column tsColumn : timestampCols) { @@ -254,7 +254,7 @@ public Map process( log.debug("done processing experiment import"); // Construct our response object - return generateStatisticsMap(importRows, phenotypeCols); + return generateStatisticsMap(importRows); } @Override @@ -271,7 +271,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program .getMutationsByObjectId(trialByNameNoScope, BrAPITrial::getTrialDbId); Map mutatedObservationByDbId = ProcessorData - .getMutationsByObjectId(observationByHash, observation -> observation.getObservationDbId()); + .getMutationsByObjectId(observationByHash, BrAPIObservation::getObservationDbId); List newLocations = ProcessorData.getNewObjects(this.locationByName) .stream() @@ -432,7 +432,7 @@ private void prepareDataForValidation(List importRows, List verifyTraits(UUID programId, List> phenotypeCols, List> timestampCols, ValidationErrors validationErrors) { + private List verifyTraits(UUID programId, List> phenotypeCols, List> timestampCols) { Set varNames = phenotypeCols.stream() .map(Column::name) .collect(Collectors.toSet()); @@ -578,8 +578,8 @@ private String getObservationHash(String observationUnitName, String variableNam return DigestUtils.sha256Hex(concat); } - private ValidationErrors validateFields(List importRows, ValidationErrors validationErrors, Map mappedBrAPIImport, List referencedTraits, Program program, - List> phenotypeCols, boolean commit, User user) throws MissingRequiredInfoException, ApiException { + private void validateFields(List importRows, ValidationErrors validationErrors, Map mappedBrAPIImport, List referencedTraits, Program program, + List> phenotypeCols, boolean commit, User user) { //fetching any existing observations for any OUs in the import CaseInsensitiveMap colVarMap = new CaseInsensitiveMap<>(); for ( Trait trait: referencedTraits) { @@ -593,7 +593,7 @@ private ValidationErrors validateFields(List importRows, Validation validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); } validateTestOrCheck(importRow, validationErrors, rowNum); - + //TODO: providing obs unit ID does not supersede import row inout data as expected and needs to be fixed //Check if existing environment. If so, ObsUnitId must be assigned // if ((mappedImportRow.getStudy().getState() == ImportObjectState.EXISTING) // && (StringUtils.isBlank(importRow.getObsUnitID()))) { @@ -604,7 +604,6 @@ private ValidationErrors validateFields(List importRows, Validation validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); validateObservations(validationErrors, rowNum, importRows.size(), importRow, phenotypeCols, colVarMap, commit, user); } - return validationErrors; } private void validateObservationUnits(ValidationErrors validationErrors, Set uniqueStudyAndObsUnit, int rowNum, ExperimentObservation importRow) { @@ -628,7 +627,7 @@ private Map fetchExistingObservations(List refe .map(PendingImportObject::getBrAPIObject) .collect(Collectors.toMap(BrAPIStudy::getStudyDbId, brAPIStudy -> Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIStudy.getStudyName(), program.getKey()))); - studyNameByDbId.keySet().stream().forEach(studyDbId -> { + studyNameByDbId.keySet().forEach(studyDbId -> { try { brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyDbId, program).forEach(ou -> { if(StringUtils.isNotBlank(ou.getObservationUnitDbId())) { @@ -640,12 +639,6 @@ private Map fetchExistingObservations(List refe throw new RuntimeException(e); } }); -// observationUnitByNameNoScope.values().forEach(ou -> { -// if(StringUtils.isNotBlank(ou.getBrAPIObject().getObservationUnitDbId())) { -// ouDbIds.add(ou.getBrAPIObject().getObservationUnitDbId()); -// } -// ouNameByDbId.put(ou.getBrAPIObject().getObservationUnitDbId(), Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getBrAPIObject().getObservationUnitName(), program.getKey())); -// }); for (Trait referencedTrait : referencedTraits) { variableDbIds.add(referencedTrait.getObservationVariableDbId()); @@ -803,6 +796,7 @@ private void validateConditionallyRequired(ValidationErrors validationErrors, in addRowError(Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); } } else { + //TODO: include this step once user-supplied obs unit id correctly supersedes other row data //validateRequiredCell(importRow.getObsUnitID(), Columns.OBS_UNIT_ID, errorMessage, validationErrors, rowNum); } } @@ -826,7 +820,7 @@ private void addIfNotNull(HashSet set, String setValue) { } } - private Map generateStatisticsMap(List importRows, List> phenotypeCols) { + private Map generateStatisticsMap(List importRows) { // Data for stats. HashSet environmentNameCounter = new HashSet<>(); // set of unique environment names HashSet obsUnitsIDCounter = new HashSet<>(); // set of unique observation unit ID's @@ -892,8 +886,8 @@ private Map generateStatisticsMap(List pi } }); } - private PendingImportObject fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) { + private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) { PendingImportObject pio; PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); String name = String.format("Observation Dataset [%s-%s]", @@ -1057,7 +1051,6 @@ private PendingImportObject fetchOrCreateDatasetPIO(Experiment obsVarDatasetByName.put(name, pio); } addObsVarsToDatasetDetails(pio, referencedTraits, program); - return pio; } private PendingImportObject fetchOrCreateStudyPIO(Program program, boolean commit, String expSequenceValue, ExperimentObservation importRow, Supplier envNextVal) { @@ -1413,9 +1406,7 @@ private void initializeStudiesForExistingObservationUnits(Program program, Map studies = fetchStudiesByDbId(studyDbIds, program); - studies.forEach(study -> { - processAndCacheStudy(study, program, studyByName); - }); + studies.forEach(study -> processAndCacheStudy(study, program, studyByName)); } private List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { From f274c7be2e0bedd8986bf0c4cde43cc0d8fc6630 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 23 Oct 2023 13:57:16 -0400 Subject: [PATCH 095/220] revert row indexing --- .../processors/ExperimentProcessor.java | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 906f73db7..b94545d91 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -602,7 +602,7 @@ private void validateFields(List importRows, ValidationErrors valid validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); - validateObservations(validationErrors, rowNum, importRows.size(), importRow, phenotypeCols, colVarMap, commit, user); + validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); } } @@ -662,7 +662,6 @@ private Map fetchExistingObservations(List refe private void validateObservations(ValidationErrors validationErrors, int rowNum, - int numRows, ExperimentObservation importRow, List> phenotypeCols, Map colVarMap, @@ -674,8 +673,8 @@ private void validateObservations(ValidationErrors validationErrors, // error if import observation data already exists and user has not selected to overwrite if(commit && importRow.getOverwrite().equals("false") && this.existingObsByObsHash.containsKey(importHash) && - StringUtils.isNotBlank(phenoCol.getString(numRows - rowNum - 1)) && - !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(numRows - rowNum - 1))) { + StringUtils.isNotBlank(phenoCol.getString(rowNum)) && + !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { addRowError( phenoCol.name(), String.format("Value already exists for ObsUnitId: %s, Phenotype: %s", importRow.getObsUnitID(), phenoCol.name()), @@ -685,8 +684,8 @@ private void validateObservations(ValidationErrors validationErrors, // preview case where observation has already been committed and the import row ObsVar data differs from what // had been saved prior to import } else if (this.existingObsByObsHash.containsKey(importHash) && - StringUtils.isNotBlank(phenoCol.getString(numRows - rowNum -1)) && - !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(numRows - rowNum -1))) { + StringUtils.isNotBlank(phenoCol.getString(rowNum)) && + !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { // add a change log entry when updating the value of an observation if (commit) { @@ -713,14 +712,14 @@ private void validateObservations(ValidationErrors validationErrors, // preview case where observation has already been committed and import ObsVar data is either empty or the // same as has been committed prior to import } else if(this.existingObsByObsHash.containsKey(importHash) && - (StringUtils.isBlank(phenoCol.getString(numRows - rowNum -1)) || - this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(numRows - rowNum -1)))) { + (StringUtils.isBlank(phenoCol.getString(rowNum)) || + this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum)))) { BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); existingObs.setObservationVariableName(phenoCol.name()); observationByHash.get(importHash).setState(ImportObjectState.EXISTING); observationByHash.get(importHash).setBrAPIObject(existingObs); } else { - validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(numRows - rowNum -1), phenoCol.name(), validationErrors, rowNum); + validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); //Timestamp validation if(timeStampColByPheno.containsKey(phenoCol.name())) { From 9a9c33552e712d2dd1afea34ce00a0d5d23daf7e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 23 Oct 2023 16:46:31 -0400 Subject: [PATCH 096/220] create ChangeLogEntry class --- .../model/imports/ChangeLogEntry.java | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 src/main/java/org/breedinginsight/brapps/importer/model/imports/ChangeLogEntry.java diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/ChangeLogEntry.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ChangeLogEntry.java new file mode 100644 index 000000000..e2ec109ed --- /dev/null +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/ChangeLogEntry.java @@ -0,0 +1,39 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapps.importer.model.imports; + +import lombok.Getter; +import lombok.Setter; + +import java.util.UUID; + +@Getter +@Setter +public class ChangeLogEntry { + private String originalValue; + private String reasonForChange; + private UUID changedBy; + private String dateOfChange; + + public ChangeLogEntry(String originalValue, String reasonForChange, UUID changedBy, String dateOfChange) { + this.originalValue = originalValue; + this.reasonForChange = reasonForChange; + this.changedBy = changedBy; + this.dateOfChange = dateOfChange; + } +} From ec839719aed30ab3748a2e05f3bbc0c265ddf182 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 7 Nov 2023 14:50:10 -0500 Subject: [PATCH 097/220] check for adding new OU to existing environment --- .../model/imports/BrAPIImportService.java | 2 +- .../ExperimentImportService.java | 2 +- .../germplasm/GermplasmImportService.java | 2 +- .../sample/SampleSubmissionImportService.java | 2 +- .../importer/services/MappingManager.java | 2 +- .../processors/ExperimentProcessor.java | 17 ++++++++++++----- .../importer/services/processors/Processor.java | 2 +- .../services/processors/ProcessorManager.java | 4 +--- .../importer/ExperimentFileImportTest.java | 3 ++- 9 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java index 2bf412955..1d520371c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java @@ -49,5 +49,5 @@ default String getWrongUserInputDataTypeMsg(String fieldName, String typeName) { return String.format("User input, \"%s\" must be an %s", fieldName, typeName); } ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) - throws UnprocessableEntityException, DoesNotExistException, ValidatorException, ApiException, MissingRequiredInfoException; + throws Exception; } diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index aa425a2da..d438da95c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -70,7 +70,7 @@ public String getMissingColumnMsg(String columnName) { @Override public ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) - throws UnprocessableEntityException, ValidatorException, ApiException, MissingRequiredInfoException { + throws Exception { ImportPreviewResponse response = null; List processors = List.of(experimentProcessorProvider.get()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java index e4084eae4..e7b128ff5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java @@ -66,7 +66,7 @@ public String getImportTypeId() { @Override public ImportPreviewResponse process(List brAPIImports, Table data, Program program, ImportUpload upload, User user, Boolean commit) - throws UnprocessableEntityException, DoesNotExistException, ValidatorException, ApiException, MissingRequiredInfoException { + throws Exception { ImportPreviewResponse response = null; List processors = List.of(germplasmProcessorProvider.get()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java index 637fa7ce3..84d8f7499 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java @@ -69,7 +69,7 @@ public ImportPreviewResponse process(List brAPIImports, Program program, ImportUpload upload, User user, - Boolean commit) throws UnprocessableEntityException, DoesNotExistException, ValidatorException, ApiException, MissingRequiredInfoException { + Boolean commit) throws Exception { List processors = List.of(sampleProcessorProvider.get()); return processorManagerProvider.get().process(brAPIImports, processors, data, program, upload, user, commit); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java b/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java index a64705452..fda6d4ef8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java @@ -351,7 +351,7 @@ private void mapUserInputField(Object parent, Field field, Map u else if (required != null && userInput.containsKey(fieldId) && userInput.get(fieldId).toString().isBlank()) { throw new UnprocessableEntityException(importService.getMissingUserInputMsg(metadata.name())); } - else if (userInput.containsKey(fieldId)) { + else if (userInput != null && userInput.containsKey(fieldId)) { String value = userInput.get(fieldId).toString(); if (!isCorrectType(type.type(), value)) { throw new UnprocessableEntityException(importService.getWrongUserInputDataTypeMsg(metadata.name(), type.type().toString().toLowerCase())); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index b94545d91..f529b792d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -30,6 +30,7 @@ import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.collections4.map.CaseInsensitiveMap; +import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.JSON; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; @@ -64,6 +65,7 @@ import org.breedinginsight.services.OntologyService; import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.services.exceptions.ValidatorException; @@ -91,6 +93,7 @@ public class ExperimentProcessor implements Processor { private static final String NAME = "Experiment"; + private static final String EXISTING_ENV = "Cannot create a new observation unit for existing environment: "; private static final String MISSING_OBS_UNIT_ID_ERROR = "Experiment Units are missing Observation Unit Id.

" + "If you’re trying to add these units to the experiment, please create a new environment" + " with all appropriate experiment units (NOTE: this will generate new Observation Unit Ids " + @@ -216,7 +219,7 @@ public Map process( Table data, Program program, User user, - boolean commit) throws ValidatorException, MissingRequiredInfoException, ApiException { + boolean commit) throws UnprocessableEntityException, ApiException, ValidatorException { log.debug("processing experiment import"); ValidationErrors validationErrors = new ValidationErrors(); @@ -490,7 +493,7 @@ private String getVariableNameFromColumn(Column column) { return column.name(); } - private void initNewBrapiData(List importRows, List> phenotypeCols, Program program, User user, List referencedTraits, boolean commit) throws ApiException { + private void initNewBrapiData(List importRows, List> phenotypeCols, Program program, User user, List referencedTraits, boolean commit) throws UnprocessableEntityException, ApiException { String expSequenceName = program.getExpSequence(); if (expSequenceName == null) { @@ -671,7 +674,7 @@ private void validateObservations(ValidationErrors validationErrors, var importHash = getImportObservationHash(importRow, phenoCol.name()); // error if import observation data already exists and user has not selected to overwrite - if(commit && importRow.getOverwrite().equals("false") && + if(commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && this.existingObsByObsHash.containsKey(importHash) && StringUtils.isNotBlank(phenoCol.getString(rowNum)) && !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { @@ -915,7 +918,7 @@ private PendingImportObject getGidPOI(ExperimentObservation impo return null; } - private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException { + private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws UnprocessableEntityException, ApiException { PendingImportObject pio; String key = createObservationUnitKey(importRow); if (this.observationUnitByNameNoScope.containsKey(key)) { @@ -944,7 +947,11 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra if (studyPIO.getBrAPIObject().getStudyDbId() != null) { List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); - pio = !matchingOU.isEmpty() ? new PendingImportObject<>(ImportObjectState.EXISTING, matchingOU.get(0)) : new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); + if (matchingOU.isEmpty()) { + throw new UnprocessableEntityException(EXISTING_ENV + Utilities.removeProgramKeyAndUnknownAdditionalData(studyPIO.getBrAPIObject().getStudyName(), program.getKey())); + } else { + pio = new PendingImportObject<>(ImportObjectState.EXISTING, matchingOU.get(0)); + } } else { pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java index e4c2e067a..38a3bf8ae 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java @@ -55,7 +55,7 @@ public interface Processor { Map process(ImportUpload upload, List importRows, Map mappedBrAPIImport, Table data, Program program, User user, boolean commit) - throws ValidatorException, MissingRequiredInfoException, ApiException; + throws Exception; /** * Given mapped brapi import with updates from prior dependencies, check if have everything needed diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java index 1f61dabe8..1961b33d1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ProcessorManager.java @@ -18,7 +18,6 @@ import io.micronaut.context.annotation.Prototype; import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; @@ -27,7 +26,6 @@ import org.breedinginsight.brapps.importer.services.ImportStatusService; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.ValidatorException; import tech.tablesaw.api.Table; @@ -53,7 +51,7 @@ public ProcessorManager(ImportStatusService statusService) { this.statusService = statusService; } - public ImportPreviewResponse process(List importRows, List processors, Table data, Program program, ImportUpload upload, User user, boolean commit) throws ValidatorException, ApiException, MissingRequiredInfoException { + public ImportPreviewResponse process(List importRows, List processors, Table data, Program program, ImportUpload upload, User user, boolean commit) throws Exception { this.processors = processors; diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 748299daf..af2759f58 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -86,6 +86,7 @@ @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) public class ExperimentFileImportTest extends BrAPITest { + private static final String OVERWRITE = "overwrite"; private FannyPack securityFp; private String mappingId; @@ -603,7 +604,7 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); - assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Units are missing Observation Unit Id.")); + assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Cannot create a new observation unit for existing environment:")); } @Test From bb66651370369b7c046100bde9c9bfb46c5ec0f0 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 8 Nov 2023 15:40:31 -0500 Subject: [PATCH 098/220] [BI-1830] format pios for display --- .../processors/ExperimentProcessor.java | 36 ++++++++++-------- .../breedinginsight/utilities/Utilities.java | 38 +++++++++++++++++++ 2 files changed, 59 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index f529b792d..deabe9901 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -76,6 +76,7 @@ import tech.tablesaw.columns.Column; import javax.inject.Inject; +import java.lang.reflect.Field; import java.math.BigDecimal; import java.math.BigInteger; import java.time.OffsetDateTime; @@ -93,7 +94,10 @@ public class ExperimentProcessor implements Processor { private static final String NAME = "Experiment"; - private static final String EXISTING_ENV = "Cannot create a new observation unit for existing environment: "; + private static final String EXISTING_ENV = "Cannot create new observation unit %s for existing environment %s." + + "If you’re trying to add these units to the experiment, please create a new environment" + + " with all appropriate experiment units (NOTE: this will generate new Observation Unit Ids " + + "for each experiment unit)."; private static final String MISSING_OBS_UNIT_ID_ERROR = "Experiment Units are missing Observation Unit Id.

" + "If you’re trying to add these units to the experiment, please create a new environment" + " with all appropriate experiment units (NOTE: this will generate new Observation Unit Ids " + @@ -948,9 +952,10 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); if (matchingOU.isEmpty()) { - throw new UnprocessableEntityException(EXISTING_ENV + Utilities.removeProgramKeyAndUnknownAdditionalData(studyPIO.getBrAPIObject().getStudyName(), program.getKey())); + throw new UnprocessableEntityException(String.format(EXISTING_ENV, importRow.getExpUnitId(), + Utilities.removeProgramKeyAndUnknownAdditionalData(studyPIO.getBrAPIObject().getStudyName(), program.getKey()))); } else { - pio = new PendingImportObject<>(ImportObjectState.EXISTING, matchingOU.get(0)); + pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); } } else { pio = new PendingImportObject<>(ImportObjectState.NEW, newObservationUnit, id); @@ -961,16 +966,17 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra } + private void fetchOrCreateObservationPIO(Program program, - User user, - ExperimentObservation importRow, - String variableName, - String value, - String timeStampValue, - boolean commit, - String seasonDbId, - PendingImportObject obsUnitPIO, - List referencedTraits) throws ApiException { + User user, + ExperimentObservation importRow, + String variableName, + String value, + String timeStampValue, + boolean commit, + String seasonDbId, + PendingImportObject obsUnitPIO, + List referencedTraits) throws ApiException { PendingImportObject pio; BrAPIObservation newObservation; String key = getImportObservationHash(importRow, variableName); @@ -982,11 +988,11 @@ private void fetchOrCreateObservationPIO(Program program, // prior observation with updated value newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); newObservation.setValue(value); - pio = new PendingImportObject<>(ImportObjectState.MUTATED, newObservation); + pio = new PendingImportObject<>(ImportObjectState.MUTATED, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); } else { // prior observation - pio = new PendingImportObject<>(ImportObjectState.EXISTING, existingObsByObsHash.get(key)); + pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(existingObsByObsHash.get(key), BrAPIObservation.class, program)); } observationByHash.put(key, pio); @@ -1566,7 +1572,7 @@ private void processAndCacheStudy(BrAPIStudy existingStudy, Program program, Map .orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId())); studyByName.put( Utilities.removeProgramKeyAndUnknownAdditionalData(existingStudy.getStudyName(), program.getKey()), - new PendingImportObject<>(ImportObjectState.EXISTING, existingStudy, UUID.fromString(xref.getReferenceID()))); + new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIStudy) Utilities.formatBrapiObjForDisplay(existingStudy, BrAPIStudy.class, program), UUID.fromString(xref.getReferenceID()))); } private void initializeTrialsForExistingObservationUnits(Program program, Map> trialByName) { diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 7aa390635..4027c1b5d 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -20,10 +20,12 @@ import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.model.Program; import org.flywaydb.core.api.migration.Context; +import java.lang.reflect.Field; import java.sql.ResultSet; import java.sql.Statement; import java.util.ArrayList; @@ -115,6 +117,42 @@ public static String removeProgramKey(String original, String programKey) { return removeProgramKey(original, programKey, null); } + /** + * Remove program key from fields visible on the front end. Mutates the original object and returns it. + * + * @param brapiInstance Object, an instance of a BrAPI Object + * @param brapiClass Class, the BrAPI class + * @param program + * @return Object, BrAPI instance formatted for display + */ + public static Object formatBrapiObjForDisplay(Object brapiInstance, Class brapiClass, Program program) throws RuntimeException { + List displayFields = new ArrayList<>(List.of( + "trialName", + "studyName", + "germplasmName", + "locationName", + "observationUnitName", + "observationVariableName")); + List fields = List.of(brapiClass.getDeclaredFields()); + for (Field field : fields) { + if (displayFields.contains(field.getName())) { + try { + field.setAccessible(true); + + // remove key of form [%s-%s] + String valueSansKeyAndInfo = removeProgramKeyAndUnknownAdditionalData((String) field.get(brapiInstance), program.getKey()); + + //remove key of form [%s] + String valueSansKey = removeProgramKey(valueSansKeyAndInfo, program.getKey()); + field.set(brapiInstance, valueSansKey); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + } + return brapiInstance; + } + public static String removeProgramKeyAndUnknownAdditionalData(String original, String programKey) { String keyValueRegEx = String.format(" \\[%s\\-.*\\]", programKey); String stripped = original.replaceAll(keyValueRegEx, ""); From 800234cbc3cc53c1ee97ab6281a783aaccb55cc9 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 9 Nov 2023 11:22:23 -0500 Subject: [PATCH 099/220] map season year to study pio --- .../importer/services/processors/ExperimentProcessor.java | 8 +++++++- .../java/org/breedinginsight/utilities/Utilities.java | 6 +++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index deabe9901..fd6af0504 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -324,7 +324,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program } List createdLocations = new ArrayList<>(locationService.create(actingUser, program.getId(), newLocations)); - // set the DbId to the for each newly created trial + // set the DbId to the for each newly created location for (ProgramLocation createdLocation : createdLocations) { String createdLocationName = createdLocation.getName(); this.locationByName.get(createdLocationName) @@ -1570,6 +1570,12 @@ private void processAndCacheObservationUnit(BrAPIObservationUnit brAPIObservatio private void processAndCacheStudy(BrAPIStudy existingStudy, Program program, Map> studyByName) { BrAPIExternalReference xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) .orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId())); + // map season dbid to year + String seasonDbId = existingStudy.getSeasons().get(0); // It is assumed that the study has only one season + if(StringUtils.isNotBlank(seasonDbId)) { + String seasonYear = this.seasonDbIdToYear(seasonDbId, program.getId()); + existingStudy.setSeasons(Collections.singletonList(seasonYear)); + } studyByName.put( Utilities.removeProgramKeyAndUnknownAdditionalData(existingStudy.getStudyName(), program.getKey()), new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIStudy) Utilities.formatBrapiObjForDisplay(existingStudy, BrAPIStudy.class, program), UUID.fromString(xref.getReferenceID()))); diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 4027c1b5d..4b26bdefa 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -139,11 +139,11 @@ public static Object formatBrapiObjForDisplay(Object brapiInstance, Class brapiC try { field.setAccessible(true); - // remove key of form [%s-%s] + // remove either of possible key formats, [%s-%s] and [%s] String valueSansKeyAndInfo = removeProgramKeyAndUnknownAdditionalData((String) field.get(brapiInstance), program.getKey()); - - //remove key of form [%s] String valueSansKey = removeProgramKey(valueSansKeyAndInfo, program.getKey()); + + // set the value without key or additional info field.set(brapiInstance, valueSansKey); } catch (IllegalAccessException e) { throw new RuntimeException(e); From 41dd3bdf235e62b22746a4b3399451b5c518cc54 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 9 Nov 2023 18:05:12 -0500 Subject: [PATCH 100/220] [BI-1830] update experiment import test --- .../brapps/importer/ExperimentFileImportTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index af2759f58..f8a022f2b 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -604,7 +604,7 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); - assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Cannot create a new observation unit for existing environment:")); + assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Cannot create a new observation unit")); } @Test From c7740711bffc18bae8b8107350bbf7bf804415fa Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 10 Nov 2023 10:15:22 -0500 Subject: [PATCH 101/220] update test --- .../brapps/importer/ExperimentFileImportTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index f8a022f2b..e6b64ca49 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -604,7 +604,7 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); - assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Cannot create a new observation unit")); + assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Cannot create new observation unit")); } @Test From 6890a6bbb6cd544b96566219b2e5a50c69fbf02a Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 21 Nov 2023 12:29:39 -0500 Subject: [PATCH 102/220] add fixes --- .../model/imports/BrAPIImportService.java | 5 --- .../ExperimentImportService.java | 9 ++--- .../germplasm/GermplasmImportService.java | 11 ++---- .../importer/services/MappingManager.java | 14 +++---- .../processors/ExperimentProcessor.java | 39 ++++++++++++------- .../services/processors/Processor.java | 1 - .../breedinginsight/utilities/Utilities.java | 1 - 7 files changed, 39 insertions(+), 41 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java index 1d520371c..0079c13b9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java @@ -17,15 +17,10 @@ package org.breedinginsight.brapps.importer.model.imports; -import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.breedinginsight.services.exceptions.ValidatorException; import tech.tablesaw.api.Table; import java.util.List; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java index d438da95c..cd795564a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentImportService.java @@ -18,18 +18,15 @@ package org.breedinginsight.brapps.importer.model.imports.experimentObservation; import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; -import org.breedinginsight.brapps.importer.model.imports.germplasm.GermplasmImport; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.brapps.importer.services.processors.*; +import org.breedinginsight.brapps.importer.services.processors.ExperimentProcessor; +import org.breedinginsight.brapps.importer.services.processors.Processor; +import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.breedinginsight.services.exceptions.ValidatorException; import tech.tablesaw.api.Table; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java index e7b128ff5..b4eac6b96 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/germplasm/GermplasmImportService.java @@ -18,24 +18,21 @@ package org.breedinginsight.brapps.importer.model.imports.germplasm; import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; -import org.breedinginsight.brapps.importer.services.processors.*; +import org.breedinginsight.brapps.importer.services.processors.GermplasmProcessor; +import org.breedinginsight.brapps.importer.services.processors.Processor; +import org.breedinginsight.brapps.importer.services.processors.ProcessorManager; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.breedinginsight.services.exceptions.ValidatorException; import tech.tablesaw.api.Table; import javax.inject.Inject; import javax.inject.Provider; import javax.inject.Singleton; -import java.util.*; +import java.util.List; @Singleton @Slf4j diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java b/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java index fda6d4ef8..a3535d13c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/MappingManager.java @@ -375,14 +375,14 @@ private void checkFieldType(ImportFieldTypeEnum expectedType, String column, Str private Boolean isCorrectType(ImportFieldTypeEnum expectedType, String value) { if (!value.isBlank()) { - try { - if (expectedType == ImportFieldTypeEnum.INTEGER) { - Integer d = Integer.parseInt(value); - } - if (expectedType == ImportFieldTypeEnum.BOOLEAN) { - Boolean b = Boolean.parseBoolean(value); + if (expectedType == ImportFieldTypeEnum.INTEGER) { + try { + Integer.parseInt(value); + } catch (NumberFormatException nfe) { + return false; } - } catch (NumberFormatException nfe) { + } + if (expectedType == ImportFieldTypeEnum.BOOLEAN && !String.valueOf(Boolean.parseBoolean(value)).equals(value)) { return false; } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index fd6af0504..5c795f123 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -19,8 +19,6 @@ import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonObject; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; import io.micronaut.context.annotation.Property; import io.micronaut.context.annotation.Prototype; import io.micronaut.http.HttpStatus; @@ -49,7 +47,6 @@ import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.base.AdditionalInfo; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.imports.PendingImport; @@ -70,31 +67,26 @@ import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.services.exceptions.ValidatorException; import org.breedinginsight.utilities.Utilities; -import org.checkerframework.checker.nullness.Opt; import org.jooq.DSLContext; import tech.tablesaw.api.Table; import tech.tablesaw.columns.Column; import javax.inject.Inject; -import java.lang.reflect.Field; import java.math.BigDecimal; import java.math.BigInteger; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.*; -import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; -import static org.breedinginsight.brapps.importer.services.FileMappingUtil.EXPERIMENT_TEMPLATE_NAME; - @Slf4j @Prototype public class ExperimentProcessor implements Processor { private static final String NAME = "Experiment"; - private static final String EXISTING_ENV = "Cannot create new observation unit %s for existing environment %s." + + private static final String EXISTING_ENV = "Cannot create new observation unit %s for existing environment %s.

" + "If you’re trying to add these units to the experiment, please create a new environment" + " with all appropriate experiment units (NOTE: this will generate new Observation Unit Ids " + "for each experiment unit)."; @@ -107,7 +99,6 @@ public class ExperimentProcessor implements Processor { private static final String TIMESTAMP_PREFIX = "TS:"; private static final String TIMESTAMP_REGEX = "^"+TIMESTAMP_PREFIX+"\\s*"; private static final String COMMA_DELIMITER = ","; - private static final String BLANK_FIELD = "Required field is blank"; private static final String BLANK_FIELD_EXPERIMENT = "Field is blank when creating a new experiment"; private static final String BLANK_FIELD_ENV = "Field is blank when creating a new environment"; private static final String BLANK_FIELD_OBS = "Field is blank when creating new observations"; @@ -401,7 +392,16 @@ public void postBrapiData(Map mappedBrAPIImport, Program mutatedObservationByDbId.forEach((id, observation) -> { try { - brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); + BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); + if (!observation.getValue().equals(updatedObs.getValue()) || !observation.getObservationTimeStamp().equals(updatedObs.getObservationTimeStamp())) { + String message; + if(!observation.getValue().equals(updatedObs.getValue())) { + message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); + } else { + message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObs.getObservationTimeStamp(), observation.getObservationTimeStamp()); + } + throw new Exception(message); + } } catch (ApiException e) { log.error("Error updating observation: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException("Error saving experiment import", e); @@ -671,7 +671,7 @@ private void validateObservations(ValidationErrors validationErrors, int rowNum, ExperimentObservation importRow, List> phenotypeCols, - Map colVarMap, + CaseInsensitiveMap colVarMap, boolean commit, User user) { phenotypeCols.forEach(phenoCol -> { @@ -705,6 +705,8 @@ private void validateObservations(ValidationErrors validationErrors, user.getId(), timestamp ); + + // create the changelog field in additonalinfo if it does not already exist if (pendingObservation.getAdditionalInfo().isJsonNull()) { pendingObservation.setAdditionalInfo(new JsonObject()); pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); @@ -713,6 +715,8 @@ private void validateObservations(ValidationErrors validationErrors, if (pendingObservation.getAdditionalInfo() != null && !pendingObservation.getAdditionalInfo().has(BrAPIAdditionalInfoFields.CHANGELOG)) { pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); } + + // add a new entry to the changelog pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); } @@ -981,13 +985,20 @@ private void fetchOrCreateObservationPIO(Program program, BrAPIObservation newObservation; String key = getImportObservationHash(importRow, variableName); existingObsByObsHash = fetchExistingObservations(referencedTraits, program); + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); + String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timeStampValue)); if (existingObsByObsHash.containsKey(key)) { if (StringUtils.isNotBlank(value) && - !existingObsByObsHash.get(key).getValue().equals(value)) { + (!existingObsByObsHash.get(key).getValue().equals(value) || + !existingObsByObsHash.get(key).getObservationTimeStamp().equals(formattedTimeStampValue))){ // prior observation with updated value newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); - newObservation.setValue(value); + if (!existingObsByObsHash.get(key).getValue().equals(value)){ + newObservation.setValue(value); + } else if (!existingObsByObsHash.get(key).getObservationTimeStamp().equals(formattedTimeStampValue)) { + newObservation.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); + } pio = new PendingImportObject<>(ImportObjectState.MUTATED, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); } else { diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java index 38a3bf8ae..820626676 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/Processor.java @@ -23,7 +23,6 @@ import org.breedinginsight.brapps.importer.model.response.ImportPreviewStatistics; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.ValidatorException; import tech.tablesaw.api.Table; diff --git a/src/main/java/org/breedinginsight/utilities/Utilities.java b/src/main/java/org/breedinginsight/utilities/Utilities.java index 4b26bdefa..c885eaa97 100644 --- a/src/main/java/org/breedinginsight/utilities/Utilities.java +++ b/src/main/java/org/breedinginsight/utilities/Utilities.java @@ -20,7 +20,6 @@ import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIExternalReference; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.model.Program; import org.flywaydb.core.api.migration.Context; From 44e7b50fbf462c74c9323bf11528cc22ce1d5c78 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 27 Nov 2023 17:31:23 -0500 Subject: [PATCH 103/220] update observation timestamps --- .../processors/ExperimentProcessor.java | 55 +++++++++++++------ 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 5c795f123..e32e42cb0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -72,6 +72,7 @@ import tech.tablesaw.columns.Column; import javax.inject.Inject; +import javax.validation.Valid; import java.math.BigDecimal; import java.math.BigInteger; import java.time.OffsetDateTime; @@ -393,7 +394,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program mutatedObservationByDbId.forEach((id, observation) -> { try { BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); - if (!observation.getValue().equals(updatedObs.getValue()) || !observation.getObservationTimeStamp().equals(updatedObs.getObservationTimeStamp())) { + if (!observation.getValue().equals(updatedObs.getValue()) || !observation.getObservationTimeStamp().isEqual(updatedObs.getObservationTimeStamp())) { String message; if(!observation.getValue().equals(updatedObs.getValue())) { message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); @@ -675,7 +676,9 @@ private void validateObservations(ValidationErrors validationErrors, boolean commit, User user) { phenotypeCols.forEach(phenoCol -> { - var importHash = getImportObservationHash(importRow, phenoCol.name()); + String importHash = getImportObservationHash(importRow, phenoCol.name()); + String importObsValue = phenoCol.getString(rowNum); + String importObsTimestamp = timeStampColByPheno.get(phenoCol.name()).getString(rowNum); // error if import observation data already exists and user has not selected to overwrite if(commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && @@ -690,9 +693,7 @@ private void validateObservations(ValidationErrors validationErrors, // preview case where observation has already been committed and the import row ObsVar data differs from what // had been saved prior to import - } else if (this.existingObsByObsHash.containsKey(importHash) && - StringUtils.isNotBlank(phenoCol.getString(rowNum)) && - !this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum))) { + } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, importObsTimestamp)) { // add a change log entry when updating the value of an observation if (commit) { @@ -700,13 +701,21 @@ private void validateObservations(ValidationErrors validationErrors, DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd:hh-mm-ssZ"); String timestamp = formatter.format(OffsetDateTime.now()); String reason = importRow.getOverwriteReason() != null ? importRow.getOverwriteReason() : ""; - ChangeLogEntry change = new ChangeLogEntry(existingObsByObsHash.get(importHash).getValue(), + String prior = ""; + if (isValueMatched(importHash, importObsValue)) { + prior.concat(existingObsByObsHash.get(importHash).getValue()); + } + if (isTimestampMatched(importHash, importObsTimestamp)) { + prior = prior.isEmpty() ? prior : prior.concat(" "); + prior.concat(existingObsByObsHash.get(importHash).getObservationTimeStamp().toString()); + } + ChangeLogEntry change = new ChangeLogEntry(prior, reason, user.getId(), timestamp ); - // create the changelog field in additonalinfo if it does not already exist + // create the changelog field in additional info if it does not already exist if (pendingObservation.getAdditionalInfo().isJsonNull()) { pendingObservation.setAdditionalInfo(new JsonObject()); pendingObservation.getAdditionalInfo().add(BrAPIAdditionalInfoFields.CHANGELOG, new JsonArray()); @@ -722,9 +731,8 @@ private void validateObservations(ValidationErrors validationErrors, // preview case where observation has already been committed and import ObsVar data is either empty or the // same as has been committed prior to import - } else if(this.existingObsByObsHash.containsKey(importHash) && - (StringUtils.isBlank(phenoCol.getString(rowNum)) || - this.existingObsByObsHash.get(importHash).getValue().equals(phenoCol.getString(rowNum)))) { + } else if(existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)) || + isObservationMatched(importHash, importObsValue, importObsTimestamp))) { BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); existingObs.setObservationVariableName(phenoCol.name()); observationByHash.get(importHash).setState(ImportObjectState.EXISTING); @@ -969,7 +977,24 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra return pio; } + boolean isTimestampMatched(String observationHash, String timeStamp) { + OffsetDateTime priorStamp = existingObsByObsHash.get(observationHash).getObservationTimeStamp(); + if (priorStamp == null) { + return timeStamp == null; + } + return priorStamp.isEqual(OffsetDateTime.parse(timeStamp)); + } + + boolean isValueMatched(String observationHash, String value) { + if (existingObsByObsHash.get(observationHash).getValue() == null) { + return value == null; + } + return existingObsByObsHash.get(observationHash).getValue().equals(value); + } + boolean isObservationMatched(String observationHash, String value, String timeStamp) { + return isTimestampMatched(observationHash, timeStamp) && isValueMatched(observationHash, value); + } private void fetchOrCreateObservationPIO(Program program, User user, @@ -985,18 +1010,16 @@ private void fetchOrCreateObservationPIO(Program program, BrAPIObservation newObservation; String key = getImportObservationHash(importRow, variableName); existingObsByObsHash = fetchExistingObservations(referencedTraits, program); - DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); + DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timeStampValue)); if (existingObsByObsHash.containsKey(key)) { - if (StringUtils.isNotBlank(value) && - (!existingObsByObsHash.get(key).getValue().equals(value) || - !existingObsByObsHash.get(key).getObservationTimeStamp().equals(formattedTimeStampValue))){ + if (StringUtils.isNotBlank(value) && !isObservationMatched(key, value, timeStampValue)){ // prior observation with updated value newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); - if (!existingObsByObsHash.get(key).getValue().equals(value)){ + if (!isValueMatched(key, value)){ newObservation.setValue(value); - } else if (!existingObsByObsHash.get(key).getObservationTimeStamp().equals(formattedTimeStampValue)) { + } else if (!isTimestampMatched(key, timeStampValue)) { newObservation.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); } pio = new PendingImportObject<>(ImportObjectState.MUTATED, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); From 4471a0abed9971d8715fab6e666ad6ca86064342 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 5 Dec 2023 14:20:07 -0500 Subject: [PATCH 104/220] fix NPEs --- .../model/imports/BrAPIImportService.java | 5 ++++ .../sample/SampleSubmissionImportService.java | 5 ---- .../processors/ExperimentProcessor.java | 30 +++++++++++-------- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java index 0079c13b9..1d520371c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/BrAPIImportService.java @@ -17,10 +17,15 @@ package org.breedinginsight.brapps.importer.model.imports; +import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.response.ImportPreviewResponse; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; +import org.breedinginsight.services.exceptions.UnprocessableEntityException; +import org.breedinginsight.services.exceptions.ValidatorException; import tech.tablesaw.api.Table; import java.util.List; diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java index 84d8f7499..434626e68 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImportService.java @@ -18,7 +18,6 @@ package org.breedinginsight.brapps.importer.model.imports.sample; import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.BrAPIImportService; @@ -28,10 +27,6 @@ import org.breedinginsight.brapps.importer.services.processors.SampleSubmissionProcessor; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.breedinginsight.services.exceptions.ValidatorException; import tech.tablesaw.api.Table; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index e32e42cb0..d35da62d8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -562,7 +562,7 @@ private void initNewBrapiData(List importRows, List> phen } //column.name() gets phenotype name String seasonDbId = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); - fetchOrCreateObservationPIO(program, user, importRow, column.name(), column.getString(rowNum), dateTimeValue, commit, seasonDbId, obsUnitPIO, referencedTraits); + fetchOrCreateObservationPIO(program, user, importRow, column, rowNum, dateTimeValue, commit, seasonDbId, obsUnitPIO, referencedTraits); } } } @@ -678,7 +678,6 @@ private void validateObservations(ValidationErrors validationErrors, phenotypeCols.forEach(phenoCol -> { String importHash = getImportObservationHash(importRow, phenoCol.name()); String importObsValue = phenoCol.getString(rowNum); - String importObsTimestamp = timeStampColByPheno.get(phenoCol.name()).getString(rowNum); // error if import observation data already exists and user has not selected to overwrite if(commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && @@ -693,7 +692,7 @@ private void validateObservations(ValidationErrors validationErrors, // preview case where observation has already been committed and the import row ObsVar data differs from what // had been saved prior to import - } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, importObsTimestamp)) { + } else if (existingObsByObsHash.containsKey(importHash) && !isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { // add a change log entry when updating the value of an observation if (commit) { @@ -705,7 +704,7 @@ private void validateObservations(ValidationErrors validationErrors, if (isValueMatched(importHash, importObsValue)) { prior.concat(existingObsByObsHash.get(importHash).getValue()); } - if (isTimestampMatched(importHash, importObsTimestamp)) { + if (timeStampColByPheno.containsKey(phenoCol.name()) && isTimestampMatched(importHash, timeStampColByPheno.get(phenoCol.name()).getString(rowNum))) { prior = prior.isEmpty() ? prior : prior.concat(" "); prior.concat(existingObsByObsHash.get(importHash).getObservationTimeStamp().toString()); } @@ -732,7 +731,7 @@ private void validateObservations(ValidationErrors validationErrors, // preview case where observation has already been committed and import ObsVar data is either empty or the // same as has been committed prior to import } else if(existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)) || - isObservationMatched(importHash, importObsValue, importObsTimestamp))) { + isObservationMatched(importHash, importObsValue, phenoCol, rowNum))) { BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); existingObs.setObservationVariableName(phenoCol.name()); observationByHash.get(importHash).setState(ImportObjectState.EXISTING); @@ -992,15 +991,20 @@ boolean isValueMatched(String observationHash, String value) { return existingObsByObsHash.get(observationHash).getValue().equals(value); } - boolean isObservationMatched(String observationHash, String value, String timeStamp) { - return isTimestampMatched(observationHash, timeStamp) && isValueMatched(observationHash, value); + boolean isObservationMatched(String observationHash, String value, Column phenoCol, Integer rowNum) { + if (timeStampColByPheno.isEmpty() || !timeStampColByPheno.containsKey(phenoCol.name())) { + return isValueMatched(observationHash, value); + } else { + String importObsTimestamp = timeStampColByPheno.get(phenoCol.name()).getString(rowNum); + return isTimestampMatched(observationHash, importObsTimestamp) && isValueMatched(observationHash, value); + } } private void fetchOrCreateObservationPIO(Program program, User user, ExperimentObservation importRow, - String variableName, - String value, + Column column, + Integer rowNum, String timeStampValue, boolean commit, String seasonDbId, @@ -1008,18 +1012,20 @@ private void fetchOrCreateObservationPIO(Program program, List referencedTraits) throws ApiException { PendingImportObject pio; BrAPIObservation newObservation; + String variableName = column.name(); + String value = column.getString(rowNum); String key = getImportObservationHash(importRow, variableName); existingObsByObsHash = fetchExistingObservations(referencedTraits, program); - DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; - String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timeStampValue)); if (existingObsByObsHash.containsKey(key)) { - if (StringUtils.isNotBlank(value) && !isObservationMatched(key, value, timeStampValue)){ + if (StringUtils.isNotBlank(value) && !isObservationMatched(key, value, column, rowNum)){ // prior observation with updated value newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); if (!isValueMatched(key, value)){ newObservation.setValue(value); } else if (!isTimestampMatched(key, timeStampValue)) { + DateTimeFormatter formatter = DateTimeFormatter.ISO_INSTANT; + String formattedTimeStampValue = formatter.format(OffsetDateTime.parse(timeStampValue)); newObservation.setObservationTimeStamp(OffsetDateTime.parse(formattedTimeStampValue)); } pio = new PendingImportObject<>(ImportObjectState.MUTATED, (BrAPIObservation) Utilities.formatBrapiObjForDisplay(newObservation, BrAPIObservation.class, program)); From 6d424c99a7c38b9ff5ba747b682a8a0337ba38b1 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 6 Dec 2023 21:17:17 +0000 Subject: [PATCH 105/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index d4fec2383..148fd5eba 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+642 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/228d6dc9abc766a7de64138d6289026f5528308f +version=v0.9.0+644 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/dadbcf32e5d73345137ab8af5361517cf89f8b95 From 81cc921c2de8fd770946f37ca382182ecb2c1c44 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 7 Dec 2023 14:26:19 +0000 Subject: [PATCH 106/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 148fd5eba..16760a984 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+644 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/dadbcf32e5d73345137ab8af5361517cf89f8b95 +version=v0.9.0+646 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/5025cd0f7f5b3abb54e0778074c48d6b1586c71b From 3a3ad650b9ccf5e4a2a4c3ccf8ec6c6196a8e3c4 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Fri, 8 Dec 2023 12:17:14 -0500 Subject: [PATCH 107/220] [BI-1965-bugfix] - replaced deprecated env var --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 892309cb1..58fba6861 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -172,7 +172,7 @@ services: aliases: - localstack environment: - - HOSTNAME_EXTERNAL=localstack + - LOCALSTACK_HOST=localstack networks: backend: From 8e4477d28396e20a2102990abac1123aeb6c477b Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Fri, 8 Dec 2023 12:45:33 -0500 Subject: [PATCH 108/220] [BI-1965-bugfix] - replaced deprecated env var in test --- .../geno/impl/GigwaGenotypeServiceImplIntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java b/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java index afa593e7c..7851d4427 100644 --- a/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java +++ b/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java @@ -251,7 +251,7 @@ public GigwaGenotypeServiceImplIntegrationTest() { .withServices(LocalStackContainer.Service.S3) .withNetwork(super.getNetwork()) .withNetworkAliases("localstack") - .withEnv("HOSTNAME_EXTERNAL", "localstack"); + .withEnv("LOCALSTACK_HOST", "localstack"); localStackContainer.start(); } From 38ef089a9b55a022d7ce2d35f9aa856464118bbf Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Fri, 8 Dec 2023 14:05:36 -0500 Subject: [PATCH 109/220] [BI-1965-bugfix] - pinned localstack image to 3.0.2 --- docker-compose.yml | 2 +- .../geno/impl/GigwaGenotypeServiceImplIntegrationTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 58fba6861..910cd1554 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -163,7 +163,7 @@ services: - gigwa_db localstack: container_name: "localstack" - image: localstack/localstack + image: localstack/localstack:3.0.2 restart: always ports: - "4566:4566" diff --git a/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java b/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java index 7851d4427..56b3c099b 100644 --- a/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java +++ b/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java @@ -247,7 +247,7 @@ public GigwaGenotypeServiceImplIntegrationTest() { gigwa.start(); localStackContainer = new LocalStackContainer(DockerImageName.parse("localstack/localstack") - .withTag("2.2.0")) + .withTag("3.0.2")) .withServices(LocalStackContainer.Service.S3) .withNetwork(super.getNetwork()) .withNetworkAliases("localstack") From 464616d69739f73498796e66f136e35900b91206 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 11 Dec 2023 15:10:37 -0500 Subject: [PATCH 110/220] [BI-2027] - improved logging in ProgramCache --- .../java/org/breedinginsight/daos/cache/ProgramCache.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java b/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java index cb28fb67f..4f9798b9b 100644 --- a/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java +++ b/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java @@ -56,6 +56,7 @@ public void populate(List keys) { public void populate(@NotNull UUID key) { String cacheKey = generateCacheKey(key); + log.debug("populate(UUID key) method called with key: " + cacheKey); RSemaphore semaphore = connection.getSemaphore(cacheKey+":semaphore"); semaphore.trySetPermits(1); @@ -72,7 +73,7 @@ public void populate(@NotNull UUID key) { next refresh will pick up data persisted by this thread */ if(queueSemaphore.tryAcquire()) { - log.debug("Attempting to refresh"); + log.debug("Attempting to refresh for: " + cacheKey); try { // block until we get the green light to refresh the cache semaphore.acquire(); @@ -139,11 +140,11 @@ public Map get(UUID key) throws ApiException { if (!connection.getBucket(cacheKey).isExists()) { RSemaphore semaphore = connection.getSemaphore(cacheKey + ":semaphore"); try { - log.debug("cache miss, populating"); + log.debug("cache miss, populating for key: " + cacheKey); populate(key); //block until any updates are done semaphore.acquire(); - log.debug("Cache loading done!!!!"); + log.debug("Cache loading done!!!! - key: " + cacheKey); } catch(Exception e){ throw new ApiException(e); } finally { From 54924ec978371314eb741eac3f68ac8316d804df Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Mon, 11 Dec 2023 15:24:08 -0500 Subject: [PATCH 111/220] [BI-2027] - added edge case log --- src/main/java/org/breedinginsight/daos/cache/ProgramCache.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java b/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java index 4f9798b9b..d231116ae 100644 --- a/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java +++ b/src/main/java/org/breedinginsight/daos/cache/ProgramCache.java @@ -111,6 +111,8 @@ public void populate(@NotNull UUID key) { RMap map = connection.getMap(cacheKey); map.clear(); map.putAll(entryMap); + } else { + log.debug("No values to cache for key: " + cacheKey); } log.debug("cache loading complete for key: " + cacheKey); } catch (Exception e) { From ee37b7c30cb0c3ae0f427c220667b1367c145c1b Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 12 Dec 2023 13:50:37 -0500 Subject: [PATCH 112/220] [BI-2019] - added explicit shm_size to bidb --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 892309cb1..6d98294f8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -85,6 +85,7 @@ services: backend: aliases: - dbserver + shm_size: 2g brapi-server: image: breedinginsight/brapi-java-server:develop container_name: brapi-server From 9dfdc18f78e273f53b1859967782ccf541d332db Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 14 Dec 2023 16:36:13 +0000 Subject: [PATCH 113/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 16760a984..c744aa15f 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+646 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/5025cd0f7f5b3abb54e0778074c48d6b1586c71b +version=v0.9.0+648 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/cedfdd85a4f25316eb889646cbab8186f9e17346 From de3d16c47dcaadeae671ad44602d41f2be17be7b Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 14 Dec 2023 16:57:38 +0000 Subject: [PATCH 114/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 16760a984..c744aa15f 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+646 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/5025cd0f7f5b3abb54e0778074c48d6b1586c71b +version=v0.9.0+648 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/cedfdd85a4f25316eb889646cbab8186f9e17346 From 2f0652998f704d5a5a7bb85d96dd19310e79995f Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 14 Dec 2023 17:26:45 +0000 Subject: [PATCH 115/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index c744aa15f..0c9a54f59 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+648 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/cedfdd85a4f25316eb889646cbab8186f9e17346 +version=v0.9.0+651 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/53370087d164567eb471e71f4377757fc1499417 From 19d876fb61fa310c6119bfb5b0cc86723cf4b4b7 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 15 Dec 2023 15:04:05 +0000 Subject: [PATCH 116/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 0c9a54f59..dfbb0d788 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+651 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/53370087d164567eb471e71f4377757fc1499417 +version=v0.9.0+653 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c7066647bf54091eaf2c51b4ffcf8b9b2b2b2b15 From b7f5473599c95b8981bd7653f31d9b6bef21c6e2 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 8 Dec 2023 11:35:18 -0500 Subject: [PATCH 117/220] assign existing germplasm to OU PIO by gid instead of by germplasm name --- .../ExperimentObservation.java | 2 ++ .../services/processors/ExperimentProcessor.java | 14 +++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java index 290c046ad..c2f5926d4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/experimentObservation/ExperimentObservation.java @@ -228,6 +228,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( String seqVal, boolean commit, String germplasmName, + String gid, String referenceSource, UUID trialID, UUID datasetId, @@ -250,6 +251,7 @@ public BrAPIObservationUnit constructBrAPIObservationUnit( germplasmName = getGermplasmName(); } observationUnit.setGermplasmName(germplasmName); + observationUnit.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GID, gid); BrAPIObservationUnitPosition position = new BrAPIObservationUnitPosition(); BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index d35da62d8..5989e88f5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -956,7 +956,7 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); UUID studyID = studyPIO.getId(); UUID id = UUID.randomUUID(); - BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); + BrAPIObservationUnit newObservationUnit = importRow.constructBrAPIObservationUnit(program, envSeqValue, commit, germplasmName, importRow.getGid(), BRAPI_REFERENCE_SOURCE, trialID, datasetId, studyID, id); // check for existing units if this is an existing study if (studyPIO.getBrAPIObject().getStudyDbId() != null) { @@ -1291,16 +1291,16 @@ private void updateStudyDbId(BrAPIStudy study, String programKey) { private void updateGermplasmDbId(BrAPIGermplasm germplasm) { this.observationUnitByNameNoScope.values() .stream() - .filter(obsUnit -> obsUnit.getBrAPIObject() - .getGermplasmName() != null && - obsUnit.getBrAPIObject() - .getGermplasmName() - .equals(germplasm.getGermplasmName())) + .filter(obsUnit -> germplasm.getAccessionNumber() != null && + germplasm.getAccessionNumber().equals(obsUnit + .getBrAPIObject() + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.GID).getAsString())) .forEach(obsUnit -> obsUnit.getBrAPIObject() .setGermplasmDbId(germplasm.getGermplasmDbId())); } - +//obsUnit.getBrAPIObject().getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.GID).getAsString() private void updateStudyDependencyValues(Map mappedBrAPIImport, String programKey) { // update location DbIds in studies for all distinct locations mappedBrAPIImport.values() From 8852119afff8b1f2b761b6228e7578b85b91aac7 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 8 Dec 2023 11:56:15 -0500 Subject: [PATCH 118/220] remove comment --- .../importer/services/processors/ExperimentProcessor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 5989e88f5..15ae356c3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1299,8 +1299,7 @@ private void updateGermplasmDbId(BrAPIGermplasm germplasm) { .forEach(obsUnit -> obsUnit.getBrAPIObject() .setGermplasmDbId(germplasm.getGermplasmDbId())); } - -//obsUnit.getBrAPIObject().getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.GID).getAsString() + private void updateStudyDependencyValues(Map mappedBrAPIImport, String programKey) { // update location DbIds in studies for all distinct locations mappedBrAPIImport.values() From 0a9181c0bab31c181fb91a078221de8e346aa2eb Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 8 Dec 2023 14:16:13 -0500 Subject: [PATCH 119/220] cleanup --- .../importer/services/processors/ExperimentProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 15ae356c3..aded65fe1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1299,7 +1299,7 @@ private void updateGermplasmDbId(BrAPIGermplasm germplasm) { .forEach(obsUnit -> obsUnit.getBrAPIObject() .setGermplasmDbId(germplasm.getGermplasmDbId())); } - + private void updateStudyDependencyValues(Map mappedBrAPIImport, String programKey) { // update location DbIds in studies for all distinct locations mappedBrAPIImport.values() From 2973f50a0870a8f138f13f5a1a1889f5789fb9a5 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 20 Dec 2023 19:13:59 +0000 Subject: [PATCH 120/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index dfbb0d788..4bf7f7da1 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+653 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c7066647bf54091eaf2c51b4ffcf8b9b2b2b2b15 +version=v0.9.0+656 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/93f983d5329d5a91546a62f6f43503b68ca4eb8a From 6c1278fda41ff67c8a5260fa557a45c3e8c39550 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 22 Dec 2023 14:58:51 -0500 Subject: [PATCH 121/220] [BI-2014] removed code that was causing a problem. We _may_ need to address legacy data in the future --- .../importer/services/processors/ExperimentProcessor.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index aded65fe1..87abc3fc6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1062,11 +1062,8 @@ private void addObsVarsToDatasetDetails(PendingImportObject pi referencedTraits.forEach(trait -> { String id = Utilities.appendProgramKey(trait.getObservationVariableName(), program.getKey()); - // Don't append the key if connected to a brapi service operating with legacy data(no appended program key) - if (trait.getFullName() == null) { - id = trait.getObservationVariableName(); - } - + // TODO - Don't append the key if connected to a brapi service operating with legacy data(no appended program key) + if (!details.getData().contains(id) && ImportObjectState.EXISTING != pio.getState()) { details.getData().add(id); } From a1d30bcc697a2b27befae7cabbe9d30f3c1afea5 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 6 Dec 2023 13:52:03 -0500 Subject: [PATCH 122/220] change OU error message --- .../importer/services/processors/ExperimentProcessor.java | 8 ++------ .../brapps/importer/ExperimentFileImportTest.java | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index aded65fe1..cfba838c7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -26,7 +26,6 @@ import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.JSON; @@ -87,14 +86,11 @@ public class ExperimentProcessor implements Processor { private static final String NAME = "Experiment"; + private static final String MISSING_OBS_UNIT_ID_ERROR = "Error(s) detected in file. (See details below.) Import cannot proceed."; private static final String EXISTING_ENV = "Cannot create new observation unit %s for existing environment %s.

" + "If you’re trying to add these units to the experiment, please create a new environment" + " with all appropriate experiment units (NOTE: this will generate new Observation Unit Ids " + "for each experiment unit)."; - private static final String MISSING_OBS_UNIT_ID_ERROR = "Experiment Units are missing Observation Unit Id.

" + - "If you’re trying to add these units to the experiment, please create a new environment" + - " with all appropriate experiment units (NOTE: this will generate new Observation Unit Ids " + - "for each experiment unit)."; private static final String MULTIPLE_EXP_TITLES = "File contains more than one Experiment Title"; private static final String MIDNIGHT = "T00:00:00-00:00"; private static final String TIMESTAMP_PREFIX = "TS:"; @@ -1066,7 +1062,7 @@ private void addObsVarsToDatasetDetails(PendingImportObject pi if (trait.getFullName() == null) { id = trait.getObservationVariableName(); } - + if (!details.getData().contains(id) && ImportObjectState.EXISTING != pio.getState()) { details.getData().add(id); } diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index e6b64ca49..4e663386e 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -604,7 +604,7 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); - assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Cannot create new observation unit")); + assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Error(s) detected in file.")); } @Test From 34c590b2c65705ce4393253127d6854ce9a97ec0 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 7 Dec 2023 14:15:15 -0500 Subject: [PATCH 123/220] update message --- .../importer/services/processors/ExperimentProcessor.java | 2 +- .../brapps/importer/ExperimentFileImportTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index cfba838c7..59eb78cec 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -86,7 +86,7 @@ public class ExperimentProcessor implements Processor { private static final String NAME = "Experiment"; - private static final String MISSING_OBS_UNIT_ID_ERROR = "Error(s) detected in file. (See details below.) Import cannot proceed."; + private static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs. You must append file with the appropriate ObsUnitIDs or specify a new environment for these entities to proceed."; private static final String EXISTING_ENV = "Cannot create new observation unit %s for existing environment %s.

" + "If you’re trying to add these units to the experiment, please create a new environment" + " with all appropriate experiment units (NOTE: this will generate new Observation Unit Ids " + diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 4e663386e..a599fd1e2 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -604,7 +604,7 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); - assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Error(s) detected in file.")); + assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experimental entities are missing ObsUnitIDs.")); } @Test From 219d848d94417807b85b9f7305b6eb7a653dbdfb Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 12 Dec 2023 11:31:22 -0500 Subject: [PATCH 124/220] [BI-1761] update and optimize --- .../importer/services/processors/ExperimentProcessor.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 59eb78cec..c691f49c8 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -62,8 +62,6 @@ import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; -import org.breedinginsight.services.exceptions.MissingRequiredInfoException; -import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.services.exceptions.ValidatorException; import org.breedinginsight.utilities.Utilities; import org.jooq.DSLContext; @@ -71,7 +69,6 @@ import tech.tablesaw.columns.Column; import javax.inject.Inject; -import javax.validation.Valid; import java.math.BigDecimal; import java.math.BigInteger; import java.time.OffsetDateTime; @@ -86,7 +83,7 @@ public class ExperimentProcessor implements Processor { private static final String NAME = "Experiment"; - private static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs. You must append file with the appropriate ObsUnitIDs or specify a new environment for these entities to proceed."; + private static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs. You must append file with the appropriate ObsUnitIDs or specify a new environment for these entities to proceed"; private static final String EXISTING_ENV = "Cannot create new observation unit %s for existing environment %s.

" + "If you’re trying to add these units to the experiment, please create a new environment" + " with all appropriate experiment units (NOTE: this will generate new Observation Unit Ids " + From 9ee490676aca8a52fbe2fc3a9ea8763ca45f6d3e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 12 Dec 2023 13:27:04 -0500 Subject: [PATCH 125/220] [BI-1761] use MissingInformationException --- .../services/processors/ExperimentProcessor.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index c691f49c8..6e4ea7033 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -61,6 +61,7 @@ import org.breedinginsight.services.OntologyService; import org.breedinginsight.services.ProgramLocationService; import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.services.exceptions.MissingRequiredInfoException; import org.breedinginsight.services.exceptions.UnprocessableEntityException; import org.breedinginsight.services.exceptions.ValidatorException; import org.breedinginsight.utilities.Utilities; @@ -84,10 +85,6 @@ public class ExperimentProcessor implements Processor { private static final String NAME = "Experiment"; private static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs. You must append file with the appropriate ObsUnitIDs or specify a new environment for these entities to proceed"; - private static final String EXISTING_ENV = "Cannot create new observation unit %s for existing environment %s.

" + - "If you’re trying to add these units to the experiment, please create a new environment" + - " with all appropriate experiment units (NOTE: this will generate new Observation Unit Ids " + - "for each experiment unit)."; private static final String MULTIPLE_EXP_TITLES = "File contains more than one Experiment Title"; private static final String MIDNIGHT = "T00:00:00-00:00"; private static final String TIMESTAMP_PREFIX = "TS:"; @@ -208,7 +205,7 @@ public Map process( Table data, Program program, User user, - boolean commit) throws UnprocessableEntityException, ApiException, ValidatorException { + boolean commit) throws ApiException, ValidatorException, MissingRequiredInfoException { log.debug("processing experiment import"); ValidationErrors validationErrors = new ValidationErrors(); @@ -491,7 +488,7 @@ private String getVariableNameFromColumn(Column column) { return column.name(); } - private void initNewBrapiData(List importRows, List> phenotypeCols, Program program, User user, List referencedTraits, boolean commit) throws UnprocessableEntityException, ApiException { + private void initNewBrapiData(List importRows, List> phenotypeCols, Program program, User user, List referencedTraits, boolean commit) throws ApiException, MissingRequiredInfoException { String expSequenceName = program.getExpSequence(); if (expSequenceName == null) { @@ -926,7 +923,7 @@ private PendingImportObject getGidPOI(ExperimentObservation impo return null; } - private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws UnprocessableEntityException, ApiException { + private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException, MissingRequiredInfoException { PendingImportObject pio; String key = createObservationUnitKey(importRow); if (this.observationUnitByNameNoScope.containsKey(key)) { @@ -956,8 +953,7 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra List existingOUs = brAPIObservationUnitDAO.getObservationUnitsForStudyDbId(studyPIO.getBrAPIObject().getStudyDbId(), program); List matchingOU = existingOUs.stream().filter(ou -> importRow.getExpUnitId().equals(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey()))).collect(Collectors.toList()); if (matchingOU.isEmpty()) { - throw new UnprocessableEntityException(String.format(EXISTING_ENV, importRow.getExpUnitId(), - Utilities.removeProgramKeyAndUnknownAdditionalData(studyPIO.getBrAPIObject().getStudyName(), program.getKey()))); + throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); } else { pio = new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIObservationUnit) Utilities.formatBrapiObjForDisplay(matchingOU.get(0), BrAPIObservationUnit.class, program)); } From bded9388239f1c41b48b3923e9a75e6993c7945e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 20 Dec 2023 12:30:20 -0500 Subject: [PATCH 126/220] update message --- .../importer/services/processors/ExperimentProcessor.java | 2 +- .../brapps/importer/ExperimentFileImportTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 6e4ea7033..73add7aed 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -84,7 +84,7 @@ public class ExperimentProcessor implements Processor { private static final String NAME = "Experiment"; - private static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs. You must append file with the appropriate ObsUnitIDs or specify a new environment for these entities to proceed"; + private static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs"; private static final String MULTIPLE_EXP_TITLES = "File contains more than one Experiment Title"; private static final String MIDNIGHT = "T00:00:00-00:00"; private static final String TIMESTAMP_PREFIX = "TS:"; diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index a599fd1e2..8db3b5bf4 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -604,7 +604,7 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); - assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experimental entities are missing ObsUnitIDs.")); + assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experimental entities are missing ObsUnitIDs")); } @Test From 7c975804931e1a394665b2cdc196783605677ff9 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 4 Jan 2024 21:07:10 +0000 Subject: [PATCH 127/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 4bf7f7da1..911e044fc 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+656 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/93f983d5329d5a91546a62f6f43503b68ca4eb8a +version=v0.9.0+658 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c11b492d5d05be8ca456550e211be801b83ebb55 From b5fe48d995fd9590d13d468bf5544ee5e6801968 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 8 Jan 2024 18:54:41 +0000 Subject: [PATCH 128/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 911e044fc..dbaa46bba 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+658 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/c11b492d5d05be8ca456550e211be801b83ebb55 +version=v0.9.0+660 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/7c975804931e1a394665b2cdc196783605677ff9 From 6753df1e709ac38202cf49fe360f758574be390a Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Thu, 11 Jan 2024 18:59:07 +0000 Subject: [PATCH 129/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index dbaa46bba..96d6b8fba 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+660 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/7c975804931e1a394665b2cdc196783605677ff9 +version=v0.9.0+664 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/12270a172908fe723960ca99111c84852f478322 From aee0658a173b31ace24ec1b9f9d19553744e94b5 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 18 Jan 2024 10:53:40 -0500 Subject: [PATCH 130/220] add season to additional info --- .../services/processors/ExperimentProcessor.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 070b2214a..29079cff1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1091,7 +1091,12 @@ private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program pr addObsVarsToDatasetDetails(pio, referencedTraits, program); } - private PendingImportObject fetchOrCreateStudyPIO(Program program, boolean commit, String expSequenceValue, ExperimentObservation importRow, Supplier envNextVal) { + private PendingImportObject fetchOrCreateStudyPIO( + Program program, + boolean commit, + String expSequenceValue, + ExperimentObservation importRow, + Supplier envNextVal) { PendingImportObject pio; if (studyByNameNoScope.containsKey(importRow.getEnv())) { pio = studyByNameNoScope.get(importRow.getEnv()); @@ -1133,8 +1138,9 @@ private void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study) { //if it is already there, don't add it. if(additionalInfo==null || additionalInfo.get(BrAPIAdditionalInfoFields.ENV_YEAR)==null) { - String seasonDbId = study.getSeasons().get(0); - String year = seasonDbIdToYear(seasonDbId, program.getId()); + String year = study.getSeasons().get(0); + //String seasonDbId = study.getSeasons().get(0); + //String year = seasonDbIdToYear(seasonDbId, program.getId()); addYearToStudyAdditionalInfo(program, study, year); } } From e2e6aa0814728c8c9d6a6c17198c725a9523fff9 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 18 Jan 2024 11:48:41 -0500 Subject: [PATCH 131/220] remove NPE --- .../services/processors/ExperimentProcessor.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 29079cff1..bfab24570 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -383,10 +383,19 @@ public void postBrapiData(Map mappedBrAPIImport, Program mutatedObservationByDbId.forEach((id, observation) -> { try { + if (observation == null) { + throw new Exception("Null observation"); + } BrAPIObservation updatedObs = brAPIObservationDAO.updateBrAPIObservation(id, observation, program.getId()); - if (!observation.getValue().equals(updatedObs.getValue()) || !observation.getObservationTimeStamp().isEqual(updatedObs.getObservationTimeStamp())) { + + if (updatedObs == null) { + throw new Exception("Null updated observation"); + } + + if (!Objects.equals(observation.getValue(), updatedObs.getValue()) + || !Objects.equals(observation.getObservationTimeStamp(), updatedObs.getObservationTimeStamp())) { String message; - if(!observation.getValue().equals(updatedObs.getValue())) { + if(!Objects.equals(observation.getValue(), updatedObs.getValue())) { message = String.format("Updated observation, %s, from BrAPI service does not match requested update %s.", updatedObs.getValue(), observation.getValue()); } else { message = String.format("Updated observation timestamp, %s, from BrAPI service does not match requested update timestamp %s.", updatedObs.getObservationTimeStamp(), observation.getObservationTimeStamp()); @@ -401,9 +410,7 @@ public void postBrapiData(Map mappedBrAPIImport, Program throw new InternalServerException(e.getMessage(), e); } }); - log.debug("experiment import complete"); - } private void prepareDataForValidation(List importRows, List> phenotypeCols, Map mappedBrAPIImport) { From b4783e8224efbad53418f0242693f5b44abb07f8 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 18 Jan 2024 15:17:25 -0500 Subject: [PATCH 132/220] remove comments --- .../importer/services/processors/ExperimentProcessor.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index bfab24570..7f8d32069 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1146,8 +1146,6 @@ private void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study) { //if it is already there, don't add it. if(additionalInfo==null || additionalInfo.get(BrAPIAdditionalInfoFields.ENV_YEAR)==null) { String year = study.getSeasons().get(0); - //String seasonDbId = study.getSeasons().get(0); - //String year = seasonDbIdToYear(seasonDbId, program.getId()); addYearToStudyAdditionalInfo(program, study, year); } } From 581767966423e32f11a42daec3df83b7ca6fabc9 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 23 Jan 2024 21:56:02 +0000 Subject: [PATCH 133/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 96d6b8fba..ce321d905 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+664 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/12270a172908fe723960ca99111c84852f478322 +version=v0.9.0+668 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/6753df1e709ac38202cf49fe360f758574be390a From 30518eab48555aac6ae4aadd0152f5973c42a91c Mon Sep 17 00:00:00 2001 From: timparsons Date: Mon, 18 Sep 2023 18:29:23 -0400 Subject: [PATCH 134/220] [BI-1921] Adding OIDC endpoints Stubbed out methods needed for Field Book to fetch data --- .../v1/controller}/ExperimentController.java | 55 +------ .../api/v1/controller/TokenController.java | 2 +- .../auth/OIDCDiscoveryController.java | 71 +++++++++ ...ler.java => BrAPIGermplasmController.java} | 4 +- .../brapi/v2/BrAPIImagesController.java | 90 ++++++++++++ ...ntroller.java => BrAPIListController.java} | 6 +- .../v2/BrAPIObservationUnitController.java | 118 +++++++++++++++ .../BrAPIObservationVariableController.java | 93 ++++++++++++ .../brapi/v2/BrAPIObservationsController.java | 136 ++++++++++++++++++ .../brapi/v2/BrAPIProgramsController.java | 106 ++++++++++++++ ...oller.java => BrAPIStudiesController.java} | 34 ++++- .../brapi/v2/BrAPITrialsController.java | 101 +++++++++++++ .../brapi/v2/BrAPIV2Controller.java | 136 +++++++++++++++--- .../brapi/v2/ServiceBuilder.java | 80 +++++++++++ .../response/mappers/StudyQueryMapper.java | 7 + 15 files changed, 960 insertions(+), 79 deletions(-) rename src/main/java/org/breedinginsight/{brapi/v2 => api/v1/controller}/ExperimentController.java (63%) create mode 100644 src/main/java/org/breedinginsight/api/v1/controller/auth/OIDCDiscoveryController.java rename src/main/java/org/breedinginsight/brapi/v2/{GermplasmController.java => BrAPIGermplasmController.java} (98%) create mode 100644 src/main/java/org/breedinginsight/brapi/v2/BrAPIImagesController.java rename src/main/java/org/breedinginsight/brapi/v2/{ListController.java => BrAPIListController.java} (96%) create mode 100644 src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java create mode 100644 src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java create mode 100644 src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java create mode 100644 src/main/java/org/breedinginsight/brapi/v2/BrAPIProgramsController.java rename src/main/java/org/breedinginsight/brapi/v2/{StudyController.java => BrAPIStudiesController.java} (71%) create mode 100644 src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java create mode 100644 src/main/java/org/breedinginsight/brapi/v2/ServiceBuilder.java diff --git a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java similarity index 63% rename from src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java rename to src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java index c819cc10c..7c64827fe 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ExperimentController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java @@ -1,4 +1,4 @@ -package org.breedinginsight.brapi.v2; +package org.breedinginsight.api.v1.controller; import io.micronaut.http.HttpHeaders; import io.micronaut.http.HttpResponse; @@ -9,28 +9,21 @@ import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; import lombok.extern.slf4j.Slf4j; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPITrial; import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; -import org.breedinginsight.api.model.v1.request.query.SearchRequest; -import org.breedinginsight.api.model.v1.response.DataResponse; import org.breedinginsight.api.model.v1.response.Response; -import org.breedinginsight.api.model.v1.validators.QueryValid; -import org.breedinginsight.brapi.v1.controller.BrapiVersion; import org.breedinginsight.brapi.v2.model.request.query.ExperimentExportQuery; -import org.breedinginsight.brapi.v2.model.request.query.ExperimentQuery; import org.breedinginsight.brapi.v2.services.BrAPITrialService; import org.breedinginsight.model.Dataset; import org.breedinginsight.model.DownloadFile; import org.breedinginsight.model.Program; import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.utilities.response.ResponseUtils; import org.breedinginsight.utilities.response.mappers.ExperimentQueryMapper; + import javax.inject.Inject; import javax.validation.Valid; -import java.util.*; +import java.util.UUID; @Slf4j @Controller @@ -47,48 +40,6 @@ public ExperimentController(BrAPITrialService experimentService, ExperimentQuery this.programService = programService; } - @Get("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/trials{?queryParams*}") - @Produces(MediaType.APPLICATION_JSON) - @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse>>> getExperiments( - @PathVariable("programId") UUID programId, - @QueryValue @QueryValid(using = ExperimentQueryMapper.class) @Valid ExperimentQuery queryParams) { - try { - log.debug("fetching trials for program: " + programId); - - List experiments = experimentService.getExperiments(programId); - SearchRequest searchRequest = queryParams.constructSearchRequest(); - return ResponseUtils.getBrapiQueryResponse(experiments, experimentQueryMapper, queryParams, searchRequest); - } catch (ApiException e) { - log.info(e.getMessage(), e); - return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving experiments"); - } catch (DoesNotExistException e) { - log.info(e.getMessage(), e); - return HttpResponse.status(HttpStatus.NOT_FOUND, e.getMessage()); - } - } - - @Get("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/trials/{trialId}") - @Produces(MediaType.APPLICATION_JSON) - @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse> getExperimentById( - @PathVariable("programId") UUID programId, - @PathVariable("trialId") UUID trialId, - @QueryValue(defaultValue = "false") Boolean stats){ - try { - String logMsg = "fetching trial id:" + trialId +" for program: " + programId; - if(stats){ - logMsg += " with stats"; - } - log.debug(logMsg); - Response response = new Response<>(experimentService.getTrialDataByUUID(programId, trialId, stats)); - return HttpResponse.ok(response); - } catch (DoesNotExistException e) { - log.info(e.getMessage(), e); - return HttpResponse.status(HttpStatus.NOT_FOUND, e.getMessage()); - } - } - @Get("/${micronaut.bi.api.version}/programs/{programId}/experiments/{experimentId}/export{?queryParams*}") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) @Produces(value={"text/csv", "application/vnd.ms-excel", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/octet-stream"}) diff --git a/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java b/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java index 9cae400fa..ca6053be7 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java @@ -61,7 +61,7 @@ public HttpResponse apiToken(@QueryValue @Nullable String returnUrl) { ApiToken apiToken = token.get(); if(returnUrl != null) { - if(StringUtils.trim(returnUrl).isEmpty()) { + if(StringUtils.trim(returnUrl).isEmpty() || "undefined".equalsIgnoreCase(returnUrl)) { return HttpResponse.badRequest("returnUrl cannot be blank"); } URI location = UriBuilder.of(returnUrl) diff --git a/src/main/java/org/breedinginsight/api/v1/controller/auth/OIDCDiscoveryController.java b/src/main/java/org/breedinginsight/api/v1/controller/auth/OIDCDiscoveryController.java new file mode 100644 index 000000000..4d58da165 --- /dev/null +++ b/src/main/java/org/breedinginsight/api/v1/controller/auth/OIDCDiscoveryController.java @@ -0,0 +1,71 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.api.v1.controller.auth; + +import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.MediaType; +import io.micronaut.http.annotation.Controller; +import io.micronaut.http.annotation.Get; +import io.micronaut.http.annotation.Produces; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; +import org.breedinginsight.dao.db.tables.pojos.ProgramEntity; +import org.breedinginsight.daos.ProgramDAO; + +import javax.inject.Inject; +import java.util.UUID; + +@Controller +@Secured(SecurityRule.IS_ANONYMOUS) +public class OIDCDiscoveryController { + + private static final String OIDC_CONFIG = """ + { + "issuer": "%s", + "authorization_endpoint": "%s/programs/%s/brapi/authorize", + "jwks_uri": "", + "token_endpoint": "", + "grant_types_supported": ["implicit"], + "response_types_supported": ["token"], + "subject_types_supported": ["public"], + "id_token_signing_alg_values_supported": [] + } + """; + + private final String webUrl; + private final ProgramDAO programDAO; + + @Inject + public OIDCDiscoveryController(@Property(name="web.base-url") String webUrl, ProgramDAO programDAO) { + this.webUrl = webUrl; + this.programDAO = programDAO; + } + + + @Get("/${micronaut.bi.api.version}/programs/{programId}/.well-known/openid-configuration") + @Produces(MediaType.APPLICATION_JSON) + public HttpResponse getOpenIdConfig(UUID programId) { + ProgramEntity programEntity = programDAO.fetchOneById(programId); + if(programEntity != null) { + return HttpResponse.ok(String.format(OIDC_CONFIG, webUrl, webUrl, programId)); + } + + return HttpResponse.notFound("unknown program"); + } +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java similarity index 98% rename from src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java rename to src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java index 568e58d99..edc89b5f0 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/GermplasmController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIGermplasmController.java @@ -51,7 +51,7 @@ @Slf4j @Controller("/${micronaut.bi.api.version}") @Secured(SecurityRule.IS_AUTHENTICATED) -public class GermplasmController { +public class BrAPIGermplasmController { private final BrAPIGermplasmService germplasmService; private final GermplasmQueryMapper germplasmQueryMapper; @@ -64,7 +64,7 @@ public class GermplasmController { @Inject - public GermplasmController(BrAPIGermplasmService germplasmService, GermplasmQueryMapper germplasmQueryMapper, ProgramDAO programDAO, BrAPIGermplasmDAO germplasmDAO, GenotypeService genoService, BrAPIEndpointProvider brAPIEndpointProvider) { + public BrAPIGermplasmController(BrAPIGermplasmService germplasmService, GermplasmQueryMapper germplasmQueryMapper, ProgramDAO programDAO, BrAPIGermplasmDAO germplasmDAO, GenotypeService genoService, BrAPIEndpointProvider brAPIEndpointProvider) { this.germplasmService = germplasmService; this.germplasmQueryMapper = germplasmQueryMapper; this.programDAO = programDAO; diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIImagesController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIImagesController.java new file mode 100644 index 000000000..1893d10d0 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIImagesController.java @@ -0,0 +1,90 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2; + +import io.micronaut.http.HttpResponse; +import io.micronaut.http.annotation.*; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; +import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.pheno.BrAPIImage; +import org.breedinginsight.api.auth.ProgramSecured; +import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; +import org.breedinginsight.brapi.v1.controller.BrapiVersion; + +import java.util.List; +import java.util.UUID; + +@Slf4j +@Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2) +@Secured(SecurityRule.IS_AUTHENTICATED) +public class BrAPIImagesController { + /* + TODO + - POST images + - PUT imagesImageDbIdImagecontent + - PUT imagesImageDbIdPut + */ + @Get("/images") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse imagesGet(@PathVariable("programId") UUID programId, + @QueryValue("imageDbId") String imageDbId, + @QueryValue("imageName") String imageName, + @QueryValue("observationUnitDbId") String observationUnitDbId, + @QueryValue("observationDbId") String observationDbId, + @QueryValue("descriptiveOntologyTerm") String descriptiveOntologyTerm, + @QueryValue("commonCropName") String commonCropName, + @QueryValue("programDbId") String programDbId, + @QueryValue("externalReferenceID") String externalReferenceID, + @QueryValue("externalReferenceId") String externalReferenceId, + @QueryValue("externalReferenceSource") String externalReferenceSource, + @QueryValue("page") Integer page, + @QueryValue("pageSize") Integer pageSize) { + return HttpResponse.notFound(); + } + + @Get("/images/{imageDbId}") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse imagesImageDbIdGet(@PathVariable("programId") UUID programId, + @PathVariable("imageDbId") String imageDbId) { + return HttpResponse.notFound(); + } + + @Put("/images/{imageDbId}/imagecontent") + @Consumes({"image/_*"}) + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse imagesImageDbIdImagecontentPut(@PathVariable("programId") UUID programId, + @PathVariable("imageDbId") String imageDbId, + Object body) { + return HttpResponse.notFound(); + } + + @Put("/images/{imageDbId}") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse imagesImageDbIdPut(@PathVariable("programId") UUID programId, + @PathVariable("imageDbId") String imageDbId, + BrAPIImage body) { + return HttpResponse.notFound(); + } + + @Post("/images") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse imagesPost(@PathVariable("programId") UUID programId, List body) { + return HttpResponse.notFound(); + } +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/ListController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIListController.java similarity index 96% rename from src/main/java/org/breedinginsight/brapi/v2/ListController.java rename to src/main/java/org/breedinginsight/brapi/v2/BrAPIListController.java index 7b33629ef..115448d58 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/ListController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIListController.java @@ -50,15 +50,15 @@ @Slf4j @Controller @Secured(SecurityRule.IS_AUTHENTICATED) -public class ListController { +public class BrAPIListController { private final ProgramService programService; private final BrAPIListService listService; private final ListQueryMapper listQueryMapper; @Inject - public ListController(ProgramService programService, BrAPIListService listService, - ListQueryMapper listQueryMapper) { + public BrAPIListController(ProgramService programService, BrAPIListService listService, + ListQueryMapper listQueryMapper) { this.programService = programService; this.listService = listService; this.listQueryMapper = listQueryMapper; diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java new file mode 100644 index 000000000..96b91df36 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java @@ -0,0 +1,118 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2; + +import io.micronaut.http.HttpResponse; +import io.micronaut.http.annotation.*; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; +import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.breedinginsight.api.auth.ProgramSecured; +import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; +import org.breedinginsight.brapi.v1.controller.BrapiVersion; + +import java.util.List; +import java.util.Map; +import java.util.UUID; + +@Slf4j +@Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2) +@Secured(SecurityRule.IS_AUTHENTICATED) +public class BrAPIObservationUnitController { + + /* + TODO + - GET observationLevels + - GET observationUnits + */ + @Get("/observationunits") + public HttpResponse observationunitsGet(@PathVariable("programId") UUID programId, + @QueryValue("observationUnitDbId") String observationUnitDbId, + @QueryValue("observationUnitName") String observationUnitName, + @QueryValue("locationDbId") String locationDbId, + @QueryValue("seasonDbId") String seasonDbId, + @QueryValue("includeObservations") Boolean includeObservations, + @QueryValue("observationUnitLevelName") String observationUnitLevelName, + @QueryValue("observationUnitLevelOrder") String observationUnitLevelOrder, + @QueryValue("observationUnitLevelCode") String observationUnitLevelCode, + @QueryValue("observationUnitLevelRelationshipName") String observationUnitLevelRelationshipName, + @QueryValue("observationUnitLevelRelationshipOrder") String observationUnitLevelRelationshipOrder, + @QueryValue("observationUnitLevelRelationshipCode") String observationUnitLevelRelationshipCode, + @QueryValue("observationUnitLevelRelationshipDbId") String observationUnitLevelRelationshipDbId, + @QueryValue("commonCropName") String commonCropName, + @QueryValue("programDbId") String programDbId, + @QueryValue("trialDbId") String trialDbId, + @QueryValue("studyDbId") String studyDbId, + @QueryValue("germplasmDbId") String germplasmDbId, + @QueryValue("externalReferenceID") String externalReferenceID, + @QueryValue("externalReferenceId") String externalReferenceId, + @QueryValue("externalReferenceSource") String externalReferenceSource, + @QueryValue("page") Integer page, + @QueryValue("pageSize") Integer pageSize) { + return HttpResponse.notFound(); + } + + @Get("/observationunits/{observationUnitDbId}") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse observationunitsObservationUnitDbIdGet(@PathVariable("programId") UUID programId, @PathVariable("observationUnitDbId") String observationUnitDbId) { + return HttpResponse.notFound(); + } + + @Put("/observationunits/{observationUnitDbId}") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse observationunitsObservationUnitDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("observationUnitDbId") String observationUnitDbId, BrAPIObservationUnit body) { + return HttpResponse.notFound(); + } + + @Post("/observationunits") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse observationunitsPost(@PathVariable("programId") UUID programId, List body) { + return HttpResponse.notFound(); + } + + @Put("/observationunits") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse observationunitsPut(@PathVariable("programId") UUID programId, Map body) { + return HttpResponse.notFound(); + } + + @Get("/observationunits/table") + @Produces({"application/json", "text/csv", "text/tsv"}) + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse observationunitsTableGet(@PathVariable("programId") UUID programId, + @Header("Accept") String accept, + @QueryValue("observationUnitDbId") String observationUnitDbId, + @QueryValue("observationVariableDbId") String observationVariableDbId, + @QueryValue("locationDbId") String locationDbId, + @QueryValue("seasonDbId") String seasonDbId, + @QueryValue("observationLevel") String observationLevel, + @QueryValue("programDbId") String programDbId, + @QueryValue("trialDbId") String trialDbId, + @QueryValue("studyDbId") String studyDbId, + @QueryValue("germplasmDbId") String germplasmDbId, + @QueryValue("observationUnitLevelName") String observationUnitLevelName, + @QueryValue("observationUnitLevelOrder") String observationUnitLevelOrder, + @QueryValue("observationUnitLevelCode") String observationUnitLevelCode, + @QueryValue("observationUnitLevelRelationshipName") String observationUnitLevelRelationshipName, + @QueryValue("observationUnitLevelRelationshipOrder") String observationUnitLevelRelationshipOrder, + @QueryValue("observationUnitLevelRelationshipCode") String observationUnitLevelRelationshipCode, + @QueryValue("observationUnitLevelRelationshipDbId") String observationUnitLevelRelationshipDbId) { + return HttpResponse.notFound(); + } +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java new file mode 100644 index 000000000..29dfa9e0b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java @@ -0,0 +1,93 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2; + +import io.micronaut.http.HttpResponse; +import io.micronaut.http.annotation.*; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; +import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.pheno.BrAPIObservationVariable; +import org.breedinginsight.api.auth.ProgramSecured; +import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; +import org.breedinginsight.brapi.v1.controller.BrapiVersion; + +import java.util.List; +import java.util.UUID; + +@Slf4j +@Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2) +@Secured(SecurityRule.IS_AUTHENTICATED) +public class BrAPIObservationVariableController { + /* + TODO + - GET /variables + */ + + @Get("/variables") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse variablesGet(@PathVariable("programId") UUID programId, + @QueryValue("observationVariableDbId") String observationVariableDbId, + @QueryValue("observationVariableName") String observationVariableName, + @QueryValue("observationVariablePUI") String observationVariablePUI, + @QueryValue("traitClass") String traitClass, + @QueryValue("methodDbId") String methodDbId, + @QueryValue("methodName") String methodName, + @QueryValue("methodPUI") String methodPUI, + @QueryValue("scaleDbId") String scaleDbId, + @QueryValue("scaleName") String scaleName, + @QueryValue("scalePUI") String scalePUI, + @QueryValue("traitDbId") String traitDbId, + @QueryValue("traitName") String traitName, + @QueryValue("traitPUI") String traitPUI, + @QueryValue("ontologyDbId") String ontologyDbId, + @QueryValue("commonCropName") String commonCropName, + @QueryValue("programDbId") String programDbId, + @QueryValue("trialDbId") String trialDbId, + @QueryValue("studyDbId") String studyDbId, + @QueryValue("externalReferenceID") String externalReferenceID, + @QueryValue("externalReferenceId") String externalReferenceId, + @QueryValue("externalReferenceSource") String externalReferenceSource, + @QueryValue("page") Integer page, + @QueryValue("pageSize") Integer pageSize) { + return HttpResponse.notFound(); + } + + @Get("/variables/{observationVariableDbId}") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse variablesObservationVariableDbIdGet(@PathVariable("programId") UUID programId, + @PathVariable("observationVariableDbId") String observationVariableDbId) { + return HttpResponse.notFound(); + } + + @Put("/variables/{observationVariableDbId}") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse variablesObservationVariableDbIdPut(@PathVariable("programId") UUID programId, + @PathVariable("observationVariableDbId") String observationVariableDbId, + BrAPIObservationVariable body) { + //DO NOT IMPLEMENT - Users are only able to update traits via the DeltaBreed UI + return HttpResponse.notFound(); + } + + @Post("/variables") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse variablesPost(@PathVariable("programId") UUID programId, List body) { + //DO NOT IMPLEMENT - Users are only able to create new traits via the DeltaBreed UI + return HttpResponse.notFound(); + } +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java new file mode 100644 index 000000000..cd35d9d5b --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java @@ -0,0 +1,136 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2; + + +import io.micronaut.http.HttpResponse; +import io.micronaut.http.annotation.*; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; +import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.api.auth.ProgramSecured; +import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; +import org.breedinginsight.brapi.v1.controller.BrapiVersion; + +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +@Slf4j +@Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2) +@Secured(SecurityRule.IS_AUTHENTICATED) +public class BrAPIObservationsController { + + @Get("/observations") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse observationsGet(@PathVariable("programId") UUID programId, + @QueryValue("observationDbId") String observationDbId, + @QueryValue("observationUnitDbId") String observationUnitDbId, + @QueryValue("observationVariableDbId") String observationVariableDbId, + @QueryValue("locationDbId") String locationDbId, + @QueryValue("seasonDbId") String seasonDbId, + @QueryValue("observationTimeStampRangeStart") Date observationTimeStampRangeStart, + @QueryValue("observationTimeStampRangeEnd") Date observationTimeStampRangeEnd, + @QueryValue("observationUnitLevelName") String observationUnitLevelName, + @QueryValue("observationUnitLevelOrder") String observationUnitLevelOrder, + @QueryValue("observationUnitLevelCode") String observationUnitLevelCode, + @QueryValue("observationUnitLevelRelationshipName") String observationUnitLevelRelationshipName, + @QueryValue("observationUnitLevelRelationshipOrder") String observationUnitLevelRelationshipOrder, + @QueryValue("observationUnitLevelRelationshipCode") String observationUnitLevelRelationshipCode, + @QueryValue("observationUnitLevelRelationshipDbId") String observationUnitLevelRelationshipDbId, + @QueryValue("commonCropName") String commonCropName, + @QueryValue("programDbId") String programDbId, + @QueryValue("trialDbId") String trialDbId, + @QueryValue("studyDbId") String studyDbId, + @QueryValue("germplasmDbId") String germplasmDbId, + @QueryValue("externalReferenceID") String externalReferenceID, + @QueryValue("externalReferenceId") String externalReferenceId, + @QueryValue("externalReferenceSource") String externalReferenceSource, + @QueryValue("page") Integer page, + @QueryValue("pageSize") Integer pageSize) { + return HttpResponse.notFound(); + } + + @Get("/observations/{observationDbId}") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse observationsObservationDbIdGet(@PathVariable("programId") UUID programId, + @PathVariable("observationDbId") String observationDbId) { + return HttpResponse.notFound(); + } + + @Put("/observations/{observationDbId}") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse observationsObservationDbIdPut(@PathVariable("programId") UUID programId, + @PathVariable("observationDbId") String observationDbId, + BrAPIObservation body) { + /* + DO NOT IMPLEMENT - users must create observations via file upload + TODO identify how observations uploaded via BrAPI will be separated from curated observations + */ + return HttpResponse.notFound(); + } + + @Post("/observations") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse observationsPost(@PathVariable("programId") UUID programId, List body) { + /* + DO NOT IMPLEMENT - users must create observations via file upload + TODO identify how observations uploaded via BrAPI will be separated from curated observations + */ + return HttpResponse.notFound(); + } + + @Put("/observations") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse observationsPut(@PathVariable("programId") UUID programId, Map body) { + /* + DO NOT IMPLEMENT - users must create observations via file upload + TODO identify how observations uploaded via BrAPI will be separated from curated observations + */ + return HttpResponse.notFound(); + } + + @Get("/observations/table") + @Produces({"application/json", "text/csv", "text/tsv"}) + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse observationsTableGet(@PathVariable("programId") UUID programId, + @Header("Accept") String accept, + @QueryValue("observationUnitDbId") String observationUnitDbId, + @QueryValue("observationVariableDbId") String observationVariableDbId, + @QueryValue("locationDbId") String locationDbId, + @QueryValue("seasonDbId") String seasonDbId, + @QueryValue("observationLevel") String observationLevel, + @QueryValue("searchResultsDbId") String searchResultsDbId, + @QueryValue("observationTimeStampRangeStart") Date observationTimeStampRangeStart, + @QueryValue("observationTimeStampRangeEnd") Date observationTimeStampRangeEnd, + @QueryValue("programDbId") String programDbId, + @QueryValue("trialDbId") String trialDbId, + @QueryValue("studyDbId") String studyDbId, + @QueryValue("germplasmDbId") String germplasmDbId, + @QueryValue("observationUnitLevelName") String observationUnitLevelName, + @QueryValue("observationUnitLevelOrder") String observationUnitLevelOrder, + @QueryValue("observationUnitLevelCode") String observationUnitLevelCode, + @QueryValue("observationUnitLevelRelationshipName") String observationUnitLevelRelationshipName, + @QueryValue("observationUnitLevelRelationshipOrder") String observationUnitLevelRelationshipOrder, + @QueryValue("observationUnitLevelRelationshipCode") String observationUnitLevelRelationshipCode, + @QueryValue("observationUnitLevelRelationshipDbId") String observationUnitLevelRelationshipDbId) { + return HttpResponse.notFound(); + } +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIProgramsController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIProgramsController.java new file mode 100644 index 000000000..309172319 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIProgramsController.java @@ -0,0 +1,106 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2; + +import io.micronaut.http.HttpResponse; +import io.micronaut.http.annotation.*; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; +import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.core.BrAPIProgram; +import org.breedinginsight.brapi.v1.controller.BrapiVersion; + +import java.util.List; +import java.util.UUID; + +@Slf4j +@Controller("/${micronaut.bi.api.version}") +@Secured(SecurityRule.IS_AUTHENTICATED) +public class BrAPIProgramsController { + /* + TODO + - GET programs + */ + + //START - endpoints at root BrAPI url + @Get(BrapiVersion.BRAPI_V2 + "/programs") + public HttpResponse rootProgramsGet(@QueryValue("abbreviation") String abbreviation, + @QueryValue("programType") String programType, + @QueryValue("commonCropName") String commonCropName, + @QueryValue("programDbId") String programDbId, + @QueryValue("programName") String programName, + @QueryValue("externalReferenceID") String externalReferenceID, + @QueryValue("externalReferenceId") String externalReferenceId, + @QueryValue("externalReferenceSource") String externalReferenceSource, + @QueryValue("page") Integer page, + @QueryValue("pageSize") Integer pageSize) { + return HttpResponse.notFound(); + } + + @Post(BrapiVersion.BRAPI_V2 + "/programs") + public HttpResponse rootProgramsPost(List body) { + //DO NOT IMPLEMENT - Users should only be able to create new programs via the DeltaBreed UI + return HttpResponse.notFound(); + } + + @Get(BrapiVersion.BRAPI_V2 + "/programs/{programDbId}") + public HttpResponse rootProgramsProgramDbIdGet(@PathVariable("programDbId") String programDbId) { + return HttpResponse.notFound(); + } + + @Put(BrapiVersion.BRAPI_V2 + "/programs/{programDbId}") + public HttpResponse rootProgramsProgramDbIdPut(@PathVariable("programDbId") String programDbId, BrAPIProgram body) { + //DO NOT IMPLEMENT - Users should only be able to update programs via the DeltaBreed UI + return HttpResponse.notFound(); + } + //END - endpoints at root BrAPI url + + + //START - endpoints for within the context of a program + @Get("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/programs") + public HttpResponse programsGet(@PathVariable("programId") UUID programId, @QueryValue("abbreviation") String abbreviation, + @QueryValue("programType") String programType, + @QueryValue("commonCropName") String commonCropName, + @QueryValue("programDbId") String programDbId, + @QueryValue("programName") String programName, + @QueryValue("externalReferenceID") String externalReferenceID, + @QueryValue("externalReferenceId") String externalReferenceId, + @QueryValue("externalReferenceSource") String externalReferenceSource, + @QueryValue("page") Integer page, + @QueryValue("pageSize") Integer pageSize) { + return HttpResponse.notFound(); + } + + @Post("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/programs") + public HttpResponse programsPost(@PathVariable("programId") UUID programId, List body) { + //DO NOT IMPLEMENT - Users should only be able to create new programs via the DeltaBreed UI + return HttpResponse.notFound(); + } + + @Get("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/programs/{programDbId}") + public HttpResponse programsProgramDbIdGet(@PathVariable("programId") UUID programId, @PathVariable("programDbId") String programDbId) { + return HttpResponse.notFound(); + } + + @Put("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/programs/{programDbId}") + public HttpResponse programsProgramDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("programDbId") String programDbId, BrAPIProgram body) { + //DO NOT IMPLEMENT - Users should only be able to update programs via the DeltaBreed UI + return HttpResponse.notFound(); + } + //END - endpoints for within the context of a program +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/StudyController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIStudiesController.java similarity index 71% rename from src/main/java/org/breedinginsight/brapi/v2/StudyController.java rename to src/main/java/org/breedinginsight/brapi/v2/BrAPIStudiesController.java index 29aeb7dd3..4f64fba1c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/StudyController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIStudiesController.java @@ -46,14 +46,14 @@ @Slf4j @Controller("/${micronaut.bi.api.version}") @Secured(SecurityRule.IS_AUTHENTICATED) -public class StudyController { +public class BrAPIStudiesController { private final BrAPIStudyService studyService; private final StudyQueryMapper studyQueryMapper; @Inject - public StudyController(BrAPIStudyService studyService, StudyQueryMapper studyQueryMapper) { + public BrAPIStudiesController(BrAPIStudyService studyService, StudyQueryMapper studyQueryMapper) { this.studyService = studyService; this.studyQueryMapper = studyQueryMapper; } @@ -80,4 +80,34 @@ public HttpResponse>>> getStudies( return HttpResponse.status(HttpStatus.UNPROCESSABLE_ENTITY, "Error parsing requested date format"); } } + + /* + TODO + - GET studies/{id} + */ + @Post("/studies") + @Consumes({"application/json"}) + @Produces({"application/json"}) + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse studiesPost(@PathVariable("programId") UUID programId, List body) { + //DO NOT IMPLEMENT - Users are only able to create new studies via the DeltaBreed UI + return HttpResponse.notFound(); + } + + @Get("/studies/{studyDbId}") + @Produces({"application/json"}) + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse studiesStudyDbIdGet(@PathVariable("programId") UUID programId, @PathVariable("studyDbId") String studyDbId) { + return HttpResponse.notFound(); + } + + @Put("/studies/{studyDbId}") + @Consumes({"application/json"}) + @Produces({"application/json"}) + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse studiesStudyDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("studyDbId") String studyDbId, + BrAPIStudy body) { + //DO NOT IMPLEMENT - Users are only able to update studies via the DeltaBreed UI + return HttpResponse.notFound(); + } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java new file mode 100644 index 000000000..71b979641 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java @@ -0,0 +1,101 @@ +package org.breedinginsight.brapi.v2; + +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; +import io.micronaut.http.annotation.*; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPITrial; +import org.breedinginsight.api.auth.ProgramSecured; +import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; +import org.breedinginsight.api.model.v1.request.query.SearchRequest; +import org.breedinginsight.api.model.v1.response.DataResponse; +import org.breedinginsight.api.model.v1.response.Response; +import org.breedinginsight.api.model.v1.validators.QueryValid; +import org.breedinginsight.brapi.v1.controller.BrapiVersion; +import org.breedinginsight.brapi.v2.model.request.query.ExperimentQuery; +import org.breedinginsight.brapi.v2.services.BrAPITrialService; +import org.breedinginsight.services.ProgramService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.utilities.response.ResponseUtils; +import org.breedinginsight.utilities.response.mappers.ExperimentQueryMapper; +import javax.inject.Inject; +import javax.validation.Valid; +import java.util.*; + +@Slf4j +@Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2) +@Secured(SecurityRule.IS_AUTHENTICATED) +public class BrAPITrialsController { + private final BrAPITrialService experimentService; + private final ExperimentQueryMapper experimentQueryMapper; + private final ProgramService programService; + + @Inject + public BrAPITrialsController(BrAPITrialService experimentService, ExperimentQueryMapper experimentQueryMapper, ProgramService programService) { + this.experimentService = experimentService; + this.experimentQueryMapper = experimentQueryMapper; + this.programService = programService; + } + + @Get("/trials{?queryParams*}") + @Produces(MediaType.APPLICATION_JSON) + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse>>> getExperiments( + @PathVariable("programId") UUID programId, + @QueryValue @QueryValid(using = ExperimentQueryMapper.class) @Valid ExperimentQuery queryParams) { + try { + log.debug("fetching trials for program: " + programId); + + List experiments = experimentService.getExperiments(programId); + SearchRequest searchRequest = queryParams.constructSearchRequest(); + return ResponseUtils.getBrapiQueryResponse(experiments, experimentQueryMapper, queryParams, searchRequest); + } catch (ApiException e) { + log.info(e.getMessage(), e); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error retrieving experiments"); + } catch (DoesNotExistException e) { + log.info(e.getMessage(), e); + return HttpResponse.status(HttpStatus.NOT_FOUND, e.getMessage()); + } + } + + @Get("/trials/{trialId}") + @Produces(MediaType.APPLICATION_JSON) + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse> getExperimentById( + @PathVariable("programId") UUID programId, + @PathVariable("trialId") UUID trialId, + @QueryValue(defaultValue = "false") Boolean stats){ + try { + String logMsg = "fetching trial id:" + trialId +" for program: " + programId; + if(stats){ + logMsg += " with stats"; + } + log.debug(logMsg); + Response response = new Response<>(experimentService.getTrialDataByUUID(programId, trialId, stats)); + return HttpResponse.ok(response); + } catch (DoesNotExistException e) { + log.info(e.getMessage(), e); + return HttpResponse.status(HttpStatus.NOT_FOUND, e.getMessage()); + } + } + + @Post("/trials") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse trialsPost(@PathVariable("programId") UUID programId, List body) { + //DO NOT IMPLEMENT - Users are only able to create new trials via the DeltaBreed UI + return HttpResponse.notFound(); + } + + + @Put("/trials/{trialDbId}") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse trialsTrialDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("trialDbId") String trialDbId, BrAPITrial body) { + //DO NOT IMPLEMENT - Users are only able to update trials via the DeltaBreed UI + return HttpResponse.notFound(); + } + +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java index 702a0ca14..3a485bd9a 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java @@ -28,6 +28,7 @@ import lombok.extern.slf4j.Slf4j; import okhttp3.*; import org.brapi.v2.model.core.BrAPIServerInfo; +import org.brapi.v2.model.core.BrAPIService; import org.brapi.v2.model.core.response.BrAPIServerInfoResponse; import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.api.auth.ProgramSecured; @@ -40,7 +41,7 @@ import javax.inject.Inject; import java.io.IOException; -import java.util.UUID; +import java.util.*; import java.util.stream.Collectors; @Slf4j @@ -63,13 +64,107 @@ public BrAPIV2Controller(SecurityService securityService, ProgramService program @Secured(SecurityRule.IS_ANONYMOUS) public BrAPIServerInfoResponse serverinfo() { BrAPIServerInfo serverInfo = new BrAPIServerInfo(); + setBrAPIServerInfo(serverInfo); + serverInfo.setServerDescription("BrAPI endpoints are not implemented at the root of this domain. Please make BrAPI calls in the context of a program (ex: https://app.breedinginsight.net/v1/programs/{programId}/brapi/v2)"); + + serverInfo.setCalls( + new ServiceBuilder().versions("2.0", "2.1") + .setBase("serverinfo").GET().build() + .setBase("programs").GET().addPath("{programDbId}").GET().build() + ); + + return new BrAPIServerInfoResponse().result(serverInfo); + } + + @Get("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/serverinfo") + @Produces(MediaType.APPLICATION_JSON) + @Secured(SecurityRule.IS_ANONYMOUS) + public BrAPIServerInfoResponse programServerinfo(@PathVariable("programId") UUID programId) { + String programBrAPIBase = String.format("v1/programs/%s%s/", programId, BrapiVersion.BRAPI_V2); + + List programServices = new ServiceBuilder() + .versions("2.0", "2.1") + //CORE + .setBase("serverinfo").GET().build() + .setBase("commoncropnames").GET().build() + .setBase("lists").GET().POST().addPath("{listDbId}").GET().PUT().withSearch() + .setBase("locations").GET().addPath("{locationDbId}").GET().withSearch() + .setBase("people").GET().addPath("{personDbId}").GET().withSearch() + .setBase("programs").GET().addPath("{programDbId}").GET().withSearch() + .setBase("seasons").GET().addPath("{seasonDbId}").GET().build() + .setBase("studies").GET().addPath("{studyDbId}").GET().withSearch() + .setBase("studytypes").GET().build() + .setBase("trials").GET().addPath("{trialDbId}").GET().withSearch() + //GERMPLASM + .setBase("attributes").GET().addPath("{attributeDbId}").GET().setPath("categories").GET().withSearch() + .setBase("attributevalues").GET().addPath("{attributeValueDbId}").GET().withSearch() + .setBase("breedingmethods").GET().addPath("{breedingMethodDbId}").GET().build() + .setBase("crosses").GET().build() + .setBase("plannedcrosses").GET().build() + .setBase("crossingprojects").GET().addPath("{crossingProjectDbId}").GET().build() + .setBase("seedlots").GET().addPath("transactions").GET().setPath("{seedLotDbId}").GET().addPath("transactions").GET().build() + .setBase("germplasm").GET().addPath("{germplasmDbId}").GET().addPath("mcpd").GET().withSearch() + //PHENOTYPING + .setBase("events").GET().build() + .setBase("images").GET().addPath("{imageDbId}").GET().addPath("imagecontent").withSearch() + .setBase("ontologies").GET().build() + .setBase("traits").GET().addPath("{traitDbId}").GET().build() + .setBase("methods").GET().addPath("{methodDbId}").GET().build() + .setBase("scales").GET().addPath("{scaleDbId}").GET().build() + .setBase("variables").GET().addPath("{observationVariableDbId}").GET().withSearch() + .setBase("observationunits").GET().addPath("{observationUnitDbId}").GET().setPath("table").GET().withSearch() + .setBase("observations").GET().addPath("{observationDbId}").GET().setPath("table").GET().withSearch() + .setBase("observationlevels").GET().build() + //GENOTYPING - TODO +// .setBase("calls").GET().withSearch() +// .setBase("callsets").GET().addPath("{callSetDbId}").GET().addPath("calls").GET().withSearch() +// .setBase("maps").GET().addPath("{mapDbId}").GET().addPath("linkagegroups").GET().build() +// .setBase("markerpositions").GET().withSearch() +// .setBase("references").GET().addPath("{referenceDbId}").GET().addPath("bases").GET().withSearch() +// .setBase("referencesets").GET().addPath("{referenceSetDbId}").GET().withSearch() +// .setBase("samples").GET().addPath("{sampleDbId}").GET().withSearch() +// .setBase("variants").GET().addPath("{variantDbId}").GET().addPath("calls").GET().withSearch() +// .setBase("variantsets").GET().addPath("extract").setPath("{variantSetDbId}").GET() +// .addPath("calls").GET().setPath("callsets").GET().setPath("variants").GET().withSearch() +// .setBase("vendor").addPath("specifications").GET().setPath("plates").addPath("{submissionId}").build() +// .setBase("vendor/orders").GET().addPath("{orderId}").addPath("plates").GET().setPath("results").GET().setPath("status").GET().build() + + //V2.0 only + .versions("2.0") + .setBase("germplasm").addPath("{germplasmDbId}").addPath("pedigree").GET().setPath("progeny").GET().build() + .setBase("lists").addPath("{listDbId}").addPath("items").build() +// .setBase("samples").addPath("{sampleDbId}").build() //TODO + //V2.1 only + .versions("2.1") +// .setBase("allelematrix").GET().withSearch() //TODO +// .setBase("calls").build() //TODO + .setBase("delete").addPath("images").setPath("observations").build() + .setBase("lists").addPath("{listDbId}").addPath("data").POST().build() + .setBase("ontologies").addPath("{ontologyDbId}").GET().build() + .setBase("pedigree").GET().withSearch() +// .setBase("plates").GET().addPath("{plateDbId}").GET().withSearch() //TODO +// .setBase("samples").build() //TODO + .build(); + + for(BrAPIService service : programServices) { + service.setService(programBrAPIBase + service.getService()); + } + + BrAPIServerInfo programServerInfo = new BrAPIServerInfo(); + setBrAPIServerInfo(programServerInfo); + programServerInfo.setCalls(programServices); + + return new BrAPIServerInfoResponse().result(programServerInfo); + } + + private void setBrAPIServerInfo(BrAPIServerInfo serverInfo) { serverInfo.setOrganizationName("Breeding Insight"); serverInfo.setServerName("DeltaBreed"); serverInfo.setContactEmail("bidevteam@cornell.edu"); - serverInfo.setOrganizationURL("breedinginsight.org"); - serverInfo.setServerDescription("BrAPI endpoints are not implemented at the root of this domain. Please make BrAPI calls in the context of a program (ex: https://app.breedinginsight.net/v1/programs//brapi/v2)"); - - return new BrAPIServerInfoResponse().result(serverInfo); + serverInfo.setOrganizationURL("https://breedinginsight.org"); + serverInfo.setServerDescription("DeltaBreed - breeding data management system"); + serverInfo.setLocation("Cornell University, Ithaca, NY, USA"); + serverInfo.setDocumentationURL("https://brapi.org/specification"); } @Get("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/{+path}") @@ -154,7 +249,23 @@ private HttpResponse makeCall(Request brapiRequest) { } private HttpUrl getUrl(UUID programId, String path, HttpRequest request) { + var programBrAPIBaseUrl = getProgramBrAPIBaseUrl(programId); + + var requestUrl = HttpUrl.parse(programBrAPIBaseUrl + "/" + path).newBuilder(); + request.getParameters() + .asMap() + .entrySet() + .stream() + .filter(param -> !param.getKey() + .equals("programId")) + .forEach(param -> param.getValue() + .forEach(val -> requestUrl.addQueryParameter(param.getKey(), val))); + + return requestUrl.build(); + } + + private String getProgramBrAPIBaseUrl(UUID programId) { ProgramBrAPIEndpoints programBrAPIEndpoints; try { programBrAPIEndpoints = programService.getBrapiEndpoints(programId); @@ -168,20 +279,7 @@ private HttpUrl getUrl(UUID programId, String path, HttpRequest request) { } var programBrAPIBaseUrl = programBrAPIEndpoints.getCoreUrl().get(); programBrAPIBaseUrl = programBrAPIBaseUrl.endsWith("/") ? programBrAPIBaseUrl.substring(0, programBrAPIBaseUrl.length() - 1) : programBrAPIBaseUrl; - String urlString = programBrAPIBaseUrl.endsWith(BrapiVersion.BRAPI_V2) ? programBrAPIBaseUrl : programBrAPIBaseUrl + BrapiVersion.BRAPI_V2; - var requestUrl = HttpUrl.parse(urlString + "/" + path) - .newBuilder(); - - request.getParameters() - .asMap() - .entrySet() - .stream() - .filter(param -> !param.getKey() - .equals("programId")) - .forEach(param -> param.getValue() - .forEach(val -> requestUrl.addQueryParameter(param.getKey(), val))); - - return requestUrl.build(); + return programBrAPIBaseUrl.endsWith(BrapiVersion.BRAPI_V2) ? programBrAPIBaseUrl : programBrAPIBaseUrl + BrapiVersion.BRAPI_V2; } private void logCall(String path, HttpRequest request) { diff --git a/src/main/java/org/breedinginsight/brapi/v2/ServiceBuilder.java b/src/main/java/org/breedinginsight/brapi/v2/ServiceBuilder.java new file mode 100644 index 000000000..c5a0464a6 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/ServiceBuilder.java @@ -0,0 +1,80 @@ +package org.breedinginsight.brapi.v2; + +import org.brapi.v2.model.BrAPIWSMIMEDataTypes; +import org.brapi.v2.model.core.BrAPIService; +import org.brapi.v2.model.core.BrAPIService.MethodsEnum; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class ServiceBuilder extends ArrayList{ + private static final long serialVersionUID = 1L; + private String path = ""; + private String base = ""; + private List methods = new ArrayList<>(); + private List versions = new ArrayList<>(); + + public ServiceBuilder setBase(String base) { + this.path = base; + this.base = base; + this.methods.clear(); + return this; + } + public ServiceBuilder setPath(String newPath) { + build(); + + String[] pathParts = this.path.split("/"); + String oldPath = pathParts[pathParts.length - 1].replaceAll("\\{", "\\\\{").replaceAll("\\}", "\\\\}"); + this.path = this.path.replaceFirst(oldPath, newPath); + this.methods.clear(); + return this; + } + public ServiceBuilder addPath(String path) { + build(); + this.path = this.path + '/' + path; + this.methods.clear(); + return this; + } + + public ServiceBuilder PUT() { + methods.add(MethodsEnum.PUT); + return this; + } + + public ServiceBuilder POST() { + methods.add(MethodsEnum.POST); + return this; + } + + public ServiceBuilder GET() { + methods.add(MethodsEnum.GET); + return this; + } + + public ServiceBuilder versions(String ... versions) { + this.versions = Arrays.asList(versions); + return this; + } + + public ServiceBuilder build() { + if(path != null && !path.isEmpty() && methods != null && !methods.isEmpty()) { + this.add(buildService(path, methods)); + } + return this; + } + public ServiceBuilder withSearch() { + build(); + this.add(buildService("search/" + base, Arrays.asList(MethodsEnum.POST))); + this.add(buildService("search/" + base + "/{searchResultsDbId}", Arrays.asList(MethodsEnum.GET))); + return this; + } + public BrAPIService buildService(String path, List methods) { + BrAPIService service = new BrAPIService(); + service.addDataTypesItem(BrAPIWSMIMEDataTypes.APPLICATION_JSON); + service.setMethods(new ArrayList<>(methods)); + service.setVersions(new ArrayList<>(this.versions)); + service.setService(path); + return service; + } +} diff --git a/src/main/java/org/breedinginsight/utilities/response/mappers/StudyQueryMapper.java b/src/main/java/org/breedinginsight/utilities/response/mappers/StudyQueryMapper.java index f15aa620f..4a12acb4e 100644 --- a/src/main/java/org/breedinginsight/utilities/response/mappers/StudyQueryMapper.java +++ b/src/main/java/org/breedinginsight/utilities/response/mappers/StudyQueryMapper.java @@ -37,6 +37,13 @@ public class StudyQueryMapper extends AbstractQueryMapper { private final Map> fields; public StudyQueryMapper() { + /* + TODO + - observationVariableDbId + - active + - programDbId + - germplasmDbId + */ fields = Map.ofEntries( Map.entry("studyType", BrAPIStudy::getStudyType), Map.entry("locationDbId", BrAPIStudy::getLocationDbId), From 6e49ae26a3a4b012b1b9f68be314100b02d82f43 Mon Sep 17 00:00:00 2001 From: timparsons Date: Mon, 25 Sep 2023 16:53:23 -0400 Subject: [PATCH 135/220] [BI-1921] fixing compilation error --- .../auth/OIDCDiscoveryController.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/breedinginsight/api/v1/controller/auth/OIDCDiscoveryController.java b/src/main/java/org/breedinginsight/api/v1/controller/auth/OIDCDiscoveryController.java index 4d58da165..8560cb445 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/auth/OIDCDiscoveryController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/auth/OIDCDiscoveryController.java @@ -35,18 +35,16 @@ @Secured(SecurityRule.IS_ANONYMOUS) public class OIDCDiscoveryController { - private static final String OIDC_CONFIG = """ - { - "issuer": "%s", - "authorization_endpoint": "%s/programs/%s/brapi/authorize", - "jwks_uri": "", - "token_endpoint": "", - "grant_types_supported": ["implicit"], - "response_types_supported": ["token"], - "subject_types_supported": ["public"], - "id_token_signing_alg_values_supported": [] - } - """; + private static final String OIDC_CONFIG = " {\n" + + " \"issuer\": \"%s\",\n" + + " \"authorization_endpoint\": \"%s/programs/%s/brapi/authorize\",\n" + + " \"jwks_uri\": \"\",\n" + + " \"token_endpoint\": \"\",\n" + + " \"grant_types_supported\": [\"implicit\"],\n" + + " \"response_types_supported\": [\"token\"],\n" + + " \"subject_types_supported\": [\"public\"],\n" + + " \"id_token_signing_alg_values_supported\": []\n" + + " }\n"; private final String webUrl; private final ProgramDAO programDAO; From 468ea83dff98907d348e8580c864c4c177984d13 Mon Sep 17 00:00:00 2001 From: timparsons Date: Wed, 4 Oct 2023 09:12:08 -0400 Subject: [PATCH 136/220] [BI-1921] Implementing various BrAPI endpoints to support Field Book --- .../api/v1/controller/TokenController.java | 4 +- .../v2/BrAPIObservationLevelsController.java | 143 ++++++++ .../v2/BrAPIObservationUnitController.java | 191 +++++++++-- .../BrAPIObservationVariableController.java | 321 ++++++++++++++++-- .../brapi/v2/BrAPIProgramsController.java | 142 ++++++-- .../brapi/v2/BrAPIStudiesController.java | 72 +++- .../brapi/v2/BrAPITrialsController.java | 37 +- .../brapi/v2/services/BrAPIStudyService.java | 13 +- .../brapi/v2/services/BrAPITrialService.java | 59 ++-- .../daos/BrAPIObservationUnitDAO.java | 150 ++++++-- .../brapps/importer/daos/BrAPIStudyDAO.java | 15 + .../processors/ObservationUnitProcessor.java | 3 + .../daos/impl/TraitDAOImpl.java | 26 +- .../org/breedinginsight/model/Dataset.java | 4 +- .../services/TraitService.java | 8 + 15 files changed, 993 insertions(+), 195 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsController.java diff --git a/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java b/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java index ca6053be7..6c396cb9d 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/TokenController.java @@ -52,7 +52,7 @@ public TokenController(SecurityService securityService, TokenService tokenServic @Get("/api-token{?returnUrl}") @Secured(SecurityRule.IS_AUTHENTICATED) - public HttpResponse apiToken(@QueryValue @Nullable String returnUrl) { + public HttpResponse apiToken(@QueryValue @Nullable String returnUrl) { AuthenticatedUser actingUser = securityService.getUser(); Optional token = tokenService.generateApiToken(actingUser); @@ -66,7 +66,7 @@ public HttpResponse apiToken(@QueryValue @Nullable String returnUrl) { } URI location = UriBuilder.of(returnUrl) .queryParam("status", 200) - .queryParam("token", apiToken.getAccessToken()) + .queryParam("access_token", apiToken.getAccessToken()) .build(); return HttpResponse.seeOther(location) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsController.java new file mode 100644 index 000000000..3fe481c73 --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsController.java @@ -0,0 +1,143 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2; + +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.annotation.Controller; +import io.micronaut.http.annotation.Get; +import io.micronaut.http.annotation.PathVariable; +import io.micronaut.http.annotation.QueryValue; +import io.micronaut.security.annotation.Secured; +import io.micronaut.security.rules.SecurityRule; +import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.ApiResponse; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; +import org.brapi.v2.model.BrAPIIndexPagination; +import org.brapi.v2.model.BrAPIMetadata; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.brapi.v2.model.pheno.BrAPIObservationUnitHierarchyLevel; +import org.brapi.v2.model.pheno.BrAPIObservationUnitLevelRelationship; +import org.brapi.v2.model.pheno.response.BrAPIObservationLevelListResponse; +import org.brapi.v2.model.pheno.response.BrAPIObservationLevelListResponseResult; +import org.breedinginsight.api.auth.ProgramSecured; +import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; +import org.breedinginsight.brapi.v1.controller.BrapiVersion; +import org.breedinginsight.brapps.importer.daos.BrAPIStudyDAO; +import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; +import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.ProgramService; +import org.breedinginsight.services.brapi.BrAPIEndpointProvider; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.utilities.BrAPIDAOUtil; +import org.breedinginsight.utilities.Utilities; + +import javax.annotation.Nullable; +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +@Slf4j +@Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2) +@Secured(SecurityRule.IS_AUTHENTICATED) +public class BrAPIObservationLevelsController { + + private final BrAPIEndpointProvider brAPIEndpointProvider; + private final ProgramDAO programDAO; + private final ProgramService programService; + private final BrAPITrialDAO trialDAO; + private final BrAPIStudyDAO studyDAO; + + @Inject + public BrAPIObservationLevelsController(BrAPIEndpointProvider brAPIEndpointProvider, ProgramDAO programDAO, ProgramService programService, BrAPITrialDAO trialDAO, BrAPIStudyDAO studyDAO) { + this.brAPIEndpointProvider = brAPIEndpointProvider; + this.programDAO = programDAO; + this.programService = programService; + this.trialDAO = trialDAO; + this.studyDAO = studyDAO; + } + + @Get("/observationlevels") + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse observationlevelsGet(@PathVariable("programId") UUID programId, + @Nullable @QueryValue("trialDbId") String experimentId, + @Nullable @QueryValue("studyDbId") String environmentId, + @Nullable @QueryValue("page") Integer page, + @Nullable @QueryValue("pageSize") Integer pageSize) { + + log.debug("fetching observation levels for programId: " + programId); + + Optional program = programService.getById(programId); + if(program.isEmpty()) { + log.warn("Program id: " + programId + " not found"); + return HttpResponse.notFound(); + } + + String programDbId = program.get().getBrapiProgram().getProgramDbId(); + String studyDbId = null; + String trialDbId = null; + + if(environmentId != null) { + try { + Optional study = studyDAO.getStudyByEnvironmentId(UUID.fromString(environmentId), program.get()); + if(study.isPresent()) { + studyDbId = study.get().getStudyDbId(); + } else { + studyDbId = environmentId; + } + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), "Error fetching environment"); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error finding observation levels"); + } + } else if(experimentId != null) { + try { + List trial = trialDAO.getTrialsByExperimentIds(List.of(UUID.fromString(experimentId)), program.get()); + if(trial.size() == 1) { + trialDbId = trial.get(0).getTrialDbId(); + } else { + trialDbId = experimentId; + } + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), "Error fetching experiments"); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error finding observation levels"); + } + } + + ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationUnitsApi.class); + try { + ApiResponse response = api.observationlevelsGet(studyDbId, trialDbId, programDbId, page == null ? 0 : page, pageSize == null ? 1000 : pageSize); + BrAPIIndexPagination responsePagination = (BrAPIIndexPagination) response.getBody().getMetadata().getPagination(); + List levels = new ArrayList<>(); + if(response.getBody() != null) { + levels = response.getBody().getResult().getData(); + } + log.debug(String.format("found %d observation levels", levels.size())); + return HttpResponse.ok(new BrAPIObservationLevelListResponse().metadata(new BrAPIMetadata().pagination(responsePagination)) + .result(new BrAPIObservationLevelListResponseResult().data(levels))); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), "Error fetching observation levels"); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "Error finding observation levels"); + } + } +} diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java index 96b91df36..0c17e31e5 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java @@ -17,24 +17,54 @@ package org.breedinginsight.brapi.v2; +import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpResponse; import io.micronaut.http.annotation.*; import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; import lombok.extern.slf4j.Slf4j; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIIndexPagination; +import org.brapi.v2.model.BrAPIMetadata; +import org.brapi.v2.model.BrAPIStatus; import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponse; +import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponseResult; +import org.brapi.v2.model.pheno.response.BrAPIObservationUnitSingleResponse; import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; import org.breedinginsight.brapi.v1.controller.BrapiVersion; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.ProgramService; +import org.breedinginsight.utilities.Utilities; +import javax.annotation.Nullable; +import javax.inject.Inject; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; @Slf4j @Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2) @Secured(SecurityRule.IS_AUTHENTICATED) public class BrAPIObservationUnitController { + private final String referenceSource; + + private final BrAPIObservationUnitDAO observationUnitDAO; + + private final ProgramService programService; + + @Inject + public BrAPIObservationUnitController(@Property(name = "brapi.server.reference-source") String referenceSource, BrAPIObservationUnitDAO observationUnitDAO, ProgramService programService) { + this.referenceSource = referenceSource; + this.observationUnitDAO = observationUnitDAO; + this.programService = programService; + } /* TODO @@ -42,60 +72,139 @@ public class BrAPIObservationUnitController { - GET observationUnits */ @Get("/observationunits") - public HttpResponse observationunitsGet(@PathVariable("programId") UUID programId, - @QueryValue("observationUnitDbId") String observationUnitDbId, - @QueryValue("observationUnitName") String observationUnitName, - @QueryValue("locationDbId") String locationDbId, - @QueryValue("seasonDbId") String seasonDbId, - @QueryValue("includeObservations") Boolean includeObservations, - @QueryValue("observationUnitLevelName") String observationUnitLevelName, - @QueryValue("observationUnitLevelOrder") String observationUnitLevelOrder, - @QueryValue("observationUnitLevelCode") String observationUnitLevelCode, - @QueryValue("observationUnitLevelRelationshipName") String observationUnitLevelRelationshipName, - @QueryValue("observationUnitLevelRelationshipOrder") String observationUnitLevelRelationshipOrder, - @QueryValue("observationUnitLevelRelationshipCode") String observationUnitLevelRelationshipCode, - @QueryValue("observationUnitLevelRelationshipDbId") String observationUnitLevelRelationshipDbId, - @QueryValue("commonCropName") String commonCropName, - @QueryValue("programDbId") String programDbId, - @QueryValue("trialDbId") String trialDbId, - @QueryValue("studyDbId") String studyDbId, - @QueryValue("germplasmDbId") String germplasmDbId, - @QueryValue("externalReferenceID") String externalReferenceID, - @QueryValue("externalReferenceId") String externalReferenceId, - @QueryValue("externalReferenceSource") String externalReferenceSource, - @QueryValue("page") Integer page, - @QueryValue("pageSize") Integer pageSize) { - return HttpResponse.notFound(); + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse observationunitsGet(@PathVariable("programId") UUID programId, + @Nullable @QueryValue("observationUnitDbId") String observationUnitDbId, + @Nullable @QueryValue("observationUnitName") String observationUnitName, + @Nullable @QueryValue("locationDbId") String locationDbId, + @Nullable @QueryValue("seasonDbId") String seasonDbId, + @Nullable @QueryValue("includeObservations") Boolean includeObservations, + @Nullable @QueryValue("observationUnitLevelName") String observationUnitLevelName, + @Nullable @QueryValue("observationUnitLevelOrder") String observationUnitLevelOrder, + @Nullable @QueryValue("observationUnitLevelCode") String observationUnitLevelCode, + @Nullable @QueryValue("observationUnitLevelRelationshipName") String observationUnitLevelRelationshipName, + @Nullable @QueryValue("observationUnitLevelRelationshipOrder") String observationUnitLevelRelationshipOrder, + @Nullable @QueryValue("observationUnitLevelRelationshipCode") String observationUnitLevelRelationshipCode, + @Nullable @QueryValue("observationUnitLevelRelationshipDbId") String observationUnitLevelRelationshipDbId, + @Nullable @QueryValue("commonCropName") String commonCropName, + @Nullable @QueryValue("programDbId") String programDbId, + @Nullable @QueryValue("trialDbId") String trialDbId, + @Nullable @QueryValue("studyDbId") String studyDbId, + @Nullable @QueryValue("germplasmDbId") String germplasmDbId, + @Nullable @QueryValue("externalReferenceID") String externalReferenceID, + @Nullable @QueryValue("externalReferenceId") String externalReferenceId, + @Nullable @QueryValue("externalReferenceSource") String externalReferenceSource, + @Nullable @QueryValue("page") Integer page, + @Nullable @QueryValue("pageSize") Integer pageSize) { + + log.debug("observationunitsGet: fetching ous by filters"); + + Optional program = programService.getById(programId); + if(program.isEmpty()) { + log.warn("Program id: " + programId + " not found"); + return HttpResponse.notFound(); + } + + try { + Optional levelOrder = Optional.empty(); + Optional levelRelationshipOrder = Optional.empty(); + if(observationUnitLevelOrder != null) { + levelOrder = Optional.of(Integer.parseInt(observationUnitLevelOrder)); + } + if(observationUnitLevelRelationshipOrder != null) { + levelOrder = Optional.of(Integer.parseInt(observationUnitLevelRelationshipOrder)); + } + + List observationUnits = observationUnitDAO.getObservationUnits(program.get(), + Optional.ofNullable(observationUnitDbId), + Optional.ofNullable(observationUnitName), + Optional.ofNullable(locationDbId), + Optional.ofNullable(seasonDbId), + Optional.ofNullable(includeObservations), + Optional.ofNullable(observationUnitLevelName), + levelOrder, + Optional.ofNullable(observationUnitLevelCode), + Optional.ofNullable(observationUnitLevelRelationshipName), + levelRelationshipOrder, + Optional.ofNullable(observationUnitLevelRelationshipCode), + Optional.ofNullable(observationUnitLevelRelationshipDbId), + Optional.ofNullable(commonCropName), + Optional.ofNullable(trialDbId), + Optional.ofNullable(studyDbId), + Optional.ofNullable(germplasmDbId)) + .stream() + .peek(this::setDbIds) + .collect(Collectors.toList()); + + return HttpResponse.ok(new BrAPIObservationUnitListResponse().metadata(new BrAPIMetadata().pagination(new BrAPIIndexPagination().currentPage(0) + .totalPages(1) + .pageSize(observationUnits.size()) + .totalCount(observationUnits.size()))) + .result(new BrAPIObservationUnitListResponseResult().data(observationUnits))); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(new BrAPIObservationUnitListResponse().metadata(new BrAPIMetadata().status(List.of(new BrAPIStatus().messageType(BrAPIStatus.MessageTypeEnum.ERROR) + .message("Error fetching observation units"))))); + } catch (Exception e) { + log.error("Error fetching OUs", e); + return HttpResponse.serverError(new BrAPIObservationUnitListResponse().metadata(new BrAPIMetadata().status(List.of(new BrAPIStatus().messageType(BrAPIStatus.MessageTypeEnum.ERROR) + .message("Error fetching observation units"))))); + } } @Get("/observationunits/{observationUnitDbId}") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse observationunitsObservationUnitDbIdGet(@PathVariable("programId") UUID programId, @PathVariable("observationUnitDbId") String observationUnitDbId) { - return HttpResponse.notFound(); + public HttpResponse observationunitsObservationUnitDbIdGet(@PathVariable("programId") UUID programId, @PathVariable("observationUnitDbId") String observationUnitDbId) { + log.debug("observationunitsObservationUnitDbIdGet: fetching ou by externalReferenceId: " + observationUnitDbId); + Optional program = programService.getById(programId); + if(program.isEmpty()) { + log.warn("Program id: " + programId + " not found"); + return HttpResponse.notFound(); + } + try { + List ous = observationUnitDAO.getObservationUnitsById(List.of(observationUnitDbId), program.get()); + if(ous.size() != 1) { + log.warn("did not find a single ou with externalReferenceId: " + observationUnitDbId); + return HttpResponse.notFound(); + } + setDbIds(ous.get(0)); + return HttpResponse.ok(new BrAPIObservationUnitSingleResponse().result(ous.get(0))); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(new BrAPIObservationUnitSingleResponse().metadata(new BrAPIMetadata().status(List.of(new BrAPIStatus().messageType(BrAPIStatus.MessageTypeEnum.ERROR) + .message("Error fetching observation unit"))))); + } catch (Exception e) { + log.error("Error fetching OU", e); + return HttpResponse.serverError(new BrAPIObservationUnitSingleResponse().metadata(new BrAPIMetadata().status(List.of(new BrAPIStatus().messageType(BrAPIStatus.MessageTypeEnum.ERROR) + .message("Error fetching observation unit"))))); + } } @Put("/observationunits/{observationUnitDbId}") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse observationunitsObservationUnitDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("observationUnitDbId") String observationUnitDbId, BrAPIObservationUnit body) { + public HttpResponse observationunitsObservationUnitDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("observationUnitDbId") String observationUnitDbId, BrAPIObservationUnit body) { + //DO NOT IMPLEMENT - Users aren't yet able to update observation units return HttpResponse.notFound(); } @Post("/observationunits") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse observationunitsPost(@PathVariable("programId") UUID programId, List body) { + public HttpResponse observationunitsPost(@PathVariable("programId") UUID programId, List body) { + //DO NOT IMPLEMENT - Users are only able to create observation units via the DeltaBreed UI return HttpResponse.notFound(); } @Put("/observationunits") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse observationunitsPut(@PathVariable("programId") UUID programId, Map body) { + public HttpResponse observationunitsPut(@PathVariable("programId") UUID programId, Map body) { + //DO NOT IMPLEMENT - Users aren't yet able to update observation units return HttpResponse.notFound(); } @Get("/observationunits/table") @Produces({"application/json", "text/csv", "text/tsv"}) @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse observationunitsTableGet(@PathVariable("programId") UUID programId, + public HttpResponse observationunitsTableGet(@PathVariable("programId") UUID programId, @Header("Accept") String accept, @QueryValue("observationUnitDbId") String observationUnitDbId, @QueryValue("observationVariableDbId") String observationVariableDbId, @@ -115,4 +224,26 @@ public HttpResponse observationunitsTableGet(@PathVariable("programId") UUID pro @QueryValue("observationUnitLevelRelationshipDbId") String observationUnitLevelRelationshipDbId) { return HttpResponse.notFound(); } + + private void setDbIds(BrAPIObservationUnit ou) { + ou.observationUnitDbId(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS)) + .orElseThrow(() -> new IllegalStateException("No BI external reference found")) + .getReferenceID()); + ou.studyDbId(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES)) + .orElseThrow(() -> new IllegalStateException("No BI external reference found")) + .getReferenceID()); + ou.trialDbId(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)) + .orElseThrow(() -> new IllegalStateException("No BI external reference found")) + .getReferenceID()); + ou.programDbId(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS)) + .orElseThrow(() -> new IllegalStateException("No BI external reference found")) + .getReferenceID()); + if (ou.getAdditionalInfo().has(BrAPIAdditionalInfoFields.GERMPLASM_UUID)) { + ou.setGermplasmDbId(ou.getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.GERMPLASM_UUID) + .getAsString()); + } + + //TODO update locationDbId + } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java index 29dfa9e0b..fd7b93b71 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java @@ -17,67 +17,176 @@ package org.breedinginsight.brapi.v2; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpRequest; import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; import io.micronaut.http.annotation.*; import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; import lombok.extern.slf4j.Slf4j; -import org.brapi.v2.model.pheno.BrAPIObservationVariable; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.BrAPIIndexPagination; +import org.brapi.v2.model.BrAPIMetadata; +import org.brapi.v2.model.BrAPIOntologyReference; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.*; +import org.brapi.v2.model.pheno.response.BrAPIObservationVariableListResponse; +import org.brapi.v2.model.pheno.response.BrAPIObservationVariableListResponseResult; +import org.brapi.v2.model.pheno.response.BrAPIObservationVariableSingleResponse; import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; import org.breedinginsight.brapi.v1.controller.BrapiVersion; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.services.BrAPITrialService; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.Program; +import org.breedinginsight.model.Trait; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.ProgramService; +import org.breedinginsight.services.TraitService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.utilities.Utilities; +import org.jetbrains.annotations.NotNull; -import java.util.List; -import java.util.UUID; +import javax.annotation.Nullable; +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; @Slf4j @Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2) @Secured(SecurityRule.IS_AUTHENTICATED) public class BrAPIObservationVariableController { - /* - TODO - - GET /variables - */ + private final String referenceSource; + + private final OntologyService ontologyService; + private final TraitService traitService; + + private final BrAPITrialService trialService; + + private final ProgramService programService; + + @Inject + public BrAPIObservationVariableController(OntologyService ontologyService, + TraitService traitService, + BrAPITrialService trialService, + ProgramService programService, + @Property(name = "brapi.server.reference-source") String referenceSource) { + this.ontologyService = ontologyService; + this.traitService = traitService; + this.trialService = trialService; + this.programService = programService; + this.referenceSource = referenceSource; + } @Get("/variables") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse variablesGet(@PathVariable("programId") UUID programId, - @QueryValue("observationVariableDbId") String observationVariableDbId, - @QueryValue("observationVariableName") String observationVariableName, - @QueryValue("observationVariablePUI") String observationVariablePUI, - @QueryValue("traitClass") String traitClass, - @QueryValue("methodDbId") String methodDbId, - @QueryValue("methodName") String methodName, - @QueryValue("methodPUI") String methodPUI, - @QueryValue("scaleDbId") String scaleDbId, - @QueryValue("scaleName") String scaleName, - @QueryValue("scalePUI") String scalePUI, - @QueryValue("traitDbId") String traitDbId, - @QueryValue("traitName") String traitName, - @QueryValue("traitPUI") String traitPUI, - @QueryValue("ontologyDbId") String ontologyDbId, - @QueryValue("commonCropName") String commonCropName, - @QueryValue("programDbId") String programDbId, - @QueryValue("trialDbId") String trialDbId, - @QueryValue("studyDbId") String studyDbId, - @QueryValue("externalReferenceID") String externalReferenceID, - @QueryValue("externalReferenceId") String externalReferenceId, - @QueryValue("externalReferenceSource") String externalReferenceSource, - @QueryValue("page") Integer page, - @QueryValue("pageSize") Integer pageSize) { - return HttpResponse.notFound(); + public HttpResponse variablesGet(@PathVariable("programId") UUID programId, + @Nullable @QueryValue("observationVariableDbId") String observationVariableDbId, + @Nullable @QueryValue("observationVariableName") String observationVariableName, + @Nullable @QueryValue("observationVariablePUI") String observationVariablePUI, + @Nullable @QueryValue("traitClass") String traitClass, + @Nullable @QueryValue("methodDbId") String methodDbId, + @Nullable @QueryValue("methodName") String methodName, + @Nullable @QueryValue("methodPUI") String methodPUI, + @Nullable @QueryValue("scaleDbId") String scaleDbId, + @Nullable @QueryValue("scaleName") String scaleName, + @Nullable @QueryValue("scalePUI") String scalePUI, + @Nullable @QueryValue("traitDbId") String traitDbId, + @Nullable @QueryValue("traitName") String traitName, + @Nullable @QueryValue("traitPUI") String traitPUI, + @Nullable @QueryValue("ontologyDbId") String ontologyDbId, + @Nullable @QueryValue("commonCropName") String commonCropName, +// @QueryValue("programDbId") Optional<@Nullable String> programDbId, //this is + @Nullable @QueryValue("trialDbId") String experimentId, + @Nullable @QueryValue("studyDbId") String environmentId, + @Nullable @QueryValue("externalReferenceID") String externalReferenceID, + @Nullable @QueryValue("externalReferenceId") String externalReferenceId, + @Nullable @QueryValue("externalReferenceSource") String externalReferenceSource, + @Nullable @QueryValue("page") Integer page, + @Nullable @QueryValue("pageSize") Integer pageSize, HttpRequest request) { + + try { + List programTraits; + if (observationVariablePUI != null || methodPUI != null || scalePUI != null || traitPUI != null || commonCropName != null || externalReferenceID != null || externalReferenceId != null || externalReferenceSource != null) { + log.debug("unsupported variable filters, returning"); + programTraits = new ArrayList<>(); + } else if(environmentId != null || experimentId != null) { + programTraits = getBrAPIObservationVariablesForExperiment(programId, Optional.ofNullable(experimentId), Optional.ofNullable(environmentId)); + } else { + log.debug("fetching variables for the program: " + programId); + programTraits = ontologyService.getTraitsByProgramId(programId, true); + + } + + List filteredObsVars = filterVariables(programTraits, + Optional.ofNullable(observationVariableDbId), + Optional.ofNullable(observationVariableName), + Optional.ofNullable(traitClass), + Optional.ofNullable(methodDbId), + Optional.ofNullable(methodName), + Optional.ofNullable(scaleDbId), + Optional.ofNullable(scaleName), + Optional.ofNullable(traitDbId), + Optional.ofNullable(traitName), + Optional.ofNullable(ontologyDbId)); + + BrAPIObservationVariableListResponse response = new BrAPIObservationVariableListResponse() + .metadata(new BrAPIMetadata() + .pagination(new BrAPIIndexPagination() + .currentPage(0) + .totalPages(1) + .pageSize(filteredObsVars.size()) + .totalCount(filteredObsVars.size()))) + .result(new BrAPIObservationVariableListResponseResult().data(filteredObsVars)); + + return HttpResponse.ok(response); + } catch (DoesNotExistException e) { + log.warn("Couldn't find object", e); + return HttpResponse.notFound(); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.status(HttpStatus.INTERNAL_SERVER_ERROR, "error fetching variables"); + } } @Get("/variables/{observationVariableDbId}") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse variablesObservationVariableDbIdGet(@PathVariable("programId") UUID programId, + public HttpResponse variablesObservationVariableDbIdGet(@PathVariable("programId") UUID programId, @PathVariable("observationVariableDbId") String observationVariableDbId) { - return HttpResponse.notFound(); + log.debug("fetching variable: " + observationVariableDbId); + UUID traitId; + try { + traitId = UUID.fromString(observationVariableDbId); + } catch (IllegalArgumentException e) { + return HttpResponse.notFound(); + } + + try { + Optional trait = traitService.getById(programId, traitId); + + if(trait.isEmpty()) { + return HttpResponse.notFound(); + } + + BrAPIObservationVariableSingleResponse response = new BrAPIObservationVariableSingleResponse() + .metadata(new BrAPIMetadata()) + .result(convertToBrAPI(trait.get())); + + return HttpResponse.ok(response); + } catch (DoesNotExistException e) { + return HttpResponse.notFound(); + } } @Put("/variables/{observationVariableDbId}") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse variablesObservationVariableDbIdPut(@PathVariable("programId") UUID programId, + public HttpResponse variablesObservationVariableDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("observationVariableDbId") String observationVariableDbId, BrAPIObservationVariable body) { //DO NOT IMPLEMENT - Users are only able to update traits via the DeltaBreed UI @@ -86,8 +195,148 @@ public HttpResponse variablesObservationVariableDbIdPut(@PathVariable("programId @Post("/variables") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse variablesPost(@PathVariable("programId") UUID programId, List body) { + public HttpResponse variablesPost(@PathVariable("programId") UUID programId, List body) { //DO NOT IMPLEMENT - Users are only able to create new traits via the DeltaBreed UI return HttpResponse.notFound(); } + + private BrAPIObservationVariable convertToBrAPI(Trait trait) { + BrAPIOntologyReference brAPIOntologyReference = new BrAPIOntologyReference().ontologyDbId(trait.getProgramOntologyId() + .toString()); + String status = trait.getActive() ? "active" : "inactive"; + return new BrAPIObservationVariable().observationVariableDbId(trait.getId().toString()) + .observationVariableName(trait.getObservationVariableName()) + .defaultValue(trait.getDefaultValue()) + .status(status) + .synonyms(trait.getSynonyms()) + .trait(new BrAPITrait().ontologyReference(brAPIOntologyReference) + .traitName(trait.getObservationVariableName()) + .traitDbId(trait.getId().toString()) + .entity(trait.getEntity()) + .attribute(trait.getAttribute()) + .status(status) + .synonyms(trait.getSynonyms())) + .method(new BrAPIMethod().ontologyReference(brAPIOntologyReference) + .methodDbId(trait.getMethod().getId().toString()) + .description(trait.getMethod().getDescription()) + .formula(trait.getMethod().getFormula())) + .scale(new BrAPIScale().ontologyReference(brAPIOntologyReference) + .scaleDbId(trait.getScale().getId().toString()) + .scaleName(trait.getScale().getScaleName()) + .dataType(BrAPITraitDataType.fromValue(trait.getScale().getDataType().getLiteral())) + .decimalPlaces(trait.getScale().getDecimalPlaces()) + .validValues(new BrAPIScaleValidValues().max(trait.getScale().getValidValueMax()) + .min(trait.getScale().getValidValueMin()) + .categories(trait.getScale().getCategories()))); + } + + @NotNull + private List filterVariables(List programTraits, + Optional observationVariableDbId, + Optional observationVariableName, + Optional traitClass, + Optional methodDbId, + Optional methodName, + Optional scaleDbId, + Optional scaleName, + Optional traitDbId, + Optional traitName, + Optional ontologyDbId) throws DoesNotExistException { + log.debug("filtering variables:\n" + + "observationVariableDbId: " + observationVariableDbId + "\n" + + "observationVariableName: " + observationVariableName + "\n" + + "traitClass: " + traitClass + "\n" + + "methodDbId: " + methodDbId + "\n" + + "methodName: " + methodName + "\n" + + "scaleDbId: " + scaleDbId + "\n" + + "scaleName: " + scaleName + "\n" + + "traitDbId: " + traitDbId + "\n" + + "traitName: " + traitName + "\n" + + "ontologyDbId: " + ontologyDbId); + + return programTraits.stream() + .filter(trait -> { + boolean matches = true; + + Map, String>> filterParams = new HashMap<>(); + filterParams.put("observationVariableDbId", + Pair.of(observationVariableDbId, + trait.getId() + .toString())); + filterParams.put("observationVariableName", Pair.of(observationVariableName, trait.getObservationVariableName())); + filterParams.put("traitClass", Pair.of(traitClass, trait.getTraitClass())); + filterParams.put("methodDbId", + Pair.of(methodDbId, + trait.getMethodId() + .toString())); + filterParams.put("methodName", + Pair.of(methodName, + trait.getMethod() + .getDescription())); + filterParams.put("scaleDbId", + Pair.of(scaleDbId, + trait.getScale() + .getId() + .toString())); + filterParams.put("scaleName", + Pair.of(scaleName, + trait.getScale() + .getScaleName())); + filterParams.put("traitDbId", + Pair.of(traitDbId, + trait.getId() + .toString())); + filterParams.put("traitName", Pair.of(traitName, trait.getObservationVariableName())); + filterParams.put("ontologyDbId", + Pair.of(ontologyDbId, + trait.getProgramOntologyId() + .toString())); + + for (Map.Entry, String>> filter : filterParams.entrySet()) { + if (filter.getValue().getLeft().isPresent()) { + log.debug("filtering traits by: " + filter.getKey()); + matches = StringUtils.equals(filter.getValue() + .getLeft() + .get(), + filter.getValue() + .getRight()); + } + if (!matches) { + break; + } + } + + return matches; + }) + .map(this::convertToBrAPI) + .collect(Collectors.toList()); + } + + private List getBrAPIObservationVariablesForExperiment(UUID programId, Optional experimentId, Optional environmentId) throws DoesNotExistException, ApiException { + log.debug(String.format("fetching variables for experiment. expId: %s, envId: %s ", experimentId.orElse(""), environmentId.orElse(""))); + Optional program = programService.getById(programId); + if(program.isEmpty()) { + throw new DoesNotExistException("Could not find program: " + programId); + } + UUID expId; + if(experimentId.isPresent()) { + expId = UUID.fromString(experimentId.get()); + } else { + UUID envId = UUID.fromString(environmentId.get()); + BrAPIStudy environment = trialService.getEnvironment(program.get(), envId); + expId = UUID.fromString(Utilities.getExternalReference(environment.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)).get().getReferenceID()); + } + + BrAPITrial experiment = trialService.getExperiment(program.get(), expId); + if(experiment + .getAdditionalInfo().getAsJsonObject() + .has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + String obsDatasetId = experiment + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); + return trialService.getDatasetObsVars(obsDatasetId, program.get()); + } + + return new ArrayList<>(); + } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIProgramsController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIProgramsController.java index 309172319..dabbebc45 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIProgramsController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIProgramsController.java @@ -22,16 +22,40 @@ import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; import lombok.extern.slf4j.Slf4j; +import org.brapi.v2.model.BrAPIIndexPagination; +import org.brapi.v2.model.BrAPIMetadata; import org.brapi.v2.model.core.BrAPIProgram; +import org.brapi.v2.model.core.response.BrAPIProgramListResponse; +import org.brapi.v2.model.core.response.BrAPIProgramListResponseResult; +import org.brapi.v2.model.core.response.BrAPIProgramSingleResponse; +import org.breedinginsight.api.auth.ProgramSecured; +import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; +import org.breedinginsight.api.auth.SecurityService; import org.breedinginsight.brapi.v1.controller.BrapiVersion; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.ProgramService; +import javax.annotation.Nullable; +import javax.inject.Inject; import java.util.List; +import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; @Slf4j @Controller("/${micronaut.bi.api.version}") @Secured(SecurityRule.IS_AUTHENTICATED) public class BrAPIProgramsController { + + private final SecurityService securityService; + private final ProgramService programService; + + @Inject + public BrAPIProgramsController(SecurityService securityService, ProgramService programService) { + this.securityService = securityService; + this.programService = programService; + } + /* TODO - GET programs @@ -39,32 +63,61 @@ public class BrAPIProgramsController { //START - endpoints at root BrAPI url @Get(BrapiVersion.BRAPI_V2 + "/programs") - public HttpResponse rootProgramsGet(@QueryValue("abbreviation") String abbreviation, - @QueryValue("programType") String programType, - @QueryValue("commonCropName") String commonCropName, - @QueryValue("programDbId") String programDbId, - @QueryValue("programName") String programName, - @QueryValue("externalReferenceID") String externalReferenceID, - @QueryValue("externalReferenceId") String externalReferenceId, - @QueryValue("externalReferenceSource") String externalReferenceSource, - @QueryValue("page") Integer page, - @QueryValue("pageSize") Integer pageSize) { - return HttpResponse.notFound(); + public HttpResponse rootProgramsGet( + @Nullable @QueryValue("abbreviation") String abbreviation, + @Nullable @QueryValue("programType") String programType, + @Nullable @QueryValue("commonCropName") String commonCropName, + @Nullable @QueryValue("programDbId") String programDbId, + @Nullable @QueryValue("programName") String programName, + @Nullable @QueryValue("externalReferenceID") String externalReferenceID, + @Nullable @QueryValue("externalReferenceId") String externalReferenceId, + @Nullable @QueryValue("externalReferenceSource") String externalReferenceSource, + @Nullable @QueryValue("page") Integer page, + @Nullable @QueryValue("pageSize") Integer pageSize) { + + Optional abbreviationOptional = Optional.ofNullable(abbreviation); + Optional programDbIdOptional = Optional.ofNullable(programDbId); + Optional programNameOptional = Optional.ofNullable(programName); + + List programs = programService.getAll(securityService.getUser()).stream().filter(program -> { + boolean matches = abbreviationOptional.map(abbr -> abbr.equals(program.getKey())).orElse(true); + matches = matches && programDbIdOptional.map(id -> id.equals(program.getId().toString())).orElse(true); + return matches && programNameOptional.map(name -> name.equals(program.getName())).orElse(true); + }).map(this::convertToBrAPI).collect(Collectors.toList()); + + return HttpResponse.ok(new BrAPIProgramListResponse().metadata(new BrAPIMetadata().pagination(new BrAPIIndexPagination().currentPage(0) + .totalPages(1) + .totalCount(programs.size()) + .pageSize(programs.size()))) + .result(new BrAPIProgramListResponseResult().data(programs))); } @Post(BrapiVersion.BRAPI_V2 + "/programs") - public HttpResponse rootProgramsPost(List body) { + public HttpResponse rootProgramsPost(List body) { //DO NOT IMPLEMENT - Users should only be able to create new programs via the DeltaBreed UI return HttpResponse.notFound(); } @Get(BrapiVersion.BRAPI_V2 + "/programs/{programDbId}") - public HttpResponse rootProgramsProgramDbIdGet(@PathVariable("programDbId") String programDbId) { - return HttpResponse.notFound(); + public HttpResponse rootProgramsProgramDbIdGet(@PathVariable("programDbId") String programDbId) { + HttpResponse brAPIProgramListResponseHttpResponse = this.rootProgramsGet(null, null, null, + programDbId, + null, + null, + null, + null, + null, + null); + Optional program = Optional.ofNullable(brAPIProgramListResponseHttpResponse.body()) + .orElse(new BrAPIProgramListResponse()) + .getResult() + .getData().stream().findFirst(); + + return HttpResponse.ok(new BrAPIProgramSingleResponse().result(program.orElse(null))); } @Put(BrapiVersion.BRAPI_V2 + "/programs/{programDbId}") - public HttpResponse rootProgramsProgramDbIdPut(@PathVariable("programDbId") String programDbId, BrAPIProgram body) { + public HttpResponse rootProgramsProgramDbIdPut(@PathVariable("programDbId") String programDbId, BrAPIProgram body) { //DO NOT IMPLEMENT - Users should only be able to update programs via the DeltaBreed UI return HttpResponse.notFound(); } @@ -73,34 +126,63 @@ public HttpResponse rootProgramsProgramDbIdPut(@PathVariable("programDbId") Stri //START - endpoints for within the context of a program @Get("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/programs") - public HttpResponse programsGet(@PathVariable("programId") UUID programId, @QueryValue("abbreviation") String abbreviation, - @QueryValue("programType") String programType, - @QueryValue("commonCropName") String commonCropName, - @QueryValue("programDbId") String programDbId, - @QueryValue("programName") String programName, - @QueryValue("externalReferenceID") String externalReferenceID, - @QueryValue("externalReferenceId") String externalReferenceId, - @QueryValue("externalReferenceSource") String externalReferenceSource, - @QueryValue("page") Integer page, - @QueryValue("pageSize") Integer pageSize) { - return HttpResponse.notFound(); + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse programsGet(@PathVariable("programId") UUID programId, + @QueryValue("abbreviation") Optional abbreviation, + @QueryValue("programType") Optional programType, + @QueryValue("commonCropName") Optional commonCropName, + @QueryValue("programDbId") Optional programDbId, + @QueryValue("programName") Optional programName, + @QueryValue("externalReferenceID") Optional externalReferenceID, + @QueryValue("externalReferenceId") Optional externalReferenceId, + @QueryValue("externalReferenceSource") Optional externalReferenceSource, + @QueryValue("page") Optional page, + @QueryValue("pageSize") Optional pageSize) { + + List programs = programService.getById(programId).stream().filter(program -> { + boolean matches = abbreviation.map(abbr -> abbr.equals(program.getKey())).orElse(true); + matches = matches && programDbId.map(id -> id.equals(program.getId().toString())).orElse(true); + return matches && programName.map(name -> name.equals(program.getName())).orElse(true); + }).map(this::convertToBrAPI).collect(Collectors.toList()); + + return HttpResponse.ok(new BrAPIProgramListResponse().metadata(new BrAPIMetadata().pagination(new BrAPIIndexPagination().currentPage(0) + .totalPages(1) + .totalCount(programs.size()) + .pageSize(programs.size()))) + .result(new BrAPIProgramListResponseResult().data(programs))); } @Post("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/programs") - public HttpResponse programsPost(@PathVariable("programId") UUID programId, List body) { + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse programsPost(@PathVariable("programId") UUID programId, List body) { //DO NOT IMPLEMENT - Users should only be able to create new programs via the DeltaBreed UI return HttpResponse.notFound(); } @Get("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/programs/{programDbId}") - public HttpResponse programsProgramDbIdGet(@PathVariable("programId") UUID programId, @PathVariable("programDbId") String programDbId) { - return HttpResponse.notFound(); + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse programsProgramDbIdGet(@PathVariable("programId") UUID programId, @PathVariable("programDbId") String programDbId) { + Optional program = programService.getById(programId) + .stream() + .filter(p -> programDbId.equals(programId.toString())) + .map(this::convertToBrAPI) + .findFirst(); + + return HttpResponse.ok(new BrAPIProgramSingleResponse().result(program.orElse(null))); } @Put("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/programs/{programDbId}") - public HttpResponse programsProgramDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("programDbId") String programDbId, BrAPIProgram body) { + @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) + public HttpResponse programsProgramDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("programDbId") String programDbId, BrAPIProgram body) { //DO NOT IMPLEMENT - Users should only be able to update programs via the DeltaBreed UI return HttpResponse.notFound(); } //END - endpoints for within the context of a program + + private BrAPIProgram convertToBrAPI(Program program) { + return new BrAPIProgram().programName(program.getName()) + .programDbId(program.getId().toString()) + .abbreviation(program.getKey()) + .commonCropName(program.getSpecies().getCommonName()); + } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIStudiesController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIStudiesController.java index 4f64fba1c..ea6f03a52 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIStudiesController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIStudiesController.java @@ -17,6 +17,7 @@ package org.breedinginsight.brapi.v2; +import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; import io.micronaut.http.MediaType; @@ -25,7 +26,10 @@ import io.micronaut.security.rules.SecurityRule; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIMetadata; +import org.brapi.v2.model.BrAPIStatus; import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.response.BrAPIStudySingleResponse; import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; import org.breedinginsight.api.model.v1.request.query.SearchRequest; @@ -35,30 +39,40 @@ import org.breedinginsight.brapi.v1.controller.BrapiVersion; import org.breedinginsight.brapi.v2.model.request.query.StudyQuery; import org.breedinginsight.brapi.v2.services.BrAPIStudyService; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.Program; +import org.breedinginsight.services.ProgramService; +import org.breedinginsight.utilities.Utilities; import org.breedinginsight.utilities.response.ResponseUtils; import org.breedinginsight.utilities.response.mappers.StudyQueryMapper; import javax.inject.Inject; import javax.validation.Valid; import java.util.List; +import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; @Slf4j -@Controller("/${micronaut.bi.api.version}") +@Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2) @Secured(SecurityRule.IS_AUTHENTICATED) public class BrAPIStudiesController { + private final String referenceSource; private final BrAPIStudyService studyService; private final StudyQueryMapper studyQueryMapper; + private final ProgramService programService; @Inject - public BrAPIStudiesController(BrAPIStudyService studyService, StudyQueryMapper studyQueryMapper) { + public BrAPIStudiesController(BrAPIStudyService studyService, StudyQueryMapper studyQueryMapper, @Property(name = "brapi.server.reference-source") String referenceSource, ProgramService programService) { this.studyService = studyService; this.studyQueryMapper = studyQueryMapper; + this.referenceSource = referenceSource; + this.programService = programService; } - @Get("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/studies{?queryParams*}") + @Get("/studies{?queryParams*}") @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) public HttpResponse>>> getStudies( @@ -67,7 +81,10 @@ public HttpResponse>>> getStudies( try { log.debug("fetching studies for program: " + programId); - List studies = studyService.getStudies(programId); + List studies = studyService.getStudies(programId) + .stream() + .peek(this::setDbIds) + .collect(Collectors.toList()); queryParams.setSortField(studyQueryMapper.getDefaultSortField()); queryParams.setSortOrder(studyQueryMapper.getDefaultSortOrder()); SearchRequest searchRequest = queryParams.constructSearchRequest(); @@ -81,33 +98,54 @@ public HttpResponse>>> getStudies( } } - /* - TODO - - GET studies/{id} - */ @Post("/studies") - @Consumes({"application/json"}) - @Produces({"application/json"}) @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse studiesPost(@PathVariable("programId") UUID programId, List body) { + public HttpResponse studiesPost(@PathVariable("programId") UUID programId, List body) { //DO NOT IMPLEMENT - Users are only able to create new studies via the DeltaBreed UI return HttpResponse.notFound(); } @Get("/studies/{studyDbId}") - @Produces({"application/json"}) @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse studiesStudyDbIdGet(@PathVariable("programId") UUID programId, @PathVariable("studyDbId") String studyDbId) { - return HttpResponse.notFound(); + public HttpResponse studiesStudyDbIdGet(@PathVariable("programId") UUID programId, @PathVariable("studyDbId") String environmentId) { + Optional program = programService.getById(programId); + if(program.isEmpty()) { + log.warn("program id: " + programId + " not found"); + return HttpResponse.notFound(); + } + try { + Optional study = studyService.getStudyByEnvironmentId(program.get(), UUID.fromString(environmentId)); + if(study.isPresent()) { + setDbIds(study.get()); + return HttpResponse.ok(new BrAPIStudySingleResponse().result(study.get())); + } else { + log.warn("studyDbId: " + environmentId + " not found"); + return HttpResponse.notFound(); + } + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + return HttpResponse.serverError(new BrAPIStudySingleResponse().metadata(new BrAPIMetadata().addStatusItem(new BrAPIStatus().message("Error fetching study") + .messageType(BrAPIStatus.MessageTypeEnum.ERROR)))); + } } @Put("/studies/{studyDbId}") - @Consumes({"application/json"}) - @Produces({"application/json"}) @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse studiesStudyDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("studyDbId") String studyDbId, + public HttpResponse studiesStudyDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("studyDbId") String studyDbId, BrAPIStudy body) { //DO NOT IMPLEMENT - Users are only able to update studies via the DeltaBreed UI return HttpResponse.notFound(); } + + private void setDbIds(BrAPIStudy study) { + study.studyDbId(Utilities.getExternalReference(study.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES)) + .orElseThrow(() -> new IllegalStateException("No BI external reference found")) + .getReferenceID()); + + study.trialDbId(Utilities.getExternalReference(study.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)) + .orElseThrow(() -> new IllegalStateException("No BI external reference found")) + .getReferenceID()); + + //TODO update locationDbId + } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java index 71b979641..77dbf2b90 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java @@ -1,5 +1,6 @@ package org.breedinginsight.brapi.v2; +import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; import io.micronaut.http.MediaType; @@ -8,7 +9,9 @@ import io.micronaut.security.rules.SecurityRule; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.core.response.BrAPITrialSingleResponse; import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; import org.breedinginsight.api.model.v1.request.query.SearchRequest; @@ -18,27 +21,31 @@ import org.breedinginsight.brapi.v1.controller.BrapiVersion; import org.breedinginsight.brapi.v2.model.request.query.ExperimentQuery; import org.breedinginsight.brapi.v2.services.BrAPITrialService; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.utilities.Utilities; import org.breedinginsight.utilities.response.ResponseUtils; import org.breedinginsight.utilities.response.mappers.ExperimentQueryMapper; import javax.inject.Inject; import javax.validation.Valid; import java.util.*; +import java.util.stream.Collectors; @Slf4j @Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2) @Secured(SecurityRule.IS_AUTHENTICATED) public class BrAPITrialsController { + + private final String referenceSource; private final BrAPITrialService experimentService; private final ExperimentQueryMapper experimentQueryMapper; - private final ProgramService programService; @Inject - public BrAPITrialsController(BrAPITrialService experimentService, ExperimentQueryMapper experimentQueryMapper, ProgramService programService) { + public BrAPITrialsController(BrAPITrialService experimentService, ExperimentQueryMapper experimentQueryMapper, @Property(name = "brapi.server.reference-source") String referenceSource) { this.experimentService = experimentService; this.experimentQueryMapper = experimentQueryMapper; - this.programService = programService; + this.referenceSource = referenceSource; } @Get("/trials{?queryParams*}") @@ -50,7 +57,7 @@ public HttpResponse>>> getExperiments( try { log.debug("fetching trials for program: " + programId); - List experiments = experimentService.getExperiments(programId); + List experiments = experimentService.getExperiments(programId).stream().peek(this::setDbIds).collect(Collectors.toList()); SearchRequest searchRequest = queryParams.constructSearchRequest(); return ResponseUtils.getBrapiQueryResponse(experiments, experimentQueryMapper, queryParams, searchRequest); } catch (ApiException e) { @@ -65,7 +72,7 @@ public HttpResponse>>> getExperiments( @Get("/trials/{trialId}") @Produces(MediaType.APPLICATION_JSON) @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse> getExperimentById( + public HttpResponse getExperimentById( @PathVariable("programId") UUID programId, @PathVariable("trialId") UUID trialId, @QueryValue(defaultValue = "false") Boolean stats){ @@ -75,8 +82,9 @@ public HttpResponse> getExperimentById( logMsg += " with stats"; } log.debug(logMsg); - Response response = new Response<>(experimentService.getTrialDataByUUID(programId, trialId, stats)); - return HttpResponse.ok(response); + BrAPITrial trial = experimentService.getTrialDataByUUID(programId, trialId, stats); + setDbIds(trial); + return HttpResponse.ok(new BrAPITrialSingleResponse().result(trial)); } catch (DoesNotExistException e) { log.info(e.getMessage(), e); return HttpResponse.status(HttpStatus.NOT_FOUND, e.getMessage()); @@ -85,7 +93,7 @@ public HttpResponse> getExperimentById( @Post("/trials") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse trialsPost(@PathVariable("programId") UUID programId, List body) { + public HttpResponse trialsPost(@PathVariable("programId") UUID programId, List body) { //DO NOT IMPLEMENT - Users are only able to create new trials via the DeltaBreed UI return HttpResponse.notFound(); } @@ -93,9 +101,20 @@ public HttpResponse trialsPost(@PathVariable("programId") UUID programId, List trialsTrialDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("trialDbId") String trialDbId, BrAPITrial body) { //DO NOT IMPLEMENT - Users are only able to update trials via the DeltaBreed UI return HttpResponse.notFound(); } + private void setDbIds(BrAPITrial trial) { + trial.trialDbId(Utilities.getExternalReference(trial.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)) + .orElseThrow(() -> new IllegalStateException("No BI external reference found")) + .getReferenceID()); + trial.programDbId(Utilities.getExternalReference(trial.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS)) + .orElseThrow(() -> new IllegalStateException("No BI external reference found")) + .getReferenceID()); + + //TODO update locationDbId + } + } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java index f2069e5bb..db5e28351 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java @@ -17,15 +17,16 @@ package org.breedinginsight.brapi.v2.services; -import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; import org.breedinginsight.brapps.importer.daos.BrAPIStudyDAO; +import org.breedinginsight.model.Program; import javax.inject.Inject; import javax.inject.Singleton; import java.util.List; +import java.util.Optional; import java.util.UUID; @Slf4j @@ -40,11 +41,11 @@ public BrAPIStudyService(BrAPIStudyDAO studyDAO) { } public List getStudies(UUID programId) throws ApiException { - try { - return studyDAO.getStudies(programId); - } catch (ApiException e) { - throw new InternalServerException(e.getMessage(), e); - } + return studyDAO.getStudies(programId); + } + + public Optional getStudyByEnvironmentId(Program program, UUID environmentId) throws ApiException { + return studyDAO.getStudyByEnvironmentId(environmentId, program); } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 95d34cd32..f1939996c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -22,11 +22,13 @@ import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.FileMappingUtil; +import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.model.BrAPIConstants; import org.breedinginsight.model.Column; import org.breedinginsight.model.DownloadFile; import org.breedinginsight.model.Program; import org.breedinginsight.model.*; +import org.breedinginsight.services.TraitService; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; import org.breedinginsight.utilities.IntOrderComparator; @@ -54,7 +56,9 @@ public class BrAPITrialService { private final BrAPITrialDAO trialDAO; private final BrAPIObservationDAO observationDAO; private final BrAPIListDAO listDAO; - private final BrAPIObservationVariableDAO obsVarDAO; +// private final BrAPIObservationVariableDAO obsVarDAO; + + private final TraitService traitService; private final BrAPIStudyDAO studyDAO; private final BrAPISeasonDAO seasonDAO; private final BrAPIObservationUnitDAO ouDAO; @@ -67,7 +71,8 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin BrAPITrialDAO trialDAO, BrAPIObservationDAO observationDAO, BrAPIListDAO listDAO, - BrAPIObservationVariableDAO obsVarDAO, +// BrAPIObservationVariableDAO obsVarDAO, + TraitService traitService, BrAPIStudyDAO studyDAO, BrAPISeasonDAO seasonDAO, BrAPIObservationUnitDAO ouDAO, @@ -78,7 +83,8 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin this.trialDAO = trialDAO; this.observationDAO = observationDAO; this.listDAO = listDAO; - this.obsVarDAO = obsVarDAO; +// this.obsVarDAO = obsVarDAO; + this.traitService = traitService; this.studyDAO = studyDAO; this.seasonDAO = seasonDAO; this.ouDAO = ouDAO; @@ -123,7 +129,7 @@ public DownloadFile exportObservations( DownloadFile downloadFile; boolean isDataset = false; List dataset = new ArrayList<>(); - List obsVars = new ArrayList<>(); + List obsVars = new ArrayList<>(); Map> rowByOUId = new HashMap<>(); Map studyByDbId = new HashMap<>(); Map studyDbIdByOUId = new HashMap<>(); @@ -301,9 +307,9 @@ public Dataset getDatasetData(Program program, UUID experimentId, UUID datsetId, log.debug("fetching observationUnits for dataset: " + datsetId); List datasetOUs = ouDAO.getObservationUnitsForDataset(datsetId.toString(), program); log.debug("fetching dataset variables dataset: " + datsetId); - List datasetObsVars = getDatasetObsVars(datsetId.toString(), program); + List datasetObsVars = getDatasetObsVars(datsetId.toString(), program); List ouDbIds = datasetOUs.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList()); - List obsVarDbIds = datasetObsVars.stream().map(BrAPIObservationVariable::getObservationVariableDbId).collect(Collectors.toList()); + List obsVarDbIds = datasetObsVars.stream().map(Trait::getObservationVariableDbId).collect(Collectors.toList()); log.debug("fetching observations for dataset: " + datsetId); List data = observationDAO.getObservationsByObservationUnitsAndVariables(ouDbIds, obsVarDbIds, program); log.debug("building dataset object for dataset: " + datsetId); @@ -333,10 +339,10 @@ private void addBrAPIObsToRecords( Map studyByDbId, Map> rowByOUId, boolean includeTimestamp, - List obsVars, + List obsVars, Map studyDbIdByOUId, Map programGermplasmByDbId) throws ApiException, DoesNotExistException { - Map varByDbId = new HashMap<>(); + Map varByDbId = new HashMap<>(); obsVars.forEach(var -> varByDbId.put(var.getObservationVariableDbId(), var)); for (BrAPIObservation obs: dataset) { @@ -345,7 +351,7 @@ private void addBrAPIObsToRecords( String ouId = getOUId(ou); // get observation variable for BrAPI observation - BrAPIObservationVariable var = varByDbId.get(obs.getObservationVariableDbId()); + Trait var = varByDbId.get(obs.getObservationVariableDbId()); // if there is a row with that ouId then just add the obs var data and timestamp to the row if (rowByOUId.get(ouId) != null) { @@ -385,11 +391,11 @@ private void addObsVarDataToRow( Map row, BrAPIObservation obs, boolean includeTimestamp, - BrAPIObservationVariable var, + Trait var, Program program) { String varName = Utilities.removeProgramKey(obs.getObservationVariableName(), program.getKey()); - if (var.getScale().getDataType().equals(BrAPITraitDataType.NUMERICAL) || - var.getScale().getDataType().equals(BrAPITraitDataType.DURATION)) { + if (var.getScale().getDataType().equals(DataType.NUMERICAL) || + var.getScale().getDataType().equals(DataType.DURATION)) { row.put(varName, Double.parseDouble(obs.getValue())); } else { row.put(varName, obs.getValue()); @@ -401,24 +407,26 @@ private void addObsVarDataToRow( } } - public List getDatasetObsVars(String datasetId, Program program) throws ApiException, DoesNotExistException { + public List getDatasetObsVars(String datasetId, Program program) throws ApiException, DoesNotExistException { List lists = listDAO.getListByTypeAndExternalRef( BrAPIListTypes.OBSERVATIONVARIABLES, program.getId(), String.format("%s/%s", referenceSource, ExternalReferenceSource.DATASET.getName()), UUID.fromString(datasetId)); if (lists == null || lists.isEmpty()) { - throw new DoesNotExistException("Dataset observation variables list not returned from BrAPI service"); +// throw new DoesNotExistException("Dataset observation variables list not returned from BrAPI service"); + log.warn(String.format("Dataset %s observation variables list not returned from BrAPI service", datasetId)); + return new ArrayList<>(); } String listDbId = lists.get(0).getListDbId(); BrAPIListsSingleResponse list = listDAO.getListById(listDbId, program.getId()); - List obsVarNames = list.getResult().getData(); + List obsVarNames = list.getResult().getData().stream().map(var -> Utilities.removeProgramKey(var, program.getKey())).collect(Collectors.toList()); log.debug("Searching for dataset obsVars: " + obsVarNames); - List obsVars = obsVarDAO.getVariableByName(obsVarNames, program.getId()); + List obsVars = traitService.getByName(program.getId(), obsVarNames); log.debug(String.format("Found %d obsVars", obsVars.size())); // sort the obsVars to match the order stored in the dataset list - return fileMappingUtil.sortByField(obsVarNames, obsVars, BrAPIObservationVariable::getObservationVariableName); + return fileMappingUtil.sortByField(obsVarNames, obsVars, Trait::getObservationVariableName); } public BrAPITrial getExperiment(Program program, UUID experimentId) throws ApiException { @@ -500,14 +508,14 @@ private Map createExportRow( private void addObsVarColumns( List columns, - List obsVars, + List obsVars, boolean includeTimestamps, Program program) { - for (BrAPIObservationVariable var: obsVars) { + for (Trait var: obsVars) { Column obsVarColumn = new Column(); obsVarColumn.setDataType(Column.ColumnDataType.STRING); - if (var.getScale().getDataType().equals(BrAPITraitDataType.NUMERICAL) || - var.getScale().getDataType().equals(BrAPITraitDataType.DURATION)) { + if (var.getScale().getDataType().equals(DataType.NUMERICAL) || + var.getScale().getDataType().equals(DataType.DURATION)) { obsVarColumn.setDataType(Column.ColumnDataType.DOUBLE); } String varName = Utilities.removeProgramKey(var.getObservationVariableName(), program.getKey()); @@ -564,4 +572,13 @@ private void sortDefaultForExportRows(@NotNull List> exportR exportRows.sort(envComparator.thenComparing(expUnitIdComparator)); } + + public BrAPIStudy getEnvironment(Program program, UUID envId) throws ApiException { + List environments = studyDAO.getStudiesByEnvironmentIds(List.of(envId), program); + if (environments.isEmpty()) { + throw new RuntimeException("A study with given experiment id was not returned"); + } + + return environments.get(0); + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java index 21fe0dbc2..f8ed51f25 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java @@ -26,16 +26,13 @@ import io.micronaut.http.server.exceptions.InternalServerException; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; -import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.brapi.v2.model.pheno.BrAPIObservationUnitLevelRelationship; import org.brapi.v2.model.pheno.request.BrAPIObservationUnitSearchRequest; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService; import org.brapi.v2.model.pheno.BrAPIObservationTreatment; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.brapi.v2.model.pheno.request.BrAPIObservationUnitSearchRequest; -import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; @@ -51,6 +48,8 @@ import javax.validation.constraints.NotNull; import java.lang.reflect.Type; import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; @Singleton public class BrAPIObservationUnitDAO { @@ -92,17 +91,18 @@ public List getObservationUnitByName(List observat observationUnitSearchRequest.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); observationUnitSearchRequest.observationUnitNames(observationUnitNames); - return searchObservationUnitsAndProcess(observationUnitSearchRequest, program.getId(), false); + return searchObservationUnitsAndProcess(observationUnitSearchRequest, program, false); } /** * Create observation units, mutates brAPIObservationUnitList */ - public List createBrAPIObservationUnits(List brAPIObservationUnitList, UUID programId, ImportUpload upload) throws ApiException { + public List createBrAPIObservationUnits(List brAPIObservationUnitList, UUID programId, ImportUpload upload) throws ApiException, DoesNotExistException { + Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationUnitsApi.class); preprocessObservationUnits(brAPIObservationUnitList); List ous = brAPIDAOUtil.post(brAPIObservationUnitList, upload, api::observationunitsPost, importDAO::update); - processObservationUnits(programId, ous, false); + processObservationUnits(program, ous, false); return ous; } @@ -130,7 +130,7 @@ public List getObservationUnitsById(Collection obs observationUnitSearchRequest.externalReferenceIDs(new ArrayList<>(observationUnitExternalIds)); observationUnitSearchRequest.externalReferenceSources(List.of(String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName()))); - return searchObservationUnitsAndProcess(observationUnitSearchRequest, program.getId(), false); + return searchObservationUnitsAndProcess(observationUnitSearchRequest, program, false); } public List getObservationUnitsForStudyDbId(@NotNull String studyDbId, Program program) throws ApiException { @@ -139,13 +139,24 @@ public List getObservationUnitsForStudyDbId(@NotNull Strin .getProgramDbId())); observationUnitSearchRequest.studyDbIds(List.of(studyDbId)); - return searchObservationUnitsAndProcess(observationUnitSearchRequest, program.getId(), false); + return searchObservationUnitsAndProcess(observationUnitSearchRequest, program, false); } public List getObservationUnitsForTrialDbId(@NotNull UUID programId, @NotNull String trialDbId) throws ApiException, DoesNotExistException { return getObservationUnitsForTrialDbId(programId, trialDbId, false); } + public List getObservationUnitsForTrialDbId(@NotNull UUID programId, @NotNull String trialDbId, boolean withGID) throws ApiException, DoesNotExistException { + Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); + + BrAPIObservationUnitSearchRequest observationUnitSearchRequest = new BrAPIObservationUnitSearchRequest(); + observationUnitSearchRequest.programDbIds(List.of(program.getBrapiProgram() + .getProgramDbId())); + observationUnitSearchRequest.trialDbIds(List.of(trialDbId)); + + return searchObservationUnitsAndProcess(observationUnitSearchRequest, program, withGID); + } + public List getObservationUnitsForDataset(@NotNull String datasetId, @NotNull Program program) throws ApiException { String datasetReferenceSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.DATASET); BrAPIObservationUnitSearchRequest ouSearchRequest = new BrAPIObservationUnitSearchRequest(); @@ -155,15 +166,99 @@ public List getObservationUnitsForDataset(@NotNull String return searchObservationUnitsAndProcess(ouSearchRequest, program.getId(), true); } - public List getObservationUnitsForTrialDbId(@NotNull UUID programId, @NotNull String trialDbId, boolean withGID) throws ApiException, DoesNotExistException { - Program program = programService.getById(programId).orElseThrow(() -> new DoesNotExistException("Program id does not exist")); - + public List getObservationUnits(Program program, + Optional observationUnitId, + Optional observationUnitName, + Optional locationDbId, + Optional seasonDbId, + Optional includeObservations, + Optional observationUnitLevelName, + Optional observationUnitLevelOrder, + Optional observationUnitLevelCode, + Optional observationUnitLevelRelationshipName, + Optional observationUnitLevelRelationshipOrder, + Optional observationUnitLevelRelationshipCode, + Optional observationUnitLevelRelationshipDbId, + Optional commonCropName, + Optional experimentId, + Optional environmentId, + Optional germplasmId +// , Integer page, +// Integer pageSize + ) throws ApiException { BrAPIObservationUnitSearchRequest observationUnitSearchRequest = new BrAPIObservationUnitSearchRequest(); observationUnitSearchRequest.programDbIds(List.of(program.getBrapiProgram() - .getProgramDbId())); - observationUnitSearchRequest.trialDbIds(List.of(trialDbId)); + .getProgramDbId())); + //TODO add pagination support +// .page(page) +// .pageSize(pageSize); - return searchObservationUnitsAndProcess(observationUnitSearchRequest, programId, withGID); + List xrefIds = new ArrayList<>(); + List xrefSources = new ArrayList<>(); + BrAPIObservationUnitLevelRelationship level = new BrAPIObservationUnitLevelRelationship(); + AtomicBoolean levelFilter = new AtomicBoolean(false); + BrAPIObservationUnitLevelRelationship relationship = new BrAPIObservationUnitLevelRelationship(); + AtomicBoolean relationshipFilter = new AtomicBoolean(false); + + observationUnitId.ifPresent(ouId -> addXRefFilter(ouId, ExternalReferenceSource.OBSERVATION_UNITS, xrefIds, xrefSources)); + observationUnitName.ifPresent(name -> observationUnitSearchRequest.setObservationUnitNames(List.of(Utilities.appendProgramKey(name, program.getKey())))); + locationDbId.ifPresent(dbid -> observationUnitSearchRequest.setLocationDbIds(List.of(dbid))); + seasonDbId.ifPresent(dbid -> observationUnitSearchRequest.setSeasonDbIds(List.of(dbid))); + includeObservations.ifPresent(observationUnitSearchRequest::includeObservations); + addLevelFilter(observationUnitLevelName, observationUnitLevelOrder, observationUnitLevelCode, level, levelFilter); + addLevelFilter(observationUnitLevelRelationshipName, observationUnitLevelRelationshipOrder, observationUnitLevelRelationshipCode, relationship, relationshipFilter); + experimentId.ifPresent(expId -> addXRefFilter(expId, ExternalReferenceSource.TRIALS, xrefIds, xrefSources)); + environmentId.ifPresent(envId -> addXRefFilter(envId, ExternalReferenceSource.STUDIES, xrefIds, xrefSources)); +// germplasmId.ifPresent(germId -> { +// xrefIds.add(germId); +// xrefSources.add(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.)); +// }); + + if(!xrefIds.isEmpty()) { + observationUnitSearchRequest.externalReferenceIDs(xrefIds); + } + if(!xrefSources.isEmpty()) { + observationUnitSearchRequest.externalReferenceSources(xrefSources); + } + + return searchObservationUnitsAndProcess(observationUnitSearchRequest, program, true).stream().filter(ou -> { + //xref search does an OR, so we need to convert the searching for ouId/expId/envId to be an AND + boolean matches = observationUnitId.map(id -> id.equals(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS)) + .get() + .getReferenceID())) + .orElse(true); + matches = matches && experimentId.map(id -> id.equals(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)) + .get() + .getReferenceID())) + .orElse(true); + matches = matches && environmentId.map(id -> id.equals(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES)) + .get() + .getReferenceID())) + .orElse(true); + + //adding filter for germplasmDbId because we can't easily search that in the stored data object + return matches && germplasmId.map(id -> id.equals(ou.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_UUID).getAsString())).orElse(true); + }).collect(Collectors.toList()); + } + + private void addXRefFilter(String ouId, ExternalReferenceSource externalReferenceSource, List xrefIds, List xrefSources) { + xrefIds.add(ouId); + xrefSources.add(Utilities.generateReferenceSource(referenceSource, externalReferenceSource)); + } + + private void addLevelFilter(Optional observationUnitLevelName, Optional observationUnitLevelOrder, Optional observationUnitLevelCode, BrAPIObservationUnitLevelRelationship level, AtomicBoolean levelFilter) { + observationUnitLevelName.ifPresent(name -> { + levelFilter.set(true); + level.setLevelName(name); + }); + observationUnitLevelOrder.ifPresent(order -> { + levelFilter.set(true); + level.setLevelOrder(order); + }); + observationUnitLevelCode.ifPresent(code -> { + levelFilter.set(true); + level.setLevelCode(code); + }); } @@ -171,24 +266,24 @@ public List getObservationUnitsForTrialDbId(@NotNull UUID * Perform observation unit search and process returned observation units to handle any modifications to the data * to be returned by bi-api */ - private List searchObservationUnitsAndProcess(BrAPIObservationUnitSearchRequest request, UUID programId, boolean withGID) throws ApiException { + private List searchObservationUnitsAndProcess(BrAPIObservationUnitSearchRequest request, Program program, boolean withGID) throws ApiException { - ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationUnitsApi.class); + ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationUnitsApi.class); List brapiObservationUnits = brAPIDAOUtil.search(api::searchObservationunitsPost, api::searchObservationunitsSearchResultsDbIdGet, request); - processObservationUnits(programId, brapiObservationUnits, withGID); + processObservationUnits(program, brapiObservationUnits, withGID); return brapiObservationUnits; } - private void processObservationUnits(UUID programId, List brapiObservationUnits, boolean withGID) throws ApiException { + private void processObservationUnits(Program program, List brapiObservationUnits, boolean withGID) throws ApiException { HashMap germplasmByDbId = new HashMap<>(); if( withGID ){ // Load germplasm for program into map. // TODO: if we use redis search, that may be more efficient than loading all germplasm for the program. - this.germplasmService.getGermplasm(programId).forEach((germplasm -> germplasmByDbId.put(germplasm.getGermplasmDbId(), germplasm))); + this.germplasmService.getGermplasm(program.getId()).forEach((germplasm -> germplasmByDbId.put(germplasm.getGermplasmDbId(), germplasm))); } // if has treatments in additionalInfo, copy to treatments property @@ -203,8 +298,23 @@ private void processObservationUnits(UUID programId, List if( withGID ){ BrAPIGermplasm germplasm = germplasmByDbId.get(ou.getGermplasmDbId()); ou.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GID, germplasm.getAccessionNumber()); + ou.putAdditionalInfoItem(BrAPIAdditionalInfoFields.GERMPLASM_UUID, + Utilities.getExternalReference(germplasm.getExternalReferences(), referenceSource) + .orElseThrow(() -> new IllegalStateException("Germplasm UUID not found")) + .getReferenceID()); } } + ou.setObservationUnitName(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); + ou.setGermplasmName(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getGermplasmName(), program.getKey())); + ou.setLocationName(Utilities.removeProgramKey(ou.getLocationName(), program.getKey())); + ou.setProgramName(ou.getProgramName().replaceAll("\\("+program.getKey()+"\\)", "").trim()); + ou.setTrialName(Utilities.removeProgramKey(ou.getTrialName(), program.getKey())); + ou.setStudyName(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getStudyName(), program.getKey())); + ou.getObservationUnitPosition() + .getObservationLevel() + .setLevelCode(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitPosition() + .getObservationLevel() + .getLevelCode(), program.getKey())); } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java index cae09d9fb..e4831dd6b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java @@ -151,6 +151,15 @@ public List getStudiesByExperimentID(@NotNull UUID experimentID, Pro ); } + public List getStudiesByEnvironmentIds(@NotNull Collection environmentIds, Program program ) throws ApiException { + return programStudyCache.get(program.getId()) + .entrySet() + .stream() + .filter(entry -> environmentIds.contains(UUID.fromString(entry.getKey()))) + .map(Map.Entry::getValue) + .collect(Collectors.toList()); + } + public List createBrAPIStudies(List brAPIStudyList, UUID programId, ImportUpload upload) throws ApiException { StudiesApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), StudiesApi.class); List createdStudies = new ArrayList<>(); @@ -208,6 +217,12 @@ public Optional getStudyByDbId(String studyDbId, Program program) th return Utilities.getSingleOptional(studies); } + public Optional getStudyByEnvironmentId(UUID environmentId, Program program) throws ApiException { + List studies = getStudiesByEnvironmentIds(List.of(environmentId), program); + + return Utilities.getSingleOptional(studies); + } + /** * Process study into a format for display diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java index daad1eecd..e931a5584 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java @@ -34,6 +34,7 @@ import org.breedinginsight.brapps.importer.model.response.PendingImportObject; import org.breedinginsight.model.Program; import org.breedinginsight.model.User; +import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.services.exceptions.ValidatorException; import tech.tablesaw.api.Table; @@ -144,6 +145,8 @@ public void postBrapiData(Map mappedBrAPIImport, Program createdObservationUnits.addAll(brAPIObservationUnitDAO.createBrAPIObservationUnits(observationUnits, program.getId(), upload)); } catch (ApiException e) { throw new InternalServerException(e.toString(), e); + } catch (DoesNotExistException e) { + throw new InternalServerException(e.toString(), e); } // Update our records diff --git a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java index 2dba86807..293a7918c 100644 --- a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java +++ b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java @@ -614,32 +614,14 @@ private SelectOnConditionStep getTraitSql(BiUserTable createdByTableAlia @Override public List getTraitsByTraitName(UUID programId, List traits){ - String[] names = traits.stream() + List names = traits.stream() .filter(trait -> trait.getObservationVariableName() != null) .map(trait -> trait.getObservationVariableName().toLowerCase()) - .collect(Collectors.toList()).toArray(String[]::new); + .collect(Collectors.toList()); List traitResults = new ArrayList<>(); - if (names.length > 0){ - - Result records = dsl.select() - .from(TRAIT) - .join(PROGRAM_ONTOLOGY).on(TRAIT.PROGRAM_ONTOLOGY_ID.eq(PROGRAM_ONTOLOGY.ID)) - .join(PROGRAM).on(PROGRAM_ONTOLOGY.PROGRAM_ID.eq(PROGRAM.ID)) - .join(SCALE).on(TRAIT.SCALE_ID.eq(SCALE.ID)) - .join(METHOD).on(TRAIT.METHOD_ID.eq(METHOD.ID)) - .where(PROGRAM.ID.eq(programId)) - .and(lower(TRAIT.OBSERVATION_VARIABLE_NAME).in(names)) - .fetch(); - - for (Record record: records) { - Trait trait = Trait.parseSqlRecord(record); - Scale scale = Scale.parseSqlRecord(record); - Method method = Method.parseSqlRecord(record); - trait.setScale(scale); - trait.setMethod(method); - traitResults.add(trait); - } + if (!names.isEmpty()) { + traitResults = getTraitsFullByProgramId(programId).stream().filter(trait -> names.contains(trait.getObservationVariableName())).collect(Collectors.toList()); } return traitResults; diff --git a/src/main/java/org/breedinginsight/model/Dataset.java b/src/main/java/org/breedinginsight/model/Dataset.java index 36bba2121..638c946f1 100644 --- a/src/main/java/org/breedinginsight/model/Dataset.java +++ b/src/main/java/org/breedinginsight/model/Dataset.java @@ -14,7 +14,7 @@ public class Dataset { public JsonObject additionalInfo; public List data; public List observationUnits; - public List observationVariables; + public List observationVariables; public enum DatasetStat { OBSERVATION_UNITS("observationUnits"), @@ -34,7 +34,7 @@ public Dataset( String experimentId, List data, List observationUnits, - List observationVariables) { + List observationVariables) { this.experimentId = experimentId; this.additionalInfo = new JsonObject(); this.data = data; diff --git a/src/main/java/org/breedinginsight/services/TraitService.java b/src/main/java/org/breedinginsight/services/TraitService.java index ff872791f..b4fcaa060 100644 --- a/src/main/java/org/breedinginsight/services/TraitService.java +++ b/src/main/java/org/breedinginsight/services/TraitService.java @@ -475,4 +475,12 @@ public List getAllTraitTags(UUID programId) { tags.add(FAVORITES_TAG); return new ArrayList<>(tags); } + + public List getByName(UUID programId, List names) throws DoesNotExistException { + if (!programService.exists(programId)) { + throw new DoesNotExistException("Program does not exist"); + } + + return traitDAO.getTraitsByTraitName(programId, names.stream().map(name -> Trait.builder().observationVariableName(name).build()).collect(Collectors.toList())); + } } From d34d5b120d60e57588d69c3e29b5f0e607723ee2 Mon Sep 17 00:00:00 2001 From: timparsons Date: Thu, 5 Oct 2023 15:22:34 -0400 Subject: [PATCH 137/220] [BI-1921] Fixing trait fetch bug --- src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java index 293a7918c..c61702622 100644 --- a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java +++ b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java @@ -621,7 +621,7 @@ public List getTraitsByTraitName(UUID programId, List traits){ List traitResults = new ArrayList<>(); if (!names.isEmpty()) { - traitResults = getTraitsFullByProgramId(programId).stream().filter(trait -> names.contains(trait.getObservationVariableName())).collect(Collectors.toList()); + traitResults = getTraitsFullByProgramId(programId).stream().filter(trait -> names.contains(trait.getObservationVariableName().toLowerCase())).collect(Collectors.toList()); } return traitResults; From 3c4536956927aa806090e737339a90afa864401f Mon Sep 17 00:00:00 2001 From: timparsons Date: Thu, 12 Oct 2023 11:56:03 -0400 Subject: [PATCH 138/220] [BI-1921] filtering out REP and BLOCK levels from the /observationlevels response --- .../brapi/v2/BrAPIObservationLevelsController.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsController.java index 3fe481c73..d5616a793 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsController.java @@ -33,9 +33,7 @@ import org.brapi.v2.model.BrAPIMetadata; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.brapi.v2.model.pheno.BrAPIObservationUnitHierarchyLevel; -import org.brapi.v2.model.pheno.BrAPIObservationUnitLevelRelationship; import org.brapi.v2.model.pheno.response.BrAPIObservationLevelListResponse; import org.brapi.v2.model.pheno.response.BrAPIObservationLevelListResponseResult; import org.breedinginsight.api.auth.ProgramSecured; @@ -44,11 +42,10 @@ import org.breedinginsight.brapps.importer.daos.BrAPIStudyDAO; import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; import org.breedinginsight.daos.ProgramDAO; +import org.breedinginsight.model.BrAPIConstants; import org.breedinginsight.model.Program; import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; -import org.breedinginsight.services.exceptions.DoesNotExistException; -import org.breedinginsight.utilities.BrAPIDAOUtil; import org.breedinginsight.utilities.Utilities; import javax.annotation.Nullable; @@ -57,6 +54,7 @@ import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; @Slf4j @Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2) @@ -130,7 +128,8 @@ public HttpResponse observationlevelsGet(@Pat BrAPIIndexPagination responsePagination = (BrAPIIndexPagination) response.getBody().getMetadata().getPagination(); List levels = new ArrayList<>(); if(response.getBody() != null) { - levels = response.getBody().getResult().getData(); + levels = response.getBody().getResult().getData().stream().filter(level -> !level.getLevelName().equals(BrAPIConstants.REPLICATE.getValue()) && !level.getLevelName().equals(BrAPIConstants.BLOCK.getValue())).collect( + Collectors.toList()); } log.debug(String.format("found %d observation levels", levels.size())); return HttpResponse.ok(new BrAPIObservationLevelListResponse().metadata(new BrAPIMetadata().pagination(responsePagination)) From 9c53b630d412ab19be5bb15d8d86e640e59fac60 Mon Sep 17 00:00:00 2001 From: timparsons Date: Thu, 12 Oct 2023 11:57:01 -0400 Subject: [PATCH 139/220] [BI-1921] Updating trait response to have the trait name be the first synonym Apps like Field Book use the first synonym as the displayed name of the synonym --- .../BrAPIObservationVariableController.java | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java index fd7b93b71..539674fd3 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java @@ -28,7 +28,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.BrAPIIndexPagination; import org.brapi.v2.model.BrAPIMetadata; import org.brapi.v2.model.BrAPIOntologyReference; @@ -102,7 +101,7 @@ public HttpResponse variablesGet(@PathVari @Nullable @QueryValue("traitPUI") String traitPUI, @Nullable @QueryValue("ontologyDbId") String ontologyDbId, @Nullable @QueryValue("commonCropName") String commonCropName, -// @QueryValue("programDbId") Optional<@Nullable String> programDbId, //this is +// @QueryValue("programDbId") Optional<@Nullable String> programDbId, //this is redundant @Nullable @QueryValue("trialDbId") String experimentId, @Nullable @QueryValue("studyDbId") String environmentId, @Nullable @QueryValue("externalReferenceID") String externalReferenceID, @@ -204,18 +203,19 @@ private BrAPIObservationVariable convertToBrAPI(Trait trait) { BrAPIOntologyReference brAPIOntologyReference = new BrAPIOntologyReference().ontologyDbId(trait.getProgramOntologyId() .toString()); String status = trait.getActive() ? "active" : "inactive"; + List synonyms = prepSynonyms(trait); return new BrAPIObservationVariable().observationVariableDbId(trait.getId().toString()) .observationVariableName(trait.getObservationVariableName()) .defaultValue(trait.getDefaultValue()) .status(status) - .synonyms(trait.getSynonyms()) + .synonyms(synonyms) .trait(new BrAPITrait().ontologyReference(brAPIOntologyReference) .traitName(trait.getObservationVariableName()) .traitDbId(trait.getId().toString()) .entity(trait.getEntity()) .attribute(trait.getAttribute()) .status(status) - .synonyms(trait.getSynonyms())) + .synonyms(synonyms)) .method(new BrAPIMethod().ontologyReference(brAPIOntologyReference) .methodDbId(trait.getMethod().getId().toString()) .description(trait.getMethod().getDescription()) @@ -230,6 +230,36 @@ private BrAPIObservationVariable convertToBrAPI(Trait trait) { .categories(trait.getScale().getCategories()))); } + /** + * Create a list of synonyms, and ensure that there the first element matches the name of the trait

+ * This is primarily needed to ensure any system using the first synonym as a display shows the actual name of the ontology term + * @param trait + * @return list of synonyms with at least one value (the name of the trait) + */ + private List prepSynonyms(Trait trait) { + List preppedSynonyms = new ArrayList<>(); + if(trait.getSynonyms() != null) { + preppedSynonyms = trait.getSynonyms(); + int traitNameIdx = -1; + for(int i = 0; i < preppedSynonyms.size(); i++) { + if(preppedSynonyms.get(i).equals(trait.getObservationVariableName())) { + traitNameIdx = i; + break; + } + } + if(traitNameIdx > -1) { + String temp = preppedSynonyms.get(traitNameIdx); + preppedSynonyms.set(traitNameIdx, preppedSynonyms.get(0)); + preppedSynonyms.set(0, temp); + } else { + preppedSynonyms.add(0, trait.getObservationVariableName()); + } + } else { + preppedSynonyms.add(trait.getObservationVariableName()); + } + return preppedSynonyms; + } + @NotNull private List filterVariables(List programTraits, Optional observationVariableDbId, From c089de86edcb9602d9a2b2422bed65bb3176388c Mon Sep 17 00:00:00 2001 From: timparsons Date: Mon, 23 Oct 2023 11:28:49 -0400 Subject: [PATCH 140/220] [BI-1921] fixing compilation error --- .../brapps/importer/daos/BrAPIObservationUnitDAO.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java index f8ed51f25..59506b012 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java @@ -163,7 +163,7 @@ public List getObservationUnitsForDataset(@NotNull String ouSearchRequest.programDbIds(List.of(program.getBrapiProgram().getProgramDbId())); ouSearchRequest.externalReferenceSources(List.of(datasetReferenceSource)); ouSearchRequest.externalReferenceIDs(List.of(datasetId)); - return searchObservationUnitsAndProcess(ouSearchRequest, program.getId(), true); + return searchObservationUnitsAndProcess(ouSearchRequest, program, true); } public List getObservationUnits(Program program, From 9cc43509ebfd5860c6aae741657ff0a8c3c34c25 Mon Sep 17 00:00:00 2001 From: timparsons Date: Wed, 29 Nov 2023 09:53:20 -0500 Subject: [PATCH 141/220] [BI-1921] fixing failing unit tests - Adding new unit tests for new BrAPI endpoints - reorganizing BrAPI DAOs to be in more appropriate package --- .../brapi/v2/BrAPIImagesController.java | 6 +- .../v2/BrAPIObservationLevelsController.java | 4 +- .../v2/BrAPIObservationUnitController.java | 47 +- .../BrAPIObservationVariableController.java | 4 +- .../brapi/v2/BrAPIObservationsController.java | 6 +- .../brapi/v2/BrAPIProgramsController.java | 8 +- .../brapi/v2/BrAPIStudiesController.java | 6 +- .../brapi/v2/BrAPITrialsController.java | 4 +- .../daos => brapi/v2/dao}/BrAPICrossDAO.java | 2 +- .../daos => brapi/v2/dao}/BrAPIListDAO.java | 20 +- .../v2/dao}/BrAPILocationDAO.java | 3 +- .../v2/dao}/BrAPIObservationDAO.java | 3 +- .../v2/dao}/BrAPIObservationUnitDAO.java | 38 +- .../v2/dao}/BrAPIObservationVariableDAO.java | 2 +- .../v2/dao}/BrAPIProgramDAO.java | 2 +- .../daos => brapi/v2/dao}/BrAPISeasonDAO.java | 19 +- .../daos => brapi/v2/dao}/BrAPIStudyDAO.java | 3 +- .../daos => brapi/v2/dao}/BrAPITrialDAO.java | 19 +- .../v2/dao}/impl/BrAPITrialDAOImpl.java | 4 +- .../v2/dao}/impl/ImportDAOImpl.java | 2 +- .../v2/dao}/impl/ImportMappingDAOImpl.java | 2 +- .../v2/services/BrAPIGermplasmService.java | 2 +- .../brapi/v2/services/BrAPIListService.java | 4 +- .../brapi/v2/services/BrAPIStudyService.java | 2 +- .../brapi/v2/services/BrAPITrialService.java | 3 +- .../processors/ExperimentProcessor.java | 3 +- .../processors/GermplasmProcessor.java | 2 +- .../processors/ObservationProcessor.java | 4 +- .../processors/ObservationUnitProcessor.java | 2 +- .../services/processors/StudyProcessor.java | 2 +- .../services/processors/TrialProcessor.java | 2 +- ...vationLevelsControllerIntegrationTest.java | 324 +++++++++++++ ...ervationUnitControllerIntegrationTest.java | 457 ++++++++++++++++++ ...tionVariableControllerIntegrationTest.java | 22 + ...ObservationsControllerIntegrationTest.java | 22 + ...BrAPIStudiesControllerIntegrationTest.java | 411 ++++++++++++++++ .../v2/BrAPIV2ControllerIntegrationTest.java | 220 +-------- .../GermplasmControllerIntegrationTest.java | 2 +- .../importer/ExperimentFileImportTest.java | 3 +- .../daos/BrAPIObservationUnitDAOTest.java | 1 + .../BrAPIGermplasmServiceUnitTest.java | 2 +- ...gwaGenotypeServiceImplIntegrationTest.java | 6 +- 42 files changed, 1404 insertions(+), 296 deletions(-) rename src/main/java/org/breedinginsight/{brapps/importer/daos => brapi/v2/dao}/BrAPICrossDAO.java (97%) rename src/main/java/org/breedinginsight/{brapps/importer/daos => brapi/v2/dao}/BrAPIListDAO.java (91%) rename src/main/java/org/breedinginsight/{brapps/importer/daos => brapi/v2/dao}/BrAPILocationDAO.java (97%) rename src/main/java/org/breedinginsight/{brapps/importer/daos => brapi/v2/dao}/BrAPIObservationDAO.java (98%) rename src/main/java/org/breedinginsight/{brapps/importer/daos => brapi/v2/dao}/BrAPIObservationUnitDAO.java (91%) rename src/main/java/org/breedinginsight/{brapps/importer/daos => brapi/v2/dao}/BrAPIObservationVariableDAO.java (97%) rename src/main/java/org/breedinginsight/{brapps/importer/daos => brapi/v2/dao}/BrAPIProgramDAO.java (98%) rename src/main/java/org/breedinginsight/{brapps/importer/daos => brapi/v2/dao}/BrAPISeasonDAO.java (80%) rename src/main/java/org/breedinginsight/{brapps/importer/daos => brapi/v2/dao}/BrAPIStudyDAO.java (99%) rename src/main/java/org/breedinginsight/{brapps/importer/daos => brapi/v2/dao}/BrAPITrialDAO.java (63%) rename src/main/java/org/breedinginsight/{brapps/importer/daos => brapi/v2/dao}/impl/BrAPITrialDAOImpl.java (99%) rename src/main/java/org/breedinginsight/{brapps/importer/daos => brapi/v2/dao}/impl/ImportDAOImpl.java (99%) rename src/main/java/org/breedinginsight/{brapps/importer/daos => brapi/v2/dao}/impl/ImportMappingDAOImpl.java (99%) create mode 100644 src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java create mode 100644 src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java create mode 100644 src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableControllerIntegrationTest.java create mode 100644 src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java create mode 100644 src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIImagesController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIImagesController.java index 1893d10d0..f760af9f0 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIImagesController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIImagesController.java @@ -70,7 +70,7 @@ public HttpResponse imagesImageDbIdGet(@PathVariable("programId") UUID programId @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) public HttpResponse imagesImageDbIdImagecontentPut(@PathVariable("programId") UUID programId, @PathVariable("imageDbId") String imageDbId, - Object body) { + @Body Object body) { return HttpResponse.notFound(); } @@ -78,13 +78,13 @@ public HttpResponse imagesImageDbIdImagecontentPut(@PathVariable("programId") UU @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) public HttpResponse imagesImageDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("imageDbId") String imageDbId, - BrAPIImage body) { + @Body BrAPIImage body) { return HttpResponse.notFound(); } @Post("/images") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse imagesPost(@PathVariable("programId") UUID programId, List body) { + public HttpResponse imagesPost(@PathVariable("programId") UUID programId, @Body List body) { return HttpResponse.notFound(); } } diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsController.java index d5616a793..9c54bed9a 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsController.java @@ -39,8 +39,8 @@ import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; import org.breedinginsight.brapi.v1.controller.BrapiVersion; -import org.breedinginsight.brapps.importer.daos.BrAPIStudyDAO; -import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.model.BrAPIConstants; import org.breedinginsight.model.Program; diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java index 0c17e31e5..29ce0cef5 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java @@ -35,7 +35,7 @@ import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; import org.breedinginsight.brapi.v1.controller.BrapiVersion; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.model.Program; import org.breedinginsight.services.ProgramService; @@ -66,11 +66,6 @@ public BrAPIObservationUnitController(@Property(name = "brapi.server.reference-s this.programService = programService; } - /* - TODO - - GET observationLevels - - GET observationUnits - */ @Get("/observationunits") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) public HttpResponse observationunitsGet(@PathVariable("programId") UUID programId, @@ -182,21 +177,21 @@ public HttpResponse observationunitsObservat @Put("/observationunits/{observationUnitDbId}") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse observationunitsObservationUnitDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("observationUnitDbId") String observationUnitDbId, BrAPIObservationUnit body) { + public HttpResponse observationunitsObservationUnitDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("observationUnitDbId") String observationUnitDbId, @Body BrAPIObservationUnit body) { //DO NOT IMPLEMENT - Users aren't yet able to update observation units return HttpResponse.notFound(); } @Post("/observationunits") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse observationunitsPost(@PathVariable("programId") UUID programId, List body) { + public HttpResponse observationunitsPost(@PathVariable("programId") UUID programId, @Body List body) { //DO NOT IMPLEMENT - Users are only able to create observation units via the DeltaBreed UI return HttpResponse.notFound(); } @Put("/observationunits") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse observationunitsPut(@PathVariable("programId") UUID programId, Map body) { + public HttpResponse observationunitsPut(@PathVariable("programId") UUID programId, @Body Map body) { //DO NOT IMPLEMENT - Users aren't yet able to update observation units return HttpResponse.notFound(); } @@ -205,23 +200,23 @@ public HttpResponse observationunitsPut(@PathVariable("programId") UUID progr @Produces({"application/json", "text/csv", "text/tsv"}) @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) public HttpResponse observationunitsTableGet(@PathVariable("programId") UUID programId, - @Header("Accept") String accept, - @QueryValue("observationUnitDbId") String observationUnitDbId, - @QueryValue("observationVariableDbId") String observationVariableDbId, - @QueryValue("locationDbId") String locationDbId, - @QueryValue("seasonDbId") String seasonDbId, - @QueryValue("observationLevel") String observationLevel, - @QueryValue("programDbId") String programDbId, - @QueryValue("trialDbId") String trialDbId, - @QueryValue("studyDbId") String studyDbId, - @QueryValue("germplasmDbId") String germplasmDbId, - @QueryValue("observationUnitLevelName") String observationUnitLevelName, - @QueryValue("observationUnitLevelOrder") String observationUnitLevelOrder, - @QueryValue("observationUnitLevelCode") String observationUnitLevelCode, - @QueryValue("observationUnitLevelRelationshipName") String observationUnitLevelRelationshipName, - @QueryValue("observationUnitLevelRelationshipOrder") String observationUnitLevelRelationshipOrder, - @QueryValue("observationUnitLevelRelationshipCode") String observationUnitLevelRelationshipCode, - @QueryValue("observationUnitLevelRelationshipDbId") String observationUnitLevelRelationshipDbId) { + @Nullable @Header("Accept") String accept, + @Nullable @QueryValue("observationUnitDbId") String observationUnitDbId, + @Nullable @QueryValue("observationVariableDbId") String observationVariableDbId, + @Nullable @QueryValue("locationDbId") String locationDbId, + @Nullable @QueryValue("seasonDbId") String seasonDbId, + @Nullable @QueryValue("observationLevel") String observationLevel, + @Nullable @QueryValue("programDbId") String programDbId, + @Nullable @QueryValue("trialDbId") String trialDbId, + @Nullable @QueryValue("studyDbId") String studyDbId, + @Nullable @QueryValue("germplasmDbId") String germplasmDbId, + @Nullable @QueryValue("observationUnitLevelName") String observationUnitLevelName, + @Nullable @QueryValue("observationUnitLevelOrder") String observationUnitLevelOrder, + @Nullable @QueryValue("observationUnitLevelCode") String observationUnitLevelCode, + @Nullable @QueryValue("observationUnitLevelRelationshipName") String observationUnitLevelRelationshipName, + @Nullable @QueryValue("observationUnitLevelRelationshipOrder") String observationUnitLevelRelationshipOrder, + @Nullable @QueryValue("observationUnitLevelRelationshipCode") String observationUnitLevelRelationshipCode, + @Nullable @QueryValue("observationUnitLevelRelationshipDbId") String observationUnitLevelRelationshipDbId) { return HttpResponse.notFound(); } diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java index 539674fd3..d94ab0541 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java @@ -187,14 +187,14 @@ public HttpResponse variablesObservation @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) public HttpResponse variablesObservationVariableDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("observationVariableDbId") String observationVariableDbId, - BrAPIObservationVariable body) { + @Body BrAPIObservationVariable body) { //DO NOT IMPLEMENT - Users are only able to update traits via the DeltaBreed UI return HttpResponse.notFound(); } @Post("/variables") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse variablesPost(@PathVariable("programId") UUID programId, List body) { + public HttpResponse variablesPost(@PathVariable("programId") UUID programId, @Body List body) { //DO NOT IMPLEMENT - Users are only able to create new traits via the DeltaBreed UI return HttpResponse.notFound(); } diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java index cd35d9d5b..2f669255d 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java @@ -79,7 +79,7 @@ public HttpResponse observationsObservationDbIdGet(@PathVariable("programId") UU @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) public HttpResponse observationsObservationDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("observationDbId") String observationDbId, - BrAPIObservation body) { + @Body BrAPIObservation body) { /* DO NOT IMPLEMENT - users must create observations via file upload TODO identify how observations uploaded via BrAPI will be separated from curated observations @@ -89,7 +89,7 @@ public HttpResponse observationsObservationDbIdPut(@PathVariable("programId") UU @Post("/observations") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse observationsPost(@PathVariable("programId") UUID programId, List body) { + public HttpResponse observationsPost(@PathVariable("programId") UUID programId, @Body List body) { /* DO NOT IMPLEMENT - users must create observations via file upload TODO identify how observations uploaded via BrAPI will be separated from curated observations @@ -99,7 +99,7 @@ public HttpResponse observationsPost(@PathVariable("programId") UUID programId, @Put("/observations") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse observationsPut(@PathVariable("programId") UUID programId, Map body) { + public HttpResponse observationsPut(@PathVariable("programId") UUID programId, @Body Map body) { /* DO NOT IMPLEMENT - users must create observations via file upload TODO identify how observations uploaded via BrAPI will be separated from curated observations diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIProgramsController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIProgramsController.java index dabbebc45..a30dbfbed 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIProgramsController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIProgramsController.java @@ -93,7 +93,7 @@ public HttpResponse rootProgramsGet( } @Post(BrapiVersion.BRAPI_V2 + "/programs") - public HttpResponse rootProgramsPost(List body) { + public HttpResponse rootProgramsPost(@Body List body) { //DO NOT IMPLEMENT - Users should only be able to create new programs via the DeltaBreed UI return HttpResponse.notFound(); } @@ -117,7 +117,7 @@ public HttpResponse rootProgramsProgramDbIdGet(@Path } @Put(BrapiVersion.BRAPI_V2 + "/programs/{programDbId}") - public HttpResponse rootProgramsProgramDbIdPut(@PathVariable("programDbId") String programDbId, BrAPIProgram body) { + public HttpResponse rootProgramsProgramDbIdPut(@PathVariable("programDbId") String programDbId, @Body BrAPIProgram body) { //DO NOT IMPLEMENT - Users should only be able to update programs via the DeltaBreed UI return HttpResponse.notFound(); } @@ -154,7 +154,7 @@ public HttpResponse programsGet(@PathVariable("program @Post("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/programs") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse programsPost(@PathVariable("programId") UUID programId, List body) { + public HttpResponse programsPost(@PathVariable("programId") UUID programId, @Body List body) { //DO NOT IMPLEMENT - Users should only be able to create new programs via the DeltaBreed UI return HttpResponse.notFound(); } @@ -173,7 +173,7 @@ public HttpResponse programsProgramDbIdGet(@PathVari @Put("/programs/{programId}" + BrapiVersion.BRAPI_V2 + "/programs/{programDbId}") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse programsProgramDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("programDbId") String programDbId, BrAPIProgram body) { + public HttpResponse programsProgramDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("programDbId") String programDbId, @Body BrAPIProgram body) { //DO NOT IMPLEMENT - Users should only be able to update programs via the DeltaBreed UI return HttpResponse.notFound(); } diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIStudiesController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIStudiesController.java index ea6f03a52..0382ecdb0 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIStudiesController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIStudiesController.java @@ -100,7 +100,7 @@ public HttpResponse>>> getStudies( @Post("/studies") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse studiesPost(@PathVariable("programId") UUID programId, List body) { + public HttpResponse studiesPost(@PathVariable("programId") UUID programId, @Body List body) { //DO NOT IMPLEMENT - Users are only able to create new studies via the DeltaBreed UI return HttpResponse.notFound(); } @@ -131,8 +131,8 @@ public HttpResponse studiesStudyDbIdGet(@PathVariable( @Put("/studies/{studyDbId}") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse studiesStudyDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("studyDbId") String studyDbId, - BrAPIStudy body) { + public HttpResponse studiesStudyDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("studyDbId") String studyDbId, + @Body BrAPIStudy body) { //DO NOT IMPLEMENT - Users are only able to update studies via the DeltaBreed UI return HttpResponse.notFound(); } diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java index 77dbf2b90..3966b1341 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java @@ -93,7 +93,7 @@ public HttpResponse getExperimentById( @Post("/trials") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse trialsPost(@PathVariable("programId") UUID programId, List body) { + public HttpResponse trialsPost(@PathVariable("programId") UUID programId, @Body List body) { //DO NOT IMPLEMENT - Users are only able to create new trials via the DeltaBreed UI return HttpResponse.notFound(); } @@ -101,7 +101,7 @@ public HttpResponse trialsPost(@PathVariable("programId") UUID programId, Lis @Put("/trials/{trialDbId}") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) - public HttpResponse trialsTrialDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("trialDbId") String trialDbId, BrAPITrial body) { + public HttpResponse trialsTrialDbIdPut(@PathVariable("programId") UUID programId, @PathVariable("trialDbId") String trialDbId, @Body BrAPITrial body) { //DO NOT IMPLEMENT - Users are only able to update trials via the DeltaBreed UI return HttpResponse.notFound(); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPICrossDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPICrossDAO.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/daos/BrAPICrossDAO.java rename to src/main/java/org/breedinginsight/brapi/v2/dao/BrAPICrossDAO.java index 5a895d8ec..edfcfb6c7 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPICrossDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPICrossDAO.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.breedinginsight.brapps.importer.daos; +package org.breedinginsight.brapi.v2.dao; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.germplasm.CrossesApi; diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIListDAO.java similarity index 91% rename from src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java rename to src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIListDAO.java index cb24fd745..801b9775c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIListDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIListDAO.java @@ -1,4 +1,21 @@ -package org.breedinginsight.brapps.importer.daos; +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2.dao; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.ApiResponse; @@ -14,6 +31,7 @@ import org.brapi.v2.model.core.request.BrAPIListSearchRequest; import org.brapi.v2.model.core.response.*; import org.brapi.v2.model.pheno.BrAPIObservation; +import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPILocationDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPILocationDAO.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/daos/BrAPILocationDAO.java rename to src/main/java/org/breedinginsight/brapi/v2/dao/BrAPILocationDAO.java index 2fe7b8b01..e45dadf5c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPILocationDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPILocationDAO.java @@ -15,12 +15,13 @@ * limitations under the License. */ -package org.breedinginsight.brapps.importer.daos; +package org.breedinginsight.brapi.v2.dao; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.core.LocationsApi; import org.brapi.v2.model.core.BrAPILocation; import org.brapi.v2.model.core.request.BrAPILocationSearchRequest; +import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.services.brapi.BrAPIEndpointProvider; diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java rename to src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java index 9c9ed184b..b52aeff22 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.breedinginsight.brapps.importer.daos; +package org.breedinginsight.brapi.v2.dao; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; @@ -26,6 +26,7 @@ import org.brapi.v2.model.pheno.request.BrAPIObservationSearchRequest; import org.brapi.v2.model.pheno.response.BrAPIObservationListResponse; import org.brapi.v2.model.pheno.response.BrAPIObservationSingleResponse; +import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.daos.ProgramDAO; import org.breedinginsight.model.Program; diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java similarity index 91% rename from src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java rename to src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java index 59506b012..ac153d893 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java @@ -15,13 +15,14 @@ * limitations under the License. */ -package org.breedinginsight.brapps.importer.daos; +package org.breedinginsight.brapi.v2.dao; import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.reflect.TypeToken; import io.micronaut.context.annotation.Property; +import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.JSON; import io.micronaut.http.server.exceptions.InternalServerException; import org.brapi.client.v2.model.exceptions.ApiException; @@ -33,6 +34,7 @@ import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService; import org.brapi.v2.model.pheno.BrAPIObservationTreatment; +import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; @@ -305,16 +307,30 @@ private void processObservationUnits(Program program, List } } ou.setObservationUnitName(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitName(), program.getKey())); - ou.setGermplasmName(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getGermplasmName(), program.getKey())); - ou.setLocationName(Utilities.removeProgramKey(ou.getLocationName(), program.getKey())); - ou.setProgramName(ou.getProgramName().replaceAll("\\("+program.getKey()+"\\)", "").trim()); - ou.setTrialName(Utilities.removeProgramKey(ou.getTrialName(), program.getKey())); - ou.setStudyName(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getStudyName(), program.getKey())); - ou.getObservationUnitPosition() - .getObservationLevel() - .setLevelCode(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitPosition() - .getObservationLevel() - .getLevelCode(), program.getKey())); + if(StringUtils.isNotBlank(ou.getGermplasmName())) { + ou.setGermplasmName(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getGermplasmName(), program.getKey())); + } + if(StringUtils.isNotBlank(ou.getLocationName())) { + ou.setLocationName(Utilities.removeProgramKey(ou.getLocationName(), program.getKey())); + } + if(StringUtils.isNotBlank(ou.getProgramName())) { + ou.setProgramName(ou.getProgramName().replaceAll("\\(" + program.getKey() + "\\)", "").trim()); + } + if(StringUtils.isNotBlank(ou.getTrialName())) { + ou.setTrialName(Utilities.removeProgramKey(ou.getTrialName(), program.getKey())); + } + if(StringUtils.isNotBlank(ou.getStudyName())) { + ou.setStudyName(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getStudyName(), program.getKey())); + } + if (ou.getObservationUnitPosition() != null + && ou.getObservationUnitPosition().getObservationLevel() != null + && StringUtils.isNotBlank(ou.getObservationUnitPosition().getObservationLevel().getLevelCode())) { + ou.getObservationUnitPosition() + .getObservationLevel() + .setLevelCode(Utilities.removeProgramKeyAndUnknownAdditionalData(ou.getObservationUnitPosition() + .getObservationLevel() + .getLevelCode(), program.getKey())); + } } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationVariableDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationVariableDAO.java similarity index 97% rename from src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationVariableDAO.java rename to src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationVariableDAO.java index 86731a0f2..0de5c19ed 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationVariableDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationVariableDAO.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.breedinginsight.brapps.importer.daos; +package org.breedinginsight.brapi.v2.dao; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.phenotype.ObservationVariablesApi; diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIProgramDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIProgramDAO.java similarity index 98% rename from src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIProgramDAO.java rename to src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIProgramDAO.java index 103585849..965de1cab 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIProgramDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIProgramDAO.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.breedinginsight.brapps.importer.daos; +package org.breedinginsight.brapi.v2.dao; import io.micronaut.context.annotation.Property; import org.brapi.client.v2.ApiResponse; diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISeasonDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPISeasonDAO.java similarity index 80% rename from src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISeasonDAO.java rename to src/main/java/org/breedinginsight/brapi/v2/dao/BrAPISeasonDAO.java index 0ca2600b8..8ce77e1e9 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPISeasonDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPISeasonDAO.java @@ -1,4 +1,21 @@ -package org.breedinginsight.brapps.importer.daos; +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2.dao; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.ApiResponse; diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java rename to src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java index e4831dd6b..2901baa4e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPIStudyDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIStudyDAO.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.breedinginsight.brapps.importer.daos; +package org.breedinginsight.brapi.v2.dao; import com.google.gson.JsonObject; import io.micronaut.context.annotation.Property; @@ -26,6 +26,7 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.request.BrAPIStudySearchRequest; +import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.daos.ProgramDAO; diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPITrialDAO.java similarity index 63% rename from src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java rename to src/main/java/org/breedinginsight/brapi/v2/dao/BrAPITrialDAO.java index 55e35f6e8..83ee3f47b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/BrAPITrialDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPITrialDAO.java @@ -1,4 +1,21 @@ -package org.breedinginsight.brapps.importer.daos; +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2.dao; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/impl/BrAPITrialDAOImpl.java b/src/main/java/org/breedinginsight/brapi/v2/dao/impl/BrAPITrialDAOImpl.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/daos/impl/BrAPITrialDAOImpl.java rename to src/main/java/org/breedinginsight/brapi/v2/dao/impl/BrAPITrialDAOImpl.java index f26d039ba..a60008dcd 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/impl/BrAPITrialDAOImpl.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/impl/BrAPITrialDAOImpl.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.breedinginsight.brapps.importer.daos.impl; +package org.breedinginsight.brapi.v2.dao.impl; import io.micronaut.context.annotation.Context; import io.micronaut.context.annotation.Property; @@ -26,7 +26,7 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.request.BrAPITrialSearchRequest; -import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/impl/ImportDAOImpl.java b/src/main/java/org/breedinginsight/brapi/v2/dao/impl/ImportDAOImpl.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/daos/impl/ImportDAOImpl.java rename to src/main/java/org/breedinginsight/brapi/v2/dao/impl/ImportDAOImpl.java index 5b28a35db..7807d573d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/impl/ImportDAOImpl.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/impl/ImportDAOImpl.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.breedinginsight.brapps.importer.daos.impl; +package org.breedinginsight.brapi.v2.dao.impl; import io.micronaut.http.server.exceptions.InternalServerException; import org.breedinginsight.brapps.importer.daos.ImportDAO; diff --git a/src/main/java/org/breedinginsight/brapps/importer/daos/impl/ImportMappingDAOImpl.java b/src/main/java/org/breedinginsight/brapi/v2/dao/impl/ImportMappingDAOImpl.java similarity index 99% rename from src/main/java/org/breedinginsight/brapps/importer/daos/impl/ImportMappingDAOImpl.java rename to src/main/java/org/breedinginsight/brapi/v2/dao/impl/ImportMappingDAOImpl.java index b0b3f7f2e..a7f1384f4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/daos/impl/ImportMappingDAOImpl.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/impl/ImportMappingDAOImpl.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.breedinginsight.brapps.importer.daos.impl; +package org.breedinginsight.brapi.v2.dao.impl; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java index cfd44f64b..b64a88c23 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java @@ -12,7 +12,7 @@ import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.germ.BrAPIGermplasmSynonyms; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapps.importer.daos.BrAPIListDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.model.Column; import org.breedinginsight.model.DownloadFile; diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIListService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIListService.java index 8ae42f92c..3353ac17c 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIListService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIListService.java @@ -9,7 +9,7 @@ import org.brapi.v2.model.core.request.BrAPIListSearchRequest; import org.brapi.v2.model.core.response.BrAPIListsSingleResponse; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; -import org.breedinginsight.brapps.importer.daos.*; +import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.model.Program; import org.breedinginsight.services.exceptions.DoesNotExistException; @@ -17,10 +17,8 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.UUID; import java.util.stream.Collectors; @Slf4j diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java index db5e28351..570609d53 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIStudyService.java @@ -20,7 +20,7 @@ import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; -import org.breedinginsight.brapps.importer.daos.BrAPIStudyDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; import org.breedinginsight.model.Program; import javax.inject.Inject; diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index f1939996c..9ebd95954 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -14,9 +14,8 @@ import org.brapi.v2.model.pheno.*; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapi.v2.dao.*; import org.breedinginsight.brapi.v2.model.request.query.ExperimentExportQuery; -import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 070b2214a..74065ffdf 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -43,8 +43,7 @@ import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; -import org.breedinginsight.brapps.importer.daos.*; +import org.breedinginsight.brapi.v2.dao.*; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java index f51f7c1fb..de5eb65ae 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/GermplasmProcessor.java @@ -33,7 +33,7 @@ import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService; -import org.breedinginsight.brapps.importer.daos.BrAPIListDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.base.Germplasm; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java index 693dda6e1..7fcefcd3d 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationProcessor.java @@ -26,8 +26,8 @@ import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.brapi.v2.model.pheno.BrAPIObservationVariable; -import org.breedinginsight.brapps.importer.daos.BrAPIObservationDAO; -import org.breedinginsight.brapps.importer.daos.BrAPIObservationVariableDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationVariableDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.base.Observation; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java index e931a5584..6f1e6723a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ObservationUnitProcessor.java @@ -24,7 +24,7 @@ import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.base.ObservationUnit; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java index 398eb2d88..e5f89e538 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/StudyProcessor.java @@ -23,7 +23,7 @@ import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; -import org.breedinginsight.brapps.importer.daos.BrAPIStudyDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIStudyDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.base.Study; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java index bce850f51..1f838f9c6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/TrialProcessor.java @@ -20,7 +20,7 @@ import io.micronaut.http.server.exceptions.InternalServerException; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.core.BrAPITrial; -import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.base.Trial; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java new file mode 100644 index 000000000..0e57050ca --- /dev/null +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java @@ -0,0 +1,324 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2; + +import com.google.gson.*; +import io.kowalski.fannypack.FannyPack; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.netty.cookies.NettyCookie; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import io.reactivex.Flowable; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.breedinginsight.BrAPITest; +import org.breedinginsight.TestUtils; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramRequest; +import org.breedinginsight.api.model.v1.request.SpeciesRequest; +import org.breedinginsight.api.v1.controller.TestTokenValidator; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.ImportTestUtils; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.dao.db.enums.DataType; +import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.SpeciesDAO; +import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.model.*; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; +import org.breedinginsight.services.writers.CSVWriter; +import org.jooq.DSLContext; +import org.junit.jupiter.api.*; + +import javax.inject.Inject; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.*; + +import static io.micronaut.http.HttpRequest.GET; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class BrAPIObservationLevelsControllerIntegrationTest extends BrAPITest { + + private Program program; + private String experimentId; + private List envIds = new ArrayList<>(); + private final List> rows = new ArrayList<>(); + private final List columns = ExperimentFileColumns.getOrderedColumns(); + private List traits; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + @Inject + private DSLContext dsl; + @Inject + private UserDAO userDAO; + @Inject + private SpeciesDAO speciesDAO; + @Inject + private OntologyService ontologyService; + @Inject + private BrAPIGermplasmDAO germplasmDAO; + + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + + private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) + (json, type, context) -> OffsetDateTime.parse(json.getAsString())) + .create(); + + @BeforeAll + void setup() throws Exception { + FannyPack fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); + ImportTestUtils importTestUtils = new ImportTestUtils(); + FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); + + // Test User + User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + // Species + super.getBrapiDsl().execute(brapiFp.get("InsertSpecies")); + SpeciesEntity validSpecies = speciesDAO.findAll().get(0); + SpeciesRequest speciesRequest = SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); + + // Test Program + ProgramRequest programRequest = ProgramRequest.builder() + .name("Test Program") + .abbreviation("Test") + .documentationUrl("localhost:8080") + .objective("To test things") + .species(speciesRequest) + .key("TEST") + .build(); + program = TestUtils.insertAndFetchTestProgram(gson, client, programRequest); + + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId()); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + // Get experiment import map + Flowable> call = client.exchange( + GET("/import/mappings?importName=ExperimentsTemplateMap") + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + String mappingId = JsonParser.parseString(Objects.requireNonNull(response.body())).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + // Add traits to program + traits = createTraits(2); + AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); + try { + ontologyService.createTraits(program.getId(), traits, user, false); + } catch (ValidatorException e) { + System.err.println(e.getErrors()); + throw e; + } + + // Add germplasm to program + List germplasm = createGermplasm(1); + BrAPIExternalReference newReference = new BrAPIExternalReference(); + newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); + newReference.setReferenceID(program.getId().toString()); + + germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); + + germplasmDAO.createBrAPIGermplasm(germplasm, program.getId(), null); + + // Make test experiment import + Map row1 = makeExpImportRow("Env1", "Plot"); + + //TODO once sub-entity support is added, update this to test for multiple levels + Map row2 = makeExpImportRow("Env2", "Plot"); + + // Add test observation data + for (Trait trait : traits) { + Random random = new Random(); + + // TODO: test for sending obs data as double. + // A float is returned from the backend instead of double. there is a separate card to fix this. + // Double val1 = Math.random(); + + Float val1 = random.nextFloat(); + row1.put(trait.getObservationVariableName(), val1); + } + + rows.add(row1); + rows.add(row2); + + // Import test experiment, environments, and any observations + JsonObject importResult = importTestUtils.uploadAndFetch( + writeDataToFile(rows, traits), + null, + true, + client, + program, + mappingId); + experimentId = importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + // Add environmentIds. + envIds.add(getEnvId(importResult, 0)); + envIds.add(getEnvId(importResult, 1)); + } + + @Test + public void testGetObservationLevels() { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/brapi/v2/observationlevels", program.getId())) + .bearerAuth("test-registered-user"), + String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); + JsonArray levels = responseObj.getAsJsonObject("result").getAsJsonArray("data"); + assertEquals(1, levels.size()); + List levelNames = new ArrayList<>(); + for(var level : levels) { + levelNames.add(level.getAsJsonObject().get("levelName").getAsString().toLowerCase()); + } + assertTrue(levelNames.contains("plot")); + } + + private File writeDataToFile(List> data, List traits) throws IOException { + File file = File.createTempFile("test", ".csv"); + + if(traits != null) { + traits.forEach(trait -> columns.add( + Column.builder() + .value(trait.getObservationVariableName()) + .dataType(Column.ColumnDataType.STRING) + .build()) + ); + } + ByteArrayOutputStream byteArrayOutputStream = CSVWriter.writeToCSV(columns, data); + FileOutputStream fos = new FileOutputStream(file); + fos.write(byteArrayOutputStream.toByteArray()); + + return file; + } + + private String getEnvId(JsonObject result, int index) { + return result + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(index).getAsJsonObject() + .get("study").getAsJsonObject() + .get("brAPIObject").getAsJsonObject() + .get("externalReferences").getAsJsonArray() + .get(2).getAsJsonObject() + .get("referenceID").getAsString(); + } + + private List createGermplasm(int numToCreate) { + List germplasm = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String gid = ""+(i+1); + BrAPIGermplasm testGermplasm = new BrAPIGermplasm(); + testGermplasm.setGermplasmName(String.format("Germplasm %s [TEST-%s]", gid, gid)); + testGermplasm.setSeedSource("Wild"); + testGermplasm.setAccessionNumber(gid); + testGermplasm.setDefaultDisplayName(String.format("Germplasm %s", gid)); + JsonObject additionalInfo = new JsonObject(); + additionalInfo.addProperty("importEntryNumber", gid); + additionalInfo.addProperty("breedingMethod", "Allopolyploid"); + testGermplasm.setAdditionalInfo(additionalInfo); + List externalRef = new ArrayList<>(); + BrAPIExternalReference testReference = new BrAPIExternalReference(); + testReference.setReferenceSource(BRAPI_REFERENCE_SOURCE); + testReference.setReferenceID(UUID.randomUUID().toString()); + externalRef.add(testReference); + testGermplasm.setExternalReferences(externalRef); + germplasm.add(testGermplasm); + } + + return germplasm; + } + + private List createTraits(int numToCreate) { + List traits = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String varName = "tt_test_" + (i + 1); + traits.add(Trait.builder() + .observationVariableName(varName) + .fullName(varName) + .entity("Plant " + i) + .attribute("height " + i) + .traitDescription("test") + .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build()) + .scale(Scale.builder() + .scaleName("test scale") + .dataType(DataType.NUMERICAL) + .validValueMin(0) + .validValueMax(100) + .build()) + .method(Method.builder() + .description("test method") + .methodClass("test method") + .build()) + .build()); + } + + return traits; + } + + private Map makeExpImportRow(String environment, String level) { + Map row = new HashMap<>(); + row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); + row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); + row.put(ExperimentObservation.Columns.EXP_TITLE, "Test Exp"); + row.put(ExperimentObservation.Columns.EXP_UNIT, level); + row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); + row.put(ExperimentObservation.Columns.ENV, environment); + row.put(ExperimentObservation.Columns.ENV_LOCATION, "Location A"); + row.put(ExperimentObservation.Columns.ENV_YEAR, "2023"); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, "a-1"); + row.put(ExperimentObservation.Columns.REP_NUM, "1"); + row.put(ExperimentObservation.Columns.BLOCK_NUM, "1"); + row.put(ExperimentObservation.Columns.ROW, "1"); + row.put(ExperimentObservation.Columns.COLUMN, "1"); + return row; + } +} diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java new file mode 100644 index 000000000..4111e58ed --- /dev/null +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java @@ -0,0 +1,457 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2; + +import com.google.gson.*; +import io.kowalski.fannypack.FannyPack; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.client.exceptions.HttpClientResponseException; +import io.micronaut.http.netty.cookies.NettyCookie; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import io.reactivex.Flowable; +import lombok.SneakyThrows; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.brapi.v2.model.pheno.response.BrAPIObservationUnitSingleResponse; +import org.breedinginsight.BrAPITest; +import org.breedinginsight.TestUtils; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramRequest; +import org.breedinginsight.api.model.v1.request.SpeciesRequest; +import org.breedinginsight.api.v1.controller.TestTokenValidator; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.ImportTestUtils; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.dao.db.enums.DataType; +import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.SpeciesDAO; +import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.model.*; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; +import org.breedinginsight.services.writers.CSVWriter; +import org.jooq.DSLContext; +import org.junit.jupiter.api.*; + +import javax.inject.Inject; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.*; +import java.util.concurrent.atomic.AtomicReference; + +import static io.micronaut.http.HttpRequest.*; +import static io.micronaut.http.HttpRequest.GET; +import static org.junit.jupiter.api.Assertions.*; + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class BrAPIObservationUnitControllerIntegrationTest extends BrAPITest { + + private Program program; + private String experimentId; + private List envIds = new ArrayList<>(); + private final List> rows = new ArrayList<>(); + private final List columns = ExperimentFileColumns.getOrderedColumns(); + private List traits; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + @Inject + private DSLContext dsl; + @Inject + private UserDAO userDAO; + @Inject + private SpeciesDAO speciesDAO; + @Inject + private OntologyService ontologyService; + @Inject + private BrAPIGermplasmDAO germplasmDAO; + + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + + private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) + (json, type, context) -> OffsetDateTime.parse(json.getAsString())) + .create(); + + @BeforeAll + void setup() throws Exception { + FannyPack fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); + ImportTestUtils importTestUtils = new ImportTestUtils(); + FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); + + // Test User + User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + // Species + super.getBrapiDsl().execute(brapiFp.get("InsertSpecies")); + SpeciesEntity validSpecies = speciesDAO.findAll().get(0); + SpeciesRequest speciesRequest = SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); + + // Test Program + ProgramRequest programRequest = ProgramRequest.builder() + .name("Test Program") + .abbreviation("Test") + .documentationUrl("localhost:8080") + .objective("To test things") + .species(speciesRequest) + .key("TEST") + .build(); + program = TestUtils.insertAndFetchTestProgram(gson, client, programRequest); + + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId()); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + // Get experiment import map + Flowable> call = client.exchange( + GET("/import/mappings?importName=ExperimentsTemplateMap") + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + String mappingId = JsonParser.parseString(Objects.requireNonNull(response.body())).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + // Add traits to program + traits = createTraits(2); + AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); + try { + ontologyService.createTraits(program.getId(), traits, user, false); + } catch (ValidatorException e) { + System.err.println(e.getErrors()); + throw e; + } + + // Add germplasm to program + List germplasm = createGermplasm(1); + BrAPIExternalReference newReference = new BrAPIExternalReference(); + newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); + newReference.setReferenceID(program.getId().toString()); + + germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); + + germplasmDAO.createBrAPIGermplasm(germplasm, program.getId(), null); + + // Make test experiment import + Map row1 = makeExpImportRow("Env1"); + Map row2 = makeExpImportRow("Env2"); + + // Add test observation data + for (Trait trait : traits) { + Random random = new Random(); + + // TODO: test for sending obs data as double. + // A float is returned from the backend instead of double. there is a separate card to fix this. + // Double val1 = Math.random(); + + Float val1 = random.nextFloat(); + row1.put(trait.getObservationVariableName(), val1); + } + + rows.add(row1); + rows.add(row2); + + // Import test experiment, environments, and any observations + JsonObject importResult = importTestUtils.uploadAndFetch( + writeDataToFile(rows, traits), + null, + true, + client, + program, + mappingId); + experimentId = importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + // Add environmentIds. + envIds.add(getEnvId(importResult, 0)); + envIds.add(getEnvId(importResult, 1)); + } + + @Test + @SneakyThrows + public void testPostObsUnitNotFound() { + BrAPIObservationUnit ou = new BrAPIObservationUnit().observationUnitName("test"); + + Flowable> postCall = client.exchange( + POST(String.format("/programs/%s/brapi/v2/observationunits", + program.getId().toString()), List.of(ou)) + .contentType(MediaType.APPLICATION_JSON) + .bearerAuth("test-registered-user"), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = postCall.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + } + + @Test + @SneakyThrows + public void testPutSingleOUNotFound() { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/brapi/v2/observationunits?trialDbId=%s", program.getId(), experimentId)) + .bearerAuth("test-registered-user"), + String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); + + BrAPIObservationUnit ou = gson.fromJson(responseObj.get("result").getAsJsonObject().get("data").getAsJsonArray().get(0).getAsJsonObject(), BrAPIObservationUnit.class); + ou.setGermplasmName("updated"); + + Flowable> postCall = client.exchange( + PUT(String.format("/programs/%s/brapi/v2/observationunits/%s", + program.getId().toString(), ou.getObservationUnitDbId()), ou) + .contentType(MediaType.APPLICATION_JSON) + .bearerAuth("test-registered-user"), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse errorResponse = postCall.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + } + + @Test + @SneakyThrows + public void testPutMultipleOUsNotFound() { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/brapi/v2/observationunits?trialDbId=%s", program.getId(), experimentId)).bearerAuth("test-registered-user"), + String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); + + BrAPIObservationUnit ou = gson.fromJson(responseObj.get("result").getAsJsonObject().get("data").getAsJsonArray().get(0).getAsJsonObject(), BrAPIObservationUnit.class); + ou.setGermplasmName("updated"); + + Flowable> postCall = client.exchange( + PUT(String.format("/programs/%s/brapi/v2/observationunits", + program.getId().toString()), Map.of(ou.getObservationUnitDbId(), ou)) + .contentType(MediaType.APPLICATION_JSON) + .bearerAuth("test-registered-user"), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse errorResponse = postCall.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + } + + @Test + @SneakyThrows + public void testGetObsUnitTableNotFound() { + Flowable> getCall = client.exchange( + GET(String.format("/programs/%s/brapi/v2/observationunits/table", + program.getId().toString())) + .contentType(MediaType.APPLICATION_JSON) + .bearerAuth("test-registered-user"), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = getCall.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + } + + @Test + public void testGetOUListByExpId() { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/brapi/v2/observationunits?trialDbId=%s", program.getId(), experimentId)) + .bearerAuth("test-registered-user"), + String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); + + JsonArray ous = responseObj.getAsJsonObject("result").getAsJsonArray("data"); + assertEquals(2, ous.size()); + List ouNames = new ArrayList<>(); + for(var study : ous) { + ouNames.add(study.getAsJsonObject().get("observationUnitName").getAsString()); + } + assertTrue(ouNames.contains("Env1 - a-1")); + assertTrue(ouNames.contains("Env2 - a-1")); + } + + @Test + public void testGetOUById() { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/brapi/v2/observationunits?trialDbId=%s", program.getId(), experimentId)) + .bearerAuth("test-registered-user"), + String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); + JsonArray ous = responseObj.getAsJsonObject("result").getAsJsonArray("data"); + assertEquals(2, ous.size()); + JsonObject ou = ous.get(0).getAsJsonObject(); + + Flowable> ouCall = client.exchange( + GET(String.format("/programs/%s/brapi/v2/observationunits/%s", program.getId(), ou.get("observationUnitDbId").getAsString())) + .bearerAuth("test-registered-user"), + String.class + ); + + HttpResponse ouResponse = ouCall.blockingFirst(); + assertEquals(HttpStatus.OK, ouResponse.getStatus()); + + AtomicReference brAPIObservationUnitSingleResponse = new AtomicReference<>(); + Assertions.assertDoesNotThrow(() -> { + brAPIObservationUnitSingleResponse.set(gson.fromJson(ouResponse.body(), BrAPIObservationUnitSingleResponse.class)); + }); + + assertNotNull(brAPIObservationUnitSingleResponse.get()); + assertNotNull(brAPIObservationUnitSingleResponse.get().getResult()); + assertEquals(ou.get("observationUnitDbId").getAsString(), brAPIObservationUnitSingleResponse.get().getResult().getObservationUnitDbId()); + } + + private File writeDataToFile(List> data, List traits) throws IOException { + File file = File.createTempFile("test", ".csv"); + + if(traits != null) { + traits.forEach(trait -> columns.add( + Column.builder() + .value(trait.getObservationVariableName()) + .dataType(Column.ColumnDataType.STRING) + .build()) + ); + } + ByteArrayOutputStream byteArrayOutputStream = CSVWriter.writeToCSV(columns, data); + FileOutputStream fos = new FileOutputStream(file); + fos.write(byteArrayOutputStream.toByteArray()); + + return file; + } + + private String getEnvId(JsonObject result, int index) { + return result + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(index).getAsJsonObject() + .get("study").getAsJsonObject() + .get("brAPIObject").getAsJsonObject() + .get("externalReferences").getAsJsonArray() + .get(2).getAsJsonObject() + .get("referenceID").getAsString(); + } + + private List createGermplasm(int numToCreate) { + List germplasm = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String gid = ""+(i+1); + BrAPIGermplasm testGermplasm = new BrAPIGermplasm(); + testGermplasm.setGermplasmName(String.format("Germplasm %s [TEST-%s]", gid, gid)); + testGermplasm.setSeedSource("Wild"); + testGermplasm.setAccessionNumber(gid); + testGermplasm.setDefaultDisplayName(String.format("Germplasm %s", gid)); + JsonObject additionalInfo = new JsonObject(); + additionalInfo.addProperty("importEntryNumber", gid); + additionalInfo.addProperty("breedingMethod", "Allopolyploid"); + testGermplasm.setAdditionalInfo(additionalInfo); + List externalRef = new ArrayList<>(); + BrAPIExternalReference testReference = new BrAPIExternalReference(); + testReference.setReferenceSource(BRAPI_REFERENCE_SOURCE); + testReference.setReferenceID(UUID.randomUUID().toString()); + externalRef.add(testReference); + testGermplasm.setExternalReferences(externalRef); + germplasm.add(testGermplasm); + } + + return germplasm; + } + + private List createTraits(int numToCreate) { + List traits = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String varName = "tt_test_" + (i + 1); + traits.add(Trait.builder() + .observationVariableName(varName) + .fullName(varName) + .entity("Plant " + i) + .attribute("height " + i) + .traitDescription("test") + .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build()) + .scale(Scale.builder() + .scaleName("test scale") + .dataType(DataType.NUMERICAL) + .validValueMin(0) + .validValueMax(100) + .build()) + .method(Method.builder() + .description("test method") + .methodClass("test method") + .build()) + .build()); + } + + return traits; + } + + private Map makeExpImportRow(String environment) { + Map row = new HashMap<>(); + row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); + row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); + row.put(ExperimentObservation.Columns.EXP_TITLE, "Test Exp"); + row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); + row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); + row.put(ExperimentObservation.Columns.ENV, environment); + row.put(ExperimentObservation.Columns.ENV_LOCATION, "Location A"); + row.put(ExperimentObservation.Columns.ENV_YEAR, "2023"); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, environment+" - a-1"); + row.put(ExperimentObservation.Columns.REP_NUM, "1"); + row.put(ExperimentObservation.Columns.BLOCK_NUM, "1"); + row.put(ExperimentObservation.Columns.ROW, "1"); + row.put(ExperimentObservation.Columns.COLUMN, "1"); + return row; + } +} diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableControllerIntegrationTest.java new file mode 100644 index 000000000..1596f26e7 --- /dev/null +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableControllerIntegrationTest.java @@ -0,0 +1,22 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2; + +public class BrAPIObservationVariableControllerIntegrationTest { + //TODO +} diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java new file mode 100644 index 000000000..cae4497e5 --- /dev/null +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java @@ -0,0 +1,22 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2; + +public class BrAPIObservationsControllerIntegrationTest { + //TODO +} diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java new file mode 100644 index 000000000..1373d9ab3 --- /dev/null +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java @@ -0,0 +1,411 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2; + +import com.google.gson.*; +import io.kowalski.fannypack.FannyPack; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.client.exceptions.HttpClientResponseException; +import io.micronaut.http.netty.cookies.NettyCookie; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import io.reactivex.Flowable; +import lombok.SneakyThrows; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.response.BrAPIStudySingleResponse; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.breedinginsight.BrAPITest; +import org.breedinginsight.TestUtils; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramRequest; +import org.breedinginsight.api.model.v1.request.SpeciesRequest; +import org.breedinginsight.api.v1.controller.TestTokenValidator; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.ImportTestUtils; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.dao.db.enums.DataType; +import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.SpeciesDAO; +import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.model.*; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; +import org.breedinginsight.services.writers.CSVWriter; +import org.jooq.DSLContext; +import org.junit.jupiter.api.*; + +import javax.inject.Inject; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.*; +import java.util.concurrent.atomic.AtomicReference; + +import static io.micronaut.http.HttpRequest.*; +import static io.micronaut.http.HttpRequest.GET; +import static org.junit.jupiter.api.Assertions.*; + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class BrAPIStudiesControllerIntegrationTest extends BrAPITest { + private Program program; + private String experimentId; + private List envIds = new ArrayList<>(); + private final List> rows = new ArrayList<>(); + private final List columns = ExperimentFileColumns.getOrderedColumns(); + private List traits; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + @Inject + private DSLContext dsl; + @Inject + private UserDAO userDAO; + @Inject + private SpeciesDAO speciesDAO; + @Inject + private OntologyService ontologyService; + @Inject + private BrAPIGermplasmDAO germplasmDAO; + + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + + private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) + (json, type, context) -> OffsetDateTime.parse(json.getAsString())) + .create(); + + @BeforeAll + void setup() throws Exception { + FannyPack fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); + ImportTestUtils importTestUtils = new ImportTestUtils(); + FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); + + // Test User + User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + // Species + super.getBrapiDsl().execute(brapiFp.get("InsertSpecies")); + SpeciesEntity validSpecies = speciesDAO.findAll().get(0); + SpeciesRequest speciesRequest = SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); + + // Test Program + ProgramRequest programRequest = ProgramRequest.builder() + .name("Test Program") + .abbreviation("Test") + .documentationUrl("localhost:8080") + .objective("To test things") + .species(speciesRequest) + .key("TEST") + .build(); + program = TestUtils.insertAndFetchTestProgram(gson, client, programRequest); + + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId()); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + // Get experiment import map + Flowable> call = client.exchange( + GET("/import/mappings?importName=ExperimentsTemplateMap") + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + String mappingId = JsonParser.parseString(Objects.requireNonNull(response.body())).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + // Add traits to program + traits = createTraits(2); + AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); + try { + ontologyService.createTraits(program.getId(), traits, user, false); + } catch (ValidatorException e) { + System.err.println(e.getErrors()); + throw e; + } + + // Add germplasm to program + List germplasm = createGermplasm(1); + BrAPIExternalReference newReference = new BrAPIExternalReference(); + newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); + newReference.setReferenceID(program.getId().toString()); + + germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); + + germplasmDAO.createBrAPIGermplasm(germplasm, program.getId(), null); + + // Make test experiment import + Map row1 = makeExpImportRow("Env1"); + Map row2 = makeExpImportRow("Env2"); + + // Add test observation data + for (Trait trait : traits) { + Random random = new Random(); + + // TODO: test for sending obs data as double. + // A float is returned from the backend instead of double. there is a separate card to fix this. + // Double val1 = Math.random(); + + Float val1 = random.nextFloat(); + row1.put(trait.getObservationVariableName(), val1); + } + + rows.add(row1); + rows.add(row2); + + // Import test experiment, environments, and any observations + JsonObject importResult = importTestUtils.uploadAndFetch( + writeDataToFile(rows, traits), + null, + true, + client, + program, + mappingId); + experimentId = importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + // Add environmentIds. + envIds.add(getEnvId(importResult, 0)); + envIds.add(getEnvId(importResult, 1)); + } + + @Test + @SneakyThrows + public void testPostGetStudiesNotFound() { + BrAPIStudy study = new BrAPIStudy().studyName("test study") + .studyCode("123") + .studyType("Phenotyping Trial") + .studyDescription("Test study description") + .active(true) + .startDate(OffsetDateTime.of(2021, 1, 5, 0, 0, 0, 0, ZoneOffset.UTC)) + .endDate(OffsetDateTime.of(2021, 2, 5, 0, 0, 0, 0, ZoneOffset.UTC)); + + Flowable> postCall = client.exchange( + POST(String.format("/programs/%s/brapi/v2/studies", + program.getId().toString()), List.of(study)) + .contentType(MediaType.APPLICATION_JSON) + .bearerAuth("test-registered-user"), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = postCall.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + } + + @Test + @SneakyThrows + public void testPutGetStudiesNotFound() { + BrAPIStudy study = new BrAPIStudy().studyName("test study") + .studyCode("123") + .studyType("Phenotyping Trial") + .studyDescription("Test study description") + .active(true) + .startDate(OffsetDateTime.of(2021, 1, 5, 0, 0, 0, 0, ZoneOffset.UTC)) + .endDate(OffsetDateTime.of(2021, 2, 5, 0, 0, 0, 0, ZoneOffset.UTC)); + + Flowable> putCall = client.exchange( + PUT(String.format("/programs/%s/brapi/v2/studies/abc", + program.getId().toString()), List.of(study)) + .contentType(MediaType.APPLICATION_JSON) + .bearerAuth("test-registered-user"), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = putCall.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + } + + @Test + public void testGetStudiesListByExpId() { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/brapi/v2/studies?trialDbId=%s", program.getId(), experimentId)) + .bearerAuth("test-registered-user"), + String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); + JsonArray studies = responseObj.getAsJsonObject("result").getAsJsonArray("data"); + assertEquals(2, studies.size()); + List studyNames = new ArrayList<>(); + for(var study : studies) { + studyNames.add(study.getAsJsonObject().get("studyName").getAsString()); + } + assertTrue(studyNames.contains("Env1")); + assertTrue(studyNames.contains("Env2")); + } + + @Test + public void testGetStudyById() { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/brapi/v2/studies?trialDbId=%s", program.getId(), experimentId)) + .bearerAuth("test-registered-user"), + String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); + JsonArray studies = responseObj.getAsJsonObject("result").getAsJsonArray("data"); + assertEquals(2, studies.size()); + JsonObject study = studies.get(0).getAsJsonObject(); + + Flowable> studyCall = client.exchange( + GET(String.format("/programs/%s/brapi/v2/studies/%s", program.getId(), study.get("studyDbId").getAsString())) + .bearerAuth("test-registered-user"), + String.class + ); + + HttpResponse studyResponse = studyCall.blockingFirst(); + assertEquals(HttpStatus.OK, studyResponse.getStatus()); + + AtomicReference brAPIStudySingleResponse = new AtomicReference<>(); + Assertions.assertDoesNotThrow(() -> { + brAPIStudySingleResponse.set(gson.fromJson(studyResponse.body(), BrAPIStudySingleResponse.class)); + }); + + assertNotNull(brAPIStudySingleResponse.get()); + assertNotNull(brAPIStudySingleResponse.get().getResult()); + assertEquals(study.get("studyDbId").getAsString(), brAPIStudySingleResponse.get().getResult().getStudyDbId()); + } + + private File writeDataToFile(List> data, List traits) throws IOException { + File file = File.createTempFile("test", ".csv"); + + if(traits != null) { + traits.forEach(trait -> columns.add( + Column.builder() + .value(trait.getObservationVariableName()) + .dataType(Column.ColumnDataType.STRING) + .build()) + ); + } + ByteArrayOutputStream byteArrayOutputStream = CSVWriter.writeToCSV(columns, data); + FileOutputStream fos = new FileOutputStream(file); + fos.write(byteArrayOutputStream.toByteArray()); + + return file; + } + + private String getEnvId(JsonObject result, int index) { + return result + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(index).getAsJsonObject() + .get("study").getAsJsonObject() + .get("brAPIObject").getAsJsonObject() + .get("externalReferences").getAsJsonArray() + .get(2).getAsJsonObject() + .get("referenceID").getAsString(); + } + + private List createGermplasm(int numToCreate) { + List germplasm = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String gid = ""+(i+1); + BrAPIGermplasm testGermplasm = new BrAPIGermplasm(); + testGermplasm.setGermplasmName(String.format("Germplasm %s [TEST-%s]", gid, gid)); + testGermplasm.setSeedSource("Wild"); + testGermplasm.setAccessionNumber(gid); + testGermplasm.setDefaultDisplayName(String.format("Germplasm %s", gid)); + JsonObject additionalInfo = new JsonObject(); + additionalInfo.addProperty("importEntryNumber", gid); + additionalInfo.addProperty("breedingMethod", "Allopolyploid"); + testGermplasm.setAdditionalInfo(additionalInfo); + List externalRef = new ArrayList<>(); + BrAPIExternalReference testReference = new BrAPIExternalReference(); + testReference.setReferenceSource(BRAPI_REFERENCE_SOURCE); + testReference.setReferenceID(UUID.randomUUID().toString()); + externalRef.add(testReference); + testGermplasm.setExternalReferences(externalRef); + germplasm.add(testGermplasm); + } + + return germplasm; + } + + private List createTraits(int numToCreate) { + List traits = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String varName = "tt_test_" + (i + 1); + traits.add(Trait.builder() + .observationVariableName(varName) + .fullName(varName) + .entity("Plant " + i) + .attribute("height " + i) + .traitDescription("test") + .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build()) + .scale(Scale.builder() + .scaleName("test scale") + .dataType(DataType.NUMERICAL) + .validValueMin(0) + .validValueMax(100) + .build()) + .method(Method.builder() + .description("test method") + .methodClass("test method") + .build()) + .build()); + } + + return traits; + } + + private Map makeExpImportRow(String environment) { + Map row = new HashMap<>(); + row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); + row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); + row.put(ExperimentObservation.Columns.EXP_TITLE, "Test Exp"); + row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); + row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); + row.put(ExperimentObservation.Columns.ENV, environment); + row.put(ExperimentObservation.Columns.ENV_LOCATION, "Location A"); + row.put(ExperimentObservation.Columns.ENV_YEAR, "2023"); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, "a-1"); + row.put(ExperimentObservation.Columns.REP_NUM, "1"); + row.put(ExperimentObservation.Columns.BLOCK_NUM, "1"); + row.put(ExperimentObservation.Columns.ROW, "1"); + row.put(ExperimentObservation.Columns.COLUMN, "1"); + return row; + } +} diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java index ebb3979d1..0ef73d284 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java @@ -25,6 +25,7 @@ import io.micronaut.http.MediaType; import io.micronaut.http.client.RxHttpClient; import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.client.exceptions.HttpClientResponseException; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import io.reactivex.Flowable; import lombok.SneakyThrows; @@ -49,7 +50,6 @@ import javax.inject.Inject; import java.time.OffsetDateTime; -import java.time.ZoneOffset; import java.util.Arrays; import java.util.Collections; import java.util.UUID; @@ -145,12 +145,15 @@ public void testRootServerInfo() { assertEquals("Breeding Insight", serverInfo.getOrganizationName()); assertEquals("DeltaBreed", serverInfo.getServerName()); assertEquals("bidevteam@cornell.edu", serverInfo.getContactEmail()); - assertEquals("breedinginsight.org", serverInfo.getOrganizationURL()); + assertEquals("https://breedinginsight.org", serverInfo.getOrganizationURL()); + assertEquals("BrAPI endpoints are not implemented at the root of this domain. Please make BrAPI calls in the context of a program (ex: https://app.breedinginsight.net/v1/programs/{programId}/brapi/v2)", serverInfo.getServerDescription()); + assertEquals("Cornell University, Ithaca, NY, USA", serverInfo.getLocation()); + assertEquals("https://brapi.org/specification", serverInfo.getDocumentationURL()); } @Test @SneakyThrows - public void testPostGetVariablesProxy() { + public void testPostVariablesNotFound() { BrAPIObservationVariable variable = generateVariable(); Flowable> postCall = biClient.exchange( @@ -161,94 +164,15 @@ public void testPostGetVariablesProxy() { .bearerAuth("test-registered-user"), String.class ); - HttpResponse postResponse; - try { - postResponse = postCall.blockingFirst(); - } catch (Exception e) { - throw new Exception(e); - } - //check the POST call was successful - assertEquals(HttpStatus.OK, postResponse.getStatus()); - - BrAPIObservationVariable createdVariable = GSON.fromJson(postResponse.body(), BrAPIObservationVariableListResponse.class) - .getResult() - .getData() - .get(0); - - //and that a variable is returned - assertNotNull(createdVariable); - //and that the variable has been assigned an ID - assertNotNull(createdVariable.getObservationVariableDbId(), "observationVariableDbId is null"); - - Flowable> getCall = biClient.exchange( - GET(String.format("%s/programs/%s/brapi/v2/variables/%s", - biApiVersion, - validProgram.getId().toString(), - createdVariable.getObservationVariableDbId())) - .bearerAuth("test-registered-user"), String.class - ); - - HttpResponse getResponse; - try { - getResponse = getCall.blockingFirst(); - } catch (Exception e) { - throw new Exception(e); - } - assertEquals(HttpStatus.OK, getResponse.getStatus()); - assertNotNull(getResponse.body(), "Response body is empty"); - BrAPIObservationVariableSingleResponse brAPIObservationVariableSingleResponse = GSON.fromJson(getResponse.body(), BrAPIObservationVariableSingleResponse.class); - - BrAPIObservationVariable fetchedVariable = brAPIObservationVariableSingleResponse.getResult(); - assertNotNull(fetchedVariable, "Observation Variable was not found"); - assertEquals(createdVariable.getObservationVariableDbId(), fetchedVariable.getObservationVariableDbId()); - //make sure the original values sent in the POST were saved correctly - assertEquals(variable.getObservationVariableName(), fetchedVariable.getObservationVariableName()); - assertEquals(variable.getExternalReferences(), fetchedVariable.getExternalReferences()); - assertEquals(variable.getCommonCropName(), fetchedVariable.getCommonCropName()); - - assertNotNull(fetchedVariable.getTrait(), "Trait is null"); - assertEquals(createdVariable.getTrait().getTraitDbId(), fetchedVariable.getTrait().getTraitDbId()); - //make sure the original values sent in the POST were saved correctly - assertEquals(variable.getTrait().getTraitName(), fetchedVariable.getTrait().getTraitName()); - assertEquals(variable.getTrait().getTraitClass(), fetchedVariable.getTrait().getTraitClass()); - - - assertNotNull(fetchedVariable.getMethod(), "Method is null"); - assertEquals(createdVariable.getMethod().getMethodDbId(), fetchedVariable.getMethod().getMethodDbId()); - //make sure the original values sent in the POST were saved correctly - assertEquals(variable.getMethod().getMethodName(), fetchedVariable.getMethod().getMethodName()); - assertEquals(variable.getMethod().getMethodClass(), fetchedVariable.getMethod().getMethodClass()); - - assertNotNull(fetchedVariable.getScale(), "Scale is null"); - assertEquals(createdVariable.getScale().getScaleDbId(), fetchedVariable.getScale().getScaleDbId()); - //make sure the original values sent in the POST were saved correctly - assertEquals(variable.getScale().getScaleName(), fetchedVariable.getScale().getScaleName()); - - Flowable> getScaleCall = biClient.exchange( - GET(String.format("%s/programs/%s/brapi/v2/scales/%s", - biApiVersion, - validProgram.getId().toString(), - createdVariable.getScale().getScaleDbId())) - .bearerAuth("test-registered-user"), String.class - ); - - HttpResponse getScaleResponse; - try { - getScaleResponse = getScaleCall.blockingFirst(); - } catch (Exception e) { - throw new Exception(e); - } - assertEquals(HttpStatus.OK, getScaleResponse.getStatus()); - - BrAPIScale scaleResponse = GSON.fromJson(JsonParser.parseString(getScaleResponse.body()).getAsJsonObject().getAsJsonObject("result"), BrAPIScale.class); - - //TODO this is not being returned -// assertEquals(variable.getScale().getDataType(), scaleResponse.getDataType()); + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = postCall.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); } @Test @SneakyThrows - public void testPutVariablesProxy() { + public void testPutVariablesNotFound() { BrAPIObservationVariable variable = generateVariable(); Flowable> postCall = biClient.exchange( @@ -259,127 +183,13 @@ public void testPutVariablesProxy() { .bearerAuth("test-registered-user"), String.class ); - HttpResponse postResponse; - try { - postResponse = postCall.blockingFirst(); - } catch (Exception e) { - throw new Exception(e); - } - //check the POST call was successful - assertEquals(HttpStatus.OK, postResponse.getStatus()); - - BrAPIObservationVariable createdVariable = GSON.fromJson(postResponse.body(), BrAPIObservationVariableListResponse.class) - .getResult() - .getData() - .get(0); - - createdVariable.setObservationVariableName("Updated variable name"); - - - Flowable> putCall = biClient.exchange( - PUT(String.format("%s/programs/%s/brapi/v2/variables/%s", - biApiVersion, - validProgram.getId().toString(), createdVariable.getObservationVariableDbId()), variable) - .contentType(MediaType.APPLICATION_JSON) - .bearerAuth("test-registered-user"), String.class - ); - - HttpResponse putResponse; - try { - putResponse = putCall.blockingFirst(); - } catch (Exception e) { - throw new Exception(e); - } - //check the PUT call was successful - assertEquals(HttpStatus.OK, putResponse.getStatus()); - - Flowable> getCall = biClient.exchange( - GET(String.format("%s/programs/%s/brapi/v2/variables/%s", - biApiVersion, - validProgram.getId().toString(), - createdVariable.getObservationVariableDbId())) - .bearerAuth("test-registered-user"), String.class - ); - - HttpResponse getResponse; - try { - getResponse = getCall.blockingFirst(); - } catch (Exception e) { - throw new Exception(e); - } - assertEquals(HttpStatus.OK, getResponse.getStatus()); - BrAPIObservationVariableSingleResponse brAPIObservationVariableSingleResponse = GSON.fromJson(getResponse.body(), BrAPIObservationVariableSingleResponse.class); - - BrAPIObservationVariable fetchedVariable = brAPIObservationVariableSingleResponse.getResult(); - //make sure the updated value persisted - assertEquals(variable.getObservationVariableName(), fetchedVariable.getObservationVariableName()); + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = postCall.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); } - @Test - @SneakyThrows - public void testPostGetStudiesProxy() { - BrAPIStudy study = new BrAPIStudy().studyName("test study") - .studyCode("123") - .studyType("Phenotyping Trial") - .studyDescription("Test study description") - .active(true) - .startDate(OffsetDateTime.of(2021, 1, 5, 0, 0, 0, 0, ZoneOffset.UTC)) - .endDate(OffsetDateTime.of(2021, 2, 5, 0, 0, 0, 0, ZoneOffset.UTC)); - - Flowable> postCall = biClient.exchange( - POST(String.format("%s/programs/%s/brapi/v2/studies", - biApiVersion, - validProgram.getId().toString()), Arrays.asList(study)) - .contentType(MediaType.APPLICATION_JSON) - .bearerAuth("test-registered-user"), String.class - ); - - HttpResponse postResponse; - try { - postResponse = postCall.blockingFirst(); - } catch (Exception e) { - throw new Exception(e); - } - //check the POST call was successful - assertEquals(HttpStatus.OK, postResponse.getStatus()); - - BrAPIStudy createdStudy = GSON.fromJson(postResponse.body(), BrAPIStudyListResponse.class) - .getResult() - .getData() - .get(0); - - //and that a study is returned - assertNotNull(createdStudy); - //and that the study has been assigned an ID - assertNotNull(createdStudy.getStudyDbId(), "studyDbId is null"); - - Flowable> getCall = biClient.exchange( - GET(String.format("%s/programs/%s/brapi/v2/studies/%s", - biApiVersion, - validProgram.getId().toString(), - createdStudy.getStudyDbId())) - .bearerAuth("test-registered-user"), String.class - ); - HttpResponse getResponse; - try { - getResponse = getCall.blockingFirst(); - } catch (Exception e) { - throw new Exception(e); - } - assertEquals(HttpStatus.OK, getResponse.getStatus()); - assertNotNull(getResponse.body(), "Response body is empty"); - - BrAPIStudy fetchedStudy = GSON.fromJson(getResponse.body(), BrAPIStudySingleResponse.class).getResult(); - - assertEquals(study.getStudyName(), fetchedStudy.getStudyName()); - assertEquals(study.getStudyCode(), fetchedStudy.getStudyCode()); - assertEquals(study.getStudyType(), fetchedStudy.getStudyType()); - assertEquals(study.getStudyDescription(), fetchedStudy.getStudyDescription()); - assertEquals(study.isActive(), fetchedStudy.isActive()); - assertEquals(study.getStartDate(), fetchedStudy.getStartDate()); - assertEquals(study.getEndDate(), fetchedStudy.getEndDate()); - } private BrAPIObservationVariable generateVariable() { var random = UUID.randomUUID() diff --git a/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java index 0c52536fc..fb95e4dd2 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java @@ -20,7 +20,7 @@ import org.breedinginsight.api.v1.controller.TestTokenValidator; import org.breedinginsight.utilities.response.mappers.GermplasmQueryMapper; import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService; -import org.breedinginsight.brapps.importer.daos.BrAPIListDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; import org.breedinginsight.dao.db.tables.pojos.BiUserEntity; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.Program; diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 8db3b5bf4..b3bcb0da1 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -45,8 +45,7 @@ import org.breedinginsight.api.model.v1.request.ProgramRequest; import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; -import org.breedinginsight.brapps.importer.daos.*; +import org.breedinginsight.brapi.v2.dao.*; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation.Columns; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.dao.db.tables.pojos.BiUserEntity; diff --git a/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java b/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java index ce742dc7c..b7a2ba564 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/daos/BrAPIObservationUnitDAOTest.java @@ -14,6 +14,7 @@ import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.api.v1.controller.TestTokenValidator; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; import org.breedinginsight.brapps.importer.model.ImportProgress; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; diff --git a/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java b/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java index 49d55d9e9..afa00df4e 100644 --- a/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java +++ b/src/test/java/org/breedinginsight/services/BrAPIGermplasmServiceUnitTest.java @@ -12,7 +12,7 @@ import org.breedinginsight.DatabaseTest; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService; -import org.breedinginsight.brapps.importer.daos.BrAPIListDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.model.exports.FileType; import org.breedinginsight.daos.ProgramDAO; diff --git a/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java b/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java index 56b3c099b..452a36268 100644 --- a/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java +++ b/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java @@ -42,11 +42,11 @@ import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponse; import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponseResult; import org.breedinginsight.DatabaseTest; -import org.breedinginsight.brapps.importer.daos.BrAPITrialDAO; +import org.breedinginsight.brapi.v2.dao.BrAPITrialDAO; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.daos.ImportMappingDAO; -import org.breedinginsight.brapps.importer.daos.impl.BrAPITrialDAOImpl; -import org.breedinginsight.brapps.importer.daos.impl.ImportMappingDAOImpl; +import org.breedinginsight.brapi.v2.dao.impl.BrAPITrialDAOImpl; +import org.breedinginsight.brapi.v2.dao.impl.ImportMappingDAOImpl; import org.breedinginsight.brapps.importer.model.ImportProgress; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.mapping.ImportMapping; From 42ea0c1b9925973b4269ffb23e700e267a20aee2 Mon Sep 17 00:00:00 2001 From: timparsons Date: Wed, 29 Nov 2023 13:26:03 -0500 Subject: [PATCH 142/220] [BI-1921] fixing unit tests --- .../TraitValidatorIntegrationTest.java | 97 +++++++++++++++---- 1 file changed, 79 insertions(+), 18 deletions(-) diff --git a/src/test/java/org/breedinginsight/services/validators/TraitValidatorIntegrationTest.java b/src/test/java/org/breedinginsight/services/validators/TraitValidatorIntegrationTest.java index d5c3dc7d6..3006bc7b5 100644 --- a/src/test/java/org/breedinginsight/services/validators/TraitValidatorIntegrationTest.java +++ b/src/test/java/org/breedinginsight/services/validators/TraitValidatorIntegrationTest.java @@ -17,20 +17,34 @@ package org.breedinginsight.services.validators; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonDeserializer; import io.kowalski.fannypack.FannyPack; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; import io.micronaut.test.extensions.junit5.annotation.MicronautTest; import lombok.SneakyThrows; -import org.breedinginsight.DatabaseTest; +import org.breedinginsight.BrAPITest; +import org.breedinginsight.TestUtils; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramRequest; +import org.breedinginsight.api.model.v1.request.SpeciesRequest; +import org.breedinginsight.api.v1.controller.TestTokenValidator; +import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.dao.db.tables.daos.ProgramDao; import org.breedinginsight.dao.db.tables.pojos.ProgramEntity; -import org.breedinginsight.model.Method; -import org.breedinginsight.model.ProgramObservationLevel; -import org.breedinginsight.model.Scale; -import org.breedinginsight.model.Trait; +import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.SpeciesDAO; +import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.model.*; +import org.breedinginsight.services.TraitService; import org.jooq.DSLContext; import org.junit.jupiter.api.*; import javax.inject.Inject; +import java.time.OffsetDateTime; +import java.util.ArrayList; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -39,7 +53,7 @@ @MicronautTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) -public class TraitValidatorIntegrationTest extends DatabaseTest { +public class TraitValidatorIntegrationTest extends BrAPITest { private ProgramEntity validProgram; @@ -49,29 +63,76 @@ public class TraitValidatorIntegrationTest extends DatabaseTest { private ProgramDao programDao; @Inject private TraitValidatorService traitValidator; + @Inject + private TraitService traitService; + @Inject + private UserDAO userDAO; + @Inject + private SpeciesDAO speciesDAO; + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + + private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) + (json, type, context) -> OffsetDateTime.parse(json.getAsString())) + .create(); @BeforeAll - public void setup() { + public void setup() throws Exception { // Insert our traits into the db var fp = FannyPack.fill("src/test/resources/sql/TraitControllerIntegrationTest.sql"); - - // Insert program - dsl.execute(fp.get("InsertProgram")); + FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); + + // Test User + User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + // Species + super.getBrapiDsl().execute(brapiFp.get("InsertSpecies")); + SpeciesEntity validSpecies = speciesDAO.findAll().get(0); + SpeciesRequest speciesRequest = SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); + + // Test Program + ProgramRequest programRequest = ProgramRequest.builder() + .name("Test Program") + .abbreviation("test") + .documentationUrl("localhost:8080") + .objective("To test things") + .species(speciesRequest) + .key("TEST") + .build(); + validProgram = TestUtils.insertAndFetchTestProgram(gson, client, programRequest); + + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), validProgram.getId()); + AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); // Insert program observation level dsl.execute(fp.get("InsertProgramObservationLevel")); - // Insert program ontology sql - dsl.execute(fp.get("InsertProgramOntology")); + Trait trait = Trait.builder() + .observationVariableName("Test Trait") + .fullName("Test Trait") + .traitDescription("test") + .entity("test") + .attribute("test") + .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build()) + .method(Method.builder() + .description("test method") + .methodClass("test method") + .build()) + .scale(Scale.builder() + .scaleName("Test Scale") + .dataType(DataType.TEXT) + .build()) + .build(); // Insert Trait - dsl.execute(fp.get("InsertMethod")); - dsl.execute(fp.get("InsertScale")); - dsl.execute(fp.get("InsertTrait")); - - // Retrieve our new data - validProgram = programDao.findAll().get(0); + traitService.createTraits(validProgram.getId(), new ArrayList<>(List.of(trait)), user, false); } @Test From 2f3309fb2f5db324de73d81ac3e9c85dc7942032 Mon Sep 17 00:00:00 2001 From: timparsons Date: Thu, 30 Nov 2023 11:36:11 -0500 Subject: [PATCH 143/220] [BI-1921] adding more unit tests --- .../BrAPIObservationVariableController.java | 1 + .../brapi/v2/BrAPIObservationsController.java | 90 ++-- ...onVariablesControllerIntegrationTest.java} | 2 +- ...tionVariableControllerIntegrationTest.java | 22 - ...ObservationsControllerIntegrationTest.java | 397 ++++++++++++++++- ...tionVariableControllerIntegrationTest.java | 420 ++++++++++++++++++ 6 files changed, 863 insertions(+), 69 deletions(-) rename src/test/java/org/breedinginsight/brapi/v1/controller/{BrapiObservationVariablesControllerIntegrationTest.java => BrapiV1ObservationVariablesControllerIntegrationTest.java} (99%) delete mode 100644 src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableControllerIntegrationTest.java create mode 100644 src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java index d94ab0541..6a748978f 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java @@ -218,6 +218,7 @@ private BrAPIObservationVariable convertToBrAPI(Trait trait) { .synonyms(synonyms)) .method(new BrAPIMethod().ontologyReference(brAPIOntologyReference) .methodDbId(trait.getMethod().getId().toString()) + .methodClass(trait.getMethod().getMethodClass()) .description(trait.getMethod().getDescription()) .formula(trait.getMethod().getFormula())) .scale(new BrAPIScale().ontologyReference(brAPIOntologyReference) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java index 2f669255d..5a6c51148 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationsController.java @@ -28,6 +28,7 @@ import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; import org.breedinginsight.brapi.v1.controller.BrapiVersion; +import javax.annotation.Nullable; import java.util.Date; import java.util.List; import java.util.Map; @@ -41,30 +42,31 @@ public class BrAPIObservationsController { @Get("/observations") @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) public HttpResponse observationsGet(@PathVariable("programId") UUID programId, - @QueryValue("observationDbId") String observationDbId, - @QueryValue("observationUnitDbId") String observationUnitDbId, - @QueryValue("observationVariableDbId") String observationVariableDbId, - @QueryValue("locationDbId") String locationDbId, - @QueryValue("seasonDbId") String seasonDbId, - @QueryValue("observationTimeStampRangeStart") Date observationTimeStampRangeStart, - @QueryValue("observationTimeStampRangeEnd") Date observationTimeStampRangeEnd, - @QueryValue("observationUnitLevelName") String observationUnitLevelName, - @QueryValue("observationUnitLevelOrder") String observationUnitLevelOrder, - @QueryValue("observationUnitLevelCode") String observationUnitLevelCode, - @QueryValue("observationUnitLevelRelationshipName") String observationUnitLevelRelationshipName, - @QueryValue("observationUnitLevelRelationshipOrder") String observationUnitLevelRelationshipOrder, - @QueryValue("observationUnitLevelRelationshipCode") String observationUnitLevelRelationshipCode, - @QueryValue("observationUnitLevelRelationshipDbId") String observationUnitLevelRelationshipDbId, - @QueryValue("commonCropName") String commonCropName, - @QueryValue("programDbId") String programDbId, - @QueryValue("trialDbId") String trialDbId, - @QueryValue("studyDbId") String studyDbId, - @QueryValue("germplasmDbId") String germplasmDbId, - @QueryValue("externalReferenceID") String externalReferenceID, - @QueryValue("externalReferenceId") String externalReferenceId, - @QueryValue("externalReferenceSource") String externalReferenceSource, - @QueryValue("page") Integer page, - @QueryValue("pageSize") Integer pageSize) { + @Nullable @QueryValue("observationDbId") String observationDbId, + @Nullable @QueryValue("observationUnitDbId") String observationUnitDbId, + @Nullable @QueryValue("observationVariableDbId") String observationVariableDbId, + @Nullable @QueryValue("locationDbId") String locationDbId, + @Nullable @QueryValue("seasonDbId") String seasonDbId, + @Nullable @QueryValue("observationTimeStampRangeStart") Date observationTimeStampRangeStart, + @Nullable @QueryValue("observationTimeStampRangeEnd") Date observationTimeStampRangeEnd, + @Nullable @QueryValue("observationUnitLevelName") String observationUnitLevelName, + @Nullable @QueryValue("observationUnitLevelOrder") String observationUnitLevelOrder, + @Nullable @QueryValue("observationUnitLevelCode") String observationUnitLevelCode, + @Nullable @QueryValue("observationUnitLevelRelationshipName") String observationUnitLevelRelationshipName, + @Nullable @QueryValue("observationUnitLevelRelationshipOrder") String observationUnitLevelRelationshipOrder, + @Nullable @QueryValue("observationUnitLevelRelationshipCode") String observationUnitLevelRelationshipCode, + @Nullable @QueryValue("observationUnitLevelRelationshipDbId") String observationUnitLevelRelationshipDbId, + @Nullable @QueryValue("commonCropName") String commonCropName, + @Nullable @QueryValue("programDbId") String programDbId, + @Nullable @QueryValue("trialDbId") String trialDbId, + @Nullable @QueryValue("studyDbId") String studyDbId, + @Nullable @QueryValue("germplasmDbId") String germplasmDbId, + @Nullable @QueryValue("externalReferenceID") String externalReferenceID, + @Nullable @QueryValue("externalReferenceId") String externalReferenceId, + @Nullable @QueryValue("externalReferenceSource") String externalReferenceSource, + @Nullable @QueryValue("page") Integer page, + @Nullable @QueryValue("pageSize") Integer pageSize) { + //TODO return HttpResponse.notFound(); } @@ -111,26 +113,26 @@ public HttpResponse observationsPut(@PathVariable("programId") UUID programId, @ @Produces({"application/json", "text/csv", "text/tsv"}) @ProgramSecured(roleGroups = {ProgramSecuredRoleGroup.ALL}) public HttpResponse observationsTableGet(@PathVariable("programId") UUID programId, - @Header("Accept") String accept, - @QueryValue("observationUnitDbId") String observationUnitDbId, - @QueryValue("observationVariableDbId") String observationVariableDbId, - @QueryValue("locationDbId") String locationDbId, - @QueryValue("seasonDbId") String seasonDbId, - @QueryValue("observationLevel") String observationLevel, - @QueryValue("searchResultsDbId") String searchResultsDbId, - @QueryValue("observationTimeStampRangeStart") Date observationTimeStampRangeStart, - @QueryValue("observationTimeStampRangeEnd") Date observationTimeStampRangeEnd, - @QueryValue("programDbId") String programDbId, - @QueryValue("trialDbId") String trialDbId, - @QueryValue("studyDbId") String studyDbId, - @QueryValue("germplasmDbId") String germplasmDbId, - @QueryValue("observationUnitLevelName") String observationUnitLevelName, - @QueryValue("observationUnitLevelOrder") String observationUnitLevelOrder, - @QueryValue("observationUnitLevelCode") String observationUnitLevelCode, - @QueryValue("observationUnitLevelRelationshipName") String observationUnitLevelRelationshipName, - @QueryValue("observationUnitLevelRelationshipOrder") String observationUnitLevelRelationshipOrder, - @QueryValue("observationUnitLevelRelationshipCode") String observationUnitLevelRelationshipCode, - @QueryValue("observationUnitLevelRelationshipDbId") String observationUnitLevelRelationshipDbId) { + @Nullable @Header("Accept") String accept, + @Nullable @QueryValue("observationUnitDbId") String observationUnitDbId, + @Nullable @QueryValue("observationVariableDbId") String observationVariableDbId, + @Nullable @QueryValue("locationDbId") String locationDbId, + @Nullable @QueryValue("seasonDbId") String seasonDbId, + @Nullable @QueryValue("observationLevel") String observationLevel, + @Nullable @QueryValue("searchResultsDbId") String searchResultsDbId, + @Nullable @QueryValue("observationTimeStampRangeStart") Date observationTimeStampRangeStart, + @Nullable @QueryValue("observationTimeStampRangeEnd") Date observationTimeStampRangeEnd, + @Nullable @QueryValue("programDbId") String programDbId, + @Nullable @QueryValue("trialDbId") String trialDbId, + @Nullable @QueryValue("studyDbId") String studyDbId, + @Nullable @QueryValue("germplasmDbId") String germplasmDbId, + @Nullable @QueryValue("observationUnitLevelName") String observationUnitLevelName, + @Nullable @QueryValue("observationUnitLevelOrder") String observationUnitLevelOrder, + @Nullable @QueryValue("observationUnitLevelCode") String observationUnitLevelCode, + @Nullable @QueryValue("observationUnitLevelRelationshipName") String observationUnitLevelRelationshipName, + @Nullable @QueryValue("observationUnitLevelRelationshipOrder") String observationUnitLevelRelationshipOrder, + @Nullable @QueryValue("observationUnitLevelRelationshipCode") String observationUnitLevelRelationshipCode, + @Nullable @QueryValue("observationUnitLevelRelationshipDbId") String observationUnitLevelRelationshipDbId) { return HttpResponse.notFound(); } } diff --git a/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiObservationVariablesControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiV1ObservationVariablesControllerIntegrationTest.java similarity index 99% rename from src/test/java/org/breedinginsight/brapi/v1/controller/BrapiObservationVariablesControllerIntegrationTest.java rename to src/test/java/org/breedinginsight/brapi/v1/controller/BrapiV1ObservationVariablesControllerIntegrationTest.java index 4da980a4e..300fcd928 100644 --- a/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiObservationVariablesControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiV1ObservationVariablesControllerIntegrationTest.java @@ -59,7 +59,7 @@ @MicronautTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestMethodOrder(MethodOrderer.OrderAnnotation.class) -public class BrapiObservationVariablesControllerIntegrationTest extends BrAPITest { +public class BrapiV1ObservationVariablesControllerIntegrationTest extends BrAPITest { @Inject @Client(BrapiVersion.BRAPI_V1) diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableControllerIntegrationTest.java deleted file mode 100644 index 1596f26e7..000000000 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableControllerIntegrationTest.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * See the NOTICE file distributed with this work for additional information - * regarding copyright ownership. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.breedinginsight.brapi.v2; - -public class BrAPIObservationVariableControllerIntegrationTest { - //TODO -} diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java index cae4497e5..84175a6d0 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java @@ -17,6 +17,399 @@ package org.breedinginsight.brapi.v2; -public class BrAPIObservationsControllerIntegrationTest { - //TODO +import com.google.gson.*; +import io.kowalski.fannypack.FannyPack; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.client.exceptions.HttpClientResponseException; +import io.micronaut.http.netty.cookies.NettyCookie; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import io.reactivex.Flowable; +import lombok.SneakyThrows; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.v2.model.pheno.BrAPIObservationUnit; +import org.brapi.v2.model.pheno.response.BrAPIObservationUnitSingleResponse; +import org.breedinginsight.BrAPITest; +import org.breedinginsight.TestUtils; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramRequest; +import org.breedinginsight.api.model.v1.request.SpeciesRequest; +import org.breedinginsight.api.v1.controller.TestTokenValidator; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.ImportTestUtils; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.dao.db.enums.DataType; +import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.SpeciesDAO; +import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.model.*; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; +import org.breedinginsight.services.writers.CSVWriter; +import org.jooq.DSLContext; +import org.junit.jupiter.api.*; + +import javax.inject.Inject; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.*; +import java.util.concurrent.atomic.AtomicReference; + +import static io.micronaut.http.HttpRequest.*; +import static org.junit.jupiter.api.Assertions.*; + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class BrAPIObservationsControllerIntegrationTest extends BrAPITest { + + private Program program; + private String experimentId; + private List envIds = new ArrayList<>(); + private final List> rows = new ArrayList<>(); + private final List columns = ExperimentFileColumns.getOrderedColumns(); + private List traits; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + @Inject + private DSLContext dsl; + @Inject + private UserDAO userDAO; + @Inject + private SpeciesDAO speciesDAO; + @Inject + private OntologyService ontologyService; + @Inject + private BrAPIGermplasmDAO germplasmDAO; + + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + + private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) + (json, type, context) -> OffsetDateTime.parse(json.getAsString())) + .create(); + + @BeforeAll + void setup() throws Exception { + FannyPack fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); + ImportTestUtils importTestUtils = new ImportTestUtils(); + FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); + + // Test User + User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + // Species + super.getBrapiDsl().execute(brapiFp.get("InsertSpecies")); + SpeciesEntity validSpecies = speciesDAO.findAll().get(0); + SpeciesRequest speciesRequest = SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); + + // Test Program + ProgramRequest programRequest = ProgramRequest.builder() + .name("Test Program") + .abbreviation("Test") + .documentationUrl("localhost:8080") + .objective("To test things") + .species(speciesRequest) + .key("TEST") + .build(); + program = TestUtils.insertAndFetchTestProgram(gson, client, programRequest); + + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId()); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + // Get experiment import map + Flowable> call = client.exchange( + GET("/import/mappings?importName=ExperimentsTemplateMap") + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + String mappingId = JsonParser.parseString(Objects.requireNonNull(response.body())).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + // Add traits to program + traits = createTraits(2); + AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); + try { + ontologyService.createTraits(program.getId(), traits, user, false); + } catch (ValidatorException e) { + System.err.println(e.getErrors()); + throw e; + } + + // Add germplasm to program + List germplasm = createGermplasm(1); + BrAPIExternalReference newReference = new BrAPIExternalReference(); + newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); + newReference.setReferenceID(program.getId().toString()); + + germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); + + germplasmDAO.createBrAPIGermplasm(germplasm, program.getId(), null); + + // Make test experiment import + Map row1 = makeExpImportRow("Env1"); + Map row2 = makeExpImportRow("Env2"); + + // Add test observation data + for (Trait trait : traits) { + Random random = new Random(); + + // TODO: test for sending obs data as double. + // A float is returned from the backend instead of double. there is a separate card to fix this. + // Double val1 = Math.random(); + + Float val1 = random.nextFloat(); + row1.put(trait.getObservationVariableName(), val1); + } + + rows.add(row1); + rows.add(row2); + + // Import test experiment, environments, and any observations + JsonObject importResult = importTestUtils.uploadAndFetch( + writeDataToFile(rows, traits), + null, + true, + client, + program, + mappingId); + experimentId = importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + // Add environmentIds. + envIds.add(getEnvId(importResult, 0)); + envIds.add(getEnvId(importResult, 1)); + } + + @Test + @SneakyThrows + public void testPostObsNotFound() { + BrAPIObservation obs = new BrAPIObservation().observationUnitName("test"); + + Flowable> postCall = client.exchange( + POST(String.format("/programs/%s/brapi/v2/observations", + program.getId().toString()), List.of(obs)) + .contentType(MediaType.APPLICATION_JSON) + .bearerAuth("test-registered-user"), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = postCall.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + } + + @Test + @SneakyThrows + public void testPutSingleObsNotFound() { + BrAPIObservation obs = new BrAPIObservation().observationDbId(UUID.randomUUID().toString()).germplasmName("updated"); + + Flowable> postCall = client.exchange( + PUT(String.format("/programs/%s/brapi/v2/observations/%s", + program.getId().toString(), obs.getObservationUnitDbId()), obs) + .contentType(MediaType.APPLICATION_JSON) + .bearerAuth("test-registered-user"), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse errorResponse = postCall.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + } + + @Test + @SneakyThrows + public void testPutMultipleObsNotFound() { + BrAPIObservation obs = new BrAPIObservation().observationDbId(UUID.randomUUID().toString()).germplasmName("updated"); + + Flowable> postCall = client.exchange( + PUT(String.format("/programs/%s/brapi/v2/observations", + program.getId().toString()), Map.of(obs.getObservationDbId(), obs)) + .contentType(MediaType.APPLICATION_JSON) + .bearerAuth("test-registered-user"), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse errorResponse = postCall.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + } + + @Test + @SneakyThrows + public void testGetObsTableNotFound() { + Flowable> getCall = client.exchange( + GET(String.format("/programs/%s/brapi/v2/observations/table", + program.getId().toString())) + .contentType(MediaType.APPLICATION_JSON) + .bearerAuth("test-registered-user"), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = getCall.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + } + + @Test + @Disabled("Disabled until fetching of observations is implemented") + public void testGetObsListByExpId() { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/brapi/v2/observations?trialDbId=%s", program.getId(), experimentId)) + .bearerAuth("test-registered-user"), + String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + } + + @Test + @Disabled("Disabled until fetching of observations is implemented") + public void testGetOUById() { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/brapi/v2/observations?trialDbId=%s", program.getId(), experimentId)) + .bearerAuth("test-registered-user"), + String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); + JsonArray observations = responseObj.getAsJsonObject("result").getAsJsonArray("data"); + JsonObject obs = observations.get(0).getAsJsonObject(); + + Flowable> ouCall = client.exchange( + GET(String.format("/programs/%s/brapi/v2/observations/%s", program.getId(), obs.get("observationDbId").getAsString())) + .bearerAuth("test-registered-user"), + String.class + ); + + HttpResponse ouResponse = ouCall.blockingFirst(); + assertEquals(HttpStatus.OK, ouResponse.getStatus()); + } + + private File writeDataToFile(List> data, List traits) throws IOException { + File file = File.createTempFile("test", ".csv"); + + if(traits != null) { + traits.forEach(trait -> columns.add( + Column.builder() + .value(trait.getObservationVariableName()) + .dataType(Column.ColumnDataType.STRING) + .build()) + ); + } + ByteArrayOutputStream byteArrayOutputStream = CSVWriter.writeToCSV(columns, data); + FileOutputStream fos = new FileOutputStream(file); + fos.write(byteArrayOutputStream.toByteArray()); + + return file; + } + + private String getEnvId(JsonObject result, int index) { + return result + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(index).getAsJsonObject() + .get("study").getAsJsonObject() + .get("brAPIObject").getAsJsonObject() + .get("externalReferences").getAsJsonArray() + .get(2).getAsJsonObject() + .get("referenceID").getAsString(); + } + + private List createGermplasm(int numToCreate) { + List germplasm = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String gid = ""+(i+1); + BrAPIGermplasm testGermplasm = new BrAPIGermplasm(); + testGermplasm.setGermplasmName(String.format("Germplasm %s [TEST-%s]", gid, gid)); + testGermplasm.setSeedSource("Wild"); + testGermplasm.setAccessionNumber(gid); + testGermplasm.setDefaultDisplayName(String.format("Germplasm %s", gid)); + JsonObject additionalInfo = new JsonObject(); + additionalInfo.addProperty("importEntryNumber", gid); + additionalInfo.addProperty("breedingMethod", "Allopolyploid"); + testGermplasm.setAdditionalInfo(additionalInfo); + List externalRef = new ArrayList<>(); + BrAPIExternalReference testReference = new BrAPIExternalReference(); + testReference.setReferenceSource(BRAPI_REFERENCE_SOURCE); + testReference.setReferenceID(UUID.randomUUID().toString()); + externalRef.add(testReference); + testGermplasm.setExternalReferences(externalRef); + germplasm.add(testGermplasm); + } + + return germplasm; + } + + private List createTraits(int numToCreate) { + List traits = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String varName = "tt_test_" + (i + 1); + traits.add(Trait.builder() + .observationVariableName(varName) + .fullName(varName) + .entity("Plant " + i) + .attribute("height " + i) + .traitDescription("test") + .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build()) + .scale(Scale.builder() + .scaleName("test scale") + .dataType(DataType.NUMERICAL) + .validValueMin(0) + .validValueMax(100) + .build()) + .method(Method.builder() + .description("test method") + .methodClass("test method") + .build()) + .build()); + } + + return traits; + } + + private Map makeExpImportRow(String environment) { + Map row = new HashMap<>(); + row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); + row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); + row.put(ExperimentObservation.Columns.EXP_TITLE, "Test Exp"); + row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); + row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); + row.put(ExperimentObservation.Columns.ENV, environment); + row.put(ExperimentObservation.Columns.ENV_LOCATION, "Location A"); + row.put(ExperimentObservation.Columns.ENV_YEAR, "2023"); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, environment+" - a-1"); + row.put(ExperimentObservation.Columns.REP_NUM, "1"); + row.put(ExperimentObservation.Columns.BLOCK_NUM, "1"); + row.put(ExperimentObservation.Columns.ROW, "1"); + row.put(ExperimentObservation.Columns.COLUMN, "1"); + return row; + } } diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java new file mode 100644 index 000000000..0058d91f3 --- /dev/null +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java @@ -0,0 +1,420 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2; + +import com.google.gson.*; +import io.kowalski.fannypack.FannyPack; +import io.micronaut.context.annotation.Property; +import io.micronaut.http.HttpResponse; +import io.micronaut.http.HttpStatus; +import io.micronaut.http.MediaType; +import io.micronaut.http.client.RxHttpClient; +import io.micronaut.http.client.annotation.Client; +import io.micronaut.http.client.exceptions.HttpClientResponseException; +import io.micronaut.http.netty.cookies.NettyCookie; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import io.reactivex.Flowable; +import lombok.SneakyThrows; +import org.brapi.v2.model.BrAPIExternalReference; +import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationVariable; +import org.breedinginsight.BrAPITest; +import org.breedinginsight.TestUtils; +import org.breedinginsight.api.auth.AuthenticatedUser; +import org.breedinginsight.api.model.v1.request.ProgramRequest; +import org.breedinginsight.api.model.v1.request.SpeciesRequest; +import org.breedinginsight.api.v1.controller.TestTokenValidator; +import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapps.importer.ImportTestUtils; +import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; +import org.breedinginsight.dao.db.enums.DataType; +import org.breedinginsight.dao.db.tables.pojos.SpeciesEntity; +import org.breedinginsight.daos.SpeciesDAO; +import org.breedinginsight.daos.UserDAO; +import org.breedinginsight.model.*; +import org.breedinginsight.services.OntologyService; +import org.breedinginsight.services.exceptions.ValidatorException; +import org.breedinginsight.services.parsers.experiment.ExperimentFileColumns; +import org.breedinginsight.services.writers.CSVWriter; +import org.jooq.DSLContext; +import org.junit.jupiter.api.*; + +import javax.inject.Inject; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.util.*; + +import static io.micronaut.http.HttpRequest.*; +import static io.micronaut.http.HttpRequest.GET; +import static org.junit.jupiter.api.Assertions.*; + +@MicronautTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class BrAPIV2ObservationVariableControllerIntegrationTest extends BrAPITest { + private Program program; + private String experimentId; + private List envIds = new ArrayList<>(); + private final List> rows = new ArrayList<>(); + private final List columns = ExperimentFileColumns.getOrderedColumns(); + private List traits; + + @Property(name = "brapi.server.reference-source") + private String BRAPI_REFERENCE_SOURCE; + @Inject + private DSLContext dsl; + @Inject + private UserDAO userDAO; + @Inject + private SpeciesDAO speciesDAO; + @Inject + private OntologyService ontologyService; + @Inject + private BrAPIGermplasmDAO germplasmDAO; + + @Inject + @Client("/${micronaut.bi.api.version}") + private RxHttpClient client; + + private final Gson gson = new GsonBuilder().registerTypeAdapter(OffsetDateTime.class, (JsonDeserializer) + (json, type, context) -> OffsetDateTime.parse(json.getAsString())) + .create(); + + @BeforeAll + void setup() throws Exception { + FannyPack fp = FannyPack.fill("src/test/resources/sql/ImportControllerIntegrationTest.sql"); + ImportTestUtils importTestUtils = new ImportTestUtils(); + FannyPack securityFp = FannyPack.fill("src/test/resources/sql/ProgramSecuredAnnotationRuleIntegrationTest.sql"); + FannyPack brapiFp = FannyPack.fill("src/test/resources/sql/brapi/species.sql"); + + // Test User + User testUser = userDAO.getUserByOrcId(TestTokenValidator.TEST_USER_ORCID).orElseThrow(Exception::new); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + // Species + super.getBrapiDsl().execute(brapiFp.get("InsertSpecies")); + SpeciesEntity validSpecies = speciesDAO.findAll().get(0); + SpeciesRequest speciesRequest = SpeciesRequest.builder() + .commonName(validSpecies.getCommonName()) + .id(validSpecies.getId()) + .build(); + + // Test Program + ProgramRequest programRequest = ProgramRequest.builder() + .name("Test Program") + .abbreviation("Test") + .documentationUrl("localhost:8080") + .objective("To test things") + .species(speciesRequest) + .key("TEST") + .build(); + program = TestUtils.insertAndFetchTestProgram(gson, client, programRequest); + + dsl.execute(securityFp.get("InsertProgramRolesBreeder"), testUser.getId().toString(), program.getId()); + dsl.execute(securityFp.get("InsertSystemRoleAdmin"), testUser.getId().toString()); + + // Get experiment import map + Flowable> call = client.exchange( + GET("/import/mappings?importName=ExperimentsTemplateMap") + .contentType(MediaType.APPLICATION_JSON) + .cookie(new NettyCookie("phylo-token", "test-registered-user")), String.class + ); + HttpResponse response = call.blockingFirst(); + String mappingId = JsonParser.parseString(Objects.requireNonNull(response.body())).getAsJsonObject() + .getAsJsonObject("result") + .getAsJsonArray("data") + .get(0).getAsJsonObject().get("id").getAsString(); + + // Add traits to program + traits = createTraits(4); + AuthenticatedUser user = new AuthenticatedUser(testUser.getName(), new ArrayList<>(), testUser.getId(), new ArrayList<>()); + try { + ontologyService.createTraits(program.getId(), traits, user, false); + } catch (ValidatorException e) { + System.err.println(e.getErrors()); + throw e; + } + + // Add germplasm to program + List germplasm = createGermplasm(1); + BrAPIExternalReference newReference = new BrAPIExternalReference(); + newReference.setReferenceSource(String.format("%s/programs", BRAPI_REFERENCE_SOURCE)); + newReference.setReferenceID(program.getId().toString()); + + germplasm.forEach(germ -> germ.getExternalReferences().add(newReference)); + + germplasmDAO.createBrAPIGermplasm(germplasm, program.getId(), null); + + // Make test experiment import + Map row1 = makeExpImportRow("Env1"); + Map row2 = makeExpImportRow("Env2"); + + // Add test observation data + List expTraits = List.of(traits.get(0), traits.get(1)); + for (int i = 0; i < 2; i++) { + var trait = expTraits.get(i); + Random random = new Random(); + + // TODO: test for sending obs data as double. + // A float is returned from the backend instead of double. there is a separate card to fix this. + // Double val1 = Math.random(); + + Float val1 = random.nextFloat(); + row1.put(trait.getObservationVariableName(), val1); + } + + rows.add(row1); + rows.add(row2); + + // Import test experiment, environments, and any observations + JsonObject importResult = importTestUtils.uploadAndFetch( + writeDataToFile(rows, expTraits), + null, + true, + client, + program, + mappingId); + experimentId = importResult + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(0).getAsJsonObject() + .get("trial").getAsJsonObject() + .get("id").getAsString(); + // Add environmentIds. + envIds.add(getEnvId(importResult, 0)); + envIds.add(getEnvId(importResult, 1)); + } + + @Test + @SneakyThrows + public void testPostVariablesNotFound() { + BrAPIObservationVariable variable = new BrAPIObservationVariable().observationVariableName(traits.get(0).getObservationVariableName()+" post"); + + Flowable> postCall = client.exchange( + POST(String.format("/programs/%s/brapi/v2/variables", + program.getId().toString()), List.of(variable)) + .contentType(MediaType.APPLICATION_JSON) + .bearerAuth("test-registered-user"), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = postCall.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + } + + @Test + @SneakyThrows + public void testPutVariablesNotFound() { + BrAPIObservationVariable variable = new BrAPIObservationVariable().observationVariableName(traits.get(0).getObservationVariableName()+" put"); + + Flowable> putCall = client.exchange( + PUT(String.format("/programs/%s/brapi/v2/variables/%s", + program.getId().toString(), traits.get(0).getId()), List.of(variable)) + .contentType(MediaType.APPLICATION_JSON) + .bearerAuth("test-registered-user"), String.class + ); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = putCall.blockingFirst(); + }); + assertEquals(HttpStatus.NOT_FOUND, e.getStatus()); + } + + @Test + public void testGetVariables() { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/brapi/v2/variables", program.getId(), experimentId)) + .bearerAuth("test-registered-user"), + String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); + JsonArray variables = responseObj.getAsJsonObject("result").getAsJsonArray("data"); + assertEquals(4, variables.size()); + List variableNames = new ArrayList<>(); + for(var variable : variables) { + variableNames.add(variable.getAsJsonObject().get("observationVariableName").getAsString()); + } + assertTrue(variableNames.contains("tt_test_1")); + assertTrue(variableNames.contains("tt_test_2")); + assertTrue(variableNames.contains("tt_test_3")); + assertTrue(variableNames.contains("tt_test_4")); + } + + @Test + public void testGetVariablesByExpId() { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/brapi/v2/variables?trialDbId=%s", program.getId(), experimentId)) + .bearerAuth("test-registered-user"), + String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); + JsonArray variables = responseObj.getAsJsonObject("result").getAsJsonArray("data"); + assertEquals(2, variables.size()); + List variableNames = new ArrayList<>(); + for(var variable : variables) { + variableNames.add(variable.getAsJsonObject().get("observationVariableName").getAsString()); + } + assertTrue(variableNames.contains("tt_test_1")); + assertTrue(variableNames.contains("tt_test_2")); + } + + @Test + public void testGetVariableById() { + Flowable> call = client.exchange( + GET(String.format("/programs/%s/brapi/v2/variables/%s", program.getId(), traits.get(0).getId())) + .bearerAuth("test-registered-user"), + String.class + ); + + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.OK, response.getStatus()); + + JsonObject responseObj = gson.fromJson(response.body(), JsonObject.class); + JsonObject variable = responseObj.getAsJsonObject("result"); + assertNotNull(variable); + assertEquals(traits.get(0).getId().toString(), variable.get("observationVariableDbId").getAsString()); + assertEquals(traits.get(0).getObservationVariableName(), variable.get("observationVariableName").getAsString()); + + JsonObject trait = variable.getAsJsonObject("trait"); + assertEquals(traits.get(0).getEntity(), trait.get("entity").getAsString()); + assertEquals(traits.get(0).getAttribute(), trait.get("attribute").getAsString()); + + JsonObject method = variable.getAsJsonObject("method"); + assertEquals(traits.get(0).getMethod().getMethodClass(), + method.get("methodClass").getAsString()); + assertEquals(traits.get(0).getMethod().getDescription(), method.get("description").getAsString()); + + JsonObject scale = variable.getAsJsonObject("scale"); + assertEquals(traits.get(0).getScale().getScaleName(), scale.get("scaleName").getAsString()); + assertEquals(traits.get(0).getScale().getDataType().getLiteral().toLowerCase(), scale.get("dataType").getAsString().toLowerCase()); + assertEquals(traits.get(0).getScale().getValidValueMin(), scale.getAsJsonObject("validValues").get("min").getAsInt()); + assertEquals(traits.get(0).getScale().getValidValueMax(), scale.getAsJsonObject("validValues").get("max").getAsInt()); + } + + private File writeDataToFile(List> data, List traits) throws IOException { + File file = File.createTempFile("test", ".csv"); + + if(traits != null) { + traits.forEach(trait -> columns.add( + Column.builder() + .value(trait.getObservationVariableName()) + .dataType(Column.ColumnDataType.STRING) + .build()) + ); + } + ByteArrayOutputStream byteArrayOutputStream = CSVWriter.writeToCSV(columns, data); + FileOutputStream fos = new FileOutputStream(file); + fos.write(byteArrayOutputStream.toByteArray()); + + return file; + } + + private String getEnvId(JsonObject result, int index) { + return result + .get("preview").getAsJsonObject() + .get("rows").getAsJsonArray() + .get(index).getAsJsonObject() + .get("study").getAsJsonObject() + .get("brAPIObject").getAsJsonObject() + .get("externalReferences").getAsJsonArray() + .get(2).getAsJsonObject() + .get("referenceID").getAsString(); + } + + private List createGermplasm(int numToCreate) { + List germplasm = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String gid = ""+(i+1); + BrAPIGermplasm testGermplasm = new BrAPIGermplasm(); + testGermplasm.setGermplasmName(String.format("Germplasm %s [TEST-%s]", gid, gid)); + testGermplasm.setSeedSource("Wild"); + testGermplasm.setAccessionNumber(gid); + testGermplasm.setDefaultDisplayName(String.format("Germplasm %s", gid)); + JsonObject additionalInfo = new JsonObject(); + additionalInfo.addProperty("importEntryNumber", gid); + additionalInfo.addProperty("breedingMethod", "Allopolyploid"); + testGermplasm.setAdditionalInfo(additionalInfo); + List externalRef = new ArrayList<>(); + BrAPIExternalReference testReference = new BrAPIExternalReference(); + testReference.setReferenceSource(BRAPI_REFERENCE_SOURCE); + testReference.setReferenceID(UUID.randomUUID().toString()); + externalRef.add(testReference); + testGermplasm.setExternalReferences(externalRef); + germplasm.add(testGermplasm); + } + + return germplasm; + } + + private List createTraits(int numToCreate) { + List traits = new ArrayList<>(); + for (int i = 0; i < numToCreate; i++) { + String varName = "tt_test_" + (i + 1); + traits.add(Trait.builder() + .observationVariableName(varName) + .fullName(varName) + .entity("Plant " + i) + .attribute("height " + i) + .traitDescription("test") + .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build()) + .scale(Scale.builder() + .scaleName("test scale") + .dataType(DataType.NUMERICAL) + .validValueMin(0) + .validValueMax(100) + .build()) + .method(Method.builder() + .description("test method") + .methodClass("test method") + .build()) + .build()); + } + + return traits; + } + + private Map makeExpImportRow(String environment) { + Map row = new HashMap<>(); + row.put(ExperimentObservation.Columns.GERMPLASM_GID, "1"); + row.put(ExperimentObservation.Columns.TEST_CHECK, "T"); + row.put(ExperimentObservation.Columns.EXP_TITLE, "Test Exp"); + row.put(ExperimentObservation.Columns.EXP_UNIT, "Plot"); + row.put(ExperimentObservation.Columns.EXP_TYPE, "Phenotyping"); + row.put(ExperimentObservation.Columns.ENV, environment); + row.put(ExperimentObservation.Columns.ENV_LOCATION, "Location A"); + row.put(ExperimentObservation.Columns.ENV_YEAR, "2023"); + row.put(ExperimentObservation.Columns.EXP_UNIT_ID, "a-1"); + row.put(ExperimentObservation.Columns.REP_NUM, "1"); + row.put(ExperimentObservation.Columns.BLOCK_NUM, "1"); + row.put(ExperimentObservation.Columns.ROW, "1"); + row.put(ExperimentObservation.Columns.COLUMN, "1"); + return row; + } +} From 2769cf35547e8d1a943787132918447b7062ec97 Mon Sep 17 00:00:00 2001 From: timparsons Date: Thu, 30 Nov 2023 11:39:23 -0500 Subject: [PATCH 144/220] [BI-1921] cleaning up imports --- .../breedinginsight/brapi/v2/BrAPITrialsController.java | 6 +++--- .../org/breedinginsight/brapi/v2/BrAPIV2Controller.java | 3 ++- .../brapi/v2/dao/BrAPIObservationUnitDAO.java | 4 ++-- .../v2/BrAPIObservationUnitControllerIntegrationTest.java | 1 - .../v2/BrAPIObservationsControllerIntegrationTest.java | 5 +---- .../brapi/v2/BrAPIStudiesControllerIntegrationTest.java | 1 - .../brapi/v2/BrAPIV2ControllerIntegrationTest.java | 8 ++------ ...APIV2ObservationVariableControllerIntegrationTest.java | 1 - .../brapi/v2/GermplasmControllerIntegrationTest.java | 5 ++--- 9 files changed, 12 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java index 3966b1341..666115efa 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPITrialsController.java @@ -9,7 +9,6 @@ import io.micronaut.security.rules.SecurityRule; import lombok.extern.slf4j.Slf4j; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPITrialSingleResponse; import org.breedinginsight.api.auth.ProgramSecured; @@ -22,14 +21,15 @@ import org.breedinginsight.brapi.v2.model.request.query.ExperimentQuery; import org.breedinginsight.brapi.v2.services.BrAPITrialService; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.Utilities; import org.breedinginsight.utilities.response.ResponseUtils; import org.breedinginsight.utilities.response.mappers.ExperimentQueryMapper; + import javax.inject.Inject; import javax.validation.Valid; -import java.util.*; +import java.util.List; +import java.util.UUID; import java.util.stream.Collectors; @Slf4j diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java index 3a485bd9a..534efbe47 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java @@ -41,7 +41,8 @@ import javax.inject.Inject; import java.io.IOException; -import java.util.*; +import java.util.List; +import java.util.UUID; import java.util.stream.Collectors; @Slf4j diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java index ac153d893..40833dc3e 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java @@ -22,18 +22,18 @@ import com.google.gson.JsonObject; import com.google.gson.reflect.TypeToken; import io.micronaut.context.annotation.Property; +import io.micronaut.http.server.exceptions.InternalServerException; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.JSON; -import io.micronaut.http.server.exceptions.InternalServerException; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; import org.brapi.v2.model.germ.BrAPIGermplasm; +import org.brapi.v2.model.pheno.BrAPIObservationTreatment; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.brapi.v2.model.pheno.BrAPIObservationUnitLevelRelationship; import org.brapi.v2.model.pheno.request.BrAPIObservationUnitSearchRequest; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService; -import org.brapi.v2.model.pheno.BrAPIObservationTreatment; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java index 4111e58ed..a45dbb6f6 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java @@ -65,7 +65,6 @@ import java.util.concurrent.atomic.AtomicReference; import static io.micronaut.http.HttpRequest.*; -import static io.micronaut.http.HttpRequest.GET; import static org.junit.jupiter.api.Assertions.*; @MicronautTest diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java index 84175a6d0..4a0a78526 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java @@ -33,8 +33,6 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservation; -import org.brapi.v2.model.pheno.BrAPIObservationUnit; -import org.brapi.v2.model.pheno.response.BrAPIObservationUnitSingleResponse; import org.breedinginsight.BrAPITest; import org.breedinginsight.TestUtils; import org.breedinginsight.api.auth.AuthenticatedUser; @@ -63,10 +61,9 @@ import java.io.IOException; import java.time.OffsetDateTime; import java.util.*; -import java.util.concurrent.atomic.AtomicReference; import static io.micronaut.http.HttpRequest.*; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; @MicronautTest @TestInstance(TestInstance.Lifecycle.PER_CLASS) diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java index 1373d9ab3..955c6e619 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java @@ -66,7 +66,6 @@ import java.util.concurrent.atomic.AtomicReference; import static io.micronaut.http.HttpRequest.*; -import static io.micronaut.http.HttpRequest.GET; import static org.junit.jupiter.api.Assertions.*; @MicronautTest diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java index 0ef73d284..ee5fbbfd7 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java @@ -33,12 +33,7 @@ import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.BrAPIPagination; import org.brapi.v2.model.core.BrAPIServerInfo; -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.response.BrAPIStudyListResponse; -import org.brapi.v2.model.core.response.BrAPIStudySingleResponse; import org.brapi.v2.model.pheno.*; -import org.brapi.v2.model.pheno.response.BrAPIObservationVariableListResponse; -import org.brapi.v2.model.pheno.response.BrAPIObservationVariableSingleResponse; import org.breedinginsight.BrAPITest; import org.breedinginsight.api.v1.controller.TestTokenValidator; import org.breedinginsight.dao.db.tables.daos.ProgramDao; @@ -54,7 +49,8 @@ import java.util.Collections; import java.util.UUID; -import static io.micronaut.http.HttpRequest.*; +import static io.micronaut.http.HttpRequest.GET; +import static io.micronaut.http.HttpRequest.POST; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java index 0058d91f3..d54aaafc1 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java @@ -63,7 +63,6 @@ import java.util.*; import static io.micronaut.http.HttpRequest.*; -import static io.micronaut.http.HttpRequest.GET; import static org.junit.jupiter.api.Assertions.*; @MicronautTest diff --git a/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java index fb95e4dd2..4fdf0cce3 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/GermplasmControllerIntegrationTest.java @@ -18,19 +18,18 @@ import org.breedinginsight.api.model.v1.request.query.FilterRequest; import org.breedinginsight.api.model.v1.request.query.SearchRequest; import org.breedinginsight.api.v1.controller.TestTokenValidator; -import org.breedinginsight.utilities.response.mappers.GermplasmQueryMapper; -import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService; import org.breedinginsight.brapi.v2.dao.BrAPIListDAO; +import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService; import org.breedinginsight.dao.db.tables.pojos.BiUserEntity; import org.breedinginsight.daos.UserDAO; import org.breedinginsight.model.Program; import org.breedinginsight.model.Species; import org.breedinginsight.services.SpeciesService; +import org.breedinginsight.utilities.response.mappers.GermplasmQueryMapper; import org.jooq.DSLContext; import org.junit.jupiter.api.*; import javax.inject.Inject; - import java.io.File; import java.time.OffsetDateTime; import java.util.ArrayList; From 5968eca4ce78826e33050d228dcede8b099940c7 Mon Sep 17 00:00:00 2001 From: timparsons Date: Fri, 1 Dec 2023 09:33:05 -0500 Subject: [PATCH 145/220] [BI-1921] fixing compilation errors --- .../api/v1/controller/ExperimentController.java | 1 + .../services/processors/SampleSubmissionProcessor.java | 4 +--- .../brapps/importer/SampleSubmissionFileImportTest.java | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java index 7c64827fe..017197219 100644 --- a/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java +++ b/src/main/java/org/breedinginsight/api/v1/controller/ExperimentController.java @@ -23,6 +23,7 @@ import javax.inject.Inject; import javax.validation.Valid; +import java.util.Optional; import java.util.UUID; @Slf4j diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java index d92ebb300..396bf8658 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java @@ -31,9 +31,7 @@ import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.api.model.v1.response.ValidationErrors; import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; -import org.breedinginsight.brapps.importer.daos.BrAPIObservationUnitDAO; -import org.breedinginsight.brapps.importer.daos.BrAPIPlateDAO; -import org.breedinginsight.brapps.importer.daos.BrAPISampleDAO; +import org.breedinginsight.brapi.v2.dao.BrAPIObservationUnitDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.PendingImport; diff --git a/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java index aa446cc55..8c5b308c3 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java @@ -43,7 +43,7 @@ import org.breedinginsight.api.model.v1.request.ProgramRequest; import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapi.v2.dao.BrAPIGermplasmDAO; +import org.breedinginsight.brapi.v2.dao.*; import org.breedinginsight.brapps.importer.daos.*; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport.Columns; From 16e1f2b069791587f4bb0640a03841d0a54e33e5 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:18:35 -0500 Subject: [PATCH 146/220] prepend serverinfo path with biapi version number --- .../java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java index 534efbe47..1e0dfba83 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java @@ -60,7 +60,7 @@ public BrAPIV2Controller(SecurityService securityService, ProgramService program } - @Get(BrapiVersion.BRAPI_V2 + "/serverinfo") + @Get("/${micronaut.bi.api.version}/" + BrapiVersion.BRAPI_V2 + "/serverinfo") @Produces(MediaType.APPLICATION_JSON) @Secured(SecurityRule.IS_ANONYMOUS) public BrAPIServerInfoResponse serverinfo() { @@ -70,8 +70,7 @@ public BrAPIServerInfoResponse serverinfo() { serverInfo.setCalls( new ServiceBuilder().versions("2.0", "2.1") - .setBase("serverinfo").GET().build() - .setBase("programs").GET().addPath("{programDbId}").GET().build() + .setBase("serverinfo").GET().build() ); return new BrAPIServerInfoResponse().result(serverInfo); From 5ce6f19322341b64b1d2f26a98d87bbc99c880b2 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:27:45 -0500 Subject: [PATCH 147/220] remove redundant query variables --- .../brapi/v2/BrAPIObservationUnitController.java | 1 - .../brapi/v2/BrAPIObservationVariableController.java | 1 - .../java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java index 29ce0cef5..c649626b1 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java @@ -82,7 +82,6 @@ public HttpResponse observationunitsGet(@PathV @Nullable @QueryValue("observationUnitLevelRelationshipCode") String observationUnitLevelRelationshipCode, @Nullable @QueryValue("observationUnitLevelRelationshipDbId") String observationUnitLevelRelationshipDbId, @Nullable @QueryValue("commonCropName") String commonCropName, - @Nullable @QueryValue("programDbId") String programDbId, @Nullable @QueryValue("trialDbId") String trialDbId, @Nullable @QueryValue("studyDbId") String studyDbId, @Nullable @QueryValue("germplasmDbId") String germplasmDbId, diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java index 6a748978f..7ec204bf5 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java @@ -101,7 +101,6 @@ public HttpResponse variablesGet(@PathVari @Nullable @QueryValue("traitPUI") String traitPUI, @Nullable @QueryValue("ontologyDbId") String ontologyDbId, @Nullable @QueryValue("commonCropName") String commonCropName, -// @QueryValue("programDbId") Optional<@Nullable String> programDbId, //this is redundant @Nullable @QueryValue("trialDbId") String experimentId, @Nullable @QueryValue("studyDbId") String environmentId, @Nullable @QueryValue("externalReferenceID") String externalReferenceID, diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java index 1e0dfba83..0d4a5449d 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java @@ -70,7 +70,7 @@ public BrAPIServerInfoResponse serverinfo() { serverInfo.setCalls( new ServiceBuilder().versions("2.0", "2.1") - .setBase("serverinfo").GET().build() + .setBase("serverinfo").GET().build() ); return new BrAPIServerInfoResponse().result(serverInfo); From 9f9b1426951119bc5ed9714d92a2cbb85a67df9b Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 22 Jan 2024 17:03:57 -0500 Subject: [PATCH 148/220] move helper methods to trait service --- .../BrAPIObservationVariableController.java | 195 +---------------- .../daos/impl/AbstractDAO.java | 2 +- .../services/TraitService.java | 198 +++++++++++++++++- 3 files changed, 205 insertions(+), 190 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java index 7ec204bf5..f7e9368f4 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java @@ -25,37 +25,30 @@ import io.micronaut.security.annotation.Secured; import io.micronaut.security.rules.SecurityRule; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; import org.brapi.client.v2.model.exceptions.ApiException; import org.brapi.v2.model.BrAPIIndexPagination; import org.brapi.v2.model.BrAPIMetadata; -import org.brapi.v2.model.BrAPIOntologyReference; -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.pheno.*; +import org.brapi.v2.model.pheno.BrAPIObservationVariable; import org.brapi.v2.model.pheno.response.BrAPIObservationVariableListResponse; import org.brapi.v2.model.pheno.response.BrAPIObservationVariableListResponseResult; import org.brapi.v2.model.pheno.response.BrAPIObservationVariableSingleResponse; import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; import org.breedinginsight.brapi.v1.controller.BrapiVersion; -import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.services.BrAPITrialService; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; -import org.breedinginsight.model.Program; import org.breedinginsight.model.Trait; import org.breedinginsight.services.OntologyService; import org.breedinginsight.services.ProgramService; import org.breedinginsight.services.TraitService; import org.breedinginsight.services.exceptions.DoesNotExistException; import org.breedinginsight.utilities.Utilities; -import org.jetbrains.annotations.NotNull; import javax.annotation.Nullable; import javax.inject.Inject; -import java.util.*; -import java.util.stream.Collectors; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.UUID; @Slf4j @Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2) @@ -115,14 +108,15 @@ public HttpResponse variablesGet(@PathVari log.debug("unsupported variable filters, returning"); programTraits = new ArrayList<>(); } else if(environmentId != null || experimentId != null) { - programTraits = getBrAPIObservationVariablesForExperiment(programId, Optional.ofNullable(experimentId), Optional.ofNullable(environmentId)); + programTraits = traitService.getBrAPIObservationVariablesForExperiment( + programId, Optional.ofNullable(experimentId), Optional.ofNullable(environmentId)); } else { log.debug("fetching variables for the program: " + programId); programTraits = ontologyService.getTraitsByProgramId(programId, true); } - List filteredObsVars = filterVariables(programTraits, + List filteredObsVars = traitService.filterVariables(programTraits, Optional.ofNullable(observationVariableDbId), Optional.ofNullable(observationVariableName), Optional.ofNullable(traitClass), @@ -174,7 +168,7 @@ public HttpResponse variablesObservation BrAPIObservationVariableSingleResponse response = new BrAPIObservationVariableSingleResponse() .metadata(new BrAPIMetadata()) - .result(convertToBrAPI(trait.get())); + .result(traitService.convertToBrAPI(trait.get())); return HttpResponse.ok(response); } catch (DoesNotExistException e) { @@ -198,175 +192,4 @@ public HttpResponse variablesPost(@PathVariable("programId") UUID programId, return HttpResponse.notFound(); } - private BrAPIObservationVariable convertToBrAPI(Trait trait) { - BrAPIOntologyReference brAPIOntologyReference = new BrAPIOntologyReference().ontologyDbId(trait.getProgramOntologyId() - .toString()); - String status = trait.getActive() ? "active" : "inactive"; - List synonyms = prepSynonyms(trait); - return new BrAPIObservationVariable().observationVariableDbId(trait.getId().toString()) - .observationVariableName(trait.getObservationVariableName()) - .defaultValue(trait.getDefaultValue()) - .status(status) - .synonyms(synonyms) - .trait(new BrAPITrait().ontologyReference(brAPIOntologyReference) - .traitName(trait.getObservationVariableName()) - .traitDbId(trait.getId().toString()) - .entity(trait.getEntity()) - .attribute(trait.getAttribute()) - .status(status) - .synonyms(synonyms)) - .method(new BrAPIMethod().ontologyReference(brAPIOntologyReference) - .methodDbId(trait.getMethod().getId().toString()) - .methodClass(trait.getMethod().getMethodClass()) - .description(trait.getMethod().getDescription()) - .formula(trait.getMethod().getFormula())) - .scale(new BrAPIScale().ontologyReference(brAPIOntologyReference) - .scaleDbId(trait.getScale().getId().toString()) - .scaleName(trait.getScale().getScaleName()) - .dataType(BrAPITraitDataType.fromValue(trait.getScale().getDataType().getLiteral())) - .decimalPlaces(trait.getScale().getDecimalPlaces()) - .validValues(new BrAPIScaleValidValues().max(trait.getScale().getValidValueMax()) - .min(trait.getScale().getValidValueMin()) - .categories(trait.getScale().getCategories()))); - } - - /** - * Create a list of synonyms, and ensure that there the first element matches the name of the trait

- * This is primarily needed to ensure any system using the first synonym as a display shows the actual name of the ontology term - * @param trait - * @return list of synonyms with at least one value (the name of the trait) - */ - private List prepSynonyms(Trait trait) { - List preppedSynonyms = new ArrayList<>(); - if(trait.getSynonyms() != null) { - preppedSynonyms = trait.getSynonyms(); - int traitNameIdx = -1; - for(int i = 0; i < preppedSynonyms.size(); i++) { - if(preppedSynonyms.get(i).equals(trait.getObservationVariableName())) { - traitNameIdx = i; - break; - } - } - if(traitNameIdx > -1) { - String temp = preppedSynonyms.get(traitNameIdx); - preppedSynonyms.set(traitNameIdx, preppedSynonyms.get(0)); - preppedSynonyms.set(0, temp); - } else { - preppedSynonyms.add(0, trait.getObservationVariableName()); - } - } else { - preppedSynonyms.add(trait.getObservationVariableName()); - } - return preppedSynonyms; - } - - @NotNull - private List filterVariables(List programTraits, - Optional observationVariableDbId, - Optional observationVariableName, - Optional traitClass, - Optional methodDbId, - Optional methodName, - Optional scaleDbId, - Optional scaleName, - Optional traitDbId, - Optional traitName, - Optional ontologyDbId) throws DoesNotExistException { - log.debug("filtering variables:\n" + - "observationVariableDbId: " + observationVariableDbId + "\n" + - "observationVariableName: " + observationVariableName + "\n" + - "traitClass: " + traitClass + "\n" + - "methodDbId: " + methodDbId + "\n" + - "methodName: " + methodName + "\n" + - "scaleDbId: " + scaleDbId + "\n" + - "scaleName: " + scaleName + "\n" + - "traitDbId: " + traitDbId + "\n" + - "traitName: " + traitName + "\n" + - "ontologyDbId: " + ontologyDbId); - - return programTraits.stream() - .filter(trait -> { - boolean matches = true; - - Map, String>> filterParams = new HashMap<>(); - filterParams.put("observationVariableDbId", - Pair.of(observationVariableDbId, - trait.getId() - .toString())); - filterParams.put("observationVariableName", Pair.of(observationVariableName, trait.getObservationVariableName())); - filterParams.put("traitClass", Pair.of(traitClass, trait.getTraitClass())); - filterParams.put("methodDbId", - Pair.of(methodDbId, - trait.getMethodId() - .toString())); - filterParams.put("methodName", - Pair.of(methodName, - trait.getMethod() - .getDescription())); - filterParams.put("scaleDbId", - Pair.of(scaleDbId, - trait.getScale() - .getId() - .toString())); - filterParams.put("scaleName", - Pair.of(scaleName, - trait.getScale() - .getScaleName())); - filterParams.put("traitDbId", - Pair.of(traitDbId, - trait.getId() - .toString())); - filterParams.put("traitName", Pair.of(traitName, trait.getObservationVariableName())); - filterParams.put("ontologyDbId", - Pair.of(ontologyDbId, - trait.getProgramOntologyId() - .toString())); - - for (Map.Entry, String>> filter : filterParams.entrySet()) { - if (filter.getValue().getLeft().isPresent()) { - log.debug("filtering traits by: " + filter.getKey()); - matches = StringUtils.equals(filter.getValue() - .getLeft() - .get(), - filter.getValue() - .getRight()); - } - if (!matches) { - break; - } - } - - return matches; - }) - .map(this::convertToBrAPI) - .collect(Collectors.toList()); - } - - private List getBrAPIObservationVariablesForExperiment(UUID programId, Optional experimentId, Optional environmentId) throws DoesNotExistException, ApiException { - log.debug(String.format("fetching variables for experiment. expId: %s, envId: %s ", experimentId.orElse(""), environmentId.orElse(""))); - Optional program = programService.getById(programId); - if(program.isEmpty()) { - throw new DoesNotExistException("Could not find program: " + programId); - } - UUID expId; - if(experimentId.isPresent()) { - expId = UUID.fromString(experimentId.get()); - } else { - UUID envId = UUID.fromString(environmentId.get()); - BrAPIStudy environment = trialService.getEnvironment(program.get(), envId); - expId = UUID.fromString(Utilities.getExternalReference(environment.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)).get().getReferenceID()); - } - - BrAPITrial experiment = trialService.getExperiment(program.get(), expId); - if(experiment - .getAdditionalInfo().getAsJsonObject() - .has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { - String obsDatasetId = experiment - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); - return trialService.getDatasetObsVars(obsDatasetId, program.get()); - } - - return new ArrayList<>(); - } } diff --git a/src/main/java/org/breedinginsight/daos/impl/AbstractDAO.java b/src/main/java/org/breedinginsight/daos/impl/AbstractDAO.java index 138cf73c4..50aee2608 100644 --- a/src/main/java/org/breedinginsight/daos/impl/AbstractDAO.java +++ b/src/main/java/org/breedinginsight/daos/impl/AbstractDAO.java @@ -161,7 +161,7 @@ public long count() throws DataAccessException { } @Override - public @Nullable P fetchOne(Field field, Z value) throws DataAccessException { + public P fetchOne(Field field, Z value) throws DataAccessException { return jooqDAO.fetchOne(field, value); } diff --git a/src/main/java/org/breedinginsight/services/TraitService.java b/src/main/java/org/breedinginsight/services/TraitService.java index b4fcaa060..d8db6a98e 100644 --- a/src/main/java/org/breedinginsight/services/TraitService.java +++ b/src/main/java/org/breedinginsight/services/TraitService.java @@ -22,10 +22,19 @@ import io.micronaut.http.server.exceptions.HttpServerException; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.text.WordUtils; -import org.brapi.v2.model.pheno.BrAPIObservation; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIOntologyReference; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.*; import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.api.model.v1.response.ValidationErrors; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.services.BrAPITrialService; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.dao.db.tables.pojos.MethodEntity; import org.breedinginsight.dao.db.tables.pojos.ProgramSharedOntologyEntity; @@ -37,6 +46,8 @@ import org.breedinginsight.services.exceptions.ValidatorException; import org.breedinginsight.services.validators.TraitValidatorError; import org.breedinginsight.services.validators.TraitValidatorService; +import org.breedinginsight.utilities.Utilities; +import org.jetbrains.annotations.NotNull; import org.jooq.DSLContext; import javax.inject.Inject; @@ -60,14 +71,15 @@ public class TraitService { private TraitValidatorService traitValidator; private DSLContext dsl; private TraitValidatorError traitValidatorError; - + private BrAPITrialService trialService; + private String referenceSource; private final static String FAVORITES_TAG = "favorites"; @Inject public TraitService(TraitDAO traitDao, MethodDAO methodDao, ScaleDAO scaleDao, ObservationDAO observationDao, ProgramService programService, ProgramOntologyService programOntologyService, ProgramObservationLevelService programObservationLevelService, UserService userService, TraitValidatorService traitValidator, DSLContext dsl, TraitValidatorError traitValidatorError, - ProgramOntologyDAO programOntologyDAO) { + ProgramOntologyDAO programOntologyDAO, BrAPITrialService trialService, String referenceSource) { this.traitDAO = traitDao; this.methodDAO = methodDao; this.scaleDAO = scaleDao; @@ -80,6 +92,8 @@ public TraitService(TraitDAO traitDao, MethodDAO methodDao, ScaleDAO scaleDao, O this.dsl = dsl; this.traitValidatorError = traitValidatorError; this.programOntologyDAO = programOntologyDAO; + this.trialService = trialService; + this.referenceSource = referenceSource; } public List getByProgramId(UUID programId, boolean getFullTrait) throws DoesNotExistException { @@ -483,4 +497,182 @@ public List getByName(UUID programId, List names) throws DoesNotE return traitDAO.getTraitsByTraitName(programId, names.stream().map(name -> Trait.builder().observationVariableName(name).build()).collect(Collectors.toList())); } + + public List getBrAPIObservationVariablesForExperiment( + UUID programId, + Optional experimentId, + Optional environmentId + ) throws DoesNotExistException, ApiException { + log.debug(String.format("fetching variables for experiment. expId: %s, envId: %s ", + experimentId.orElse(""), environmentId.orElse(""))); + Optional program = programService.getById(programId); + if(program.isEmpty()) { + throw new DoesNotExistException("Could not find program: " + programId); + } + UUID expId; + if(experimentId.isPresent()) { + expId = UUID.fromString(experimentId.get()); + } else { + UUID envId = UUID.fromString(environmentId.get()); + BrAPIStudy environment = trialService.getEnvironment(program.get(), envId); + expId = UUID.fromString(Utilities.getExternalReference(environment.getExternalReferences(), + Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)).get().getReferenceId()); + } + + BrAPITrial experiment = trialService.getExperiment(program.get(), expId); + if(experiment + .getAdditionalInfo().getAsJsonObject() + .has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + String obsDatasetId = experiment + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); + return trialService.getDatasetObsVars(obsDatasetId, program.get()); + } + + return new ArrayList<>(); + } + + @NotNull + public List filterVariables(List programTraits, + Optional observationVariableDbId, + Optional observationVariableName, + Optional traitClass, + Optional methodDbId, + Optional methodName, + Optional scaleDbId, + Optional scaleName, + Optional traitDbId, + Optional traitName, + Optional ontologyDbId) throws DoesNotExistException { + log.debug("filtering variables:\n" + + "observationVariableDbId: " + observationVariableDbId + "\n" + + "observationVariableName: " + observationVariableName + "\n" + + "traitClass: " + traitClass + "\n" + + "methodDbId: " + methodDbId + "\n" + + "methodName: " + methodName + "\n" + + "scaleDbId: " + scaleDbId + "\n" + + "scaleName: " + scaleName + "\n" + + "traitDbId: " + traitDbId + "\n" + + "traitName: " + traitName + "\n" + + "ontologyDbId: " + ontologyDbId); + + return programTraits.stream() + .filter(trait -> { + boolean matches = true; + + Map, String>> filterParams = new HashMap<>(); + filterParams.put("observationVariableDbId", + Pair.of(observationVariableDbId, + trait.getId() + .toString())); + filterParams.put("observationVariableName", Pair.of(observationVariableName, trait.getObservationVariableName())); + filterParams.put("traitClass", Pair.of(traitClass, trait.getTraitClass())); + filterParams.put("methodDbId", + Pair.of(methodDbId, + trait.getMethodId() + .toString())); + filterParams.put("methodName", + Pair.of(methodName, + trait.getMethod() + .getDescription())); + filterParams.put("scaleDbId", + Pair.of(scaleDbId, + trait.getScale() + .getId() + .toString())); + filterParams.put("scaleName", + Pair.of(scaleName, + trait.getScale() + .getScaleName())); + filterParams.put("traitDbId", + Pair.of(traitDbId, + trait.getId() + .toString())); + filterParams.put("traitName", Pair.of(traitName, trait.getObservationVariableName())); + filterParams.put("ontologyDbId", + Pair.of(ontologyDbId, + trait.getProgramOntologyId() + .toString())); + + for (Map.Entry, String>> filter : filterParams.entrySet()) { + if (filter.getValue().getLeft().isPresent()) { + log.debug("filtering traits by: " + filter.getKey()); + matches = StringUtils.equals(filter.getValue() + .getLeft() + .get(), + filter.getValue() + .getRight()); + } + if (!matches) { + break; + } + } + + return matches; + }) + .map(this::convertToBrAPI) + .collect(Collectors.toList()); + } + + public BrAPIObservationVariable convertToBrAPI(Trait trait) { + BrAPIOntologyReference brAPIOntologyReference = new BrAPIOntologyReference().ontologyDbId(trait.getProgramOntologyId() + .toString()); + String status = trait.getActive() ? "active" : "inactive"; + List synonyms = prepSynonyms(trait); + return new BrAPIObservationVariable().observationVariableDbId(trait.getId().toString()) + .observationVariableName(trait.getObservationVariableName()) + .defaultValue(trait.getDefaultValue()) + .status(status) + .synonyms(synonyms) + .trait(new BrAPITrait().ontologyReference(brAPIOntologyReference) + .traitName(trait.getObservationVariableName()) + .traitDbId(trait.getId().toString()) + .entity(trait.getEntity()) + .attribute(trait.getAttribute()) + .status(status) + .synonyms(synonyms)) + .method(new BrAPIMethod().ontologyReference(brAPIOntologyReference) + .methodDbId(trait.getMethod().getId().toString()) + .methodClass(trait.getMethod().getMethodClass()) + .description(trait.getMethod().getDescription()) + .formula(trait.getMethod().getFormula())) + .scale(new BrAPIScale().ontologyReference(brAPIOntologyReference) + .scaleDbId(trait.getScale().getId().toString()) + .scaleName(trait.getScale().getScaleName()) + .dataType(BrAPITraitDataType.fromValue(trait.getScale().getDataType().getLiteral())) + .decimalPlaces(trait.getScale().getDecimalPlaces()) + .validValues(new BrAPIScaleValidValues().max(trait.getScale().getValidValueMax()) + .min(trait.getScale().getValidValueMin()) + .categories(trait.getScale().getCategories()))); + } + + /** + * Create a list of synonyms, and ensure that there the first element matches the name of the trait

+ * This is primarily needed to ensure any system using the first synonym as a display shows the actual name of the ontology term + * @param trait + * @return list of synonyms with at least one value (the name of the trait) + */ + private List prepSynonyms(Trait trait) { + List preppedSynonyms = new ArrayList<>(); + if(trait.getSynonyms() != null) { + preppedSynonyms = trait.getSynonyms(); + int traitNameIdx = -1; + for(int i = 0; i < preppedSynonyms.size(); i++) { + if(preppedSynonyms.get(i).equals(trait.getObservationVariableName())) { + traitNameIdx = i; + break; + } + } + if(traitNameIdx > -1) { + String temp = preppedSynonyms.get(traitNameIdx); + preppedSynonyms.set(traitNameIdx, preppedSynonyms.get(0)); + preppedSynonyms.set(0, temp); + } else { + preppedSynonyms.add(0, trait.getObservationVariableName()); + } + } else { + preppedSynonyms.add(trait.getObservationVariableName()); + } + return preppedSynonyms; + } } From f408da381c3af7cf7e06352ad57e93404dbf0f0d Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 22 Jan 2024 17:08:16 -0500 Subject: [PATCH 149/220] remove vestigial comments --- .../breedinginsight/brapi/v2/services/BrAPITrialService.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 9ebd95954..98c07a23e 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -55,7 +55,6 @@ public class BrAPITrialService { private final BrAPITrialDAO trialDAO; private final BrAPIObservationDAO observationDAO; private final BrAPIListDAO listDAO; -// private final BrAPIObservationVariableDAO obsVarDAO; private final TraitService traitService; private final BrAPIStudyDAO studyDAO; @@ -70,7 +69,6 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin BrAPITrialDAO trialDAO, BrAPIObservationDAO observationDAO, BrAPIListDAO listDAO, -// BrAPIObservationVariableDAO obsVarDAO, TraitService traitService, BrAPIStudyDAO studyDAO, BrAPISeasonDAO seasonDAO, @@ -82,7 +80,6 @@ public BrAPITrialService(@Property(name = "brapi.server.reference-source") Strin this.trialDAO = trialDAO; this.observationDAO = observationDAO; this.listDAO = listDAO; -// this.obsVarDAO = obsVarDAO; this.traitService = traitService; this.studyDAO = studyDAO; this.seasonDAO = seasonDAO; @@ -413,7 +410,6 @@ public List getDatasetObsVars(String datasetId, Program program) throws A String.format("%s/%s", referenceSource, ExternalReferenceSource.DATASET.getName()), UUID.fromString(datasetId)); if (lists == null || lists.isEmpty()) { -// throw new DoesNotExistException("Dataset observation variables list not returned from BrAPI service"); log.warn(String.format("Dataset %s observation variables list not returned from BrAPI service", datasetId)); return new ArrayList<>(); } From 09f8c47869ea9a07e06da449c41a256fb4777427 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 23 Jan 2024 11:12:39 -0500 Subject: [PATCH 150/220] move helper methods to new ObservationVariableService class --- .../BrAPIObservationVariableController.java | 11 +- .../BrAPIObservationVariableService.java | 233 ++++++++++++++++++ 2 files changed, 240 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java index f7e9368f4..568a2d286 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java @@ -35,6 +35,7 @@ import org.breedinginsight.api.auth.ProgramSecured; import org.breedinginsight.api.auth.ProgramSecuredRoleGroup; import org.breedinginsight.brapi.v1.controller.BrapiVersion; +import org.breedinginsight.brapi.v2.services.BrAPIObservationVariableService; import org.breedinginsight.brapi.v2.services.BrAPITrialService; import org.breedinginsight.model.Trait; import org.breedinginsight.services.OntologyService; @@ -57,19 +58,21 @@ public class BrAPIObservationVariableController { private final String referenceSource; private final OntologyService ontologyService; + private final BrAPIObservationVariableService observationVariableService; private final TraitService traitService; - private final BrAPITrialService trialService; private final ProgramService programService; @Inject public BrAPIObservationVariableController(OntologyService ontologyService, + BrAPIObservationVariableService observationVariableService, TraitService traitService, BrAPITrialService trialService, ProgramService programService, @Property(name = "brapi.server.reference-source") String referenceSource) { this.ontologyService = ontologyService; + this.observationVariableService = observationVariableService; this.traitService = traitService; this.trialService = trialService; this.programService = programService; @@ -108,7 +111,7 @@ public HttpResponse variablesGet(@PathVari log.debug("unsupported variable filters, returning"); programTraits = new ArrayList<>(); } else if(environmentId != null || experimentId != null) { - programTraits = traitService.getBrAPIObservationVariablesForExperiment( + programTraits = observationVariableService.getBrAPIObservationVariablesForExperiment( programId, Optional.ofNullable(experimentId), Optional.ofNullable(environmentId)); } else { log.debug("fetching variables for the program: " + programId); @@ -116,7 +119,7 @@ public HttpResponse variablesGet(@PathVari } - List filteredObsVars = traitService.filterVariables(programTraits, + List filteredObsVars = observationVariableService.filterVariables(programTraits, Optional.ofNullable(observationVariableDbId), Optional.ofNullable(observationVariableName), Optional.ofNullable(traitClass), @@ -168,7 +171,7 @@ public HttpResponse variablesObservation BrAPIObservationVariableSingleResponse response = new BrAPIObservationVariableSingleResponse() .metadata(new BrAPIMetadata()) - .result(traitService.convertToBrAPI(trait.get())); + .result(observationVariableService.convertToBrAPI(trait.get())); return HttpResponse.ok(response); } catch (DoesNotExistException e) { diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java new file mode 100644 index 000000000..76f879c3e --- /dev/null +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java @@ -0,0 +1,233 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.breedinginsight.brapi.v2.services; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.brapi.client.v2.model.exceptions.ApiException; +import org.brapi.v2.model.BrAPIOntologyReference; +import org.brapi.v2.model.core.BrAPIStudy; +import org.brapi.v2.model.core.BrAPITrial; +import org.brapi.v2.model.pheno.*; +import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; +import org.breedinginsight.model.*; +import org.breedinginsight.services.ProgramService; +import org.breedinginsight.services.exceptions.DoesNotExistException; +import org.breedinginsight.utilities.Utilities; +import org.jetbrains.annotations.NotNull; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +@Singleton +public class BrAPIObservationVariableService { + private final ProgramService programService; + private final BrAPITrialService trialService; + private final String referenceSource; + + @Inject + public BrAPIObservationVariableService( + ProgramService programService, BrAPITrialService trialService, String referenceSource) { + this.programService = programService; + this.trialService = trialService; + this.referenceSource = referenceSource; + } + + public List getBrAPIObservationVariablesForExperiment( + UUID programId, + Optional experimentId, + Optional environmentId + ) throws DoesNotExistException, ApiException { + log.debug(String.format("fetching variables for experiment. expId: %s, envId: %s ", + experimentId.orElse(""), environmentId.orElse(""))); + Optional program = programService.getById(programId); + if(program.isEmpty()) { + throw new DoesNotExistException("Could not find program: " + programId); + } + UUID expId; + if(experimentId.isPresent()) { + expId = UUID.fromString(experimentId.get()); + } else { + UUID envId = UUID.fromString(environmentId.get()); + BrAPIStudy environment = trialService.getEnvironment(program.get(), envId); + expId = UUID.fromString(Utilities.getExternalReference(environment.getExternalReferences(), + Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)).get().getReferenceId()); + } + + BrAPITrial experiment = trialService.getExperiment(program.get(), expId); + if(experiment + .getAdditionalInfo().getAsJsonObject() + .has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + String obsDatasetId = experiment + .getAdditionalInfo().getAsJsonObject() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); + return trialService.getDatasetObsVars(obsDatasetId, program.get()); + } + + return new ArrayList<>(); + } + + @NotNull + public List filterVariables(List programTraits, + Optional observationVariableDbId, + Optional observationVariableName, + Optional traitClass, + Optional methodDbId, + Optional methodName, + Optional scaleDbId, + Optional scaleName, + Optional traitDbId, + Optional traitName, + Optional ontologyDbId) throws DoesNotExistException { + log.debug("filtering variables:\n" + + "observationVariableDbId: " + observationVariableDbId + "\n" + + "observationVariableName: " + observationVariableName + "\n" + + "traitClass: " + traitClass + "\n" + + "methodDbId: " + methodDbId + "\n" + + "methodName: " + methodName + "\n" + + "scaleDbId: " + scaleDbId + "\n" + + "scaleName: " + scaleName + "\n" + + "traitDbId: " + traitDbId + "\n" + + "traitName: " + traitName + "\n" + + "ontologyDbId: " + ontologyDbId); + + return programTraits.stream() + .filter(trait -> { + boolean matches = true; + + Map, String>> filterParams = new HashMap<>(); + filterParams.put("observationVariableDbId", + Pair.of(observationVariableDbId, + trait.getId() + .toString())); + filterParams.put("observationVariableName", Pair.of(observationVariableName, trait.getObservationVariableName())); + filterParams.put("traitClass", Pair.of(traitClass, trait.getTraitClass())); + filterParams.put("methodDbId", + Pair.of(methodDbId, + trait.getMethodId() + .toString())); + filterParams.put("methodName", + Pair.of(methodName, + trait.getMethod() + .getDescription())); + filterParams.put("scaleDbId", + Pair.of(scaleDbId, + trait.getScale() + .getId() + .toString())); + filterParams.put("scaleName", + Pair.of(scaleName, + trait.getScale() + .getScaleName())); + filterParams.put("traitDbId", + Pair.of(traitDbId, + trait.getId() + .toString())); + filterParams.put("traitName", Pair.of(traitName, trait.getObservationVariableName())); + filterParams.put("ontologyDbId", + Pair.of(ontologyDbId, + trait.getProgramOntologyId() + .toString())); + + for (Map.Entry, String>> filter : filterParams.entrySet()) { + if (filter.getValue().getLeft().isPresent()) { + log.debug("filtering traits by: " + filter.getKey()); + matches = StringUtils.equals(filter.getValue() + .getLeft() + .get(), + filter.getValue() + .getRight()); + } + if (!matches) { + break; + } + } + + return matches; + }) + .map(this::convertToBrAPI) + .collect(Collectors.toList()); + } + + public BrAPIObservationVariable convertToBrAPI(Trait trait) { + BrAPIOntologyReference brAPIOntologyReference = new BrAPIOntologyReference().ontologyDbId(trait.getProgramOntologyId() + .toString()); + String status = trait.getActive() ? "active" : "inactive"; + List synonyms = prepSynonyms(trait); + return new BrAPIObservationVariable().observationVariableDbId(trait.getId().toString()) + .observationVariableName(trait.getObservationVariableName()) + .defaultValue(trait.getDefaultValue()) + .status(status) + .synonyms(synonyms) + .trait(new BrAPITrait().ontologyReference(brAPIOntologyReference) + .traitName(trait.getObservationVariableName()) + .traitDbId(trait.getId().toString()) + .entity(trait.getEntity()) + .attribute(trait.getAttribute()) + .status(status) + .synonyms(synonyms)) + .method(new BrAPIMethod().ontologyReference(brAPIOntologyReference) + .methodDbId(trait.getMethod().getId().toString()) + .methodClass(trait.getMethod().getMethodClass()) + .description(trait.getMethod().getDescription()) + .formula(trait.getMethod().getFormula())) + .scale(new BrAPIScale().ontologyReference(brAPIOntologyReference) + .scaleDbId(trait.getScale().getId().toString()) + .scaleName(trait.getScale().getScaleName()) + .dataType(BrAPITraitDataType.fromValue(trait.getScale().getDataType().getLiteral())) + .decimalPlaces(trait.getScale().getDecimalPlaces()) + .validValues(new BrAPIScaleValidValues().max(trait.getScale().getValidValueMax()) + .min(trait.getScale().getValidValueMin()) + .categories(trait.getScale().getCategories()))); + } + + /** + * Create a list of synonyms, and ensure that there the first element matches the name of the trait

+ * This is primarily needed to ensure any system using the first synonym as a display shows the actual name of the ontology term + * @param trait + * @return list of synonyms with at least one value (the name of the trait) + */ + private List prepSynonyms(Trait trait) { + List preppedSynonyms = new ArrayList<>(); + if(trait.getSynonyms() != null) { + preppedSynonyms = trait.getSynonyms(); + int traitNameIdx = -1; + for(int i = 0; i < preppedSynonyms.size(); i++) { + if(preppedSynonyms.get(i).equals(trait.getObservationVariableName())) { + traitNameIdx = i; + break; + } + } + if(traitNameIdx > -1) { + String temp = preppedSynonyms.get(traitNameIdx); + preppedSynonyms.set(traitNameIdx, preppedSynonyms.get(0)); + preppedSynonyms.set(0, temp); + } else { + preppedSynonyms.add(0, trait.getObservationVariableName()); + } + } else { + preppedSynonyms.add(trait.getObservationVariableName()); + } + return preppedSynonyms; + } +} From 8a8dfed65f719fae0589c249ee1347c2c3f27a5d Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 23 Jan 2024 11:40:21 -0500 Subject: [PATCH 151/220] add orElse for optionals --- .../BrAPIObservationVariableService.java | 5 +- .../services/TraitService.java | 178 ------------------ 2 files changed, 3 insertions(+), 180 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java index 76f879c3e..091bcaab6 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java @@ -68,10 +68,11 @@ public List getBrAPIObservationVariablesForExperiment( if(experimentId.isPresent()) { expId = UUID.fromString(experimentId.get()); } else { - UUID envId = UUID.fromString(environmentId.get()); + UUID envId = UUID.fromString(environmentId.orElseThrow(() -> new IllegalStateException("no environment id found"))); BrAPIStudy environment = trialService.getEnvironment(program.get(), envId); expId = UUID.fromString(Utilities.getExternalReference(environment.getExternalReferences(), - Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)).get().getReferenceId()); + Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)) + .orElseThrow(() -> new IllegalStateException("no external reference found")).getReferenceId()); } BrAPITrial experiment = trialService.getExperiment(program.get(), expId); diff --git a/src/main/java/org/breedinginsight/services/TraitService.java b/src/main/java/org/breedinginsight/services/TraitService.java index d8db6a98e..d4f0f00d9 100644 --- a/src/main/java/org/breedinginsight/services/TraitService.java +++ b/src/main/java/org/breedinginsight/services/TraitService.java @@ -497,182 +497,4 @@ public List getByName(UUID programId, List names) throws DoesNotE return traitDAO.getTraitsByTraitName(programId, names.stream().map(name -> Trait.builder().observationVariableName(name).build()).collect(Collectors.toList())); } - - public List getBrAPIObservationVariablesForExperiment( - UUID programId, - Optional experimentId, - Optional environmentId - ) throws DoesNotExistException, ApiException { - log.debug(String.format("fetching variables for experiment. expId: %s, envId: %s ", - experimentId.orElse(""), environmentId.orElse(""))); - Optional program = programService.getById(programId); - if(program.isEmpty()) { - throw new DoesNotExistException("Could not find program: " + programId); - } - UUID expId; - if(experimentId.isPresent()) { - expId = UUID.fromString(experimentId.get()); - } else { - UUID envId = UUID.fromString(environmentId.get()); - BrAPIStudy environment = trialService.getEnvironment(program.get(), envId); - expId = UUID.fromString(Utilities.getExternalReference(environment.getExternalReferences(), - Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)).get().getReferenceId()); - } - - BrAPITrial experiment = trialService.getExperiment(program.get(), expId); - if(experiment - .getAdditionalInfo().getAsJsonObject() - .has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { - String obsDatasetId = experiment - .getAdditionalInfo().getAsJsonObject() - .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID).getAsString(); - return trialService.getDatasetObsVars(obsDatasetId, program.get()); - } - - return new ArrayList<>(); - } - - @NotNull - public List filterVariables(List programTraits, - Optional observationVariableDbId, - Optional observationVariableName, - Optional traitClass, - Optional methodDbId, - Optional methodName, - Optional scaleDbId, - Optional scaleName, - Optional traitDbId, - Optional traitName, - Optional ontologyDbId) throws DoesNotExistException { - log.debug("filtering variables:\n" + - "observationVariableDbId: " + observationVariableDbId + "\n" + - "observationVariableName: " + observationVariableName + "\n" + - "traitClass: " + traitClass + "\n" + - "methodDbId: " + methodDbId + "\n" + - "methodName: " + methodName + "\n" + - "scaleDbId: " + scaleDbId + "\n" + - "scaleName: " + scaleName + "\n" + - "traitDbId: " + traitDbId + "\n" + - "traitName: " + traitName + "\n" + - "ontologyDbId: " + ontologyDbId); - - return programTraits.stream() - .filter(trait -> { - boolean matches = true; - - Map, String>> filterParams = new HashMap<>(); - filterParams.put("observationVariableDbId", - Pair.of(observationVariableDbId, - trait.getId() - .toString())); - filterParams.put("observationVariableName", Pair.of(observationVariableName, trait.getObservationVariableName())); - filterParams.put("traitClass", Pair.of(traitClass, trait.getTraitClass())); - filterParams.put("methodDbId", - Pair.of(methodDbId, - trait.getMethodId() - .toString())); - filterParams.put("methodName", - Pair.of(methodName, - trait.getMethod() - .getDescription())); - filterParams.put("scaleDbId", - Pair.of(scaleDbId, - trait.getScale() - .getId() - .toString())); - filterParams.put("scaleName", - Pair.of(scaleName, - trait.getScale() - .getScaleName())); - filterParams.put("traitDbId", - Pair.of(traitDbId, - trait.getId() - .toString())); - filterParams.put("traitName", Pair.of(traitName, trait.getObservationVariableName())); - filterParams.put("ontologyDbId", - Pair.of(ontologyDbId, - trait.getProgramOntologyId() - .toString())); - - for (Map.Entry, String>> filter : filterParams.entrySet()) { - if (filter.getValue().getLeft().isPresent()) { - log.debug("filtering traits by: " + filter.getKey()); - matches = StringUtils.equals(filter.getValue() - .getLeft() - .get(), - filter.getValue() - .getRight()); - } - if (!matches) { - break; - } - } - - return matches; - }) - .map(this::convertToBrAPI) - .collect(Collectors.toList()); - } - - public BrAPIObservationVariable convertToBrAPI(Trait trait) { - BrAPIOntologyReference brAPIOntologyReference = new BrAPIOntologyReference().ontologyDbId(trait.getProgramOntologyId() - .toString()); - String status = trait.getActive() ? "active" : "inactive"; - List synonyms = prepSynonyms(trait); - return new BrAPIObservationVariable().observationVariableDbId(trait.getId().toString()) - .observationVariableName(trait.getObservationVariableName()) - .defaultValue(trait.getDefaultValue()) - .status(status) - .synonyms(synonyms) - .trait(new BrAPITrait().ontologyReference(brAPIOntologyReference) - .traitName(trait.getObservationVariableName()) - .traitDbId(trait.getId().toString()) - .entity(trait.getEntity()) - .attribute(trait.getAttribute()) - .status(status) - .synonyms(synonyms)) - .method(new BrAPIMethod().ontologyReference(brAPIOntologyReference) - .methodDbId(trait.getMethod().getId().toString()) - .methodClass(trait.getMethod().getMethodClass()) - .description(trait.getMethod().getDescription()) - .formula(trait.getMethod().getFormula())) - .scale(new BrAPIScale().ontologyReference(brAPIOntologyReference) - .scaleDbId(trait.getScale().getId().toString()) - .scaleName(trait.getScale().getScaleName()) - .dataType(BrAPITraitDataType.fromValue(trait.getScale().getDataType().getLiteral())) - .decimalPlaces(trait.getScale().getDecimalPlaces()) - .validValues(new BrAPIScaleValidValues().max(trait.getScale().getValidValueMax()) - .min(trait.getScale().getValidValueMin()) - .categories(trait.getScale().getCategories()))); - } - - /** - * Create a list of synonyms, and ensure that there the first element matches the name of the trait

- * This is primarily needed to ensure any system using the first synonym as a display shows the actual name of the ontology term - * @param trait - * @return list of synonyms with at least one value (the name of the trait) - */ - private List prepSynonyms(Trait trait) { - List preppedSynonyms = new ArrayList<>(); - if(trait.getSynonyms() != null) { - preppedSynonyms = trait.getSynonyms(); - int traitNameIdx = -1; - for(int i = 0; i < preppedSynonyms.size(); i++) { - if(preppedSynonyms.get(i).equals(trait.getObservationVariableName())) { - traitNameIdx = i; - break; - } - } - if(traitNameIdx > -1) { - String temp = preppedSynonyms.get(traitNameIdx); - preppedSynonyms.set(traitNameIdx, preppedSynonyms.get(0)); - preppedSynonyms.set(0, temp); - } else { - preppedSynonyms.add(0, trait.getObservationVariableName()); - } - } else { - preppedSynonyms.add(trait.getObservationVariableName()); - } - return preppedSynonyms; - } } From 46d01f063b21f7d9eff979aff1b9f54190b0c902 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 23 Jan 2024 12:08:16 -0500 Subject: [PATCH 152/220] remove circular dependency --- .../java/org/breedinginsight/services/TraitService.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/services/TraitService.java b/src/main/java/org/breedinginsight/services/TraitService.java index d4f0f00d9..5f240e025 100644 --- a/src/main/java/org/breedinginsight/services/TraitService.java +++ b/src/main/java/org/breedinginsight/services/TraitService.java @@ -71,15 +71,13 @@ public class TraitService { private TraitValidatorService traitValidator; private DSLContext dsl; private TraitValidatorError traitValidatorError; - private BrAPITrialService trialService; - private String referenceSource; private final static String FAVORITES_TAG = "favorites"; @Inject public TraitService(TraitDAO traitDao, MethodDAO methodDao, ScaleDAO scaleDao, ObservationDAO observationDao, ProgramService programService, ProgramOntologyService programOntologyService, ProgramObservationLevelService programObservationLevelService, UserService userService, TraitValidatorService traitValidator, DSLContext dsl, TraitValidatorError traitValidatorError, - ProgramOntologyDAO programOntologyDAO, BrAPITrialService trialService, String referenceSource) { + ProgramOntologyDAO programOntologyDAO) { this.traitDAO = traitDao; this.methodDAO = methodDao; this.scaleDAO = scaleDao; @@ -92,8 +90,6 @@ public TraitService(TraitDAO traitDao, MethodDAO methodDao, ScaleDAO scaleDao, O this.dsl = dsl; this.traitValidatorError = traitValidatorError; this.programOntologyDAO = programOntologyDAO; - this.trialService = trialService; - this.referenceSource = referenceSource; } public List getByProgramId(UUID programId, boolean getFullTrait) throws DoesNotExistException { From 003481b6fa9d071e030de337c08529664df00f4e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 23 Jan 2024 14:41:38 -0500 Subject: [PATCH 153/220] [BI-1921] update tests --- .../brapi/v2/BrAPIObservationVariableController.java | 7 +------ .../org/breedinginsight/brapi/v2/BrAPIV2Controller.java | 2 +- .../brapi/v2/services/BrAPIObservationVariableService.java | 4 +++- .../brapi/v2/BrAPIV2ControllerIntegrationTest.java | 2 +- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java index 568a2d286..45d6983f3 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationVariableController.java @@ -17,7 +17,6 @@ package org.breedinginsight.brapi.v2; -import io.micronaut.context.annotation.Property; import io.micronaut.http.HttpRequest; import io.micronaut.http.HttpResponse; import io.micronaut.http.HttpStatus; @@ -55,13 +54,11 @@ @Controller("/${micronaut.bi.api.version}/programs/{programId}" + BrapiVersion.BRAPI_V2) @Secured(SecurityRule.IS_AUTHENTICATED) public class BrAPIObservationVariableController { - private final String referenceSource; private final OntologyService ontologyService; private final BrAPIObservationVariableService observationVariableService; private final TraitService traitService; private final BrAPITrialService trialService; - private final ProgramService programService; @Inject @@ -69,14 +66,12 @@ public BrAPIObservationVariableController(OntologyService ontologyService, BrAPIObservationVariableService observationVariableService, TraitService traitService, BrAPITrialService trialService, - ProgramService programService, - @Property(name = "brapi.server.reference-source") String referenceSource) { + ProgramService programService) { this.ontologyService = ontologyService; this.observationVariableService = observationVariableService; this.traitService = traitService; this.trialService = trialService; this.programService = programService; - this.referenceSource = referenceSource; } @Get("/variables") diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java index 0d4a5449d..d6e4dde43 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIV2Controller.java @@ -60,7 +60,7 @@ public BrAPIV2Controller(SecurityService securityService, ProgramService program } - @Get("/${micronaut.bi.api.version}/" + BrapiVersion.BRAPI_V2 + "/serverinfo") + @Get("/${micronaut.bi.api.version}" + BrapiVersion.BRAPI_V2 + "/serverinfo") @Produces(MediaType.APPLICATION_JSON) @Secured(SecurityRule.IS_ANONYMOUS) public BrAPIServerInfoResponse serverinfo() { diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java index 091bcaab6..f9742fb74 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIObservationVariableService.java @@ -17,6 +17,7 @@ package org.breedinginsight.brapi.v2.services; +import io.micronaut.context.annotation.Property; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -47,7 +48,8 @@ public class BrAPIObservationVariableService { @Inject public BrAPIObservationVariableService( - ProgramService programService, BrAPITrialService trialService, String referenceSource) { + ProgramService programService, BrAPITrialService trialService, + @Property(name = "brapi.server.reference-source") String referenceSource) { this.programService = programService; this.trialService = trialService; this.referenceSource = referenceSource; diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java index ee5fbbfd7..1e1cb922b 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ControllerIntegrationTest.java @@ -127,7 +127,7 @@ public void setup() { @Test public void testRootServerInfo() { - Flowable> call = biClient.exchange(GET("/brapi/v2/serverinfo"), String.class); + Flowable> call = biClient.exchange(GET("/v1/brapi/v2/serverinfo"), String.class); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.OK, response.getStatus()); From 9753a8b3a836233513515451236bcacd172daad4 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 24 Jan 2024 16:49:59 +0000 Subject: [PATCH 154/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index ce321d905..4ce495dd6 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+668 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/6753df1e709ac38202cf49fe360f758574be390a +version=v0.9.0+671 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f882017bcb77640faf8cd6af9ffb0935caaa3236 From 984e75de85ba72716a971a93b1243b49fd326081 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 24 Jan 2024 18:04:16 +0000 Subject: [PATCH 155/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 4ce495dd6..ac6fc860d 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+671 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/f882017bcb77640faf8cd6af9ffb0935caaa3236 +version=v0.9.0+673 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/fbaef308cd2c1a13939bb7bb5379b5bb57c7091d From 025ab04dbbb1b8e01ab0b7005987fd15d87209cf Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 1 Feb 2024 16:35:00 -0500 Subject: [PATCH 156/220] [BI-2043] - moved expensive call outside nested loops --- .../importer/services/processors/ExperimentProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 4d93ad4f9..525a5e5b5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -509,7 +509,7 @@ private void initNewBrapiData(List importRows, List> phen throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, "Program is not properly configured for environment import"); } Supplier envNextVal = () -> dsl.nextval(envSequenceName.toLowerCase()); - + existingObsByObsHash = fetchExistingObservations(referencedTraits, program); for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); @@ -1010,7 +1010,7 @@ private void fetchOrCreateObservationPIO(Program program, String variableName = column.name(); String value = column.getString(rowNum); String key = getImportObservationHash(importRow, variableName); - existingObsByObsHash = fetchExistingObservations(referencedTraits, program); + if (existingObsByObsHash.containsKey(key)) { if (StringUtils.isNotBlank(value) && !isObservationMatched(key, value, column, rowNum)){ From 0edb9e337e5978e8de78338173a74029b4082600 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Mon, 12 Feb 2024 17:27:06 +0000 Subject: [PATCH 157/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index ac6fc860d..0e86bc81a 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+673 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/fbaef308cd2c1a13939bb7bb5379b5bb57c7091d +version=v0.9.0+681 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/d15e597ede75c582aad72a310cab6bafdd9de2f9 From 315c3cddd1239f6c4661b3bd7ed47a7589d08f09 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Tue, 13 Feb 2024 16:43:47 -0500 Subject: [PATCH 158/220] [BI-2053] - fixed scale units bug --- .../daos/impl/TraitDAOImpl.java | 1 + .../java/org/breedinginsight/model/Scale.java | 2 ++ .../services/OntologyService.java | 2 +- .../services/TraitService.java | 15 +++--------- .../parsers/trait/TraitFileColumns.java | 2 +- .../parsers/trait/TraitFileParser.java | 4 +++- .../migration/V1.18.0__alter_scale_table.sql | 23 +++++++++++++++++++ 7 files changed, 34 insertions(+), 15 deletions(-) create mode 100644 src/main/resources/db/migration/V1.18.0__alter_scale_table.sql diff --git a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java index c61702622..983c739c2 100644 --- a/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java +++ b/src/main/java/org/breedinginsight/daos/impl/TraitDAOImpl.java @@ -380,6 +380,7 @@ public List createTraitsBrAPI(List traits, User actingUser, Progra .min(trait.getScale().getValidValueMin()); BrAPIScale brApiScale = new BrAPIScale() .scaleName(String.format("%s [%s]", trait.getScale().getScaleName(), program.getKey())) + .units(trait.getScale().getUnits()) .externalReferences(List.of(scaleReference)) .dataType(brApiTraitDataType) .decimalPlaces(trait.getScale().getDecimalPlaces()) diff --git a/src/main/java/org/breedinginsight/model/Scale.java b/src/main/java/org/breedinginsight/model/Scale.java index f1c3ebf8f..b2458e80b 100644 --- a/src/main/java/org/breedinginsight/model/Scale.java +++ b/src/main/java/org/breedinginsight/model/Scale.java @@ -58,6 +58,7 @@ public class Scale extends ScaleEntity { public Scale(ScaleEntity scaleEntity){ this.setId(scaleEntity.getId()); this.setScaleName(scaleEntity.getScaleName()); + this.setUnits(scaleEntity.getUnits()); this.setProgramOntologyId(scaleEntity.getProgramOntologyId()); this.setDataType(scaleEntity.getDataType()); this.setCreatedAt(scaleEntity.getCreatedAt()); @@ -70,6 +71,7 @@ public static Scale parseSqlRecord(Record record) { return Scale.builder() .id(record.getValue(SCALE.ID)) .scaleName(record.getValue(SCALE.SCALE_NAME)) + .units(record.getValue(SCALE.UNITS)) .programOntologyId(record.getValue(SCALE.PROGRAM_ONTOLOGY_ID)) .dataType(record.getValue(SCALE.DATA_TYPE)) .createdAt(record.getValue(SCALE.CREATED_AT)) diff --git a/src/main/java/org/breedinginsight/services/OntologyService.java b/src/main/java/org/breedinginsight/services/OntologyService.java index 90de17ec8..a2a711535 100644 --- a/src/main/java/org/breedinginsight/services/OntologyService.java +++ b/src/main/java/org/breedinginsight/services/OntologyService.java @@ -505,7 +505,7 @@ public List> processData(List traits) { if(scale!=null) { row.put(TraitFileColumns.SCALE_CLASS.toString(), DataTypeTranslator.getDisplayNameFromType(scale.getDataType())); - row.put(TraitFileColumns.SCALE_NAME.toString(), scale.getScaleName()); + row.put(TraitFileColumns.UNITS.toString(), scale.getUnits()); row.put(TraitFileColumns.SCALE_DECIMAL_PLACES.toString(), scale.getDecimalPlaces()); row.put(TraitFileColumns.SCALE_LOWER_LIMIT.toString(), scale.getValidValueMin()); row.put(TraitFileColumns.SCALE_UPPER_LIMIT.toString(), scale.getValidValueMax()); diff --git a/src/main/java/org/breedinginsight/services/TraitService.java b/src/main/java/org/breedinginsight/services/TraitService.java index 5f240e025..c36296122 100644 --- a/src/main/java/org/breedinginsight/services/TraitService.java +++ b/src/main/java/org/breedinginsight/services/TraitService.java @@ -22,19 +22,10 @@ import io.micronaut.http.server.exceptions.HttpServerException; import io.micronaut.http.server.exceptions.InternalServerException; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.text.WordUtils; -import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.BrAPIOntologyReference; -import org.brapi.v2.model.core.BrAPIStudy; -import org.brapi.v2.model.core.BrAPITrial; -import org.brapi.v2.model.pheno.*; +import org.brapi.v2.model.pheno.BrAPIObservation; import org.breedinginsight.api.auth.AuthenticatedUser; import org.breedinginsight.api.model.v1.response.ValidationErrors; -import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; -import org.breedinginsight.brapi.v2.services.BrAPITrialService; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.dao.db.tables.pojos.MethodEntity; import org.breedinginsight.dao.db.tables.pojos.ProgramSharedOntologyEntity; @@ -46,8 +37,6 @@ import org.breedinginsight.services.exceptions.ValidatorException; import org.breedinginsight.services.validators.TraitValidatorError; import org.breedinginsight.services.validators.TraitValidatorService; -import org.breedinginsight.utilities.Utilities; -import org.jetbrains.annotations.NotNull; import org.jooq.DSLContext; import javax.inject.Inject; @@ -203,6 +192,7 @@ public List createTraits(UUID programId, List traits, Authenticate // Create scale ScaleEntity jooqScale = ScaleEntity.builder() .scaleName(trait.getScale().getScaleName()) + .units(trait.getScale().getUnits()) .dataType(trait.getScale().getDataType()) .programOntologyId(programOntology.getId()) .createdBy(actingUser.getId()) @@ -419,6 +409,7 @@ public List updateTraits(UUID programId, List traits, Authenticate // Jump to scale ScaleEntity existingScaleEntity = scaleDAO.fetchOneById(existingTraitEntity.getScaleId()); existingScaleEntity.setScaleName(updatedTrait.getScale().getScaleName()); + existingScaleEntity.setUnits(updatedTrait.getScale().getUnits()); existingScaleEntity.setDataType(updatedTrait.getScale().getDataType()); existingScaleEntity.setUpdatedBy(user.getId()); scaleDAO.update(existingScaleEntity); diff --git a/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileColumns.java b/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileColumns.java index f3782adf7..303671c76 100644 --- a/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileColumns.java +++ b/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileColumns.java @@ -38,7 +38,7 @@ public enum TraitFileColumns { METHOD_CLASS("Method class", Column.ColumnDataType.STRING), METHOD_FORMULA("Method formula", Column.ColumnDataType.STRING), SCALE_CLASS("Scale class", Column.ColumnDataType.STRING), - SCALE_NAME("Units", Column.ColumnDataType.STRING), + UNITS("Units", Column.ColumnDataType.STRING), SCALE_DECIMAL_PLACES("Scale decimal places", Column.ColumnDataType.INTEGER), SCALE_LOWER_LIMIT("Scale lower limit", Column.ColumnDataType.INTEGER), SCALE_UPPER_LIMIT("Scale upper limit", Column.ColumnDataType.INTEGER), diff --git a/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java b/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java index 66c7134f6..e93f91760 100644 --- a/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java +++ b/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java @@ -242,8 +242,10 @@ private List excelRecordsToTraits(List records) throws Valid } } + String units = parseExcelValueAsString(record, TraitFileColumns.UNITS); Scale scale = Scale.builder() - .scaleName(parseExcelValueAsString(record, TraitFileColumns.SCALE_NAME)) + .scaleName(units == null ? parseExcelValueAsString(record, TraitFileColumns.SCALE_CLASS) : units) + .units(units) .dataType(dataType) .decimalPlaces(decimalPlaces) .validValueMin(validValueMin) diff --git a/src/main/resources/db/migration/V1.18.0__alter_scale_table.sql b/src/main/resources/db/migration/V1.18.0__alter_scale_table.sql new file mode 100644 index 000000000..e27cfef5a --- /dev/null +++ b/src/main/resources/db/migration/V1.18.0__alter_scale_table.sql @@ -0,0 +1,23 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Add units to scale table, units is an optional BrAPI field. +ALTER TABLE scale ADD COLUMN units text; +-- Update existing scale records. +UPDATE scale +SET units = scale_name +WHERE data_type = 'NUMERICAL'; \ No newline at end of file From 03a42f7440e0c2cb3e46ab6f25fbc46308d2a97f Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 14 Feb 2024 14:43:29 -0500 Subject: [PATCH 159/220] [BI-2053] - added error message for non-numeric scale with units --- .../services/parsers/ParsingExceptionType.java | 3 ++- .../services/parsers/trait/TraitFileParser.java | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/services/parsers/ParsingExceptionType.java b/src/main/java/org/breedinginsight/services/parsers/ParsingExceptionType.java index 31e7356d2..4ae0d0c1d 100644 --- a/src/main/java/org/breedinginsight/services/parsers/ParsingExceptionType.java +++ b/src/main/java/org/breedinginsight/services/parsers/ParsingExceptionType.java @@ -36,7 +36,8 @@ public enum ParsingExceptionType { INVALID_SCALE_LOWER_LIMIT("Invalid scale lower limit value. Value must be numeric and be a whole number."), INVALID_SCALE_UPPER_LIMIT("Invalid scale upper limit value. Value must be numeric and be a whole number."), INVALID_SCALE_CATEGORIES("Invalid scale categories format"), - INVALID_TERM_TYPE("Invalid term type"); + INVALID_TERM_TYPE("Invalid term type"), + SCALE_UNIT_NOT_ALLOWED("Units identified for non-numeric scale class."); private String value; diff --git a/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java b/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java index e93f91760..17a9e46f6 100644 --- a/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java +++ b/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java @@ -243,6 +243,11 @@ private List excelRecordsToTraits(List records) throws Valid } String units = parseExcelValueAsString(record, TraitFileColumns.UNITS); + if (dataType != DataType.NUMERICAL && units != null && !units.isEmpty()) { + ValidationError error = new ValidationError(TraitFileColumns.UNITS.toString(), + ParsingExceptionType.SCALE_UNIT_NOT_ALLOWED.toString(), HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(traitValidatorError.getRowNumber(i), error); + } Scale scale = Scale.builder() .scaleName(units == null ? parseExcelValueAsString(record, TraitFileColumns.SCALE_CLASS) : units) .units(units) From 16516115e87a5d9fa1273871ef4d4cc416f10d27 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 15 Feb 2024 10:08:26 -0500 Subject: [PATCH 160/220] [BI-2045] added error for 'Experiment Title already exist' --- .../importer/services/processors/ExperimentProcessor.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 525a5e5b5..9f789e172 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -84,6 +84,7 @@ public class ExperimentProcessor implements Processor { private static final String NAME = "Experiment"; private static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs"; + private static final String DUPLICATE_EXPERIMENT_TITLE = "Experiment Title already exists"; private static final String MULTIPLE_EXP_TITLES = "File contains more than one Experiment Title"; private static final String MIDNIGHT = "T00:00:00-00:00"; private static final String TIMESTAMP_PREFIX = "TS:"; @@ -1177,6 +1178,9 @@ private PendingImportObject fetchOrCreateTrialPIO(Program program, U PendingImportObject pio; if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { pio = trialByNameNoScope.get(importRow.getExpTitle()); + if (pio.getState() == ImportObjectState.EXISTING && StringUtils.isBlank( importRow.getObsUnitID() ) ){ + throw new UnprocessableEntityException(DUPLICATE_EXPERIMENT_TITLE); + } } else if (!trialByNameNoScope.isEmpty()) { throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); } else { From 72130a937c6eb8b225a0e141778b9088781adad7 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Thu, 15 Feb 2024 16:36:25 -0500 Subject: [PATCH 161/220] [BI-2053] - fixed tests --- .../parsers/trait/TraitFileParser.java | 11 +++++++---- .../validators/TraitFileValidatorError.java | 2 +- .../validators/TraitValidatorError.java | 4 ++-- .../TraitValidatorErrorInterface.java | 2 +- .../validators/TraitValidatorService.java | 5 ++--- .../TraitControllerIntegrationTest.java | 2 ++ .../TraitUploadControllerIntegrationTest.java | 2 +- ...ionVariablesControllerIntegrationTest.java | 1 + ...vationLevelsControllerIntegrationTest.java | 1 + ...ervationUnitControllerIntegrationTest.java | 1 + ...ObservationsControllerIntegrationTest.java | 1 + ...BrAPIStudiesControllerIntegrationTest.java | 1 + ...tionVariableControllerIntegrationTest.java | 1 + .../ExperimentControllerIntegrationTest.java | 1 + .../brapps/importer/ImportTestUtils.java | 1 + .../parsers/TraitFileParserUnitTest.java | 2 +- .../resources/files/data_mismatched_cases.csv | 2 +- .../resources/files/data_multiple_rows.csv | 6 +++--- .../resources/files/data_multiple_rows.xls | Bin 21504 -> 21504 bytes .../resources/files/data_multiple_rows.xlsx | Bin 10589 -> 10541 bytes src/test/resources/files/data_one_row.csv | 2 +- src/test/resources/files/data_one_row.xls | Bin 24576 -> 24576 bytes src/test/resources/files/data_one_row.xlsx | Bin 12644 -> 12636 bytes .../files/data_one_row_blank_active_value.csv | 2 +- .../files/data_one_row_blank_scale_class.csv | 2 +- .../files/data_one_row_computation.csv | 2 +- .../files/data_one_row_empty_active_value.xls | Bin 24576 -> 24576 bytes .../data_one_row_empty_formula_headers.xls | Bin 24576 -> 24576 bytes .../files/data_one_row_empty_headers.csv | 2 +- .../files/data_one_row_empty_headers.xls | Bin 24576 -> 24576 bytes .../data_one_row_invalid_active_value.csv | 2 +- .../data_one_row_invalid_scale_class.csv | 2 +- .../data_one_row_scale_categories_blank.csv | 2 +- .../data_one_row_scale_decimal_blank.csv | 2 +- .../data_one_row_scale_decimal_invalid.csv | 2 +- .../data_one_row_trait_level_not_exist.csv | 2 +- src/test/resources/files/duplicatesInFile.csv | 4 ++-- .../files/missing_category_label_nominal.csv | 2 +- .../resources/files/missing_scale_unit.xlsx | Bin 17927 -> 17926 bytes .../files/multiple_rows_parsing_errors.csv | 6 +++--- .../data_nominal_missing_categories.xlsx | Bin 18004 -> 17936 bytes .../data_ordinal_missing_categories.xlsx | Bin 17944 -> 17891 bytes .../resources/files/parsing_exceptions.csv | 2 +- 43 files changed, 47 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java b/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java index 17a9e46f6..2dc710483 100644 --- a/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java +++ b/src/main/java/org/breedinginsight/services/parsers/trait/TraitFileParser.java @@ -243,10 +243,13 @@ private List excelRecordsToTraits(List records) throws Valid } String units = parseExcelValueAsString(record, TraitFileColumns.UNITS); - if (dataType != DataType.NUMERICAL && units != null && !units.isEmpty()) { - ValidationError error = new ValidationError(TraitFileColumns.UNITS.toString(), - ParsingExceptionType.SCALE_UNIT_NOT_ALLOWED.toString(), HttpStatus.UNPROCESSABLE_ENTITY); - validationErrors.addError(traitValidatorError.getRowNumber(i), error); + // Note: if method class is "Computation", scale class will be overwritten to "Numerical", so treat accordingly. + if (dataType != DataType.NUMERICAL && (method.getMethodClass() == null || !method.getMethodClass().equalsIgnoreCase(Method.COMPUTATION_TYPE))) { + if (units != null && !units.isEmpty()) { + ValidationError error = new ValidationError(TraitFileColumns.UNITS.toString(), + ParsingExceptionType.SCALE_UNIT_NOT_ALLOWED.toString(), HttpStatus.UNPROCESSABLE_ENTITY); + validationErrors.addError(traitValidatorError.getRowNumber(i), error); + } } Scale scale = Scale.builder() .scaleName(units == null ? parseExcelValueAsString(record, TraitFileColumns.SCALE_CLASS) : units) diff --git a/src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java b/src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java index 0eab0ad37..4ba0f99d7 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitFileValidatorError.java @@ -56,7 +56,7 @@ public ValidationError getMissingScaleMsg() { } @Override - public ValidationError getMissingScaleNameMsg() { + public ValidationError getMissingScaleUnitMsg() { return new ValidationError("Unit", "Missing unit", HttpStatus.UNPROCESSABLE_ENTITY); } diff --git a/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java b/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java index 107f5e28f..b2db156f6 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitValidatorError.java @@ -56,8 +56,8 @@ public ValidationError getMissingScaleMsg() { } @Override - public ValidationError getMissingScaleNameMsg() { - return new ValidationError("scale.scaleName", "Missing scale name", HttpStatus.BAD_REQUEST); + public ValidationError getMissingScaleUnitMsg() { + return new ValidationError("scale.units", "Missing units", HttpStatus.BAD_REQUEST); } @Override diff --git a/src/main/java/org/breedinginsight/services/validators/TraitValidatorErrorInterface.java b/src/main/java/org/breedinginsight/services/validators/TraitValidatorErrorInterface.java index a6bb314f4..49ef88da5 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitValidatorErrorInterface.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitValidatorErrorInterface.java @@ -28,7 +28,7 @@ public interface TraitValidatorErrorInterface { ValidationError getMissingMethodMsg(); ValidationError getMissingMethodClassMsg(); ValidationError getMissingScaleMsg(); - ValidationError getMissingScaleNameMsg(); + ValidationError getMissingScaleUnitMsg(); ValidationError getMissingScaleDataTypeMsg(); ValidationError getMissingObsVarNameMsg(); ValidationError getMissingTraitEntityMsg(); diff --git a/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java b/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java index 1b893c8bc..20d811125 100644 --- a/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java +++ b/src/main/java/org/breedinginsight/services/validators/TraitValidatorService.java @@ -18,7 +18,6 @@ import org.breedinginsight.api.model.v1.response.ValidationError; import org.breedinginsight.api.model.v1.response.ValidationErrors; -import org.breedinginsight.brapi.v1.model.TraitDataType; import org.breedinginsight.dao.db.enums.DataType; import org.breedinginsight.daos.TraitDAO; import org.breedinginsight.model.Method; @@ -65,8 +64,8 @@ public ValidationErrors checkRequiredTraitFields(List traits, TraitValida errors.addError(traitValidatorErrors.getRowNumber(i), error); } else { if (scale.getDataType() != null & scale.getDataType() == DataType.NUMERICAL && - (isBlank(scale.getScaleName()) || scale.getScaleName() == null )) { - ValidationError error = traitValidatorErrors.getMissingScaleNameMsg(); + (isBlank(scale.getUnits()))) { + ValidationError error = traitValidatorErrors.getMissingScaleUnitMsg(); errors.addError(traitValidatorErrors.getRowNumber(i), error); } if (scale.getDataType() == null || scale.getDataType() == null) { diff --git a/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java index 9f43c37b8..71f471f1b 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/TraitControllerIntegrationTest.java @@ -1367,6 +1367,8 @@ public void postTraitComputation() { // Set scale class to computation trait1.getMethod().setMethodClass("Computation"); + // Since method class of "Computation" will change scale class to "Numerical", units is reauired. + trait1.getScale().setUnits("test unit"); List traits = List.of(trait1); diff --git a/src/test/java/org/breedinginsight/api/v1/controller/TraitUploadControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/TraitUploadControllerIntegrationTest.java index 4763b7af8..f52cd917a 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/TraitUploadControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/TraitUploadControllerIntegrationTest.java @@ -581,7 +581,7 @@ private void checkValidTraitUpload(JsonObject traitUpload) { assertEquals("a^2 + b^2 = c^2", method.get("formula").getAsString(), "wrong method formula"); JsonObject scale = trait.get("scale").getAsJsonObject(); - assertEquals("1-4 Parlier field response score", scale.get("scaleName").getAsString(), "wrong scale name"); + assertEquals("Ordinal", scale.get("scaleName").getAsString(), "wrong scale name"); assertEquals(DataType.ORDINAL.getLiteral(), scale.get("dataType").getAsString(), "wrong scale dataType"); assertEquals(2, scale.get("decimalPlaces").getAsInt(), "wrong scale decimal places"); assertEquals(2, scale.get("validValueMin").getAsInt(), "wrong scale min value"); diff --git a/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiV1ObservationVariablesControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiV1ObservationVariablesControllerIntegrationTest.java index 300fcd928..038a691ec 100644 --- a/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiV1ObservationVariablesControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v1/controller/BrapiV1ObservationVariablesControllerIntegrationTest.java @@ -163,6 +163,7 @@ Trait buildTestTrait(String name, String traitClass, String description, Scale scale = Scale.builder() .scaleName(name + " scale") + .units("test unit") .dataType(DataType.NUMERICAL) .build(); diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java index 0e57050ca..85f6c6773 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationLevelsControllerIntegrationTest.java @@ -290,6 +290,7 @@ private List createTraits(int numToCreate) { .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build()) .scale(Scale.builder() .scaleName("test scale") + .units("test unit") .dataType(DataType.NUMERICAL) .validValueMin(0) .validValueMax(100) diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java index a45dbb6f6..3a3d8ffa3 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitControllerIntegrationTest.java @@ -422,6 +422,7 @@ private List createTraits(int numToCreate) { .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build()) .scale(Scale.builder() .scaleName("test scale") + .units("test unit") .dataType(DataType.NUMERICAL) .validValueMin(0) .validValueMax(100) diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java index 4a0a78526..16d7261fb 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIObservationsControllerIntegrationTest.java @@ -378,6 +378,7 @@ private List createTraits(int numToCreate) { .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build()) .scale(Scale.builder() .scaleName("test scale") + .units("test unit") .dataType(DataType.NUMERICAL) .validValueMin(0) .validValueMax(100) diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java index 955c6e619..ac276003c 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIStudiesControllerIntegrationTest.java @@ -376,6 +376,7 @@ private List createTraits(int numToCreate) { .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build()) .scale(Scale.builder() .scaleName("test scale") + .units("test unit") .dataType(DataType.NUMERICAL) .validValueMin(0) .validValueMax(100) diff --git a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java index d54aaafc1..58c20194f 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/BrAPIV2ObservationVariableControllerIntegrationTest.java @@ -386,6 +386,7 @@ private List createTraits(int numToCreate) { .scale(Scale.builder() .scaleName("test scale") .dataType(DataType.NUMERICAL) + .units("test unit") .validValueMin(0) .validValueMax(100) .build()) diff --git a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java index c3496ba41..de17556aa 100644 --- a/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/brapi/v2/ExperimentControllerIntegrationTest.java @@ -314,6 +314,7 @@ private List createTraits(int numToCreate) { .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build()) .scale(Scale.builder() .scaleName("test scale") + .units("test unit") .dataType(DataType.NUMERICAL) .validValueMin(0) .validValueMax(100) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java index 3a521aa55..445542e6c 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ImportTestUtils.java @@ -182,6 +182,7 @@ public List createTraits(int numToCreate) { .programObservationLevel(ProgramObservationLevel.builder().name("Plot").build()) .scale(Scale.builder() .scaleName("test scale") + .units("test unit") .dataType(DataType.NUMERICAL) .validValueMin(0) .validValueMax(100) diff --git a/src/test/java/org/breedinginsight/services/parsers/TraitFileParserUnitTest.java b/src/test/java/org/breedinginsight/services/parsers/TraitFileParserUnitTest.java index af195b790..e99587fd2 100644 --- a/src/test/java/org/breedinginsight/services/parsers/TraitFileParserUnitTest.java +++ b/src/test/java/org/breedinginsight/services/parsers/TraitFileParserUnitTest.java @@ -295,7 +295,7 @@ private void assertTestTraitEquals(Trait trait) { assertEquals("Estimation", method.getMethodClass(), "wrong method class"); assertEquals("a^2 + b^2 = c^2", method.getFormula(), "wrong method formula"); Scale scale = trait.getScale(); - assertEquals("1-4 Parlier field response score", scale.getScaleName(), "wrong scale name"); + assertEquals("Ordinal", scale.getScaleName(), "wrong scale name"); assertEquals(DataType.ORDINAL, scale.getDataType(), "wrong scale dataType"); assertEquals(2, scale.getDecimalPlaces(), "wrong scale decimal places"); assertEquals(2, scale.getValidValueMin(), "wrong scale min value"); diff --git a/src/test/resources/files/data_mismatched_cases.csv b/src/test/resources/files/data_mismatched_cases.csv index 4e31b2eb0..c74ab1152 100644 --- a/src/test/resources/files/data_mismatched_cases.csv +++ b/src/test/resources/files/data_mismatched_cases.csv @@ -1,2 +1,2 @@ name,synoNyms,deScription,Trait entity,Trait attribute,StaTus,Trait lists,Method name,Method description,Method class,Method formula,Units,Scale class,Scale decimal places,Scale lower limit,ScALe Upper limit,Scale CAtegories,Tags,Full Name,Term Type -PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,1-4 Parlier field response score,Ordinal,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",,,Germplasm PASSPORT \ No newline at end of file +PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,,Ordinal,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",,,Germplasm PASSPORT \ No newline at end of file diff --git a/src/test/resources/files/data_multiple_rows.csv b/src/test/resources/files/data_multiple_rows.csv index 636c6e376..7812d6a3b 100644 --- a/src/test/resources/files/data_multiple_rows.csv +++ b/src/test/resources/files/data_multiple_rows.csv @@ -1,4 +1,4 @@ Name,Synonyms,Description,Trait entity,Trait attribute,Status,Trait lists,Method name,Method description,Method class,Method formula,Units,Scale class,Scale decimal places,Scale lower limit,Scale upper limit,Scale categories,Tags,Full Name,Term Type -PM_leaf,Powdery Mildew Leaf; Powdery Mildew Severity Leaf,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew deverity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,1-4 Parlier field response score,Ordinal,,,,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",,Powdery Mildew Severity Leaf,Germplasm Attribute -PM_bud,Powdery Mildew Bud; Powdery Mildew Severity Bud,"Powdery mildew (PM) due to Erysiphe necator severity in field, bud only ",bud,powdery mildew deverity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, bud",Estimation,a^2 + b^2 = c^2,1-4 Parlier field response score,Ordinal,,,,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",,Powdery Mildew Severity Bud, -PM_stem,Powdery Mildew Stem; Powdery Mildew Severity Stem,"Powdery mildew (PM) due to Erysiphe necator severity in field, stem only ",stem,powdery mildew deverity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, stem",Estimation,a^2 + b^2 = c^2,1-4 Parlier field response score,Ordinal,,,,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",,Powdery Mildew Severity Stem,phenotype \ No newline at end of file +PM_leaf,Powdery Mildew Leaf; Powdery Mildew Severity Leaf,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew deverity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,,Ordinal,,,,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",,Powdery Mildew Severity Leaf,Germplasm Attribute +PM_bud,Powdery Mildew Bud; Powdery Mildew Severity Bud,"Powdery mildew (PM) due to Erysiphe necator severity in field, bud only ",bud,powdery mildew deverity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, bud",Estimation,a^2 + b^2 = c^2,,Ordinal,,,,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",,Powdery Mildew Severity Bud, +PM_stem,Powdery Mildew Stem; Powdery Mildew Severity Stem,"Powdery mildew (PM) due to Erysiphe necator severity in field, stem only ",stem,powdery mildew deverity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, stem",Estimation,a^2 + b^2 = c^2,,Ordinal,,,,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",,Powdery Mildew Severity Stem,phenotype \ No newline at end of file diff --git a/src/test/resources/files/data_multiple_rows.xls b/src/test/resources/files/data_multiple_rows.xls index 33e6590117d89e774572d4d5174c459aa5ed0880..be871c7da37f37cf8db549843be4c7f1bccc8809 100644 GIT binary patch delta 707 zcmZXS&ubGw6vyA2*`}LKP12V2SKRc6_D5=Kk;ESXY0-lLw;KYbc$Bt^J+z7#5j_Q@ z;4e^k7teywi%{9(kH0|BgW#p0(z`cLwmnGwPT7tJ4{v5ZGjHE#c4k^D+*;w)MW?+n zH~RR&m2Yy}S>{)=?dLk8UYVYd;^MI%_%UcB5pRO80X*MYOcIR5@Fg^WJ4?czJo@UEnSG zj+UgD3I1yf8aAw;XNuP{_lNnpj8|sbSDi_y$KofI9gbh0{-pfQF`{S4j!qDL)O$}V{z{t1rdC-1Kois_kmQ0=F37LbOpaYf|bkLFl9kQfBhvm!E8S7Vx z-;a8HRw-Od207;66m?+aOB4mQy?rP$`{+yR;dqcyuw=|fEE#iP$(W1sCi-XAiFq?I zYirhtdAuG}_sn|N+;8id6Bq@pBlkBpa*jOyN*gqxy67!lNJ^ILgIPvRb4N_5o{2xy MYb?L&)t3$DFW63fM*si- delta 873 zcmZ{j&ubGw6vw|ayG_$gY~pHTG$yXCtxYv8mEaGQE%acZNot`KdQ{R41cK3B@gRld zB1lCL#)D^1ISN|@4_eQHCj}89NdJo-G`>l8$BMYao7wNYH}8FRX13F1oi2Ob)JJcB z+57a+_$EC4G3$u~z0W@Q2P0+9;}LN)I>(-i`_atxO#ny8w#kl>Jt5;{$@4&t#xz;A zQFB{ftNy^cRj)T{j(h|=6SVOFNwF?yrD z7{oHfNu`uZk~*Q(Q7qxCQZnnsK*=+)2d5$VIPuvyseGK&DWx=08Krd5trSWj7awvd z?-<3(b020kNF7%yMk?gy#BC!b($#D!<^0X9x6B?+@iNkVy_WF3u~jWc4q zdZit5X@7BPILxKdnm=3`4s&TkF3S9#-ZZ+)_Mjz#N=Z^7uQuSFQWUhmIncepS6Uj^ lfg3Q`5A4I{Cf-EoClHm@nQjABZ zN!Gp}Q%Uno7dWj1EG8}N)aUS{L#7FG`P27f#y2C+S!d*n>(SO%cUx4%(I!*B~`)06Y$D_ zIP=m1vsHcy472tE03 z;*)4jd2aAfrcKqkc@u5L8)ZUO7DF$+X^v$NxZfdwTDQ@KlKsv59$6MUSiT`uM`vDc zB@SB+tB7f&;0@mMk)Rs?f+lR23RFCzW4_uM8v@;pOLsM3}FvrZnHFRc{O(%stz@TL-^SngM=U2jShF$dAUfkO~OnQU~DD8sL>2NGVTdlO7eZhSp7p2<_Qj1CKM6H^5 zD7(##c87=fv7NL-Z-h^9r{(MRdM>0OFVHz((xFgS?BVU4Br||YAewF5x3W`&Dwk0R_3IG6HT>$}DN)RnA0W_NxLk0kFQvm>s0Bp4YGwOp` zpZAOeT?@;G@(psCCnHe5TnY_Op^fwvl4xWe7jT#~$*+_jHTn_Q%^ZsW?D+G|n>o#xZxAwC7vd?`pL* zPQylW@k*(VnFG~yJHt5ql(Z}nMc36D#N13O?w#XEqh>lrw$wJzA+#{8l|u1@DYy5m zE{{&0IV-@ENk!OA45^wa9m1&dCRVbnvw^654B`_J$gf&tmS&mu#;oyEiX~{COus1U zeN@1m2k4$3-_mhX7k}jkt|<^ zYITlj3eC-OmFlR2{mZpmD_rOsHq)Q`{89?jUPGF@g${M`c7!)Pp0`G+BuiJbPtO{_ z-zChPpe*2l*|2`RNE<4~=z=S2t z4mDX{tUQ`)6S{NUz7IC~z!)-7rB3OSyCuZd=C%uDx5PNa0u@j?M=mlU-ZZJxMXT{c z!A{}dL+ewfPMM`#aNBHO!O9;#q*P6|hjj7I;XHP&ecq#qSnm>_RZG!&vBaND>>F-V z@5CoZiS7oC(u6WaH}-eNWy$Xodi<_NE*6xY=nQ;{BtMM}Od>}5n#q}u$ts?h`BH>$ zhY5z3xIYm0bBq8kjJBqOm}GNfgK==n)s zFIURyA&q@fV5X}^#x45(1L!I#`SO^vuU@TC795|-x8BT=H)Q!WS7Vhki)8aByd&a2 zO6De(dto^x1YrUhfV-&BQ2K;_}k!iAgc1cjushd#jNus<_H#jo*lAnf5KUj}$ zyFYf^=N0ZD9qN}0a>L0Er9M2y6vmKpG|vZ3u?OYlKD{+7xH+9Lx_5G7@eWLq3%0u< zB}h5dSHqy=q$>)dYNK@>5pmeR8IIl>j%)difMc2DWFS|MXS^DVmZ_J!y)SEpK+ zvPNgK7bTRFCn9Ab#ib6mUVQeq;xK_ZOUXE7|1niB-SfKT>W=4Z5l&*C*l5+Se9iwf zCI+F*quTr3;qQ^Q$9K-=arL~}bDzJ-i+>sX##dQp< z=W*%;h4a70Zt-T0;qI z7$Zd|sqgD17`q|fqM%-xn!FFZ83A;7PA8P)0BrbfMF3!F*l-l8H}AIF1XSCeu|5eQ zqq{uMqn~ApK@#gQ&$9>=c&MKRG@g zzq%;6dvz|9_>{DVE3^L|hl1~ASO7kCk(My#mi~m-b?QDYl1r59A?(v+H_OB-lAnlp zOb{rQO!#fAae4QxurKagVw3MXdK|>C7rygK9Pi3?jDw$*#a>F~Jv9v&Nj`WrB>!2b zoI`LI0l3v@tRiG~4K~gw2zOE0l%#x!iRspk5=ApOQQ*K8=$axh%1Dk-2cg*!<-c$o zfq%r;DnHxd(7|vBs!|t{3!xcGYc>edMvB0?U$h+aTLSB9yBlnk5d170=??dqzQk$D0P zI`NZ=^S%ZEWY_+<&36qoVh}CC_)LDETYKITJVeE~ zqIgSM?C@F|e(8+ixG|(j=(TnsQ=@1#-DMrUAscd&8I3KOaj}%+z#J`;0{$mx6-wWJ+ z!20NG+u=O!3V@B|pZr&Uwil3L|Nrjr>n(oc9=1wA75sx100{9E_YVm5boR%h1sFMg z$2mO!0QtTBx2UJb?$NVhwFQOArkJt3EKJyVLE&o_Sc0Gm*#ZytNRW^HxADK~%CGx> Pp&BbK1ify~|M%#>Tx6*S delta 3088 zcmY*bcRbXO8^7aRl+D?NID2PZ`_(_azQ6Y$@8|t`UeEK_GhXj1Q>tk#jDf-EL69daEeN!PKgR$EIJD@c%13R< z=?^S#)}&@Rw=U$cmG#hh}RyE*GDS$ub&~73m_2VuGyzM-b0wsl4^&8aTU*gS}&sEKEGJcon34; zH}0Dx4-LZTUq;7AJF0IV?1<7kc=JF=Mc46$d_=;SYqYwgB6B)5m5p5d7H10lbb*>_Ub zE^IBDp8<2~IW zHlOMTD6){$@WPbK8JU<%s&LHCKr$c;*>bp)YUgot(K}K<7I)AxCnOg$#^&7RY4GTc z5KYa}z0{ABaql%VQ}T`UGL0+K-d;~H#c-Nk#Cab06$!n-t})Oy<~BX~#^>RP9eG6G zu2wBz=jwtQ(AUF0iCLGj?btC;zR*oDp|48xv`it$#6+1UGDa_~^SD^JYxV+KrXgVt z*(c0Bkoj3F-a;QY3&iy4>c@S)bj-Io2h;~dD-KyXaZ7WhMG~&s?&f^=q{ruUXX>Md zYf-km9M5avy!GkawkY z3A*=%BKyO&qDNnjc5pVZ-_8<kAtkr|qk?>Qv zxXqWpQy$8o6}yG=#ZLM0Asbavr%R5?hkrYV{tRl|_^_wpJZ#tC84er__HWQ9T}kK` z7-f37bIdYuzvLTfM#wgptN(ic0?^qNY;8)Z!cJRjSGFZn0g4YP=FG z3z3x!y{w-}9#6S2!}-eN zB&^$5AT4dWj2@fZ5r9BCSj`EPCKTqX+SbF`eR1a}OmF>sU{);n@w>dMgiWy# zvMdKGFex}Lwy?c8FV5`2(-q?v6FZ`L40qnV{_tZ6qKbMtf_fwvK&89&f)=O3J&O!6 zmyoOC_>5?-2>e=d!v&-YM5o@V_eLgoY2O>!^_`>@j+TEJ^phiqI|ZZ zc~^YZ+C*;cr~-z&*u}mS=RN-3zVv~!Ird)z?TTNoZoZ1ZQQ=qY+gpH@;M;z=wd+!& zmE|L+nyp|GFYWga4;Q`e8Be6c*I^vV)=)kk9gF)#R!;W_tg!vl{6J=gv;pMcD?0;G zJ+`umvnDIpKa?fHNOLOioVZch(v8YxE4%jS%^90WIIyd)EAn(5y*9UeoRl*0w$=t7 zzBTP#Ac4b7q!rYP6V;(V91n=ofOBYF{iaiZms~*7{$2|2>Bv*@iaS5d8AhT!MPG(s zg=-f-t+(d%{Zt>WQ|{(XdtdO}0Ln3RoDicKB02`XZQE5Rsb9OCZgR?*@Y2|xRT@e^ z239Ki0ekQ&ZgeX&A?2?7Pl=camvs);7djbLhpp!rRhcYK-jN7kr#?D&`hk*=P%F;&P zHsov46_U!YH*0A#tj%0HH@X4%b{6aoW)fPsB1?m+*3E7f_9-J;!?shL*wF*Bz`WNZEVx_F!ZFI?wgjq`_QdVR+Z~JW$Rtuz(H?b@&T_I zKHWPKn|D^1O>2wov+vXBdg zry@*fZd)lG{z39hp4AzgUa9_3`2x94e76}<+;zHHMDgL(KJq-jGxt@?K|=# z&AtZp9f@C_ODT=rF}=$lbV%^6sPn$rtiWyIr@nqo)qPrQB6Sq)6u8q)wwB^juLjmV(9 zJ!pF%+PXGxrnYg55|V-P9;yw@G*M<{#o*YKC()UO6#53=lW^9w3$7wt=)>!JP0Q0! z0qEhmOCLA_ny6?sM%Ct%%+}$;<@qtI3RP+5QW?nb%{xpknm2BoV2Q)>pQfq~-q6m{ zXy3g=61*(b<}1y#ALn&(^Ui!o8KHwvQ<))jj(N!l`>dc^A-G(sSJ?VuE!~l!*Cldo z*4^Jr5(@k84p@Ab25+kWU|6)Xe?2m@x#gh*g-b8HbKWea4?l`G;%x2ysm&HWm;l&`aLW zKgi3)4}VjflmFj)X9IzF{#E}KWB4O+9U3}r{51&;a11_1;wlZJFuqITJpaF$|8X>b R+WrG7o>o#68YlKQ^*`mzs}ukL diff --git a/src/test/resources/files/data_one_row.csv b/src/test/resources/files/data_one_row.csv index 53e09c795..39839d30b 100644 --- a/src/test/resources/files/data_one_row.csv +++ b/src/test/resources/files/data_one_row.csv @@ -1,2 +1,2 @@ Name,Synonyms,Description,Trait entity,Trait attribute,Status,Trait lists,Method name,Method description,Method class,Method formula,Units,Scale class,Scale decimal places,Scale lower limit,Scale upper limit,Scale categories,Tags,Full Name,Term Type -PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,1-4 Parlier field response score,Ordinal,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",,Powdery Mildew Severity,germplasm passport \ No newline at end of file +PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,,Ordinal,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",,Powdery Mildew Severity,germplasm passport \ No newline at end of file diff --git a/src/test/resources/files/data_one_row.xls b/src/test/resources/files/data_one_row.xls index c08d525b4c1cdac4638abe67abacfcfcdcb7c7c9..8567ec081ea82fe4f4bc7fddadd22c8d644b1aef 100644 GIT binary patch delta 285 zcmZoTz}RqraYGIZJ43(Nc!9VCh=qZ;0f?D^SU{bDK@LO%v2S8YNk(e9f^TA8N@a4!NOM43qbB zXo@w+2Y_s0WCp4Ofq(xQ7Oue__VS;^9{&{|k&HHzmqeNIF@b%?2=o__CA|4Xlmz2s0~Ve|F)RQv5>Cwk diff --git a/src/test/resources/files/data_one_row.xlsx b/src/test/resources/files/data_one_row.xlsx index 57a616386e9553323540b3e385ccf40578156ec9..9514a98158fd7d74e68e9c95e4d8e692e54a8f58 100644 GIT binary patch delta 3634 zcmV-24$bl8V%%b|A_;#mAb}ty0{{RT2LJ#M0001ZY%h0ja%*C5Z)+}iZEU1f-%r~x z5PqMu{~_{waGWM-i3lkDr6G+80fWcMbzEZM#F_2TBEMsr%2l5<;090oyxD=KQ#Bc%Ex7;3#1 zjL#KoD_+|kL~}_k&^M)M%z#$0kHA$_KeUa{RMh~)N=RX!I#A?QEIq%km8L5RZf{BK zZ8dxo_>lmHaJYX7Ob1t0f@x*c+8zNk?nxi`PXbKHyAW&-WAHJg2y6Z*Tn_iFbn>69 zl08=P3PyrYqY;o+r6+g`dJZJHmna+#9IlhoKnAuKO4G=`3L-?lw=W!*gXfndrs@06+mGEgJ zu+a=#a2n!eI)lM3S_Z#SEj=uef6y&uE*Kc1(KGI!|Beh0*Xf|mORGJ2KA(Sw{9MyV z$QEd#*cE^Cxl70BSIu-vem%#FAfM*b*dH%~u^;78~?`1o#k#}>b#+cjRG-OXOz70cBkpI;Ikb$96VqN|Iy zr*C-GK&$0%&O;a6pAS1h4A(F!85EP79B|-_d~3fO+s(U@kqY(>nt>E6&HJv|y*UsF zUVua2<0MRGlO*)RoQ!-z7O_7aMX|qJESD1k@|YbjU!Yhz)D)0sJqcqqyw9%q`ueH1 zP6cj1@BG7X3Zy$Q!^3Os^bfOP3}p`m-_Un1jg!k8GZwX|V;5bTYz5-fSlsPj3tBqc z1SnD?lN1~wH+7RKTCsJ~=waumci17;iAkki3_I_q=TBdB$a$etp^WBXr)1Z8HTw3O z6X!hPUG0XQrT6*ElWZI-f5r0-6@O(t;eCKvc)a`(_F-Os~26pb)j zn6xKoYXn(Eug8<|0i{LE<(Vy=%)v{|iSWj5xreOn(2tB(Wh2n*B2aKj)jhfh#)|MH z9oV82Cy^DN9_Q99tjH+kxzwRby2Ljx>C;3B?YlHl2EXSOfB(OJ`TO%hk1kOf@uFLr zC2N|2ybu#|IVTdJQ%Vgx{q@UV9*0Ws63)2NdCm?Rr5EnoLmw}E(IC%NwC?bPK8eK3 z6hr6+@n82DZo2XC?Be(+7jvvwX}pBZsGRo_g8z{Q7QP-WVBcrC(4K*p!e-x4NW?SnIs>9J`VNrY~aj4d)GXwg?GHaX+skm#NT8T>8S{fW5vtOF@ zziMU{-+Gunv@7XGL+L42}A<_a9f8XT@<)*R8sQp>E!syf?_3y*Kvl?soENaP;H7KrSvQ3o+# zTVF8t#w_eO)gGrJ&-X$RH}^xSIQ2gR!g#<7G?`}7e#3Yv?YhX;$*&St`Prk3(1Ve) zf4rtvaey*$=msT}n6ZkrUvY~bDL;tDO}AZRC?75S096{+p#nOn!Fs&j*|8ba(EiHC zd^tKAUK(1-Nn=G^RfWmLXgEHgABN#h`tk5P7%|6*1rO-i zFyOgsYiI{;V-)JWy|af=+L!HX!h0)cVuf#L!#y4K-@Dm;t>zxhnTkcDa?#A@b?Eu- z+`0LzX>^G&j$gKXMm=zH4j%$7x9|C=ilaaSYTNZ!dp;kk39#nVGI=t!Mj0vZUg`T_Y9McAs+!$ldd5af2wI(yY1OH z7qDu_nQaPe>UZBcB!Gm}Mva2+F!}xQ-G%w}PAl>P)=5+4ltl?8Pzh7WYL(NUPj`bW zN}T6a!Ih~Xr+aYpdhz+wd~58pTLXB443*32+WUGI$4;z4a~GK!D*UB1R&$SE?J9P) z1zt2Dt>QFEvRHFj(F4QGfA--pWhn*R8nMw(`2&ds#Xb6WYgxOFO^ZHsn%n11JrG9M zNU@Yk`h7!6i59c_Rb?z+Dm?d&jkq8?i+75jVJFk1JCcQR+1W-GIsKQkf7bvb3rjLcSXj=k zSGt%N5<47DMJ$v#{laEXDUBENW`2H2*t)Zvc)olD1;PVrFiO0sAE5L%pj1w8aO1Br z`cZHN-@-)VCTda=rg&-0bKtv2WP-x3;R(WAc=-Ycjt4A=`_ov#6n(@;BjaLml z@0KjNgBzoM%fhdNe<&pdlzgN7*(Qs|$L$|x`xWGB?eXDIwlH_IVt)%x;J~B9h>e51 zFbcJBC0d3z7YCN_nz(JHCK1a{wAuI>MhWy z)4`+zO{1$m<;1qO9{cLUg$+rmfNDKQyC@<7e?d;eFc7>W@rTIw)J_^uvC>om#ED8s z6oI%Idua`cW7%6me~;so&r&7$$(Pt!o5^ zf6}d~74luIf#!^z(*{r|fr9pesT~!`UQ)?X(r#TH1(c*vLjwjq%_bTB;Uc)k|M?*z z2h2v}&W4%xb`Q0c9KezN>GeqLx^B`W)bf0>4Hj9+84%XwQ21lThFgqVM^^H7EudY`&q{>;&Qg)uQi&a+e;)0*$ z&;7hd|NC7Gjb?B1f5jqYd72fgoN+eed2yilEfCjR3$xyaBdWlp5BG7+V=GQ{1bm;> z7n5NO6|;CNp#uppAb}ty0{{RTlOiup0cMkNFFFBildLa50qc|PFDo10(04A41poj^ z4*&oZ0000000000000000Jf7eFee+qyFq$x1ONc{3;+Na0000000000000000Ckgf zFhv0wle;h`8}wjed5Qr50PF$)01*HH00000000000001?ll(9~0b-LoF(U>)E&u=k E00j-fL;wH) delta 3721 zcmV;44tDX}V&r15A_;%fWzw1_0{{RW2LJ#M0001ZY%h0ja%*C5Z)+}iZEU1fO>d(x z5WTOo{{i_J04=X7j z8~`YFj51@Y2xHAMUQm4`D_(+SEmc7cJgR%FE5&K5GtNzsW8d>aTu@P>79mof!I0~< zV0|+#x3cC_=$%Jd6$CiVGcfr)W?cH3VVioRx11= zt8kAMzJd|&+h_!&)o3Mx*mLvI$?;m`{HB$zL#ygHE$m6<$jPZTNh%CaV>EzA zx#8~?R9%g0k%N`M)kFAXFXv0;qZQ!41Vc_*-tGOz`~ZCwbqc*S%7Od& z^c+sx9ZK=-Sa_#VrDt^PY>Ak?YnVqt!mql_my-1+R+rHdnA2}+rd#HIz zY_M;=m%ON;+wwR5&m^<*at>{%n>Lgtdcykc*Y#ZR2b{ z9tcJ>FCgwg=!cVG=)3+I>A8d?ft5wz&XU<|NJf+CWHft$dTCH2K%TTDv{CR$Tk!SO zQ)#RO?D|;~9|kYbZO06EU$&Ef0F$8u6ti>;P!9zn=I@z|lgS%1f32uv7hRfc1>)3L z-0fcrS~}YVC{iUUCqhx6udx@{gY`-Fk+S1hlCuohA45J;8)`u3X>=RDzE?S`GD_xX!{-z7`V zgzK4{HCE=vX2RpIlY1N*e>rX`wES4`X=V8Ns53foa&+QHr$!Z-KI!|>Nk3RB>uYw2 zmAMD(`qJ5r$a2MX^OLC+(vuchW9hdd6Pf}P%O2?SfOa4v4{(Ydrp;I&kWX2DAqo>LNzLSA;x>(LeG!eX6P zLKW=rmS57K=3gMg3zv%oBtfvL4_Vux9~rI6MxfV4px~6Mdvp5UcWI&we$OlZe}Db*_veEiU7|GNMYl9d z)-(foAtvNzBVg4wc{~oN=Y|oEK3@2uL7uB<-Qfv+5{Z{9 zxFBq@#6$YIM}LM=fOn!$K2%M$CDa*eX$d*p!tK@XulBa1Qt+qAXt6R-4AfFw-YN~b zr72VzYEq;rek1hoj5FqWDVUP_0pC2K1|C);Jkbao0Sx5|y&GG&nwHzclH8 z)yyou^)P*CSJI7!(o;HfUZRgcw3=jI!+UrQwjvw7okg`ux5WhUugL(eZSN zu94@{U<}-=F7Dw}=Va(HYbh0$=~H!o^s!507Ot?Ie?h3b6uTj~*S!;lV9oLZv_?$2 z6pF3U0dHegUs;0w+}|HbI+;mbC5WU3NDB5kg&wq=A=P zYqVVa7HGXO)0ZZL^r|jh7Fmc`czkOysfC9r+9Oo#ikW_~uO@GzTE#GCK!^lY<5=kZ zb@$e1)XkL>DI_`r5pspj-r)DUO|EVsQS=&de?c4_dO(rQ5H(c5?mdXTW6@%I z^m?nj$Cc>bZ4eF26*!DFI9$oBIiiiFmRXTib+#QB9^bMn6&@3j$UTZJ5ZPO!4r0Kz zzF_Q)S=e!^Jx)cQ?}Z|6?uSxw>VF1=@qia-GR>s@j`333b&;);UnQ*avqu-92P0>B ze@(680A=FP4N536V-;(^;ubwpeh`hDZo9@%K3ezzsx+)a1$0n@^?1FrV>76s{gsXR za&$DjG_;bHe1gxIO~B>>9S{Eq?QYK3U2>qR3X_Y`aC|^N48xuD(DU87 zbMsl#=n`QZzij!8df?<7J_K5B-}6xwM}Y>^w(GCHJUvJtl5PzSv?_l|z zBn}V?B2-hWYL%vG8r}A6oC{dB^1 zZX;{gv1!qVPILRbtp~#B8Yxy%Nq=ZaDbZr~xUP)lD~0FYvk@0$Z}Cp?H|%7Z{VOFc zg*DEUK0@aB==u8i(|8(labmhYe=iy~iY>gz(7-cQ`aXY4Y<%KM&t$`Ql4U2dP%b;$ z$|9$KllB^5WMN4L2@A``^^Y#*g~SerQxOYgPCv2PQ%d8-yqTY$5_aw)C!VjCpg?#) z4MvGK^%p4p9VnI4N8I>hj9v<^;A@yj+(b=E!W6HJc@BJ!h)ht}H9SF>3ol>b$gvC` z!tcfkrs#hoJ{h^pJ5vwM+<&uV#U0!k^-C6h6GSN~pyXTSFE&{;K5c(DyU!rk8;>uC zvW2;u6^DCp0tX%)Mr<79g;A)5E73B1xj3+V-^5)jHHla@O0pEW#nySFI}a`V2*f@& zK#VPLG>L}8G#Rotkm#;KVr-FF8f85sv62lCW9LP6*vsp*1!C;HCcP9x*^lK3J#XuS zzF(;eJ5>~u4l)jndJQz{bTH{a)99v8IkBy+$Lx&F-V&U2AWY6KU+F^J49GDfZ_!O@tA5MAzy zX=}9kk4?zT2rBK03dAW9!R?HQa8YwI6WlIbF@~Zni3p`%5fO{(iV4KGR?W)=9$U$t zZf>W2`=McNNz4$|j;suW_O94%pS(3GY~P}@B_)aj3E0m1oR0wj0P?fAC?WxW!A|Qi z5WGK$e~5ff?S#SuD-9Anae5UX$L5jKtX%K+>MH4C#mEp>DInY z0!mV7p#y_~7L$U0aS?nMe*BP-J!YqI>*7p1yZhQo4&cas4|*hSHk--jIN6B)3;Hns zf0sthwdu2xK zX)ihfZj+=hKmzR+laV16llw0f8zScKnT!Ph07(x302BZK00000000000002ElPfSM z95DgQd~gH+02B=X02lxO00000000000001XA(N3I8k2}HDghjmvoIzb*v|T#j{yJx n@&W(=5dZ)H0000000000005+u^e{dFYLhcDBL+t<00000^R@Jw diff --git a/src/test/resources/files/data_one_row_blank_active_value.csv b/src/test/resources/files/data_one_row_blank_active_value.csv index 5c6021a72..a5060e5c8 100644 --- a/src/test/resources/files/data_one_row_blank_active_value.csv +++ b/src/test/resources/files/data_one_row_blank_active_value.csv @@ -1,2 +1,2 @@ Name,Synonyms,Description,Trait entity,Trait attribute,Status,Tags,Method name,Method description,Method class,Method formula,Units,Scale class,Scale decimal places,Scale lower limit,Scale upper limit,Scale categories,Full Name,Term Type -PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",Leaf,powdery mildew severity,,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,1-4 Parlier field response score,Ordinal,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",, \ No newline at end of file +PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",Leaf,powdery mildew severity,,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,,Ordinal,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",, \ No newline at end of file diff --git a/src/test/resources/files/data_one_row_blank_scale_class.csv b/src/test/resources/files/data_one_row_blank_scale_class.csv index 10274194b..b2b6c8064 100644 --- a/src/test/resources/files/data_one_row_blank_scale_class.csv +++ b/src/test/resources/files/data_one_row_blank_scale_class.csv @@ -1,2 +1,2 @@ Name,Synonyms,Description,Trait entity,Trait attribute,Status,Trait lists,Method name,Method description,Method class,Method formula,Units,Scale class,Scale decimal places,Scale lower limit,Scale upper limit,Scale categories,Full Name,Term Type -PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,1-4 Parlier field response score,,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",, \ No newline at end of file +PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,,,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",, \ No newline at end of file diff --git a/src/test/resources/files/data_one_row_computation.csv b/src/test/resources/files/data_one_row_computation.csv index 99d968aed..50eb06167 100644 --- a/src/test/resources/files/data_one_row_computation.csv +++ b/src/test/resources/files/data_one_row_computation.csv @@ -1,2 +1,2 @@ Name,Synonyms,Description,Trait entity,Trait attribute,Status,Trait lists,Method name,Method description,Method class,Method formula,Units,Scale class,Scale decimal places,Scale lower limit,Scale upper limit,Scale categories,Tags,Full Name,Term Type -PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation","Powdery Mildew severity, leaf",Computation,a^2 + b^2 = c^2,1-4 Parlier field response score,Ordinal,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",,, \ No newline at end of file +PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation","Powdery Mildew severity, leaf",Computation,a^2 + b^2 = c^2,test unit,Ordinal,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",,, \ No newline at end of file diff --git a/src/test/resources/files/data_one_row_empty_active_value.xls b/src/test/resources/files/data_one_row_empty_active_value.xls index a799cd957cf8c664f769b837f7d883d59d55064c..a77a8e470e1d9bf5f23a74642327864316048207 100644 GIT binary patch delta 239 zcmZoTz}RqraYGIZ`vN-~gXZ{KlZ#ne87EBs$?~6Z$L61`-b_q<%#+eNHl*;6c$;UY~IUC-+&>Dlb(97V YH_#c3V93qAIX`M5<75LCmPIiv0AaU0vH$=8 delta 264 zcmZoTz}RqraYGIZyTaG5tL0obCl|A{GESNNljT3-fz3Zzy_uMJ*e1JiXfaNhT*x79 z9l^lBzzf7RKr9Tz9YD+k!~*IJ401qP0*HMxlZ*0;^V3Qc{L|7hlT#H!i&Kjj7$)!M z&=h0P^#<9($P82m0{{LqFz~Q#X5&<1ln}+Cb+RE>#N-)V4y^y{zcBvae3$F3CS%Cv z(Olkp}U-=0$ZhjH=gNgSq0~Z4$12=;MgBsAbmyx!Dd_bp(FmN&2F$pnjV*|Q; ZvU!vSj}XvaMlci-+MFFVk#SKB3jn9=MDhRt diff --git a/src/test/resources/files/data_one_row_empty_formula_headers.xls b/src/test/resources/files/data_one_row_empty_formula_headers.xls index c52f828ad33b5f6785232e05311e56fe6e86f9e4..3def6652920d7b5c71798199d74a47f4f1921fa9 100644 GIT binary patch delta 37 tcmZoTz}RqraYGIZ`-WXQ63y|qHW#yOW#wdJV93qMHL)<>Y{+#!6aWYx4E_KB delta 33 pcmZoTz}RqraYGIZdqTTcq8Ing&BZKRSs6tquj3Hk?7(#(6adis3)TPt diff --git a/src/test/resources/files/data_one_row_empty_headers.csv b/src/test/resources/files/data_one_row_empty_headers.csv index 212c8db5d..2f1514357 100644 --- a/src/test/resources/files/data_one_row_empty_headers.csv +++ b/src/test/resources/files/data_one_row_empty_headers.csv @@ -1,2 +1,2 @@ Name,Synonyms,Description,Trait entity,Trait attribute,Status,Tags,Method name,Method description,Method class,Method formula,Units,Scale class,Scale decimal places,Scale lower limit,Scale upper limit,Scale categories,Full Name,Term Type -PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,1-4 Parlier field response score,Ordinal,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",, \ No newline at end of file +PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,,Ordinal,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",, \ No newline at end of file diff --git a/src/test/resources/files/data_one_row_empty_headers.xls b/src/test/resources/files/data_one_row_empty_headers.xls index ec41889c20e0e7bf84019f3b25aebaae5a5bd493..f4b24f44a57d476a1b4ef1df8b32db4f79db590d 100644 GIT binary patch delta 277 zcmZoTz}RqraYGIZyTa#im*)6elZ#ne88=M+$?~6Z$L61`-b_q<%#+eNHl*;6c$;UY~IU8QbQ@CmRDpZceU=h4ExZuJFkLTmqA4a5=F4sQ<$FWAa@tCq{}nv5ZvPn&QkGC2ioe&r|3xcNmG7c(a#6B`2~12==i3mJtlOxi|Mm6*5jXU}0Gl!vX;1vO$sn delta 302 zcmZoTz}RqraYGIZdxEm$r*f{FlZ#ne8MjRS$?~7^z~-N<-b_q9Y?IwMv=}E$F65B5 zj$mM5;059uAQlGV4j^U%VgYpq200)t0mQzU$wm3a`DrBz{%L8M$*BsV#i>OM43qbB zXo@kUKK}@`g^?Mk4g~)FXJFuA+swwP#3&((L+fNiu87GqxExsj*MDLBKlv_~6Jz~k z1#Ve1vI;69&CJA5zj+(Ch$ds`=F=t|icD?+n_u|}Gj4tn#>LG0mw}6ck%61Rfk6%E urr(j?f_gv?h%iVn+A#?+Y-0oZWpa3w29F5Pb&OyrBC@$Vs*rI}3=0535KSil diff --git a/src/test/resources/files/data_one_row_invalid_active_value.csv b/src/test/resources/files/data_one_row_invalid_active_value.csv index eb181ba35..cc1ebd433 100644 --- a/src/test/resources/files/data_one_row_invalid_active_value.csv +++ b/src/test/resources/files/data_one_row_invalid_active_value.csv @@ -1,2 +1,2 @@ Name,Synonyms,Description,Trait entity,Trait attribute,Status,Trait lists,Method name,Method description,Method class,Method formula,Units,Scale class,Scale decimal places,Scale lower limit,Scale upper limit,Scale categories,Full Name,Term Type -PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powder mildew severity,blah,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,1-4 Parlier field response score,Ordinal,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",, \ No newline at end of file +PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powder mildew severity,blah,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,,Ordinal,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",, \ No newline at end of file diff --git a/src/test/resources/files/data_one_row_invalid_scale_class.csv b/src/test/resources/files/data_one_row_invalid_scale_class.csv index ba37dcf37..942743d9c 100644 --- a/src/test/resources/files/data_one_row_invalid_scale_class.csv +++ b/src/test/resources/files/data_one_row_invalid_scale_class.csv @@ -1,2 +1,2 @@ Name,Synonyms,Description,Trait entity,Trait attribute,Status,Trait lists,Method name,Method description,Method class,Method formula,Units,Scale class,Scale decimal places,Scale lower limit,Scale upper limit,Scale categories,Full Name,Term Type -PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,1-4 Parlier field response score,blah,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",, \ No newline at end of file +PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,,blah,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",, \ No newline at end of file diff --git a/src/test/resources/files/data_one_row_scale_categories_blank.csv b/src/test/resources/files/data_one_row_scale_categories_blank.csv index b4b0906a8..41c726109 100644 --- a/src/test/resources/files/data_one_row_scale_categories_blank.csv +++ b/src/test/resources/files/data_one_row_scale_categories_blank.csv @@ -1,2 +1,2 @@ Name,Synonyms,Description,Trait entity,Trait attribute,Status,Tags,Method name,Method description,Method class,Method formula,Units,Scale class,Scale decimal places,Scale lower limit,Scale upper limit,Scale categories,Full Name,Term Type -PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,1-4 Parlier field response score,Ordinal,2,2,9999,,, \ No newline at end of file +PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,,Ordinal,2,2,9999,,, \ No newline at end of file diff --git a/src/test/resources/files/data_one_row_scale_decimal_blank.csv b/src/test/resources/files/data_one_row_scale_decimal_blank.csv index 585d81ba2..4fda9344b 100644 --- a/src/test/resources/files/data_one_row_scale_decimal_blank.csv +++ b/src/test/resources/files/data_one_row_scale_decimal_blank.csv @@ -1,2 +1,2 @@ Name,Synonyms,Description,Trait entity,Trait attribute,Status,Tags,Method name,Method description,Method class,Method formula,Units,Scale class,Scale decimal places,Scale lower limit,Scale upper limit,Scale categories,Full Name,Term Type -PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,1-4 Parlier field response score,Ordinal,,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",, \ No newline at end of file +PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,,Ordinal,,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",, \ No newline at end of file diff --git a/src/test/resources/files/data_one_row_scale_decimal_invalid.csv b/src/test/resources/files/data_one_row_scale_decimal_invalid.csv index a7d774994..e504fb732 100644 --- a/src/test/resources/files/data_one_row_scale_decimal_invalid.csv +++ b/src/test/resources/files/data_one_row_scale_decimal_invalid.csv @@ -1,2 +1,2 @@ Name,Synonyms,Description,Trait entity,Trait attribute,Status,Trait lists,Method name,Method description,Method class,Method formula,Units,Scale class,Scale decimal places,Scale lower limit,Scale upper limit,Scale categories,Full Name,Term Type -PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,1-4 Parlier field response score,Ordinal,2t,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",, \ No newline at end of file +PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,,Ordinal,2t,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",, \ No newline at end of file diff --git a/src/test/resources/files/data_one_row_trait_level_not_exist.csv b/src/test/resources/files/data_one_row_trait_level_not_exist.csv index edcbb3a3f..a8f8770a1 100644 --- a/src/test/resources/files/data_one_row_trait_level_not_exist.csv +++ b/src/test/resources/files/data_one_row_trait_level_not_exist.csv @@ -1,2 +1,2 @@ Name,Synonyms,Description,Trait entity,Trait attribute,Status,Trait lists,Method name,Method description,Method class,Method formula,Units,Scale class,Scale decimal places,Scale lower limit,Scale upper limit,Scale categories,Tags,Full Name,Term Type -PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",bud,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,1-4 Parlier field response score,Ordinal,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",,Powdery Mildew Severity,Phenotype \ No newline at end of file +PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",bud,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,,Ordinal,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",,Powdery Mildew Severity,Phenotype \ No newline at end of file diff --git a/src/test/resources/files/duplicatesInFile.csv b/src/test/resources/files/duplicatesInFile.csv index 55e7b9605..3f88503f3 100644 --- a/src/test/resources/files/duplicatesInFile.csv +++ b/src/test/resources/files/duplicatesInFile.csv @@ -1,3 +1,3 @@ Name,Synonyms,Description,Trait entity,Trait attribute,Status,Trait lists,Method name,Method description,Method class,Method formula,Units,Scale class,Scale decimal places,Scale lower limit,Scale upper limit,Scale categories,Tags,Full Name,Term Type -PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,1-4 Parlier field response score,Ordinal,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",,, -PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,1-4 Parlier field response score,Ordinal,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",,, \ No newline at end of file +PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,,Ordinal,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",,, +PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,,Ordinal,2,2,9999,"1=No visible infection; 2=Very few, small colonies; 3=< 50% coverage; 4=>50% coverage",,, \ No newline at end of file diff --git a/src/test/resources/files/missing_category_label_nominal.csv b/src/test/resources/files/missing_category_label_nominal.csv index 7998bf97b..5e67bb65a 100644 --- a/src/test/resources/files/missing_category_label_nominal.csv +++ b/src/test/resources/files/missing_category_label_nominal.csv @@ -1,2 +1,2 @@ Name,Synonyms,Description,Trait entity,Trait attribute,Status,Trait lists,Method name,Method description,Method class,Method formula,Units,Scale class,Scale decimal places,Scale lower limit,Scale upper limit,Scale categories,Tags,Full Name,Term Type -PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,1-4 Parlier field response score,Nominal,2,2,9999,"No visible infection; Very few, small colonies; < 50% coverage; >50% coverage",,, \ No newline at end of file +PM_Leaf,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,active,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,,Nominal,2,2,9999,"No visible infection; Very few, small colonies; < 50% coverage; >50% coverage",,, \ No newline at end of file diff --git a/src/test/resources/files/missing_scale_unit.xlsx b/src/test/resources/files/missing_scale_unit.xlsx index 466b531c0fffb90672eadc613e65e19ddc7799af..ae300d40d66464ffb896505fc9d9aec9f1ac0e08 100644 GIT binary patch delta 4212 zcmY+HXHe6L)5jBfA{|1n(pw0EbOj<+0a4n4^d?o1E(n1LLVzIThk!stX(COK-UO+k zx2Q)i8U&TzQOKX?Ufj*IuXc83%gpY~em@B*;HN3zYA*e zU->ZqKpz)>e;;_bpKpbwiGLx2x&2)o8GK@jg$P5W-_E3Yi)>UCL<`9DtA2z!eO5Lp zDNOs>oso#el4z-wEg zlqL3Tf(`X$&G*mQ?7Dv00dlaO5>%Xe$HN%rTw&99sqD5}@2zdwV(C(_d1ZAK`gZu?;|!yyZosq9JG$>%mWsV}*vH<2Pu1C;Z;nF`S(?ygA@rdI8dIjSvmtQ`IY7I!wxyI8XR-(AEF?BqZx@m2<3l5Iv9xGk9Ps&@YpA7M| zJS&N>v`Ed%^Hwy8U-n9&aT;XM(u^s!u(u~(rj_4$jARzrB57JXX>W`Lbz2%S0Bo#V|v($pg+{qj>_j2swyyz4l3kp$`{jm zP4#7;1xn$5JM4FehvGt{Dkt87pT(zle{#+9gyKczB5f%taAAM^^`H%=cW@o*9ny zjWn*lQ68cH!KR%?wQ$nOOI@Y1aWLpyS4lTo>>-Lf)tsZ8gtpEk6UlYP3TZ{GTA2aXR$9cmpz)B_UfEvTlOH`IV7 zdS|c~fMgIuUEdoyHDIhAQOb8*{AMk&G*7WRzacwI|26qnRg=Ke&$+qfxoyw<#xW0+ z`OL4d`W?}@IZ1EXggcdEC9>m7#JYZ8g}DB-+MCfxE7(n?B9A{)`&mGAK6ToR^`?3- z@?(ly82uU4IfD%A zyJNA*E_dU$gcNV}lkHRc;GpAqojTVRJSVrteD{E+^T)UYWCH1m(6zeDb9OxrbCB>*gOv7OozMhYx&cu;v6_WrpD_NQAC8(Kn(oCwhh9pO(z>6xOCoS7&)jXqH;1&@X)(cHtuFkggj zS>1##B!O!*Xu-APaCcSm$I=U_5qEPTzCKAOhUa4b4b-QHAyFY%3O_@0oUL+&wnWLq z$d8R&l1`aKyX!EOwr6l`XkXnUcCQv!^qN^i1u9nJxdoc*xV$0#0ZgR$7HddG&qEGK z7cY?|@M^qpd6Idk#JWRuUodm8w~Kbi!j4sB&MWf(bm^O@NWEfDaLw{n6$x6s*jFe0 zej>1G;`kBE7#%|@^aJG2UB^CbN2#c)<=vsdIPFyR5&2n1-Q>ja*!}lpBQDKk_Q=&x zFv`Fp@P(+R;9moDx8&p8DTht729Mfg$G4iub~-(Wtj{>fznE<0ZZXREwnEvl8fsgahL>oy7NWGoD|I?7GMQl}eAtjM#IR@E(hx>3Nxsos_t;a6q4U&GpcmKj24; zgV5>oL`MZ!@^$`#+qb)e1OD#OuQ1n8oW@T!d(kXp=UfkGdC->96Z(2&;N@F{CX9r~ zj8&+0(33VW2B;Z*$9Ds>u>6w)v=QpCp+0wRZ1UtGA|$CTWM@r--qn1n{pg-m!R^w%WH*G6(7M*49>!#E^5d3+p0z-rjyMt8@4T zW*zqa#3S@DuQu7H(dONeUah~!^n>59jgr2c(n$XWb@G?z_+xoGWDwLqD4x1My3;$)Jc)O{n6jig~$tHq#fs!08$7>Uq6 zNnI3a))Hed6jRhwNTnHJfl{WRe92BlAD)>=Nof_x5q4MQko!8#bbl)K$zsvrHt|?f z5XeE#sbLk2uQHL?*aThd3DK`2r66Xm!jNRC*NSSx33r)-v?TkV(#cT3TH(&qwuDoI-ZxBDVd8!$HDG5n+9R{$=J|(R7K-j%zJ9PJa_Fx zO1D_GCI7Vm+K6n8B_VsY)}1A7$@`3*xn>0Z7)h~MM#C=Tu#9@~lY3S&2Wv69cY}q5 zdcf~H`L;VUy7||f13;9cdRF5<>02PD7CqWrF?RGI|VInLSvT0FCEL>n^iJ*;J zGvUZyIRE(1gjW`X%toU@LOZYX*5rN{2Rylnd9{SAPFqf5BrQQd-Mj{^8peQ`=veY( zsXl(4LM3&8YP;F*(n1St*u3FXAW$14Ajk(p`S};BGM`6Jp3#2Wipv7UOS;v2CDlzX zEZp2MI{E|#{_^p4%Y!#hogeY?8hYl=7d2a@+s=l2ynOMT#E zDBGQW!@$Yo^BMt# zCom?pvBZ#;DT?!VFqX5Z;KID(woF5~VaYEE}h-5;K& zJd_-T?|Ll0nCHbDy%cnSkJIU2!ny?cgiji%?H!DjatlwLbWrsM6<<4Hb(1PP`Dyw9 zCyHs8O#LdF9iii7Tp3B9!C54xmsGHXWr_1o?2MyeZQ*zy1J@v~vZJTvEDB7YDO87CuFkz15SRzgSxol~&l_$|+60PIa`1EyV6C!!r=!^m&)4 zLB}z_k(ki|nW$guD@+6L0VCX8&%}NoL&=n6(){-5CgIWOiFN6P2DbK8=d7|WJ+Erq^N*-Je-lX|?8XWE3 zp#Kt|x;(pfeIzdH&eVNum2&C3JG0@FN{qdp>zAyOKCC5qP-5#}^ScM63J(5k?#$P) z+#)6(iGUaFv>B+TszG`a0=b!9v^O1hTi^yTWhIl<-VS9hFKJqMoLpbA#sKT4r zk1<~6n+>pi8p;uIE-eeq40Gm4txmAMZfMuP*YXBSBRI-lNmuxo@G0%}j}hvGb`ck6`3hv*W0mSKV2e z%y+g7I4`%zq6h&zz2cqypr_e7j3bvEEtN*dk+t#dvYU3{) zF{4pII>k$>FNf1ialAL};= zmhphmuBESNyp#^F36eb-N^QxH5Dz#8Ta==F6!E{{)9BknHfO~{Hh+iUzr+_Z{9Npp zU+fK7?lhX~$c4vWHFafPns5xpZF zbdMW*aQ#ATIs$R7l@n5gM?rL3VbcO)U`XOEWjLfZ#tdchQg(85yGj zhuEQ>+t`KQvUbzaq{K}ek&X;MfB`Tcd+iupg&FX_EC7gV$eSXoDn5BAEGW5hXzoUDT){qD2ic zf+&d)5j~>wc+RudIq!P+r@i*t*WUMsYu)$ti%SB>CxL6csc!h zmzJF$aA@nzti1H~6zI6SxEC^V6rt8ux;GT&Y=$-M=~oX(eNQb4bws-a2BcCU@feXXEJkCZycy<8$i!Zr1cvl`I4^Ov@x1|Dr^JJXh>BauSvUBp+BHk zVAdO$cY73z$YFMT&cZ|3>SA1%FF}iXrPm>vxC!n$B$4U6W1(sG9S`8*t*0TsF-|pL(d)_Vol7_U3zDa`t;kUm8C) z=-Hg}zZtNIt4I=7H)!@iC5_w`3I25mY9m-GSA~*WUA1Lc8rK>zprYwNb1!}>p5Vc3 z>XkZ!%RBaT|Kx0J2Ig(7n$CY@K<-JwPr9jT?4@Jng9`MqlzGh5VYy-knR zMt|p-Y325#*`noJs7Lal6fvd+gftO!cfmHzJ;FrYHUmUpvWG(3&~|>~6cZ8Mk)*;T z*--D?krfrMbIQw*%AyzbTBCu6`CvX$%YAtsr$H7qM)PidGA~K0uW|Lz^<;OB>iM(b zzVrI6(feJpN__7UsLj>_^RXlxd|V&HU5>SWj(k-yeGl+G5%OF>%XX}(%+_J2CmZcX zf`69M)z$x0b>w#;hW?%&CZRZ;m5)=+`yr?6xUj=8+9BP6ijnXdmMzf?i?d8}tbN*u z#;uH@C)!=^JB@jZ3Xl2dmU~&?hIgT5-1l8DKYf{9#_qWV;~)26%%&stP`a5PWwiHP ze$dTmzx6qDSQ}^59pj>it=iI5An({13X9n|pve1j5T2*LQ!+2Elz-Fn+grE3Iq>*J z>4RbEgq9EQYYY=7iG(o;dz3X=U66Wtl17D8P3sAV?_8w(F~B5uh0L)s)K~|Ir?nuh z3z(G!7HDsSZv$F%0=S$bk2Ajb&mP&M)qcC>%-eUsN1MSJn7AEeu(dsd_WJPfrxs;%YUYdYOk1<6_$<@Yu?^i1#Fh{gRl$q91&ITi^& z(3FQXt!N$Cys-|i=lBkKH=kKGYU8ZE|JZSpe_~Q2<6(VwI~h*-H$K+VG*v@_)d&lJ z)qypb3fN`g<1$1xzRZN~f}LZM?Z#={J1`C7m5*L8K>4_AXg-i?6{Ae9?Jq{(Ostp= ze?(|SmffV1D%u^&GFT~iJ31{xo17{B6Y8m@d!Hi2*2I1;rc;x%${f+`n=GUxw#V30 z57pQ4ndZ#-m5p0*@6KVe>V2dHX$o8U5OU^HAx$b^UV)fyx-itwAA|*o&UgC2e~Xb9 zo^Cm_#}DQb+)LMr+(?31+w_zUS>>D7hquZK+9x);`lmM_N^Gj)xi$VjYVMNvGF4JG zu?cxxfnk`vkE(hX(V)_?GQl6#TFg3gtS8oa-iH!~d4ROkmAWH(2$=pfuPBR~3Xz=+mgxSTWxzZp0zZcAuG<5q_sR45f3Yh_;U)BdzLn2Xpe?ffbEDTW$7gIlO& z7Q%~SL@51ZzwPB5Ojp;nDT^dJHrFDEVul1-K1W%H#IV-xg^qcrVfgFhsVnI%D`F8V zEDQFXxF$89J#o~3lBqI?b? zZ(qYr7w&%G$9wk;hTp>SRQz&|dNcHuyX?w&ZbQCULXXqyb7=lq7x9DsTl$l#)TL)v z&h=PQ1dOUPyEGQaG-kTg{R7W>1U!?q>0-yFDnBt75^#UW)8H8~4ej1UTaA<76j@nX z9M?WZa17>zE;ZSA|5k-eY>b-BENCy)m#ekmQbafJk9V@x7iTQ^9BEtJS^eN*u|ub> zM`Rcw*x)_M21>rXyAgTX^FDZ$n@sn9CCjIyWIOWf6faSm@^Fb4-1d0rd0XtwHIW`y zI|c%At|fd4#aeqm$QCIrQGkl$pdVm#-HqTz^OH3ym~c0ysTt@g59lxC26^JbRxV&T z8E+;=yPkTz-UG1geMkc4SWTob8t;%MLE`d3_(V`w=6XGCf`sd?2gi0JUAtydIHCQv zh;y#CN>7#}KXSDJ|0Hx&p>g*{@y;hwdl`lH0D(Cjwl`Y^K_s*(NMs5j233;zqQDfj zHF>x9T6~9=dA%6h;eISKxpBWwz3L-w`#n53u^w4eKU| zhqb^4bzSGbTYdiVh%W?+8d3JI4yBH-<5&V};*ZjNiLN94b zTPgMQ)%4#Mdb;z$7m-?TqjA=WN^oZpPwb5#*Zd`({_DZw{TG4_@j|_;_w;I{)>T7lLb`)-bD!S!w&#|bXEXU>iFGy0UPJA6V-^8d3D;b zcLD>SpMi`K!Uxq8FizVz z zyp|uMu%`wn^-#pb=y%4M*>-6fCAB0K4O`yC3wHY#`mb(ur7KG>f_Y+rhuonNr^Y)J zbwe;S3Y?r#WuPo62z2!_aMArQa|Hz6L%Rq3H*wWl+xZnL!J?eJFGzM(m!QuXA0`Q6 z@VeQ#9IqU_ojBl$^1ZsfYpD*yxCK`dDvT^3H!E!Bdo6z}V$~=7{6%HfzK)wVnK51w zi${h^QV$YEj@PB@T;1fU_aL%es7JxLA}ubK&I<*<$z*&y9nTRM{psP zT9Mh5nchjCN36nGlxSpb5c^n{#&T%KLD65-Q28gLhh-GK-ox)r7|d!xS#5#XVCf8- zxSQqCuZe7$L~Gj8M#t!Y@_pVlyGr)7tWVZqqS*%H31&AzL?(Asz;5=Pul)455eB{J z2sXl1m8y%OKUR(jFeQSTDaZBttjyT*-of`7;kCy*->Tvg zu)-rFdmamkUwjio^0@M(#wc}ZhFv0kcZ&}ESJv}m*rX?RG~(YIj9ov1x`~wS5p(DG zQP6Nx_C6GRKlW{@?LND(l%ByO7_v)MmU00fgC!#k6mSDO1eXqrDAnQYY=_uHuJD%? ztv@-h)}g~$BSe1aJie#3V|#Zf(1s4`iuy4!mSA3_Fs2u;5@ku2Z1r(n$-H}8I&cIA z`bx4TE1<$BRbOj-6-SwoYe-lYOfFWu1JBhp{S|i*oS9RSgd>@xs%`_XrM1q3_Qf>* zph!frn0q+`U!OW`?v3H%)4UBH`;}LiuP%VEQeZbA$a=3q; zmGgm`+fs{1tJJIy(#=DXO!t}1S^tl0>mG~9&Yg1Dqvb(+1AFsD*YjL?K5exFUW1B8 zHE*a1mmy>;>8vJm6FqSQjSD>=QlB7Up=G$WSe`tvSOPA;L7s{8S-#UK*S;KabM%&z zC9}P9l{a8C;>z}>iAo_WLM62HS<0?~Q}p8XiI>mz9ljpp>$RtI8r8}iW8U#&UHq{Mj*VhA3##r|+xX-B=*T8$VvysU$1m38^gFp)s^8tC`vWWLu`A|0;4%+nz0jg= zspMujad27#OvrhW@jIT8J>mFGkJNztYfAW~&W3b3Nu z>leJ)Vg8GI;(BY7hHiD%#ZH`bilknkRF*ugqsnW=MxhJiySEI+y*`dBwzGPryb+38 zj-pj3jucmUQZOZBloGU6Y1A(ysWT34vm`giU+oC8@7rs!1_ZfAV3m}G5ec6dU+jpI zQWbZ?yV-b`*d6NuXYQ-O34;Ez%OL9B^HqQWIVDaYj8~!#rvy*{DpIsS2sa;)#V1kM z$DIzQl6>vkW(x*^uytVqa4-&Q1Q8#1HUsU<}ESv`2U$9eqkQ1f8 zTDT0toenaojsWLEz zjCB-Wgt`U1Xm7X~TYEda^c7W(wo0u@OYsy(P~x@0>t0k!yl!#6HEB3=11!bwXZQpo zy%Alvy;tR&Z`tCgeNv|u;gmZm#Mr9g{HtN^Un?G}D^yL0Y^Du==1$(c&s128nSK7` zgjEuk;6zN3inMgix*TVi;Hf#k+HuN%r1*C{Jrh!71nhY^0W&2DpoF&@ zfNG9W{R7y*x~3fa-*|C}+m@HO4ZCdp|E?ffAeEK{kk?@Zj_4Tytdv8)v^X_ z1I!3Pq5nTS{(XRg{cn2z-y{K~bzqmC_zUa~q#(qAPUK&g81o4scKyAWL)-dC0s`e!Xppd*3p1a zD*zT!BESb7QOL3qkb-0Z=#X-dpDKTE_2sC6t4Ky50QnbWNA2No visible infection; 2->Very few, small colonies; 3->50% coverage; 4->50% coverage",, -PM_Bud,Powdery Mildew Bud: Powdery Mildew Severity Bud,"Powdery mildew (PM) due to Erysiphe necator severity in field, bud only ",bud,powdery mildew severity,FALSE,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, bud",Estimation,a^2 + b^2 = c^2,1-4 Parlier field response score,Txt,,,,"1=No visible infection: 2=Very few, small colonies: 3=< 50% coverage: 4=>50% coverage",, -PM_Stem,Powdery Mildew Stem. Powdery Mildew Severity Stem,"Powdery mildew (PM) due to Erysiphe necator severity in field, stem only ",stem,powdery mildew severity,actv,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, stem",Estimation,a^2 + b^2 = c^2,1-4 Parlier field response score,Nmber,Four,One,Ten,"No visible infection; Very few, small colonies; 50% coverage; 50% coverage",,NotATermType \ No newline at end of file +PM_Leaf,Powdery Mildew Leaf Powdery Mildew Severity Leaf,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powdery mildew severity,TRUE,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, leaf",Estimation,a^2 + b^2 = c^2,,Ordinl,,,,"1->No visible infection; 2->Very few, small colonies; 3->50% coverage; 4->50% coverage",, +PM_Bud,Powdery Mildew Bud: Powdery Mildew Severity Bud,"Powdery mildew (PM) due to Erysiphe necator severity in field, bud only ",bud,powdery mildew severity,FALSE,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, bud",Estimation,a^2 + b^2 = c^2,,Txt,,,,"1=No visible infection: 2=Very few, small colonies: 3=< 50% coverage: 4=>50% coverage",, +PM_Stem,Powdery Mildew Stem. Powdery Mildew Severity Stem,"Powdery mildew (PM) due to Erysiphe necator severity in field, stem only ",stem,powdery mildew severity,actv,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation ","Powdery Mildew severity, stem",Estimation,a^2 + b^2 = c^2,,Nmber,Four,One,Ten,"No visible infection; Very few, small colonies; 50% coverage; 50% coverage",,NotATermType \ No newline at end of file diff --git a/src/test/resources/files/ontology/data_nominal_missing_categories.xlsx b/src/test/resources/files/ontology/data_nominal_missing_categories.xlsx index dee1c7dc24bb97f237e89a1548a9dd5c275a5571..52de1dd023f909a06257693ce7e75bb7276c5f02 100644 GIT binary patch delta 4464 zcmZ8lcQD-R_udt&v#Ug1qL;-+kmzFd9$oY$dhaC(i;$>Ge6*-Rlpvyo5S^%3=Ss9i zh!S05mEh;z@6652@BH!3nRm`J@64I^obx(QnfhSy+#~L# z;pLZjD`EnVkj+^YbYIiJffSx?Zk-{UGbr!lgkt&~PA4|s-~oHYphLp0=lgki-}(Xv zO+T|kMvo?pDns`kZe+&b-FJg3IEX(WLG)2M$)jOWYrRljL2E^O4r3H0#Wc+iO`oi- z1V6*oFZdQ1gHfvU3)tHvbpK~X5&85=WUxyhd))dIC^R=#v(XuHPGX6gF0ZoWjVELO z&dK077tUFV9``rXR9TnmUres5)88jwejw}oJCGMclB;b@Qt~F9+>dB1SnW*t{(%UG zmqE4EaA#jKLFEZheOI>HV42S>nz(YGM78C15p}i6w=|LzyhiMQjvnS+{P5BaBTogneUZM)?SrmpezqAUDb`96UcF-Y#>Q zF;~xG^3F^>rqM~Fq`ZMur}^gmRBpsVgubtfg$n)Tig^88)A~qoE@Lm^TS(4WpK`f< zmdqp1Fd=aJTn)BLrdc2qZy+6KBpI?qHlCy@ZEpIW5GNTp8b)yl%WQo*#|scZ5U!z; zVuKOqnpfPj!34#&Ws2W>^v)d#(^i$n5jd;>#sI zYBv{&FhNroU`1Uq8s^#VHvZ(SMIgjenllv%kEmdM;R&upCofD>17@bahC%R)rx!l* z{jYJhzzugc3m$h^7`c$#UE+cN$~(G*?;j~Fad{LlAJ*?Y;y74i zb!4{zRqrEPXE`#=E+M`}#;#7VSG!J0Fq7@y>^XMp#Pj~f$u2+RIpFCfn#XNr!TE$< z`Rb9Hd%DsTn`pX&YFd+V>boW9=I`;aDA)ZdUD!TEtD(~8e0u*j@RIW;S@Bf3pP!|9l$$waV^ zH18Tm)B4Hm`Wf8e=_NblxALPV6JTRDK*`! zK$fyoDYfKhjAYJzdikWn%`br4))(%LD7o>7&l@+Y*JhMIg|Wz$iU>ZlGX-tnXV zqIY4JAN=fo24XawUP{b;H0irOjW@$HF;ZJi{!CL?8WQ^ zDfla>TET*GkwRm(L9@bysJ5Y}A2K~JqKipJtV|JgOQFR$yYbk_ZY!?o<|qd2VNWKx zf+HHTA@sX&Xya*47+rtdh-$O9bLFOkXIx@lDEm(;<@){9;kb*ocPzf!$VP0 zT$T09%Tz==94OM*w%1){JtsL7(^jWdHuubFQ?+Z!Y;l*skbIiP)Qq0g{{MOd459I)XEHjgMUUi}^eZ75f)7dXvq!SI} zt;IZcn|KPtck5qv#c$~e6PgFWV%Soif16ycw>tAz&&TBd#H-qtEsJuTbpi3yrL2T@ z#ILN_FAXy&55&(Y$M>WB3poOrj>duKHqDkXZOf85KTQ}ztp7IHd26B7*q5i>mk7sW zBcf-}t)AH9CgSWzD#0^Lv~%GQByq?U`r&UcLZ04Wg_JvBHctk7@COvN){9SZy8}sAS4ed>5PAu0y)WQolAz z5i85TMqt4IL0NXl5@m-?)f5iJ8fnsxt?3VluA1eg3N4LyaojoJCrrv)Q=w+uFJ#A{ zJ1Qc>o3+fe{PkxV<(_)l&jnV#yxN(f(oNi1_bXLhQdga0%=6gl+Nsa!el#YPArYP* z%}wypsfvBff=5^^k`uwhCL5fhPN69T7ZtyBiOEAbL5g&)XYpf|thNR~Bs1&ob8cynIlRiVf*lD@X_+6^5-+?b;37JE3jXA&hFUB8r zjj`4E2U8_0fn<6C>l^^n&LekzK=#lu_@W3?s3@s|z9GX%Iz_z&(hF3W996A=Li=i( z@NQTL95_c55}Gf;>^gDakYOv;@mDK0m-?4=j5iI5F@L(T#HFq@d(_rACo4b7uvT5=X&%&ba zoW1a-$>gQ^T(D!_s}M)`(fjo`>)2x5%a#NVd6BgwqbjIVUKNa-ePBYRs;xDeS5gBr zo+rN1y@JPmPn;F=ZRb%odmSkuFpDrAg_jOAzUdlPo{H+uFz6=}%p6RdoJv_L99O*Q z_jD#Phs+QwO<9L%SJsGLYqauMFD$PqbQ>(2WwGinKm1xs$w2d-Nx`EPv~Kc4Wunns zit$dx?QJ9Rj6&+-tZ-|!RG#4ht&g!7KvMF)E5Y!L%b{Z5ul$q6LBYEaEO(MoDOC7- z@ltt>Z_PJ^3K%raks11JMP~FTU%z*|+a0gV8n!#nZbGHvLy(!t>5)^Y)xXI4Zf*w7 z+o{f^Yb=Xes*ITEcr}sm*aHcqk)SMl3w>?kr;U7UBI^td}u*Bl?ZdoJ8NL0xa z?fl$=C)A{levOtfaZH}J6Mpt9mOG`f7brjdq3fB#9ox!}{l=a77=_k;_k<#q;SG;U zT;Wn8b(B|3M-%}~GktA1QfBHqhA#4}u<-YURI(VvrenpGpnf;*gC3dmxSh3L3pHHK z%H&zkLz-qW@fEta!r~JJSR12=qQsx~kf|cU6glJ&&`J2YV(iF(VCeSd3Ur_r&`6o@ z8UNW!Su)8Vx3E95+aEG=YFZ{un`Y`P%}`}pc4iEJG6914b$w~&9m1_)B3{Vrl?V&B zRn%&RM%qOl*+}f!k_5~{Wkv#MmsUoj*jbxTe#tYkMa$YRH#?r+{^Y7-?}&u-W|>{| zZ#*~cF$!%zsGy5m9kkT4G+cW0J6D2BEu{ipGkaL67XEafBvOp38`bQmlV5%D^|Wq7 zJE<8RJOwG(N~W-7_WzZI;V3lsld#A#@og&RE5l9G%u6m#@(m9gM#-KKvQXq=skgp! zJ>EQrS~^KLnBjP;ZmG+>8Fm?pA6)j|CHisZALh`YnwN`Rp4*Tgj>^!VbV{tcSN2|i zE_gzm>a+7Exk+5xMx5h~#193m4&K;XgFl)&@@0(oreg41V4S4_hLUiy8l*DLmYwcG ztZ8%j>3w$rnF+bU3acUXMUsFt_KF>{VBC=&He-6mpQ`_|@Y_k9{EJrgMmV~6od~OQ z;Udd?b^ZGN#+LHc0hsZp}4~r)qEUl2PvNWuH=aOF}r-TEvY} zR?U9T9eR2BF}-kH`^_^iX5J3AM-P7YUobwd%GI$NrQ(Fod7D2Eqsk zFqK!g>^o{MO(eQCLidiO=xP6Sa=`(Do3y0eyNgM58J?dTfAtUZ1}ucRfNgf+x(Z=7 zFwny(3>+i#>e{(}gGp0t*7l6SAdp*K4-Xd@Kv%c0qB{J`*wH=jL zW2s@?NJb+bj+*;<%uCNo`|i2gLf~r;NUtBaL*@pJk@?)6ghc*=WnOQxwe&s{iTUPI z4AQ&rv&?Ofp(5ve_*#kuR_!&;DrO@gbtQ6J-d_&YgW=bJ5rZubk800ym%QQ#8(_;TDGMv^ts;Cao*qd`Io}|?(3u?+ zUhXa28+_-MOl4EetBUi75iKke2^JQLI(rVfGBKjP?L1gn$g2Z(Ij6*7RGY+$=?f&T zD3wO@Y3;!PqBJU+x@i2z#?n?;y)-ndF6k=PEnn}x<+`{%w_Ox61{dys7TL^GbMgSyA%wPeaN?8G{2<#~v5x_-(mn!st qg^DCTBMZj4iy}wk>)2|Dj_K#J+whchf>lb5{iU? zNX+N^-uu1xK6n3d&UyB;*ZFI$v-UoziQwr(aGe_kc3973{TBoT!jpnPv>*^D)KenF z$KT_wkB^6VsF!E8*+O!jBwcvzG*;uGOHv!3Cc5&odD67`M6g6khC}`esbtd#e^n)6 zNlJ8Zcf;v{W_%Khf|_)IbJ*#s5#uP4h1GyC68}aUTAMwLXwskfK9Zq~NS15-;L0OX z_~Hik)kN(6SxDA{ieL|+5Gcr03E3tU6jAT2C{fXsDqGw0I<2yVG}ApHs77HVaI{T` z>e~a`M6|H*G!J}RjM=||K5>lCCGB3uQs}wd&WI4I_|j($lG)WMIyF?$8%UsI zA9dK1E1?s)Rj*w$4})A=QZJ@YHLuDbjyTD3ok+_|>C0ln?6n)$CnKCfdThP#NGcT2Ki*KhLqrI>#k0{cYQsu6dy90 z*ypi2X=Pu-EVsfzj2c2oy{e~4=+n5(=zkVhwDq)*={=WV#&tG9>fSB(&0RY(%5x*# z+P(}7^s~g*BJOK;TSRy=(^S=rACvr8?qB9o8L2+%OjF#)sEKJBev8M-*)dch3`rZi z+_{E2k_GRlR`-|`ir1oa=d;g+ju@4YJowY$pE;cOsrJ)RjsmReoiB%)4y&HGT)c)Z z@P`QevMoZ~%^F>`HO={ao8~=F=uxQES>5NBPY=>^RFG;c@B3Z2+GaO2a#9Dnu;d@@ zD@r#>O7Gp+q7O`ZOroGd=*qkI?u2$b(?u43GIp(@< zRhQxaZMuv3toRlDS8v#)a~OqmIR>Q3a36RqPif;rkMLGtKi!q24hvx%y+F!X4l_*+ zlnoS;4-z&LyfE+!jKi%B(;PM=b`A0j_uaog7U9{oTA>0{+rVqM5$Wf+Te(?JSnN zV<^~lk#SSbQ}>C2-#nz^3LN@`<7!?K$>v+2Jm~TSn0mE<6%`O!i5xjWq!}9iT-kmJ z+apT)VntJYIMqOIbQOnpmrs;C{e}-9&de6h8K;cJM9Se?hJ&ZX$?u-Yx~@GiG$38F6&*lbDHqLwbex-OBkYB;pdq z6FmnLk5KE_)9P^h0x>y7xLfYnl%`&>=I1rT{6TlWBF)#VEQ&(DCN*x%VQ+}!St@Fg4N7)G-5^B zC7y{3rwGWQzkDRS0GaZ@Q0kT9A_4vhWu@Ql%;K<J{fXC_50>qB0Jut63p^GzA`R|xpQPvQGE^+q+jN;=Q$_G4cB|trEK*LKf>La-TV-f2V66 zQFkatJEhML;le*YHLA^K_=0`;ZYrw2+XZ>k(2&e#H)z)IdC!HF|9i-s^VWq$E1SzQ z#OFoYcyffekwv-85lFfF#j(ND9;o8)-_fssCf78b4)N9#+LIoF|3g&ZjwUO>OnX2v zkn+_{l>`K81g_BWU~PQnC1Ei?1iq^yx_lVOa={ViCROg8Ib}gyGkuDW%#@v*h3ec0 z;r;vuyg^#e$^8?7!~B!kTIp_8!JLjq^;$DjT2uLOqX3)u`<>j&-Ky65I(Ip$MEHYf zsQii1-abp+c|V%&HM4=TxSuyuz4r4~9=bjx|14Yoh|#^H0XvD3ip7p7HQXkurqsRZ zj4aH`x%+}&;LR}goz0IoeOqV3UOo#JhyxF@Z?~}3H>H*RqRYR#aj&Iw&B}|u9%kI? z@F*RzKevwi6eGrW9RAFq4~XpbdEAW=y|Q6vW*ouhb@ZWBXVf6qu0?&MjAdPHD~J#& z8JN!!O8<~Z*%Z4kr~4Dbj4cno#nZ-jG7pqVjYqV(G41xk{HBs`7 zZmp|AUVq3xKZ~=Ic!x7eAU*0w^_1`vcOwDv-GE2-iK*8)3X_%DCZ&Q|ahKJ~zo(>C zMecl-l*C@9_@Pv`sXt2f4NtrmF_NQ8q;d4`g*-EiAWppaCjDYRiS@dq980STQ3zEL zbsMw8+j~mx-(^zQyzcoYt6ve=fvp#cjg`Nw+3}^5eReOmpl|Rw?~6m%vHPx%hcqW5 z8(rR9R$3b*CYPw9ZBIVAxF$>#T^jW_;oe!x+@GSw^4f>hzjc$T;0ny%&4zv#u*wQv zI;tr!9DC!j!}aGCB@q5z5;HO05gK$cwOYp@q&>!CyhKM#t4U0rd-@$y6TQd6LWp10 z?zyyEnDJnXbi?6@IRE$iO-hbO&kNyai&-sPRT}rc-{|xjX~9U!sP_q*+aAtw2ItLr zW4kM81)LYde`cI;Y~olt3653Gx4cH)F4>gx>rO@A-@Kb-R<3f`A#0cdNZveOhwta9 z!<%&Qyt%w4$?w3(>pU2}v>QB_B3@}QixEEl8koDyXebqxLp7j^q%0KN#+2YOyqiM} zbbKWkMlG0cV`E$}J91?csY490B_YNVcTkWAwlKK21<%M!hp{+H2Ro(`3Z|i)BGO}7 zx=B4c+sRVp+4EmE8m%+?$G;}JJR<#)PY%kVA|Ezi&8&DHXKYCeWn`Zik1s`%j(QWh z>I9*}0zTOYZ9H2RFO95#(cCjWS$nivH2(!bsTbTQt)c91ASncicsVC#qD_x2t>Y{u zFVGNT9((iQsrLdM#z zG!=4{{0v3H$U|v=_6JF_arHra1eBD-m#T!cZOL&@t!apZM1?bI=wGll!M|W${{>rV zHPdz>zM4Hm2qxj6rnL7oIdO`>x_$Hamj$QqQ`gPpU(utyp_M%l@sxr?A{r)$+$6qP zPo4&8uTfyIqpb^%{53~2_unUG;+3~B?X+hOed$s z((rtMkS~?E7MrRDRa7>9?s|s(1_P@@Csd0aZu5TeoZ2pJsowxG$WGo`^Hn z2hGqs=>{C+dauEu*il-YM`0}KOiHf^!^h^faWI8+6%2=3V|G?VRo{pAnN=_n4byh} zOzh|AyLJ1uweMX0K&MsYO;OEy8Y5vf!7%VxwExlF|CYb4Kj! zelMbZ*#fiIX&We$l01S|{Sl3u#HCnr&s4#j=BCeOvZ@4Suw`~{o%#Z3T9dCg(>j(r zkWwWmI(G1;yGYjm-d}$e6Xz#5I(q259KYxhA5zFwC^1fsq8hmy?r~6Z;=i#|^q56z zYF|C}J$hX2l)+J`@^Gf$GhCP90fxO_SMn&Lr`+m@T~I;?{RD>WRhA}SPJ8S}LLA8F z2(}3>9}!Y$z}~hRW)ZpyuPoU~x~xUeU@f$SHW3N;H21B1hXZfXFgWOLj*iEfl*oc_f7LfKdPI0Ijw+w2~WV`HDXp%Z+ns*Q`DWv`sij67i0 z;y8IL{uM`oUmJ_!X2<TwiXJx>@cO!^1PG-wB4f>wW-p)Bb&9~ zX^q;m`OPX-oXP#(ra2>*U%5Y2F9Y_O2VR-NeJ2l5Aqr#710ZilT{UA`FddqgI-LY zKsuuR;=xpK6~~j5wy*dh#jKJcZJXy3lIHFitAZuWkJqep%$$EYwL=-1Z}mU5&oED2 zy=K`lTw80r1h-}jy^aH~!!7Z;ANV^4_mJCqQSj_AA-C|SAo`c%e@mM&j|L{`v67ly+3@AOO`k5P4A_xm)-!M3~w z3;DnpMt9?+_rt_&v)AnCZ)P6$I zJBRr&F3B#0NAoksVG8;m6K=_&sCnE;M7WZ3?9`1)f2|BgohLQ@k0rv+j) zW!eAfeXoE(%>Q5U_s&lT%xWUQIRJ+iKimI$$$u(s09!3?z)BPf{Z9@AYUc)cw6Fdl z%T>S-!3xRd1y;1wA@$e(WQE!PApc+2FMOzi3DhQNmt3XIZ0K7II@KB2t zqA&LMeM*R{z#I?sFE{%q3IKP61|&uTu$2%3J|cv{_UTA8#90OaBN+h)qzuGM?(btQ RQU)9d;F0{KwiMxzuabukdNVraP>KX9{e#W87p2=@Gj0*X1%i11ts%V zwS)QEn2K3x=}GH#tM{fZl=ZwWFJ4tm#b9+!&5xOJq4KqlN=-O4M5$cdc2CP~nJBH7 zckWXaGi~@}a1xw9gaOvee85Rav*q0u3*U99wkK!0lha#X{_Nzmr(!nA%<86vrd}N% zWc+>6_Fh3E{tmN|sf>@W*m~dRSWUTBvS<~gVAm|I?-?rWi0FewfW4DMZ;2De-+F#f zUPM&BLkjcMr0(X@>AUnSkZlT;oDdwlnY%3&@vgLh$u&kw2GD-%B~JR4tpR%djP`=y zO3c1=M|!Jq$1a8erORB+v-f^9aL~js$Af#lI`1sTq?KfVb*!*;H~M^T_%MtBMWW88*opz1#^CO&kF71>KrpDn~4h*n)!7lVGpls$u=;&AO=lK!^0X zm-ppu@?W*HtdJeu^DotNoGs|8g~vGSu+qOSwJ2jaHpyKJR6C9_;0u+0oti1QGyT^Q z0!w>&gLs=z$FPUr&ap@~4PnoSI>Zn1@AVb7hT7c}1V)++k)*tl4v+a_4V_~w2mILQ z*}$nX;a*ol!P{FRe`XG2%kQl8~vKh5{b}B^3WZu9_UP-8Dd$H+Ydgvhg zxAw}GX`^Zd`R{>64J{cqMJg}bG11S3E-U8I_L;3VXFToaLi{j(3GC`pZQvxI36^QE zp{VPb!1IfwB{hTMzyreVPR6#p4=PsA#|B(RlTJ+1m6dd%8dG&2Rr-bZCMxm&cdwJdLvr6O%b_>J_OLFf1~yCz7O-Q_#|if*VF@v4%xyu`FXJQ3cz? z$x!YfL+r5@(!Gg)euH#VHn?a#!z-Na+(;Z?Z~hc&ybgo4KBqoZ-e^Uh&_N(4Co~WX zQyN+XgdW0p%7h4HTYp*U;wdGVQ%Vs3pd{GaEy&Zuni%9mzz6?Ag@iK;n8L8OuulNO z&AlP^(>zl{H;j{1Z%)EqSc}6$Y&|4-giW*cDt6j(ekZzSg4{zLOF$SWNCvl|Xs*m0EDpY1CQ7egs-tFzHH@jn#BK?ZJc_WSmqO9keI(tMm=guS9 z%6Zwmm9hsX>B{`8fdLWYk??G|JCbdlM=MaWx4VHgZBaGr(`V0%hO;|}nw^L=xuY$g z!_=9OTnWeFu|Wk%^l+`KHu6nS?{Rd-`-j3?amKL?ES!_+59SNW!rcQ7oX5_4eFYkY zKQ6M~Wx=^fK0bM}ppTarmfI#k2DkZK4Kp2}|)z(YQy?B^zTjR3tCB zFGz8Orr`dBnH$~lx*|@-&g1=;`sc(~i(GClO_hW$I5IIOPi+@d{|YwQ4Cvot#8;vs z6#Mlhq6wT@_W;e1O)B$ZMaxINk5eqtsDl*46@~ zpK&QeiQjWU^*mcCz52C#ytkdk`#d%4P=2mDXFop&LZ)Nqfx_n=v%Nr^Hr^B`1i}je zXR%1&_jKogNem+X+qrcu`_=$XhG$q>oOPLR3)Z?R#czT4rSIE56*`smwr>-OTm|Mf zQqcknU(}a1cKHVh4^8sU63T~m1VHi5gz+nOd7t~X_5$|oxJc#sxAY4u4+L@=yDuiE zpu8AvWUpri-;cDIy7c*SgUrZ>8noR9$-8g;fJX9|fh7abFIlJ7>BF6teq~!$#!6rR z0l$tF6OZ*}X3?$3icK z_OnCy5jWohvO&b;Y;GT@-~@WMvSNE@;?FLTECz#2zbU3JH8DNmb@ zUo}tbks34*Z`&Yf{@Hm}PQ{5q8p9uMV_U37;Nul6kbLzm+lPs6E-8IeUUolSNLRgf zr^winM1Gxb_lXiVm&jY)a?4TQ5?valT?U0I=p=dEzU)shLhY@Lkbi!bVpZ0W1iI{R z6RQvN;9ZDalHHkX_ol7sG~*n=o+PgNM1j< zJ&^_uC}0wh&EwNOs9?+`oF0+w5}D0@Hpt=!dsN4X?P8E9)q8BYFKryZs#x3o_3DX^ zf#>5V$$9~YSXz>Nh-#wbP7EM4f)dd^N*t4D361azC@!9(9x(ixjHkozdEe>HB^J-HR1wytZLc?(Hd!~YDu#Xvojvd z!dwuvaNPZ4TWs_|1Bqn?EL(xi}9mraY`1d8$9xZXIF$c}ol`Dx7h zDSPvB8iYUC*nW-E(VA5aaY4vKQ@WVmW98A$MGu*pc4Cv^{@rOdR!VM@q6+6V(fzbzz6X5AYH@3#bz#Y?KJqVf1!~ zI`#kDsA^xmbrJJ5egqHqImh;OcTTUNkAbZGnD)=i*tk2@*OWWpaU{d(cP}g(3=;^y zK7|c$*`Qyda6K916I^3r9ZpO+*P0QFVG2ro9w@=Id$CNAxwwI3IQI^|zqHGE=4ZoD zi0gYz`=89OceUo&A>=tk84DOJD+X2xqsg7J@1Wp)Q8bw%m%~7g7PEjd21AE1*XL%VCA4u+tdP{_1QTwiy90Ya>F>O`AYnf(Wn{rle`91S>q$fUMWoQJ zk?4AE-oZ!dxy@!4J~vDiRGMjfuFN9dq1hi~_ECUhxxn6i1{sK9YItfyAC^#RUB6X(t#vlF)&jWP5vZY1_d9;p}|_w z0kVTi7BkboB|Z3;wi5sUN|#fB|9Qf|D_AV-3mi#^AD!HjUzW2>3{hPu9W|3PYc&q6L zqkwcs3rNZ1`f9y*{y1l^Q+w^T_HV~x5;!LrT<=bXc|}!|b_@Z5=7~Wd8W0E+>?I!L z>+k90>+2~N?CpiKunoXUAj0OJU(guLx`QHxxF0oc;H0-w$G7^N`QrqvBI|W4lnQCFV)_u91F$3q&7!WFlIUnP!x-J00u(swuI;h`r= z8cW3p21@??Rr8M4za8%64OTC9%J8tg$t;}PkY6E%$Io8MS^<| zYl*cjjY_7g;OQb#Pum|tJ%zt9{Wu**`X~ly6Bi(N}ow=af#A|8~T#;^}SQSi+W~!zBGdj9pm5$it%(3jb$dmio*inZj zjOt%5`vD>6j%IdCGQd6hnQ1(*j{QkePUk@v%c+dw?`g>rLus#;^KV~_{Q z_`2WJrctk*?y8oM3r| z+lBZPrQ9CMnk}d}-K$E5t(mjYhwAfXsVcjua>;#2pR&qX*H0m$mfE?!;Xj@hVu~3` zEd1{N|Ty`l)Q+rMWLvjpTElj7KfSFLiwEVDH#bY58TP|1+|eyry^TWZeo@@kT6m zCZ)tsQtbp8$y=t!hiUrr!2~v?mfv?S8OOW(IFn6>0e!%mDq2^6|HxdvLS?_gaJc_6 z*VcS(N>o074%BAN`QNe}S7~n_^ri01V_C{OwlhB5X8yr#65E*)ZW?B} zy&;;L+7R!!sYh5HsxRJH6Qm*5Unw#53->3>l1|V#;*-=tXMd6mjf)>689*Ge-R}_U zkPS5nD|Kk)!1PVBD%|P&_KAv7xTrx%p#O_5*$GHOYcRzvj?4bFUUf4y)}Ef5=?+CV zOrz|U3G0MInqflnloEMVt4>uQnT}?NV`=(y=N^UJ@l0U)p>193QAAe?E3bfo&&L;> z$CH9xbFzKV{MhjQ_BMDEY0J0i^)mNVdefV4#&5=4r^L*(lhdbP#JoW#>{+umwC>x^ zc{TD0SPefSB^-%+A6MeajDiT4=?$keg4vfD4aWEDj>84?7t3W?KZF_fKJe$uI1jfI zyB#DS6fz2XCx5}BO2Xo3b%8wi_#RyMwl*GOXgDVKmGUifq4BD{I3%U0vY2p4-t$W% z_x#H1E`dcFTKqmW1Wf)%L_+d)A`nQ49{5GWjxo3OowR84vYtbye(7{$G5*9ZkG~b{N5(brNJDPK+tup+2{8g+Ijflx zgt43vSlO+!a((0(VNrZ*0M~yYWo9r@s*&@YllhM(I;4;6oM445)Ja_`{&ef6mp95R zlr-OI#r<{rytOy2lVGV|A4rexz3cI&HEVOm)Ny1gi!n9vJEC7VpYh?|IhjVWdLMpx z`6OR9*Z#HIa2d-Yf9d1}O;qvXP$zyD`W;i0c&v-f=Eq^7tnKtTmB^DVDSSZ%(ku5K=CB^>pk?VTtw!_hwaPldFU7d{IvmvS!P$b>eg&du}mWaevBWC4@O>fyGCCVNpJN?QAZR#jJGLsO>_<1M`ML z?(%**FW#?YSseW~g`ZcKNVplOwaG69(9hD`A`x4|B<+_Fr}hVTqN=;h@f*PuStWyL zSLR-!YU|AGAd(z z*FWD&fIVn=+z#Ul=hms{VNTG&xK@kXz>`J$mnipM%y=YUY{;V)-I-C&UczP3LDybT zHZePQhbSpoInKAWz09qAtS0X)-rL=rUxors8jy}kBP;Vx2$t0nHpE1FS(UEnm{ zY5$$AB0H^%%-J}8$;B5F*UWseZr@+=9T_mTX7~aXu{TX!K2~~T-JyjWbI(m}tfldw z)^}@bUj(Treia%oFMm(TS!kaqb$DEz<`GES0qr?j>Qq#I4ttSrs zl6DU@9gxQ_hD*2d95z=O;TOye(iguCDyLc$wVjqsO|B3Gm~+{k8HDT3amMjq&;7nI zl&jq*g#O4pW?REzknNX4RV}!q6~`ShbD-4AR8{6}uaJ3Rh+SU!s;=LQpR2kH*-Si5 zw@^>(SGmkmdy12Werh_-bEW;M`P4;><5}Kv0(a!2XR#&}9QxpduOh<5Ssh9DG_xX; zR4AoHc*uOtrzo-uBonGK9vR*>@X(=UvHbY!m8lgffN zmqa6phlXY`zaQoE3#vkkY|7>GndeU&b{O*=h+(iq>oKg1!F|4-5^$;nl(Co?a!0_I z*4#ntBY{DtzFNIouAF#%MBt5}w*tEJpld}`p$kDunhz4T&WIZuiU~I7IQKFY@| z;fqMpTdM ztid4A1AMM97Z`)q@0LKs&JkT|4Lr`110=Nwk*TVR2%JG^(ttOU7m$u>y?8wCQ_Yr` z_I7n?P%*o7FwBg^-T^luq@{}}0U;(etAw<~-z(iM>QpL&RX0UZvUOrz!oxE6v;4j1 z2i6#q0SBnDY1Y1+mh!4~;;Ut4x15wVt&mp}WsW247{gebUbQt@We*Fndp)Wj)2^i1ChatkXr`yyzwTcXf%Y?<0y zyNUIY>)Ab_KW_;MzFrk)df2)Uv2X78fl8v>up{c&fudpG#$fN>>`wjt&)(-P`eX~+ zPknjhXUsM}5v92ZD#FAW4H7Y|s+fv^P#9_+`pLNf-%#+iE>ZS_uEBzYUCbViL(*_1 zV`%Oa$#i_y*{2bpccAtxhHP(?>+MVV7&ggpo(fBfKv?t3~R%QxzzvG`VVR780r83 diff --git a/src/test/resources/files/parsing_exceptions.csv b/src/test/resources/files/parsing_exceptions.csv index 83f273e30..6590cc858 100644 --- a/src/test/resources/files/parsing_exceptions.csv +++ b/src/test/resources/files/parsing_exceptions.csv @@ -1,2 +1,2 @@ Name,Synonyms,Description,Trait entity,Trait attribute,Status,Trait lists,Method name,Method description,Method class,Method formula,Units,Scale class,Scale decimal places,Scale lower limit,Scale upper limit,Scale categories,Tags,Full Name,Term Type -,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powder mildew severity,inactive,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation","Powdery Mildew severity, leaf",bad,a^2 + b^2 = c^2,1-4 Parlier field response score,paragraph,two,one,nine-nine-nine-nine,"1=No visible infection... 2=Very few, small colonies... 3=< 50% coverage... 4=>50% coverage",,, +,Powdery Mildew; Powdery Mildew Severity,"Powdery mildew (PM) due to Erysiphe necator severity in field, leaves only ",leaf,powder mildew severity,inactive,Nursery evaluation; Greenhouse evaluation; Trial evaluation,"Powdery Mildew severity, leaves - Estimation","Powdery Mildew severity, leaf",bad,a^2 + b^2 = c^2,,paragraph,two,one,nine-nine-nine-nine,"1=No visible infection... 2=Very few, small colonies... 3=< 50% coverage... 4=>50% coverage",,, \ No newline at end of file From a4a8091c9a76815debec2fe00fcbf3372a45f627 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Fri, 16 Feb 2024 11:49:11 -0500 Subject: [PATCH 162/220] [BI-2053] - added test --- .../parsers/ParsingExceptionType.java | 2 +- .../TraitUploadControllerIntegrationTest.java | 21 ++++++++++++++++++ .../ontology/non_numerical_with_unit.xlsx | Bin 0 -> 17673 bytes 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 src/test/resources/files/ontology/non_numerical_with_unit.xlsx diff --git a/src/main/java/org/breedinginsight/services/parsers/ParsingExceptionType.java b/src/main/java/org/breedinginsight/services/parsers/ParsingExceptionType.java index 4ae0d0c1d..56ce72e69 100644 --- a/src/main/java/org/breedinginsight/services/parsers/ParsingExceptionType.java +++ b/src/main/java/org/breedinginsight/services/parsers/ParsingExceptionType.java @@ -37,7 +37,7 @@ public enum ParsingExceptionType { INVALID_SCALE_UPPER_LIMIT("Invalid scale upper limit value. Value must be numeric and be a whole number."), INVALID_SCALE_CATEGORIES("Invalid scale categories format"), INVALID_TERM_TYPE("Invalid term type"), - SCALE_UNIT_NOT_ALLOWED("Units identified for non-numeric scale class."); + SCALE_UNIT_NOT_ALLOWED("Units identified for non-numeric scale class"); private String value; diff --git a/src/test/java/org/breedinginsight/api/v1/controller/TraitUploadControllerIntegrationTest.java b/src/test/java/org/breedinginsight/api/v1/controller/TraitUploadControllerIntegrationTest.java index f52cd917a..01a478bdf 100644 --- a/src/test/java/org/breedinginsight/api/v1/controller/TraitUploadControllerIntegrationTest.java +++ b/src/test/java/org/breedinginsight/api/v1/controller/TraitUploadControllerIntegrationTest.java @@ -811,6 +811,27 @@ void putNominalCategoryLabelPresentError() { assertEquals("Scale label cannot be populated for Nominal scale type", labelError.get("errorMessage").getAsString()); } + @Test + void putNonNumericalClassWithUnitError() { + + File file = new File("src/test/resources/files/ontology/non_numerical_with_unit.xlsx"); + + HttpClientResponseException e = Assertions.assertThrows(HttpClientResponseException.class, () -> { + HttpResponse response = uploadFile(validProgram.getId().toString(), file, "test-registered-user"); + }); + assertEquals(HttpStatus.UNPROCESSABLE_ENTITY, e.getStatus()); + + JsonArray rowErrors = JsonParser.parseString((String) e.getResponse().getBody().get()).getAsJsonObject().getAsJsonArray("rowErrors"); + assertTrue(rowErrors.size() == 1, "Wrong number of row errors returned"); + + JsonObject rowError1 = rowErrors.get(0).getAsJsonObject(); + JsonArray errors = rowError1.getAsJsonArray("errors"); + assertTrue(errors.size() == 1, "Not enough errors were returned"); + JsonObject error = errors.get(0).getAsJsonObject(); + assertEquals(422, error.get("httpStatusCode").getAsInt(), "Incorrect http status code"); + assertEquals("Units identified for non-numeric scale class", error.get("errorMessage").getAsString(), "Incorrect http status code"); + } + @Test void putOrdinalMissingCategories() { diff --git a/src/test/resources/files/ontology/non_numerical_with_unit.xlsx b/src/test/resources/files/ontology/non_numerical_with_unit.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..f038a57aadd21b5308a775694166a730adeeb8eb GIT binary patch literal 17673 zcmeHuWmp|+lP$sBEkJ<{GP9Q zXxQwo#qR1=_10S=BMt(J3Iq-W2?PZ67O3Xi=19UTARsshARrVVNMLn-D@%JrOM7hv z7i&X1O=@QgbNn1oV6rSAV8HeNKK>7DpiNO+yq6cT1LirXO4+1A2{EPs*BwVuru#TG zgIWBxZ>+B2Wo=L?OvHkUTnjmN^4Bbw|*thbN$sz{W9lP;1X@av$$FXKG zu&7J9`GrgfaaI%78$7s=Nk~;2rcnvT!CU$9Dl8kQiEn-P6Kq7Jka!A0!b>tspa$M| zdQl&5Kyh9d^&#eN!iCR{$l0`R%u^6s^@nY_)>r12_=irWsv*H7*j0K(f9-KLH=W4N zWDBIGBay+v2C{7kAEecH1twq)%987Bdgc0bt>!keA0bwQ5 zfDtw7-DJD%i6(WKJA%mrdQaU7e2o<&Th0{ucBj47=6IUy-twVOK;1L_1^k;&Mz97Q zj!DtFj)(+QR2$j)<5{iA*a^HmNFUb*yTRN)uNCBGz6HBIDq{paqYDULwbS-%aE>16 zdZ=Br|Hf>LX>Tdw1ZVV8?(eA7Qnw<`6BKQ%Q$AjR1VMVyHA_z;+v9G6}*M7-P0UbyBCE!)cR zrzU>!#&i%?>htAl)-YjJNwCFK`znS|LadRn`zCN5OJyT2jhRKV z(q!1xYA~OWBDz3eEL!z0dXeww3sa6#c0N&emFLi5@X#ZHJ_s(U+SA8_bF)b~IU(YJ zRFI6l`x92My0$z)I~1>#6e0J0+BSNGLKRDz!JJ#VYA*dT<#86|v9+3qKbp*Xo)H-q zz+?mcCT|Jz4fyrfw#Eo|Vq-xI|4 zV-C@33;oeodhmA3*C(bpyMFzSRkIfNw+yev<;fO_zv@N%uisYtx2<{{Cwk<3iD2?2 zl^uDFWf<14;M=)RBzp_Gp%|g| zFyLs+mtQ9(4xL?^6?cSB;m;U3zfpk2$uVv4$Ydxh&|RM zcCdr7u-bEP+#^ItBk6W^i$;3ND!DT4E0_@G3sO!;pK{XV#e@2=n=j}jFpN{5&EV3X z;1>Vvj77F^4$lFOcn%y0=&vgP%=(#AWs1v|xwMEL+9ofr?lYE1{1Fx8@|%q*9F%1% zqbMNKb--xN(nw@yz1HcVM3>v_mjy#cX|3Hq>^Zx8R+HQAhfq)%ye4!NitZg2D9F+1 zKX|r03LKaWOduaN9MrGd#nOcB@Z!)GuTK?kwgt+Xp&JS>CJ1+kz4Sd~P`yOvgeivp z(4`<=M$xBQ!Z0QoqaPIwgdq`c9E4aLr8CgSN90Z!*Rqkn=tGvt0P@v=dZ75NTTB2OF3%stC@%$p$9oR)7D0rnNlm@9d8iB0bZ~z;K zaT}orG7oX`6bt!6`rd}V2)EKV(FH7AmEw)KGG7oSYtY=CVPn+1Sz}_~9S`Jkw~f8* z?t8wSZ@Q$KL5j09N(*AU1A{5daSE83{Y4N38eg_Sg2iTM4XJD|?e0$H&1`MXqknq>E##AJX?fXZjQr z)^^bg-;q!+#+nI}qzG)5paWv_Ud(=bVu3_JAD_~$d1Xr(+OTkW6OVBSIgJ3;WKni~ zwNn~NAS*{Hh#?oJsS`_^pUQF(MFITi;~C2tS|0ber;|w?`qPNQvnmdvv*f!D?3+N3 z#&b8qEeEI!sURA}0O%Znnk9orQ}q$ddzmGadFcidu6TiZk*H&yxToEv`_+jDE%UsF zN-oHbIblte=|#>^`h;W5YZIVJbM?5ux5>x-_hfQ$eFHEdUd}guVTR!0uWfjm-AnQvBWw+A5yKAIzUlSf z9h6{mG}VG_<33$zy6@L8n;yyHc?DW59mq9xt4V)ougP=bW_jSm>9iBoF51ZQTAI4G zf$s)3_*t0Lyy5)}E_I?|e|)n%rD}JKeJ@6mgkPfLt5@35dc#6#%b``nPS^8SB~_8pzw*nphgy{pEDS6_z8X0R|$xg(u|c3&9NbNsw%?I7_URGR!nT52_sO|}dr9M(DXNL?-vX7EKw~&0v z;MaOYkEt74lO(}o7|qwkH>Xvoy8Va=O6Uca{t*sOCB;p3PeDp~c(cp0eNmZKow8(> zV7zFg_LbMpNMqVZvt*0H$}YQxR2Z#7@tL^0CpwQirfEKF)&|md!Wb3KQe{Ua#v6>pf{dl3x07l`YLT+o%&J;IZ5= zSQ+8>VnY#9{IgD?*;eY|W7I%sYot?rQBTPIyU4fQ+B1}AX;*+}|4&wMSkQ}i1K7X^ zc$e^p$FwsxG_<$-*RuTm1>Jux%KlN^p*^&yg0~>g{LXH1a{;jY@+<^Riu)jX4@=<7 zNzu2cPj0n1hF!b6*5kX(X^))6*dO3yC}&!L19tQ=jH@o(mZ&=4SF;|iph8h`M(a*f zccg+yNr{Dgj#fc~sSc1Odcy4W{Ipmd2~`};y|*YD;zB!kABH_qR$^*#r43O>BUOJ+ zx#OYi$Id0E`g{`UfyZf?+c_q3HSRCU7OlgbBf>TyczWfn_x{-FCezFV9<;z>WcbSs z)Somk5a?bM2C#`fcpxCme`w&xy{E@(TBXsV2A-3i^5Cyy&=OIG@Rb%T)``h8)ju{l zhKrWzjhGjTGM{#yFX-@=)J-aof2Ly_cZqshI!=o#^2I1%Omc{{#?1AeRX}fB+dWvG zADsmTTZ(SMPbgE39FxdRJH9=i6E4oY&ZI*D%AR9e!k#Y65t-&f5eTZ$WojeONB-2M zTpwnii%G5#Ta-f-Su295m77>4Cy!Y}$!6#jiwjyR2_suZ0RHuu%c`yriXDtJ*p-`6 zSs@f0QC-W%&QS&xvZZBxRxZW@%;V!l_{2uuMqHQI(Qwnfasp_fn5k+i;(%=Fy~E?M z@oS`|IIUYAC_keZJ(?BE4R0~(IRAYviq|@a0aax|i@S2AAT+jLInJId@bw;pgMz$n zAv7yt;5p1&p;0O9z9LoP%A;ylW_WejInLzT=nUM1D28K!Gtokan{(Grt<;nZO@fnn zijrrN6)*TX5J_xdwpnVnC0s3DoxNoU>{+rg*X4ur(xuwg1&7!yj05JMK)UAZ*QYVv zc7G8BQ_oo}FY~DfV(^$^6bB6xOl5AzqR0mOZI36Fo0@}ouV(+>Fh%7E&m7fb|)^>#D)3LI4QW;`&P z_4<^`QUCNX;Wvlky#EV4oZC`IqYNv-$ec4f*8qO+IqJZN1QT90+LpNtV_3RN_ir_O_$hRXt zN=78iPwkR<)JWmxM0Smv-kdF(tc|UdPy!1BabXhXdV(TQ<_41Z*%9WDz7DozQPnuvJcl1nHkx*yWN(Cv=nhEpDmr z>U1jPx@AgZQrXhcj(Y{=i8>2F${#M6yr>01rvWkTKV7&3cZx|IAm9zh`YWIKJ?OPJ zHncFL`F{THw@*~n!FC!wZ4efEvPY%a8F_H9~9N^FVgv7-VTb^>wnl@ZeM{K5&H|aFyZZ|f`f}z=F zPg5|4B*msHx?HYR)C4cpZ_XF1y4=o$6Q5eq3noSqTm#y;O1yWLzZ%8U66`48uYcn0 z(c>hu9O0jQ9zX03I~uI5W0&4lOD?WYw85N_KuXRI-bqI_|1Kd5hcLED%2&_Y&4Zm- z{{%K)+{tOTO?s0VNrO4u#1&stzn4#!G)OUILmYQ%hK;Z8l#d6+k@M57|FO&`FizuE zwCnvuCz`}bDNBJqboEiotY_QS6&+g@d5?nH$rkzXFYQ^iHDedcg8+%(3;Gg8s*GKtMr4Qb3k;GGkKv{W$~fD_)54D?s2R!>_j=ubnMnwR zTFaqS~re$$KjS5)TmpP?k+8TpULCqcy)0&*sCp#%f;?;XSdd9Yq+5Keo&&( z?Y>vp%j35B%RbF|r~5aDxoZ>7`AZtlhpj%V&dwpO@FvKDp}i9%-BycqKl!%#yL66EhXi_XgVEvPeDA_5qA{;( z>X7^uu&Wn4PwijVN3;Y*Nf5Q62dfPr&@uHmzor5WwV)0nD{$?)jSe;^LP$JR`F!2% zEiq1ODCOnL*0`u4t=pzHRq5NaFx?@YR2?D###VQ~!XJJ(fXe|(h&o=LE{c4(Abs~X zp3K&gR@d*G;R{^MZ8Su@x#X>dZ-o6vD$qt=4F@qnp8?Xanb_gscOXkv zBCZm#4A)QbHIFahHOov7IE<@jcbYS8amdFU4!iMzR=rOU39UisgK)4k!+Rsw@0)la z8y`C{&MeNa?uQ++C-B$$)HKUiw##8|$+N%mQj)K2o1Tv8YNf(v$y&Ud8MAGVNfWkC zu|k1ep%fhI7frbf4?e)>vs2kno@KW0x0`1;8+je%&}piKt>-?}xWq-d<-t67Q#xC+ zlJ;S%JBDnD;<@F`4Hf}EM1&qfgJY#*Jwv^+7E*MB6(8U62$5`Kv~d~MTbN3MY#~Q# zN;PzJ$wSYC5=Hd+Y#`I)vaX&%pIxQp>-G~IpazBDt&kCUN202s3W>4i53cI8#ageb zLRA!bV-6E{S{$5h?Ny80g$m%1)tXa?>gdBKK=)^h@zT?cD%1tV;9TiZOB=JZCXG9y zG{lkE^5GaK6MRX=Y?k4UxYHvdCfiP>bk`A*E^$nQhQVlZ7+5*6!x-H**M(H~$T7G+ z^5CUd+$(qU65~Rj&v4y{IIf2UfjrjPc7$6YZZS$A#fKRSOHZ zcZ7ldXcBN;Oz-7ZRc8Pra{Hc<%~8lcW=%jw&olQO2QtdOBPp`U^2|`OfZzc_aAY|=Ryj*2Zu|H~W&8jX3(1i_4cxaJRY#n_OuWxwz62{MKe+E(i zzI5#fJYT7ufy|}dvUUdA&$1Z9KVmHl<6(4e!{-=>ylTEUNNd6>(bp6+u`P?>kVQQ| zP`T8hO{j43mL1{B5H_4ME{YQ!QLE4F*6RDh`Fj>UM-|1C8W4oH0fKP2zt=E3dlz%V z@4M1EWeclZR>Wt|sV9h?bN?iu05XGmlhC@UwKcI*(c9TqaVlo!2I1@sA2L&1KA4a5)S=blhL{&^kYABd*FcU7zP~3C^x`sb3@;PX)zTqh6`jmv zJm}rJUEhmXVzBKb7EsOdWg%aBr#OyNIM|riGb%9S-i08NoR6fr#$yOA@=hi2zK zACV3%AIT0^`>sG(y`!FqZ%6~ zT`A3p0%KARQhutpmT)}r=%9F0NE|CGk16ozyvY^c=j)GmYeN)FV39;o3WW$%d-)qB z)wb2!?*v}~&Av_c*xuq8|4KSw-DyT=^<0fh=ZA*C7Un{o9G&Pp^L}Rp`ZL<0L!|I%cPvqX%3+^=BA!l!>yQ%6`jHZ z5<6IsZwybO2W0kP(b4U6+6ZEeXsaJCdpiQdZOF!`uRg4&tYU}UX5wj4Pe2O6MCjR> zIZp|*eYqYldi{3#z7w?1E}!cj*?_#{{#eQ~mMEm1EOMJD+4a4SM7bMmJVqX|U}(-p z1d_j1aF;(YatnHmH#Nu19;)vm71dsNJaq88!w;n|6tcbR^)k@xlX_Y*ZAdr_no543 zRFGR37`IT1sTilSBefNLawEPnQ^(Uf!=}T40o{2&@`a2IQh1ykZXpcT#EHhPe46~) z4_7cv7P@*db~fNLcCTE-j*_5k%#Kl^T=bxb<2D5ZX4kXOPBOf{>l4H+Q$R)4QCcJB z;j8ESMVsYZ7t&z`fn&@8Wn*jHAOB6u-8DBRZUCpjX3K|1npRUugAdeT{i&)?0|$Z1 zpAR4-kX346O=K>9Mk_r7njll)lqw0K{8AQ9D~ z<2(bFeH0C=i)DYE`Zh<+jzv9H$+kJ4v}k=6d4+Xtnsjtj-kU zLBoi>L3N$D0#Ae^Z`5QsY-rQ|4E*@P-qWx_C^I8+bNPUK)IUjf+9;xexu{Zh*?Ec) zzR&m&Mk%!8AoK%BM1A%fL+ePqq2uPRj~t5UGr_lnuhunq;XtRWfCMLT(-KV}#I&zk zD|@q{m0;wy-pC^aMUk^dzTktcDt0A$&MQBV#mmOzY~Reped;^_46gGWlvJT>PDu6a4#E2^@0unMSDddsV{$P)f9CIP3-^xHqBXB7~Q{x!iwf!XNn%XdUj1)bJCujw~&K4ygdy#@N+v3TR_Is^f78y+Bk?e58d z^d0Q%tt?b5%xO&Q4J~ABt*q_n|E{uRgCWCI0XHVsbmW~fUW(o1X1?uwlZT`=8%E(5(b z9)8PQj3(Aa3`5k0ZaDOrU)XspCWfCmA%GLe- zSG{eZM@A=;XkY8xdDBUoObrX!Uqk~*26;%`lSR0qeJ&ATU1&pDHt;n2Doxa_(AboCJ0rIIhEQOh)6ujnbU1Hr)?Jtq}cN^hayaFJiN!%#h|2LBHg33n`3!G3N(tX%r51tLL1sMf`#Q{^3`gh zXj{M3Ngo~f#q8Ejv!E(G)MDO)O$nX2CR&@!d>!qY;;xG}z z;lg(d@E8Bo=Q%F>h5|sBW`GPk5}NYH;5ECGabkCin8WTFDr zAs!K8oh0k}&}3oM52yXRao)HM$Ew3klZi4PcWTefsWRUryNy4w?ngVjl~C-%#G^+@ zM4uz=cpJcB)QC)ocQ~;C3T$Fhr9ynZt`o*-Yipm%B2;oWM470_2zfaYi5xyHfhP3V zA@!#w$jG0n2GkAf=bD z*QDb$8BzU&Dl=VV4{<*BNW-#}@(mcGA}IFS=P z<)Z728C9yU26NDCFZFsZBmOI!!G7XWPW%`=q4#&ptW7O=Wm0|*C)-|IOT<3MLHFLR zVRPNS&ad)v&yUYzJpV|=G&l~qSO68@0NzFXnTopB*8gTAfP()z(ql(0muV6G&%qwx zfzRo7^B|%9^ki7cZH;9>9Ag>8%P9#^MXgzUR}0U@_;A=xiyNO_!}7ovTf%41xDFiX z6@9Xm4F&d%sE2tJ<2>uoc1Ht?g{r@#<0X#t-V*Y9&SPdH8Q1}WX@FD-NhB-J6IkXm zvhF_aN+a1DLu_=R(Vu38X`JkEL=@3V+^zQqd0&*>AwtSe$5!F`pgqmL>DB?>>|@&+i-GvIXf9FUg-EmiiLmz*$L2|v_!^?{g^zU` zIi5_dfTzDlGX7n@Pzze&iK^(?aKky`%siWM;becSC z!QyS2Je<0^w3Ahec`hzW4BmQ`|2aU5m?z#q0Py%PE(5wDe{!kcBej21`9FvZ1e6{< zZ0SdfD0rLjfRK0_3FiY&$S*)Evg#1y7!@uWeZjww{a*2(fK9}6+r`~QWt&Y1F0yjcS$FtL> zr7ZOtygO*{-OnA~c%0>I(xRbTfpcU8ldmSJR^P@VA>(bec}?vX{!{qdyri=Ai%5iK8fhxk$Obir6jFp*6HvqkI_|9m?nB_?9Ks}{={lCY4nm@<2~=(H-hKH*|0**C?i zi-cu@ZP%@qv;#I#p1MB)u=umXWSW`J9{i;kX;PQZE+N3Vrwv z8+sbbW1TH`vnHblV)GDfA-_JTIhivZHJ?6*mwTy1PJtI@Rv8m#TsP#asCfum!<5D7 zq)UmIkxghm9mIrS@Skd8PQ>uh2ovH!iro1OmM|W>f1)|w+-l-vrD@#~lh8$O(Suym00&if=mwFqNOY)?e<9GRPdI{H< zWv>nKV^_@kTkgCQ^V{(Oh`Cw_7Ze3%IPj=_(&~CGT4R+cbb;reS4VwbM~maJqoj7tWDcKj_vT$x(UP~Lg_T&MCzuL?nY1unV+S_K;D*Z+ zjFR-NHAN0g)e~r{(u3y4l9XnwZ}(8UhY(q`N2)ar)XDvtb%V*?>20tjGjyA?b&W72 zTW};}Wt~r56bXQe8lVElyt|UY6MO}6+-zSI9KS9ynBhQHJW?Z$?5fU`j?F4(F_ z!HodezkBmcXx~h*dn?@MJT{2&!I)!)m@fc6BE}HGh`}H>#zZY{(ms@QZlpuNfU|*i zj!8RwD?CIW^23LlwvG;XEn$#;mUjk;m*c(jtjklZrvi^oes-(jryzFS6pX6!Fw!&? zfwjY7mk-dI5~PIgJ~V}U6a6zJsD|&I+PrOw?_HYIL$D7EWRptxQOH#e0=f!mct3zq zuNLG@3#e-tZ}7-|~j|Ws-UM)Om^^W6^hA zcIxBUF1(TFx!5kcz#!UDg-*0?Q$9#}Xiw~fB393iI3WH8aZN4^ZKM=g2+!+ud#7-f$ELJm#cBu0>NJ+n8m12rB6Nv(`redT~R@F!8X1v10G3 zNVF|uGomg&k(7HD5|i+T%eU`B6vtE86L)iD>@--4N6m+_Zuq}|``;w(H`d)OXBDR==n0WU5^R{2h$49P{L|9=) zhk;C)j!g8Eb?Yi@=I4MH7l6()-C< zj=F$R&finFNTn5)$7s6$oR)$3&A0cDYwJT=sA1Cp>tXGPhP(^3Kfu4AK=NOSAP}vJ z5fH_k>!QDM+EhUo%FN;YDtSZVhm@n6EGj>*V`l5?BYN}XZ*jSCov>+KHLgUwx#ZxD zyvuk+{(2l1vW(W#Cy16a0@ltvncr{YmHi^yi!8?~j@=zqVwA6PL-xuQ1q&|2l+66h zO4EduG7p8e0fU0Bv-CkxZ<72@X>J5$roLc5z1AiSkc%U^g~9BE5U4k_E%98I&?WNx zlvMAE&!I$#Y)39wXt(+Sam&r8D^W(mCA0!2FBH1mL1(C5;0<$=bcNi8ocoz9<>My} z_Nj7Y0mDXpps2uwWJ;Ryn$VpnJKHJ-0*x2(YJ!6>9Hrr=*mUdj0(|%rhSg4bsgfPV zzE)OY7}@gcKEY=a(1SC5m~__`U3V|J8|xsmfxw-2n-=jl4_if#DUnMx`iGAaLdl8T zqAn?4vz5MB)jJ5R^chF@MMB3+>?E3c9+Z>MCznHg1s=zomm}=sRqw_ogVZ3-f`pL) z)|BpY-vnyTd0#%;14PZY;qTpG4rS0QI#1TKEU~!JYxP+ojIFl><5FJXsv!l`1`Q7G zRx@<}(f#I3Si%WfRV$}KnZ1AE0u&qLh8?fmc%fC!FtHs3%?w;xa!IpLZij8q>(A`HufWV=r~#oRQ_drbG38##RU>p zADW0WDNxPE+q@fZ>ySYczTxZ&b;yJI;;9Wj`$-C8Y)jCIgo@?+R%Es#T=aPa-S^64=ZN3P7 zuU4lgygx>HQd%F169CDV2|^$sz+(SbgReE9d(qy|R^HIw{;xL1-(S;F>zP=dsEt}q z^CCXaiO!aY;1dyfN@Z&xie$xKIZ-yt#N-4k5i|}VQm7C(?#fKzB61?obj4sxPeNpI z*=?YAsJdYXEyR?9#xu7jwjW=goSm@L#$^n0^Mghy!*B!F$##Q(phZP`b*b}g z1y5}rW$$vKc5))Q3NFt(`$G1OEZI_)V=$GN<^b`}!%j%rVp;Bk zIW569D{ydS$&f`oLRD{?A@0fg4ER5lHp^!zEbJp*`e|%UkzE#GpEOsOy_hPYU#4Zk zh<;f?!A2EO|3EkjjmH2SWY|lfgOm&cTDF>25N)WA|D>u=epv-M!T}6_>TSNetx0Xs z(^nT2r%On$k!6!8(2>CVSq>#6?QJ5dNj8Q09ca92yn5}_3oK5^V-8-wmZ~Nf1tb}# zFi)H+p-arm#@td;FdMmemhN6`k&&1e)8}fEj|!iO4i81szJk_2&=A=%Q0yCAOLFcz z^n1JaJJPp*<7>aI6=VT3VPz-i362GoXLWdOvZ0BMtzexc%aKRKH*1fTZ-zWLMQ+Fu zeqj9JWNzP!vRr=XOBp81aG_uq0WA_R$BKQ0u%{yt`(ZQsnP3NoqLyp>NOi+gxbj)%uz`$aH>CbZU)uTS7N2up}*NGRMP3G<_sYKW}Fz zVS6siNtny~tC+~~d9R~MlIPs5^~e7DbUUcm<4f;K1UKwkvJm{-Q*i339bX3=Ubaah zL{$a4ai-LcyG8Gegb>9|OKf~@>^*Y0y|O3=Gp^bG#OwIgA3RP|0q=a zJsa^?x#C9;CSbJa-+4xWw*1GtLxJgF!8ckrIO10_#+(sj4xLre+d`T)>h0+n`b^|g z$sTUdr70*+=%DVO^HzC!%FdD(%cwF#s+7#{po0DPTMXe8VmKT*qEJ{f1U7WLo_74zmzUJC?Ah zg#NIKn8~_lK_NOghZx^F4O5U?@+=&KTPQfQA{=yX?%L9>5r z{Y8W8p!KE_JW3q0*9WQSs3gMjB5Pgyhj(oN8cZ7~iy3eD0_uP!(_R~<`P7%l^ra<4 z%vHfV?Qrv#hVlwr70UHEkt5&8Imb zf~)iZfXJ@;=9$_@##+Cu! z|Ez2NuPS%uksR=VPF1i<->EpNgNZFhA86TK;C|N7{IdBgYP@w^i1$A6u9+Wj9f{f; zaAt+yh+SoLqZnav<}a2cjfU-!1$~f{bOW-9Bc-4MOHy^7nWOWOV1E4c_8q6=MZHOV zw~FQz=82zzHXT58@`FDWdJHvhepkn{6ng8q)`(5(@?@-%);TF8CQm?(Od=-F$ zGuDr2@OP|VgNUE82s!_Z^&_PC9qZSX=x3~V+&^ReWlH~|FBo7X0iXiF*9IWQLE-u9 G)&B$cO^I#* literal 0 HcmV?d00001 From ec7419489211d39641fe2b5347f736d4dd633cc1 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 19 Feb 2024 12:29:55 -0500 Subject: [PATCH 163/220] [BI-2045] improved variable name --- .../importer/services/processors/ExperimentProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 9f789e172..847647eec 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -84,7 +84,7 @@ public class ExperimentProcessor implements Processor { private static final String NAME = "Experiment"; private static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs"; - private static final String DUPLICATE_EXPERIMENT_TITLE = "Experiment Title already exists"; + private static final String PREEXISTING_EXPERIMENT_TITLE = "Experiment Title already exists"; private static final String MULTIPLE_EXP_TITLES = "File contains more than one Experiment Title"; private static final String MIDNIGHT = "T00:00:00-00:00"; private static final String TIMESTAMP_PREFIX = "TS:"; @@ -1179,7 +1179,7 @@ private PendingImportObject fetchOrCreateTrialPIO(Program program, U if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { pio = trialByNameNoScope.get(importRow.getExpTitle()); if (pio.getState() == ImportObjectState.EXISTING && StringUtils.isBlank( importRow.getObsUnitID() ) ){ - throw new UnprocessableEntityException(DUPLICATE_EXPERIMENT_TITLE); + throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); } } else if (!trialByNameNoScope.isEmpty()) { throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); From 71a7cf61a093743b5d3c6925dcaf1b20a7b236ee Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 19 Feb 2024 12:15:01 -0500 Subject: [PATCH 164/220] [BI-2026] If studyYear is null, change it to a blank, to avoid a null-poiner-exception --- .../importer/services/processors/ExperimentProcessor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 525a5e5b5..c3346730e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -578,7 +578,7 @@ private String getImportObservationHash(ExperimentObservation importRow, String private String getObservationHash(String observationUnitName, String variableName, String studyName) { String concat = DigestUtils.sha256Hex(observationUnitName) + DigestUtils.sha256Hex(variableName) + - DigestUtils.sha256Hex(studyName); + DigestUtils.sha256Hex(StringUtils.defaultString(studyName)); return DigestUtils.sha256Hex(concat); } @@ -792,12 +792,12 @@ private void validateConditionallyRequired(ValidationErrors validationErrors, in } } if(validateRequiredCell(importRow.getEnvYear(), Columns.ENV_YEAR, errorMessage, validationErrors, rowNum)) { - String studyYear = this.studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getSeasons().get(0); + String studyYear = StringUtils.defaultString( this.studyByNameNoScope.get(importRow.getEnv()).getBrAPIObject().getSeasons().get(0) ); String rowYear = importRow.getEnvYear(); if(commit) { rowYear = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); } - if(!studyYear.equals(rowYear)) { + if(StringUtils.isNotBlank(studyYear) && !studyYear.equals(rowYear)) { addRowError(Columns.ENV_YEAR, ENV_YEAR_MISMATCH, validationErrors, rowNum); } } From 52a56557861fad481306d5296451bd1b97a10834 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Thu, 22 Feb 2024 14:10:47 -0500 Subject: [PATCH 165/220] [BI-2045] WIP (improve unit testing) --- .../importer/ExperimentFileImportTest.java | 122 +++++++++++------- 1 file changed, 78 insertions(+), 44 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index b3bcb0da1..dab0e1fc9 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -216,7 +216,7 @@ public void importNewExpNewLocNoObsSuccess() { @Test @SneakyThrows - public void importNewExpMultiNewEnvNoObsSuccess() { + public void importNewExpMultiNewEnvError() { Program program = createProgram("New Exp and Multi New Env", "MULENV", "MULENV", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); Map firstEnv = new HashMap<>(); firstEnv.put(Columns.GERMPLASM_GID, "1"); @@ -282,8 +282,8 @@ public void importNewExpMultiNewEnvNoObsSuccess() { @Test @SneakyThrows - public void importNewEnvExistingExpNoObsSuccess() { - Program program = createProgram("New Env Existing Exp", "NEWENV", "NEWENV", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + public void importNewEnvExistingExpErrorMessage() { + Program program = createProgram("New Env Existing Exp", "DUPENV", "DUPENV", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); @@ -316,19 +316,52 @@ public void importNewEnvExistingExpNoObsSuccess() { newEnv.put(Columns.ROW, "1"); newEnv.put(Columns.COLUMN, "1"); + Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, false, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); + } + + @Test + @SneakyThrows + public void importNewEnvNoObsSuccess() { + Program program = createProgram("New Env", "NEWENV", "NEWENV", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + + Map newEnv = new HashMap<>(); + newEnv.put(Columns.GERMPLASM_GID, "1"); + newEnv.put(Columns.TEST_CHECK, "T"); + newEnv.put(Columns.EXP_TITLE, "Test Exp"); + newEnv.put(Columns.EXP_UNIT, "Plot"); + newEnv.put(Columns.EXP_TYPE, "Phenotyping"); + newEnv.put(Columns.ENV, "New Trial Existing Exp"); + newEnv.put(Columns.ENV_LOCATION, "Location A"); + newEnv.put(Columns.ENV_YEAR, "2023"); + newEnv.put(Columns.EXP_UNIT_ID, "a-1"); + newEnv.put(Columns.REP_NUM, "1"); + newEnv.put(Columns.BLOCK_NUM, "1"); + newEnv.put(Columns.ROW, "1"); + newEnv.put(Columns.COLUMN, "1"); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, true, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); JsonObject row = previewRows.get(0).getAsJsonObject(); - assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); - assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("trial").get("state").getAsString()); + assertEquals("NEW", row.getAsJsonObject("location").get("state").getAsString()); assertEquals("NEW", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("NEW", row.getAsJsonObject("observationUnit").get("state").getAsString()); assertRowSaved(newEnv, program, null); } + @ParameterizedTest @ValueSource(booleans = {true, false}) @SneakyThrows @@ -566,45 +599,46 @@ public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); } - @ParameterizedTest - @ValueSource(booleans = {true, false}) - @SneakyThrows - public void verifyFailureNewOuExistingEnv(boolean commit) { - Program program = createProgram("New OU Exising Env "+(commit ? "C" : "P"), "FLOU"+(commit ? "C" : "P"), "FLOU"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); - Map newExp = new HashMap<>(); - newExp.put(Columns.GERMPLASM_GID, "1"); - newExp.put(Columns.TEST_CHECK, "T"); - newExp.put(Columns.EXP_TITLE, "Test Exp"); - newExp.put(Columns.EXP_UNIT, "Plot"); - newExp.put(Columns.EXP_TYPE, "Phenotyping"); - newExp.put(Columns.ENV, "New Env"); - newExp.put(Columns.ENV_LOCATION, "Location A"); - newExp.put(Columns.ENV_YEAR, "2023"); - newExp.put(Columns.EXP_UNIT_ID, "a-1"); - newExp.put(Columns.REP_NUM, "1"); - newExp.put(Columns.BLOCK_NUM, "1"); - newExp.put(Columns.ROW, "1"); - newExp.put(Columns.COLUMN, "1"); - - importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); - - Map newOU = new HashMap<>(newExp); - newOU.put(Columns.EXP_UNIT_ID, "a-2"); - newOU.put(Columns.ROW, "1"); - newOU.put(Columns.COLUMN, "2"); - - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); - HttpResponse response = call.blockingFirst(); - assertEquals(HttpStatus.ACCEPTED, response.getStatus()); - - String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); - - HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); - JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); - assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); - - assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experimental entities are missing ObsUnitIDs")); - } + // NO Longer needed, but may be needed in the future. +// @ParameterizedTest +// @ValueSource(booleans = {true, false}) +// @SneakyThrows +// public void verifyFailureNewOuExistingEnv(boolean commit) { +// Program program = createProgram("New OU Exising Env "+(commit ? "C" : "P"), "FLOU"+(commit ? "C" : "P"), "FLOU"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); +// Map newExp = new HashMap<>(); +// newExp.put(Columns.GERMPLASM_GID, "1"); +// newExp.put(Columns.TEST_CHECK, "T"); +// newExp.put(Columns.EXP_TITLE, "Test Exp"); +// newExp.put(Columns.EXP_UNIT, "Plot"); +// newExp.put(Columns.EXP_TYPE, "Phenotyping"); +// newExp.put(Columns.ENV, "New Env"); +// newExp.put(Columns.ENV_LOCATION, "Location A"); +// newExp.put(Columns.ENV_YEAR, "2023"); +// newExp.put(Columns.EXP_UNIT_ID, "a-1"); +// newExp.put(Columns.REP_NUM, "1"); +// newExp.put(Columns.BLOCK_NUM, "1"); +// newExp.put(Columns.ROW, "1"); +// newExp.put(Columns.COLUMN, "1"); +// +// importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); +// +// Map newOU = new HashMap<>(newExp); +// newOU.put(Columns.EXP_UNIT_ID, "a-2"); +// newOU.put(Columns.ROW, "1"); +// newOU.put(Columns.COLUMN, "2"); +// +// Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); +// HttpResponse response = call.blockingFirst(); +// assertEquals(HttpStatus.ACCEPTED, response.getStatus()); +// +// String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); +// +// HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); +// JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); +// assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); +// +// assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experimental entities are missing ObsUnitIDs")); +// } @Test @SneakyThrows From d7e3dcd70437d606c1c63a7d24f32580955a28d3 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 4 Mar 2024 09:38:33 -0500 Subject: [PATCH 166/220] [BI-2045] Check for existing Env when validating for 'Experiment Title already exist' --- .../services/processors/ExperimentProcessor.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 847647eec..7396ca664 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1175,10 +1175,12 @@ private void fetchOrCreateLocationPIO(ExperimentObservation importRow) { } private PendingImportObject fetchOrCreateTrialPIO(Program program, User user, boolean commit, ExperimentObservation importRow, Supplier expNextVal) throws UnprocessableEntityException { - PendingImportObject pio; + PendingImportObject trialPio; if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { - pio = trialByNameNoScope.get(importRow.getExpTitle()); - if (pio.getState() == ImportObjectState.EXISTING && StringUtils.isBlank( importRow.getObsUnitID() ) ){ + PendingImportObject envPio; + trialPio = trialByNameNoScope.get(importRow.getExpTitle()); + envPio = this.studyByNameNoScope.get(importRow.getEnv()); + if (trialPio.getState() == ImportObjectState.EXISTING && (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); } } else if (!trialByNameNoScope.isEmpty()) { @@ -1190,10 +1192,10 @@ private PendingImportObject fetchOrCreateTrialPIO(Program program, U expSeqValue = expNextVal.get().toString(); } BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); - pio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); - this.trialByNameNoScope.put(importRow.getExpTitle(), pio); + trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); + this.trialByNameNoScope.put(importRow.getExpTitle(), trialPio); } - return pio; + return trialPio; } private void updateObservationDependencyValues(Program program) { From 1a2ff0927fa1ac6a99a4a24114582bb63da4bcfe Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 4 Mar 2024 10:55:03 -0500 Subject: [PATCH 167/220] [BI-2045] fixed test --- .../importer/ExperimentFileImportTest.java | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index dab0e1fc9..02d8125fd 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -290,7 +290,7 @@ public void importNewEnvExistingExpErrorMessage() { newExp.put(Columns.EXP_TITLE, "Test Exp"); newExp.put(Columns.EXP_UNIT, "Plot"); newExp.put(Columns.EXP_TYPE, "Phenotyping"); - newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV, "Existing Env"); newExp.put(Columns.ENV_LOCATION, "Location A"); newExp.put(Columns.ENV_YEAR, "2023"); newExp.put(Columns.EXP_UNIT_ID, "a-1"); @@ -301,22 +301,22 @@ public void importNewEnvExistingExpErrorMessage() { JsonObject expResult = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); - Map newEnv = new HashMap<>(); - newEnv.put(Columns.GERMPLASM_GID, "1"); - newEnv.put(Columns.TEST_CHECK, "T"); - newEnv.put(Columns.EXP_TITLE, "Test Exp"); - newEnv.put(Columns.EXP_UNIT, "Plot"); - newEnv.put(Columns.EXP_TYPE, "Phenotyping"); - newEnv.put(Columns.ENV, "New Trial Existing Exp"); - newEnv.put(Columns.ENV_LOCATION, "Location A"); - newEnv.put(Columns.ENV_YEAR, "2023"); - newEnv.put(Columns.EXP_UNIT_ID, "a-1"); - newEnv.put(Columns.REP_NUM, "1"); - newEnv.put(Columns.BLOCK_NUM, "1"); - newEnv.put(Columns.ROW, "1"); - newEnv.put(Columns.COLUMN, "1"); - - Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newEnv), null), null, false, client, program, mappingId); + Map dupExp = new HashMap<>(); + dupExp.put(Columns.GERMPLASM_GID, "1"); + dupExp.put(Columns.TEST_CHECK, "T"); + dupExp.put(Columns.EXP_TITLE, "Test Exp"); + dupExp.put(Columns.EXP_UNIT, "Plot"); + dupExp.put(Columns.EXP_TYPE, "Phenotyping"); + dupExp.put(Columns.ENV, "Existing Env"); + dupExp.put(Columns.ENV_LOCATION, "Location A"); + dupExp.put(Columns.ENV_YEAR, "2023"); + dupExp.put(Columns.EXP_UNIT_ID, "a-1"); + dupExp.put(Columns.REP_NUM, "1"); + dupExp.put(Columns.BLOCK_NUM, "1"); + dupExp.put(Columns.ROW, "1"); + dupExp.put(Columns.COLUMN, "1"); + + Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(dupExp), null), null, false, client, program, mappingId); HttpResponse response = call.blockingFirst(); assertEquals(HttpStatus.ACCEPTED, response.getStatus()); From 1102222714d0506a71a6bbb6f7ea7b508bfefa50 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 4 Mar 2024 12:07:21 -0500 Subject: [PATCH 168/220] [BI-2045] fixed test name --- .../brapps/importer/ExperimentFileImportTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 02d8125fd..940260cf3 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -282,7 +282,7 @@ public void importNewExpMultiNewEnvError() { @Test @SneakyThrows - public void importNewEnvExistingExpErrorMessage() { + public void importExistingExpAndEnvErrorMessage() { Program program = createProgram("New Env Existing Exp", "DUPENV", "DUPENV", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); From 1ae021c73365fee0c9306a11c33dbe546091ab00 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Mon, 4 Mar 2024 15:31:28 -0500 Subject: [PATCH 169/220] [BI-2046] address PR comments --- .../importer/services/processors/ExperimentProcessor.java | 2 +- .../brapps/importer/ExperimentFileImportTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 7396ca664..2113a0398 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1180,7 +1180,7 @@ private PendingImportObject fetchOrCreateTrialPIO(Program program, U PendingImportObject envPio; trialPio = trialByNameNoScope.get(importRow.getExpTitle()); envPio = this.studyByNameNoScope.get(importRow.getEnv()); - if (trialPio.getState() == ImportObjectState.EXISTING && (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ + if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); } } else if (!trialByNameNoScope.isEmpty()) { diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 940260cf3..7a797a815 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -216,7 +216,7 @@ public void importNewExpNewLocNoObsSuccess() { @Test @SneakyThrows - public void importNewExpMultiNewEnvError() { + public void importNewExpMultiNewEnvSuccess() { Program program = createProgram("New Exp and Multi New Env", "MULENV", "MULENV", BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); Map firstEnv = new HashMap<>(); firstEnv.put(Columns.GERMPLASM_GID, "1"); From ab579d63083490cd06efe20bbec9412e9dc01008 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 5 Mar 2024 14:44:05 +0000 Subject: [PATCH 170/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 0e86bc81a..cdb0bb87d 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+681 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/d15e597ede75c582aad72a310cab6bafdd9de2f9 +version=v0.9.0+683 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1c88caa4d1014b848ce35aa61d8df75c75b00676 From f955c284ab79773e04b9b16ddcab9901bfa1de6f Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 5 Mar 2024 16:41:43 +0000 Subject: [PATCH 171/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index cdb0bb87d..bd8088a32 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+683 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/1c88caa4d1014b848ce35aa61d8df75c75b00676 +version=v0.9.0+685 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/014b9e212da112169f6613f1ddf4950dd1a278c1 From fc9c1c6db6b576802b709e37ee981aeed741e371 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Fri, 8 Mar 2024 10:17:13 -0500 Subject: [PATCH 172/220] [BI-2026] protect against year = null --- .../importer/services/processors/ExperimentProcessor.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index c3346730e..58937983f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1824,7 +1824,11 @@ private String yearToSeasonDbIdFromDatabase(String year, UUID programId) { } if (targetSeason == null) { BrAPISeason newSeason = new BrAPISeason(); - newSeason.setYear(Integer.parseInt(year)); + Integer intYear = null; + if( StringUtils.isNotBlank(year) ){ + intYear = Integer.parseInt(year); + } + newSeason.setYear(intYear); newSeason.setSeasonName(year); targetSeason = this.brAPISeasonDAO.addOneSeason(newSeason, programId); } From 34c257df6a7e2c52672f8e10023dbd7dd5bcddf0 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 12 Mar 2024 13:39:17 +0000 Subject: [PATCH 173/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index bd8088a32..4dd377520 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+685 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/014b9e212da112169f6613f1ddf4950dd1a278c1 +version=v0.9.0+689 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/17861f1bc04a9d3b1f7c9a43d6859e08e10e9025 From 79552a069e62bd2976de4930d96cebaa0d4a68d6 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Tue, 19 Mar 2024 15:48:11 +0000 Subject: [PATCH 174/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 4dd377520..55429bbd0 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+689 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/17861f1bc04a9d3b1f7c9a43d6859e08e10e9025 +version=v0.9.0+699 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/34c257df6a7e2c52672f8e10023dbd7dd5bcddf0 From 5bda44a80e40f6d2f1266321fb911ae53f6d5774 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 13 Feb 2024 16:32:51 -0500 Subject: [PATCH 175/220] [BI-2046] add validator for exp import on obsUnitID for existing studies --- .../processors/ExperimentProcessor.java | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 4ea10c07a..9fcd1b501 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -118,7 +118,7 @@ public class ExperimentProcessor implements Processor { private final Map seasonDbIdToYearCache = new HashMap<>(); //These BrapiData-objects are initially populated by the getExistingBrapiData() method, - // then updated by the getNewBrapiData() method. + // then updated by the initNewBrapiData() method. private Map> trialByNameNoScope = null; private Map> locationByName = null; private Map> studyByNameNoScope = null; @@ -129,7 +129,7 @@ public class ExperimentProcessor implements Processor { private final Map> observationByHash = new HashMap<>(); private Map existingObsByObsHash = new HashMap<>(); - // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the getNewBrapiData() method + // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the initNewBrapiData() method private Map> existingGermplasmByGID = null; // Associates timestamp columns to associated phenotype column name for ease of storage @@ -598,13 +598,6 @@ private void validateFields(List importRows, ValidationErrors valid validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); } validateTestOrCheck(importRow, validationErrors, rowNum); - //TODO: providing obs unit ID does not supersede import row inout data as expected and needs to be fixed - //Check if existing environment. If so, ObsUnitId must be assigned -// if ((mappedImportRow.getStudy().getState() == ImportObjectState.EXISTING) -// && (StringUtils.isBlank(importRow.getObsUnitID()))) { -// throw new MissingRequiredInfoException(MISSING_OBS_UNIT_ID_ERROR); -// } - validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); @@ -810,8 +803,14 @@ private void validateConditionallyRequired(ValidationErrors validationErrors, in addRowError(Columns.OBS_UNIT_ID, "ObsUnitID cannot be specified when creating a new environment", validationErrors, rowNum); } } else { - //TODO: include this step once user-supplied obs unit id correctly supersedes other row data - //validateRequiredCell(importRow.getObsUnitID(), Columns.OBS_UNIT_ID, errorMessage, validationErrors, rowNum); + //Check if existing environment. If so, ObsUnitId must be assigned + validateRequiredCell( + importRow.getObsUnitID(), + Columns.OBS_UNIT_ID, + "Experimental entities are missing ObsUnitIDs. Import cannot proceed", + validationErrors, + rowNum + ); } } @@ -1380,7 +1379,7 @@ private Map> initializeObserva BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); - ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceID()); + ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); row.setExpTitle(Utilities.removeProgramKey(brAPIObservationUnit.getTrialName(), program.getKey())); row.setEnv(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getStudyName(), program.getKey())); row.setEnvLocation(Utilities.removeProgramKey(brAPIObservationUnit.getLocationName(), program.getKey())); @@ -1600,12 +1599,12 @@ private void processAndCacheObservationUnit(BrAPIObservationUnit brAPIObservatio BrAPIExternalReference idRef = Utilities.getExternalReference(brAPIObservationUnit.getExternalReferences(), refSource) .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); - ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceID()); + ExperimentObservation row = rowByObsUnitId.get(idRef.getReferenceId()); row.setExpUnitId(Utilities.removeProgramKeyAndUnknownAdditionalData(brAPIObservationUnit.getObservationUnitName(), program.getKey())); observationUnitByName.put(createObservationUnitKey(row), new PendingImportObject<>(ImportObjectState.EXISTING, brAPIObservationUnit, - UUID.fromString(idRef.getReferenceID()))); + UUID.fromString(idRef.getReferenceId()))); } private void processAndCacheStudy(BrAPIStudy existingStudy, Program program, Map> studyByName) { From ba789170a3cce9cd17c0ebbf05188647df5f962e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 14 Feb 2024 11:17:54 -0500 Subject: [PATCH 176/220] [BI-2046] update ref to obsolete brapi-client field --- .../importer/ExperimentFileImportTest.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 7a797a815..175f8584e 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -666,7 +666,7 @@ public void importNewObsVarExisingOu() { BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program).get(0); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); @@ -686,7 +686,7 @@ public void importNewObsVarExisingOu() { newObsVar.put(Columns.BLOCK_NUM, "1"); newObsVar.put(Columns.ROW, "1"); newObsVar.put(Columns.COLUMN, "1"); - newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); + newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId); @@ -731,7 +731,7 @@ public void importNewObsExisingOu(boolean commit) { BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program).get(0); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); @@ -751,7 +751,7 @@ public void importNewObsExisingOu(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); + newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, commit, client, program, mappingId); @@ -798,7 +798,7 @@ public void verifyFailureImportNewObsExisingOuWithExistingObs(boolean commit) { BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program).get(0); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); @@ -818,7 +818,7 @@ public void verifyFailureImportNewObsExisingOuWithExistingObs(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); + newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "2"); uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName(), commit); @@ -925,7 +925,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program).get(0); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); @@ -945,7 +945,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); + newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "2"); @@ -999,7 +999,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program).get(0); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); @@ -1021,7 +1021,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceID()); + newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), ""); newObservation.put(traits.get(1).getObservationVariableName(), "2"); @@ -1053,7 +1053,7 @@ private Map assertRowSaved(Map expected, Program Optional trialIdXref = Utilities.getExternalReference(trial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); assertTrue(trialIdXref.isPresent()); - List studies = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceID()), program); + List studies = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program); assertFalse(studies.isEmpty()); BrAPIStudy study = null; for(BrAPIStudy s : studies) { From ef4fea6c3ce999a7f4fbb47ee7c65e8d12ed291b Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 15 Feb 2024 10:55:44 -0500 Subject: [PATCH 177/220] [BI-2046] fix typos and obsolete refs --- .../importer/ExperimentFileImportTest.java | 86 +++++++++---------- 1 file changed, 42 insertions(+), 44 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 175f8584e..056e8b297 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -361,7 +361,6 @@ public void importNewEnvNoObsSuccess() { assertRowSaved(newEnv, program, null); } - @ParameterizedTest @ValueSource(booleans = {true, false}) @SneakyThrows @@ -599,50 +598,49 @@ public void verifyFailureImportNewExpWithInvalidObs(boolean commit) { uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newExp), traits), traits.get(0).getObservationVariableName(), commit); } - // NO Longer needed, but may be needed in the future. -// @ParameterizedTest -// @ValueSource(booleans = {true, false}) -// @SneakyThrows -// public void verifyFailureNewOuExistingEnv(boolean commit) { -// Program program = createProgram("New OU Exising Env "+(commit ? "C" : "P"), "FLOU"+(commit ? "C" : "P"), "FLOU"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); -// Map newExp = new HashMap<>(); -// newExp.put(Columns.GERMPLASM_GID, "1"); -// newExp.put(Columns.TEST_CHECK, "T"); -// newExp.put(Columns.EXP_TITLE, "Test Exp"); -// newExp.put(Columns.EXP_UNIT, "Plot"); -// newExp.put(Columns.EXP_TYPE, "Phenotyping"); -// newExp.put(Columns.ENV, "New Env"); -// newExp.put(Columns.ENV_LOCATION, "Location A"); -// newExp.put(Columns.ENV_YEAR, "2023"); -// newExp.put(Columns.EXP_UNIT_ID, "a-1"); -// newExp.put(Columns.REP_NUM, "1"); -// newExp.put(Columns.BLOCK_NUM, "1"); -// newExp.put(Columns.ROW, "1"); -// newExp.put(Columns.COLUMN, "1"); -// -// importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); -// -// Map newOU = new HashMap<>(newExp); -// newOU.put(Columns.EXP_UNIT_ID, "a-2"); -// newOU.put(Columns.ROW, "1"); -// newOU.put(Columns.COLUMN, "2"); -// -// Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); -// HttpResponse response = call.blockingFirst(); -// assertEquals(HttpStatus.ACCEPTED, response.getStatus()); -// -// String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); -// -// HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); -// JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); -// assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); -// -// assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experimental entities are missing ObsUnitIDs")); -// } + @ParameterizedTest + @ValueSource(booleans = {true, false}) + @SneakyThrows + public void verifyFailureNewOuExistingEnv(boolean commit) { + Program program = createProgram("New OU Existing Env "+(commit ? "C" : "P"), "FLOU"+(commit ? "C" : "P"), "FLOU"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), null); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + + Map newOU = new HashMap<>(newExp); + newOU.put(Columns.EXP_UNIT_ID, "a-2"); + newOU.put(Columns.ROW, "1"); + newOU.put(Columns.COLUMN, "2"); + + Flowable> call = importTestUtils.uploadDataFile(importTestUtils.writeExperimentDataToFile(List.of(newOU), null), null, commit, client, program, mappingId); + HttpResponse response = call.blockingFirst(); + assertEquals(HttpStatus.ACCEPTED, response.getStatus()); + + String importId = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("result").get("importId").getAsString(); + + HttpResponse upload = importTestUtils.getUploadedFile(importId, client, program, mappingId); + JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); + assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); + + assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experimental entities are missing ObsUnitIDs")); + } @Test @SneakyThrows - public void importNewObsVarExisingOu() { + public void importNewObsVarExistingOu() { List traits = importTestUtils.createTraits(2); Program program = createProgram("New ObsVar Existing OU", "OUVAR", "OUVAR", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -708,7 +706,7 @@ public void importNewObsVarExisingOu() { @ParameterizedTest @ValueSource(booleans = {true, false}) @SneakyThrows - public void importNewObsExisingOu(boolean commit) { + public void importNewObsExistingOu(boolean commit) { List traits = importTestUtils.createTraits(1); Program program = createProgram("New Obs Existing OU "+(commit ? "C" : "P"), "OUOBS"+(commit ? "C" : "P"), "OUOBS"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); @@ -774,7 +772,7 @@ public void importNewObsExisingOu(boolean commit) { @ParameterizedTest @ValueSource(booleans = {true, false}) @SneakyThrows - public void verifyFailureImportNewObsExisingOuWithExistingObs(boolean commit) { + public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { List traits = importTestUtils.createTraits(1); Program program = createProgram("New Obs Existing Obs "+(commit ? "C" : "P"), "FEXOB"+(commit ? "C" : "P"), "FEXOB"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); From b7836323ee4f7639ee268a7a408cd3ec9cb09653 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 26 Feb 2024 16:04:26 -0500 Subject: [PATCH 178/220] [BI-2046] set pendingObsUnitByOUId --- .../importer/services/FileMappingUtil.java | 14 +- .../processors/ExperimentProcessor.java | 258 +++++++++++++++++- .../importer/ExperimentFileImportTest.java | 64 +++++ 3 files changed, 318 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java index e603a7d21..a3e96164a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java @@ -27,10 +27,7 @@ import javax.inject.Inject; import javax.inject.Singleton; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; @Singleton public class FileMappingUtil { @@ -94,4 +91,13 @@ public List sortByField(List sortedFields, List unsortedItems, return unsortedItems; } + + public boolean isValidUUID(String id) { + try { + UUID.fromString(id); + return true; + } catch (IllegalArgumentException e) { + return false; + } + } } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 9fcd1b501..148e139ce 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -34,6 +34,7 @@ import org.brapi.v2.model.core.*; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; +import org.brapi.v2.model.geno.BrAPIReference; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; @@ -83,7 +84,7 @@ public class ExperimentProcessor implements Processor { private static final String NAME = "Experiment"; - private static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs"; + private static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs. Import cannot proceed"; private static final String PREEXISTING_EXPERIMENT_TITLE = "Experiment Title already exists"; private static final String MULTIPLE_EXP_TITLES = "File contains more than one Experiment Title"; private static final String MIDNIGHT = "T00:00:00-00:00"; @@ -120,12 +121,17 @@ public class ExperimentProcessor implements Processor { //These BrapiData-objects are initially populated by the getExistingBrapiData() method, // then updated by the initNewBrapiData() method. private Map> trialByNameNoScope = null; + private Map> pendingTrialByOUId = null; private Map> locationByName = null; + private Map> pendingLocationByOUId = null; private Map> studyByNameNoScope = null; + private Map> pendingStudyByOUId = null; private Map> obsVarDatasetByName = null; + private Map> pendingObsDatasetByOUId = null; // It is assumed that there are no preexisting Observation Units for the given environment (so this will not be // initialized by getExistingBrapiData() ) private Map> observationUnitByNameNoScope = null; + private Map> pendingObsUnitByOUId = null; private final Map> observationByHash = new HashMap<>(); private Map existingObsByObsHash = new HashMap<>(); @@ -135,6 +141,9 @@ public class ExperimentProcessor implements Processor { // Associates timestamp columns to associated phenotype column name for ease of storage private final Map> timeStampColByPheno = new HashMap<>(); private final Gson gson; + private boolean hasAllReferenceUnitIds = true; + private boolean hasNoReferenceUnitIds = true; + private Set referenceOUIds = new HashSet<>(); @Inject public ExperimentProcessor(DSLContext dsl, @@ -179,12 +188,47 @@ public void getExistingBrapiData(List importRows, Program program) .map(trialImport -> (ExperimentObservation) trialImport) .collect(Collectors.toList()); - this.observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); - this.trialByNameNoScope = initializeTrialByNameNoScope(program, experimentImportRows); - this.studyByNameNoScope = initializeStudyByNameNoScope(program, experimentImportRows); - this.locationByName = initializeUniqueLocationNames(program, experimentImportRows); - this.obsVarDatasetByName = initializeObsVarDatasetByName(program, experimentImportRows); - this.existingGermplasmByGID = initializeExistingGermplasmByGID(program, experimentImportRows); + Map referenceUnitById = new HashMap<>(); + Map expTitleByRefUnitId = new HashMap<>(); + // check for references to Deltabreed-generated observation units + setReferenceOUIds(importRows); + + if (hasAllReferenceUnitIds) { + + // get all prior units referenced in import + try { + List referenceOUs = brAPIObservationUnitDAO.getObservationUnitsById(new ArrayList(referenceOUIds), program); + for (BrAPIObservationUnit unit: referenceOUs) { + setPendingObsUnitByOUId(program); + setPendingTrialByOUId(program); + + // set pendingStudiesByOUId + // set pendingLocationsByOUId + // set pendingObsDatasetByOUId + + BrAPIExternalReference xref = Utilities.getExternalReference( + unit.getExternalReferences(), + ExternalReferenceSource.OBSERVATION_UNITS.getName() + ).orElseThrow(() -> new IllegalStateException("No BI external reference found")); + referenceUnitById.put(xref.getReferenceId(), unit); + expTitleByRefUnitId.put(xref.getReferenceId(), unit.getTrialName()); + } + } catch (ApiException e) { + throw new InternalServerException(e.toString(), e); + } + } else if (hasNoReferenceUnitIds) { + observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); + trialByNameNoScope = initializeTrialByNameNoScope(program, experimentImportRows); + studyByNameNoScope = initializeStudyByNameNoScope(program, experimentImportRows); + locationByName = initializeUniqueLocationNames(program, experimentImportRows); + obsVarDatasetByName = initializeObsVarDatasetByName(program, experimentImportRows); + existingGermplasmByGID = initializeExistingGermplasmByGID(program, experimentImportRows); + } else { + + // can't proceed if you have a mix of ObsUnitId for some but not all rows + throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, MISSING_OBS_UNIT_ID_ERROR); + } + } /** @@ -495,7 +539,14 @@ private String getVariableNameFromColumn(Column column) { return column.name(); } - private void initNewBrapiData(List importRows, List> phenotypeCols, Program program, User user, List referencedTraits, boolean commit) throws ApiException, MissingRequiredInfoException { + private void initNewBrapiData( + List importRows, + List> phenotypeCols, + Program program, + User user, + List referencedTraits, + boolean commit + ) throws ApiException, MissingRequiredInfoException { String expSequenceName = program.getExpSequence(); if (expSequenceName == null) { @@ -807,7 +858,7 @@ private void validateConditionallyRequired(ValidationErrors validationErrors, in validateRequiredCell( importRow.getObsUnitID(), Columns.OBS_UNIT_ID, - "Experimental entities are missing ObsUnitIDs. Import cannot proceed", + MISSING_OBS_UNIT_ID_ERROR, validationErrors, rowNum ); @@ -1106,7 +1157,7 @@ private PendingImportObject fetchOrCreateStudyPIO( PendingImportObject pio; if (studyByNameNoScope.containsKey(importRow.getEnv())) { pio = studyByNameNoScope.get(importRow.getEnv()); - if (! commit){ + if (!commit){ addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); } } else { @@ -1173,7 +1224,13 @@ private void fetchOrCreateLocationPIO(ExperimentObservation importRow) { } } - private PendingImportObject fetchOrCreateTrialPIO(Program program, User user, boolean commit, ExperimentObservation importRow, Supplier expNextVal) throws UnprocessableEntityException { + private PendingImportObject fetchOrCreateTrialPIO( + Program program, + User user, + boolean commit, + ExperimentObservation importRow, + Supplier expNextVal + ) throws UnprocessableEntityException { PendingImportObject trialPio; if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { PendingImportObject envPio; @@ -1397,6 +1454,60 @@ private Map> initializeObserva } } + /** + * Retrieves a list of pending Observation Units based on their IDs. + * + * This function retrieves Observation Units based on a list of reference Observation Unit IDs + * and the associated program. It then sets pending Observation Units for each ID. + * + * @return List of BrAPIObservationUnit: The list of reference Observation Units retrieved. + * + * @throws InternalServerException if an error occurs during the process. + */ + private List setPendingObsUnitByOUId(Program program) { + try { + // Retrieve reference Observation Units based on IDs + List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( + new ArrayList(referenceOUIds), + program + ); + + // Construct the DeltaBreed observation unit source for external references + String deltabreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + + if (referenceObsUnits.size() == referenceOUIds.size()) { + // Iterate through reference Observation Units + referenceObsUnits.forEach(unit -> { + // Get external reference for the Observation Unit + BrAPIExternalReference unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltabreedOUSource) + .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); + + // Set pending Observation Unit by its ID + pendingObsUnitByOUId.put( + unitXref.getReferenceId(), + new PendingImportObject( + ImportObjectState.EXISTING, unit, UUID.fromString(unitXref.getReferenceId())) + ); + }); + } else { + // Handle missing Observation Unit IDs + List missingIds = new ArrayList<>(referenceOUIds); + Set fetchedIds = referenceObsUnits.stream().map(unit -> + Utilities.getExternalReference(unit.getExternalReferences(), deltabreedOUSource) + .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")) + .getReferenceId()) + .collect(Collectors.toSet()); + missingIds.removeAll(fetchedIds); + throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(COMMA_DELIMITER, missingIds)); + } + + return referenceObsUnits; + } catch (ApiException e) { + log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + private Map> initializeTrialByNameNoScope(Program program, List experimentImportRows) { Map> trialByName = new HashMap<>(); @@ -1461,12 +1572,26 @@ private void initializeStudiesForExistingObservationUnits(Program program, Map processAndCacheStudy(study, program, studyByName)); } + /** + * Fetches a list of BrAPI studies by studyDbIds belonging to a specific + * program. + * If not all studyDbIds are found in the database, it throws an + * IllegalStateException. + * + * @param studyDbIds A set of studyDbIds to fetch studies for + * @param program The program for which the studies are associated + * @return A list of BrAPIStudy objects representing the fetched studies + * @throws ApiException if an error occurs during the database fetch + * operation + * @throws IllegalStateException if not all studyDbIds are found in the database + */ private List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { List studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); - if(studies.size() != studyDbIds.size()) { + if (studies.size() != studyDbIds.size()) { List missingIds = new ArrayList<>(studyDbIds); missingIds.removeAll(studies.stream().map(BrAPIStudy::getStudyDbId).collect(Collectors.toList())); - throw new IllegalStateException("Study not found for studyDbId(s): " + String.join(COMMA_DELIMITER, missingIds)); + throw new IllegalStateException( + "Study not found for studyDbId(s): " + String.join(COMMA_DELIMITER, missingIds)); } return studies; } @@ -1621,6 +1746,52 @@ private void processAndCacheStudy(BrAPIStudy existingStudy, Program program, Map new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIStudy) Utilities.formatBrapiObjForDisplay(existingStudy, BrAPIStudy.class, program), UUID.fromString(xref.getReferenceID()))); } + private void setPendingTrialByOUId(Program program) { + if(observationUnitByNameNoScope.size() > 0) { + Set trialDbIds = new HashSet<>(); + Set studyDbIds = new HashSet<>(); + + observationUnitByNameNoScope.values() + .forEach(pio -> { + BrAPIObservationUnit existingOu = pio.getBrAPIObject(); + if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { + throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); + } + + if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { + trialDbIds.add(existingOu.getTrialDbId()); + } else { + studyDbIds.add(existingOu.getStudyDbId()); + } + }); + + //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId + if(!studyDbIds.isEmpty()) { + try { + trialDbIds.addAll(fetchTrialDbidsForStudies(studyDbIds, program)); + } catch (ApiException e) { + log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + try { + List trials = brapiTrialDAO.getTrialsByDbIds(trialDbIds, program); + if (trials.size() != trialDbIds.size()) { + List missingIds = new ArrayList<>(trialDbIds); + missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); + throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(COMMA_DELIMITER, missingIds)); + } + + String trialRefSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName()); + trials.forEach(trial -> processAndCachePendingTrial(trial, trialRefSource, "foo")); + } catch (ApiException e) { + log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + } + private void initializeTrialsForExistingObservationUnits(Program program, Map> trialByName) { if(observationUnitByNameNoScope.size() > 0) { Set trialDbIds = new HashSet<>(); @@ -1680,7 +1851,12 @@ private Set fetchTrialDbidsForStudies(Set studyDbIds, Program pr return trialDbIds; } - private void processAndCacheTrial(BrAPITrial existingTrial, Program program, String trialRefSource, Map> trialByNameNoScope) { + private void processAndCacheTrial( + BrAPITrial existingTrial, + Program program, + String trialRefSource, + Map> trialByNameNoScope + ) { //get TrialId from existingTrial BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), trialRefSource) @@ -1692,6 +1868,60 @@ private void processAndCacheTrial(BrAPITrial existingTrial, Program program, Str new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); } + /** + * Cache pending trial with reference to observation unit. + * + * This method processes the given existing trial, associates it with the + * specified + * program, retrieves the experiment ID from the trial's external references, + * then caches it with the provided organizational unit reference ID. + * + * @param existingTrial The existing trial to process and cache. + * @param trialRefSource The source to retrieve the trial's reference. + * @param referenceOUId The ID of the reference observation unit. + */ + private void processAndCachePendingTrial( + BrAPITrial existingTrial, + String trialRefSource, + String referenceOUId) { + + // Retrieve experiment ID from existingTrial + BrAPIExternalReference experimentIDRef = Utilities + .getExternalReference(existingTrial.getExternalReferences(), trialRefSource) + .orElseThrow(() -> new InternalServerException( + "An Experiment ID was not found in any of the external references")); + UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); + + // Cache pending trial by observation unit ID + pendingTrialByOUId.put( + referenceOUId, + new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); + } + + /** + * Sets the reference Observation Unit IDs based on the import rows. + * Checks for references to existing observation units and populates the + * referenceOUIds list. + * Sets flags hasAllReferenceUnitIds and hasNoReferenceUnitIds accordingly. + */ + private void setReferenceOUIds(List importRows) { + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { + ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); + + // Check if ObsUnitID is blank + if (importRow.getObsUnitID() == null || importRow.getObsUnitID().isBlank()) { + hasAllReferenceUnitIds = false; + } else if (referenceOUIds.contains(importRow.getObsUnitID())) { + // Throw exception if ObsUnitID is repeated + throw new IllegalStateException("ObsUnitId is repeated: " + importRow.getObsUnitID()); + } else { + // Add ObsUnitID to referenceOUIds + referenceOUIds.add(importRow.getObsUnitID()); + hasNoReferenceUnitIds = false; + } + } + } + private void validateTimeStampValue(String value, String columnHeader, ValidationErrors validationErrors, int row) { if (StringUtils.isBlank(value)) { diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 056e8b297..58299ece5 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -702,6 +702,70 @@ public void importNewObsVarExistingOu() { assertRowSaved(newObsVar, program, traits); } + @Test + @SneakyThrows + public void importNewObsVarByObsUnitId() { + List traits = importTestUtils.createTraits(2); + Program program = createProgram("New ObsVar Existing OU", "OUVAR", "OUVAR", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + newExp.put(traits.get(0).getObservationVariableName(), null); + + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + + BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); + Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); + assertTrue(trialIdXref.isPresent()); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); + + BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); + Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); + assertTrue(ouIdXref.isPresent()); + + Map newObsVar = new HashMap<>(); +// newObsVar.put(Columns.GERMPLASM_GID, "1"); +// newObsVar.put(Columns.TEST_CHECK, "T"); +// newObsVar.put(Columns.EXP_TITLE, "Test Exp"); +// newObsVar.put(Columns.EXP_UNIT, "Plot"); +// newObsVar.put(Columns.EXP_TYPE, "Phenotyping"); +// newObsVar.put(Columns.ENV, "New Env"); +// newObsVar.put(Columns.ENV_LOCATION, "Location A"); +// newObsVar.put(Columns.ENV_YEAR, "2023"); +// newObsVar.put(Columns.EXP_UNIT_ID, "a-1"); +// newObsVar.put(Columns.REP_NUM, "1"); +// newObsVar.put(Columns.BLOCK_NUM, "1"); +// newObsVar.put(Columns.ROW, "1"); +// newObsVar.put(Columns.COLUMN, "1"); + newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObsVar.put(traits.get(1).getObservationVariableName(), null); + + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); + assertTrue(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().has("observationDatasetId")); + assertTrue(importTestUtils.UUID_REGEX.matcher(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().get("observationDatasetId").getAsString()).matches()); + assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowSaved(newObsVar, program, traits); + } + @ParameterizedTest @ValueSource(booleans = {true, false}) From 9917eb5fe054979a5d5fa329e8b708f10ec7c7f4 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 27 Feb 2024 15:47:36 -0500 Subject: [PATCH 179/220] [BI-2046] refactor initializing OUs --- .../processors/ExperimentProcessor.java | 122 ++++++++++-------- 1 file changed, 69 insertions(+), 53 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 148e139ce..384ec8260 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -188,47 +188,32 @@ public void getExistingBrapiData(List importRows, Program program) .map(trialImport -> (ExperimentObservation) trialImport) .collect(Collectors.toList()); - Map referenceUnitById = new HashMap<>(); - Map expTitleByRefUnitId = new HashMap<>(); // check for references to Deltabreed-generated observation units - setReferenceOUIds(importRows); + referenceOUIds = collateReferenceOUIds(importRows); if (hasAllReferenceUnitIds) { - - // get all prior units referenced in import try { - List referenceOUs = brAPIObservationUnitDAO.getObservationUnitsById(new ArrayList(referenceOUIds), program); - for (BrAPIObservationUnit unit: referenceOUs) { - setPendingObsUnitByOUId(program); - setPendingTrialByOUId(program); - - // set pendingStudiesByOUId - // set pendingLocationsByOUId - // set pendingObsDatasetByOUId - - BrAPIExternalReference xref = Utilities.getExternalReference( - unit.getExternalReferences(), - ExternalReferenceSource.OBSERVATION_UNITS.getName() - ).orElseThrow(() -> new IllegalStateException("No BI external reference found")); - referenceUnitById.put(xref.getReferenceId(), unit); - expTitleByRefUnitId.put(xref.getReferenceId(), unit.getTrialName()); - } + + // get all prior units referenced in import + pendingObsUnitByOUId = fetchReferenceObservationUnits(referenceOUIds, program); + observationUnitByNameNoScope = mapPendingObservationUnitByName(pendingObsUnitByOUId, program); } catch (ApiException e) { + log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); } } else if (hasNoReferenceUnitIds) { observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); - trialByNameNoScope = initializeTrialByNameNoScope(program, experimentImportRows); - studyByNameNoScope = initializeStudyByNameNoScope(program, experimentImportRows); - locationByName = initializeUniqueLocationNames(program, experimentImportRows); - obsVarDatasetByName = initializeObsVarDatasetByName(program, experimentImportRows); - existingGermplasmByGID = initializeExistingGermplasmByGID(program, experimentImportRows); + } else { // can't proceed if you have a mix of ObsUnitId for some but not all rows throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, MISSING_OBS_UNIT_ID_ERROR); } - + trialByNameNoScope = initializeTrialByNameNoScope(program, experimentImportRows); + studyByNameNoScope = initializeStudyByNameNoScope(program, experimentImportRows); + locationByName = initializeUniqueLocationNames(program, experimentImportRows); + obsVarDatasetByName = initializeObsVarDatasetByName(program, experimentImportRows); + existingGermplasmByGID = initializeExistingGermplasmByGID(program, experimentImportRows); } /** @@ -562,6 +547,7 @@ private void initNewBrapiData( } Supplier envNextVal = () -> dsl.nextval(envSequenceName.toLowerCase()); existingObsByObsHash = fetchExistingObservations(referencedTraits, program); + for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); @@ -1454,6 +1440,28 @@ private Map> initializeObserva } } + private Map> mapPendingObservationUnitByName( + Map> pendingUnitById, + Program program + ) { + Map> pendingUnitByName = new HashMap<>(); + for (Map.Entry> entry : pendingUnitById.entrySet()) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + entry.getValue().getBrAPIObject().getStudyName(), + program.getKey() + ); + String observationUnitName = Utilities.removeProgramKeyAndUnknownAdditionalData( + entry.getValue().getBrAPIObject().getObservationUnitName(), + program.getKey() + ); + pendingUnitByName.put(createObservationUnitKey(studyName, observationUnitName), entry.getValue()); + } + return pendingUnitByName; + } + + + + /** * Retrieves a list of pending Observation Units based on their IDs. * @@ -1464,7 +1472,11 @@ private Map> initializeObserva * * @throws InternalServerException if an error occurs during the process. */ - private List setPendingObsUnitByOUId(Program program) { + private Map> fetchReferenceObservationUnits( + Set referenceOUIds, + Program program + ) throws ApiException { + Map> pendingUnitById = new HashMap<>(); try { // Retrieve reference Observation Units based on IDs List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( @@ -1480,10 +1492,10 @@ private List setPendingObsUnitByOUId(Program program) { referenceObsUnits.forEach(unit -> { // Get external reference for the Observation Unit BrAPIExternalReference unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltabreedOUSource) - .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")); + .orElseThrow(() -> new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID")); // Set pending Observation Unit by its ID - pendingObsUnitByOUId.put( + pendingUnitById.put( unitXref.getReferenceId(), new PendingImportObject( ImportObjectState.EXISTING, unit, UUID.fromString(unitXref.getReferenceId())) @@ -1494,17 +1506,17 @@ private List setPendingObsUnitByOUId(Program program) { List missingIds = new ArrayList<>(referenceOUIds); Set fetchedIds = referenceObsUnits.stream().map(unit -> Utilities.getExternalReference(unit.getExternalReferences(), deltabreedOUSource) - .orElseThrow(() -> new InternalServerException("An ObservationUnit ID was not found in any of the external references")) + .orElseThrow(() -> new InternalServerException("External reference does not exist for Deltabreed ObservationUnit ID")) .getReferenceId()) .collect(Collectors.toSet()); missingIds.removeAll(fetchedIds); throw new IllegalStateException("Observation Units not found for ObsUnitId(s): " + String.join(COMMA_DELIMITER, missingIds)); } - return referenceObsUnits; + return pendingUnitById; } catch (ApiException e) { log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); + throw new ApiException(e); } } @@ -1562,11 +1574,12 @@ private Map> initializeStudyByNameNoScop } private void initializeStudiesForExistingObservationUnits(Program program, Map> studyByName) throws ApiException { - Set studyDbIds = this.observationUnitByNameNoScope.values() - .stream() - .map(pio -> pio.getBrAPIObject() - .getStudyDbId()) - .collect(Collectors.toSet()); + Map> observationUnitMap = hasAllReferenceUnitIds ? pendingObsUnitByOUId : observationUnitByNameNoScope; + Set studyDbIds = observationUnitMap.values() + .stream() + .map(pio -> pio.getBrAPIObject() + .getStudyDbId()) + .collect(Collectors.toSet()); List studies = fetchStudiesByDbId(studyDbIds, program); studies.forEach(study -> processAndCacheStudy(study, program, studyByName)); @@ -1796,20 +1809,21 @@ private void initializeTrialsForExistingObservationUnits(Program program, Map 0) { Set trialDbIds = new HashSet<>(); Set studyDbIds = new HashSet<>(); - - observationUnitByNameNoScope.values() - .forEach(pio -> { - BrAPIObservationUnit existingOu = pio.getBrAPIObject(); - if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { - throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); - } - - if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { - trialDbIds.add(existingOu.getTrialDbId()); - } else { - studyDbIds.add(existingOu.getStudyDbId()); - } - }); + Map> observationUnitMap = hasAllReferenceUnitIds ? pendingObsUnitByOUId : observationUnitByNameNoScope; + + observationUnitMap.values() + .forEach(pio -> { + BrAPIObservationUnit existingOu = pio.getBrAPIObject(); + if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { + throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); + } + + if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { + trialDbIds.add(existingOu.getTrialDbId()); + } else { + studyDbIds.add(existingOu.getStudyDbId()); + } + }); //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId if(!studyDbIds.isEmpty()) { @@ -1904,7 +1918,8 @@ private void processAndCachePendingTrial( * referenceOUIds list. * Sets flags hasAllReferenceUnitIds and hasNoReferenceUnitIds accordingly. */ - private void setReferenceOUIds(List importRows) { + private Set collateReferenceOUIds(List importRows) { + Set referenceOUIds = new HashSet<>(); for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); @@ -1920,6 +1935,7 @@ private void setReferenceOUIds(List importRows) { hasNoReferenceUnitIds = false; } } + return referenceOUIds; } private void validateTimeStampValue(String value, From 9db276a95ccf731266396403e6484ceb0194b0c7 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 27 Feb 2024 19:56:02 -0500 Subject: [PATCH 180/220] [BI-2046] get trial pio from reference obs unit --- .../processors/ExperimentProcessor.java | 58 ++++++++++++------- .../importer/ExperimentFileImportTest.java | 2 +- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 384ec8260..e756c6954 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1107,7 +1107,12 @@ private void addObsVarsToDatasetDetails(PendingImportObject pi } private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) { PendingImportObject pio; - PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); + PendingImportObject trialPIO; + if (hasAllReferenceUnitIds) { + trialPIO = trialByNameNoScope.values().iterator().next(); + } else { + trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); + } String name = String.format("Observation Dataset [%s-%s]", program.getKey(), trialPIO.getBrAPIObject() @@ -1218,24 +1223,35 @@ private PendingImportObject fetchOrCreateTrialPIO( Supplier expNextVal ) throws UnprocessableEntityException { PendingImportObject trialPio; - if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { - PendingImportObject envPio; - trialPio = trialByNameNoScope.get(importRow.getExpTitle()); - envPio = this.studyByNameNoScope.get(importRow.getEnv()); - if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ - throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); + + // use the prior trial if observation unit IDs are supplied + if (hasAllReferenceUnitIds) { + if (trialByNameNoScope.size() != 1) { + throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); } - } else if (!trialByNameNoScope.isEmpty()) { - throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); + trialPio = trialByNameNoScope.values().iterator().next(); + + // otherwise create a new trial, but there can be only one allowed } else { - UUID id = UUID.randomUUID(); - String expSeqValue = null; - if (commit) { - expSeqValue = expNextVal.get().toString(); + if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { + PendingImportObject envPio; + trialPio = trialByNameNoScope.get(importRow.getExpTitle()); + envPio = this.studyByNameNoScope.get(importRow.getEnv()); + if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ + throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); + } + } else if (!trialByNameNoScope.isEmpty()) { + throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); + } else { + UUID id = UUID.randomUUID(); + String expSeqValue = null; + if (commit) { + expSeqValue = expNextVal.get().toString(); + } + BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); + trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); + this.trialByNameNoScope.put(importRow.getExpTitle(), trialPio); } - BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); - trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); - this.trialByNameNoScope.put(importRow.getExpTitle(), trialPio); } return trialPio; } @@ -1482,11 +1498,11 @@ private Map> fetchReferenceObs List referenceObsUnits = brAPIObservationUnitDAO.getObservationUnitsById( new ArrayList(referenceOUIds), program - ); + ); // Construct the DeltaBreed observation unit source for external references String deltabreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - + if (referenceObsUnits.size() == referenceOUIds.size()) { // Iterate through reference Observation Units referenceObsUnits.forEach(unit -> { @@ -1574,8 +1590,7 @@ private Map> initializeStudyByNameNoScop } private void initializeStudiesForExistingObservationUnits(Program program, Map> studyByName) throws ApiException { - Map> observationUnitMap = hasAllReferenceUnitIds ? pendingObsUnitByOUId : observationUnitByNameNoScope; - Set studyDbIds = observationUnitMap.values() + Set studyDbIds = observationUnitByNameNoScope.values() .stream() .map(pio -> pio.getBrAPIObject() .getStudyDbId()) @@ -1809,9 +1824,8 @@ private void initializeTrialsForExistingObservationUnits(Program program, Map 0) { Set trialDbIds = new HashSet<>(); Set studyDbIds = new HashSet<>(); - Map> observationUnitMap = hasAllReferenceUnitIds ? pendingObsUnitByOUId : observationUnitByNameNoScope; - observationUnitMap.values() + observationUnitByNameNoScope.values() .forEach(pio -> { BrAPIObservationUnit existingOu = pio.getBrAPIObject(); if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 58299ece5..59607a577 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -706,7 +706,7 @@ public void importNewObsVarExistingOu() { @SneakyThrows public void importNewObsVarByObsUnitId() { List traits = importTestUtils.createTraits(2); - Program program = createProgram("New ObsVar Existing OU", "OUVAR", "OUVAR", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Program program = createProgram("New ObsVar Referring to OU by ID", "OUVAR", "VAROU", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); From fd40f55bf61071f99c77299fadd378910ffbdb8f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 29 Feb 2024 14:56:05 -0500 Subject: [PATCH 181/220] [BI-2046] update fetchOrCreate methods --- .../processors/ExperimentProcessor.java | 106 ++++++++++++------ 1 file changed, 74 insertions(+), 32 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index e756c6954..adbef9852 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -234,7 +234,7 @@ public Map process( Table data, Program program, User user, - boolean commit) throws ApiException, ValidatorException, MissingRequiredInfoException { + boolean commit) throws ApiException, ValidatorException, MissingRequiredInfoException, UnprocessableEntityException { log.debug("processing experiment import"); ValidationErrors validationErrors = new ValidationErrors(); @@ -531,7 +531,7 @@ private void initNewBrapiData( User user, List referencedTraits, boolean commit - ) throws ApiException, MissingRequiredInfoException { + ) throws UnprocessableEntityException, ApiException, MissingRequiredInfoException { String expSequenceName = program.getExpSequence(); if (expSequenceName == null) { @@ -594,9 +594,23 @@ private void initNewBrapiData( dateTimeValue += MIDNIGHT; } } - //column.name() gets phenotype name - String seasonDbId = this.yearToSeasonDbId(importRow.getEnvYear(), program.getId()); - fetchOrCreateObservationPIO(program, user, importRow, column, rowNum, dateTimeValue, commit, seasonDbId, obsUnitPIO, referencedTraits); + + // get the study year either referenced from the observation unit or listed explicitly on the import row + String studyYear = hasAllReferenceUnitIds ? studyPIO.getBrAPIObject().getSeasons().get(0) : importRow.getEnvYear(); + String seasonDbId = yearToSeasonDbId(studyYear, program.getId()); + fetchOrCreateObservationPIO( + program, + user, + importRow, + column, //column.name() gets phenotype name + rowNum, + dateTimeValue, + commit, + seasonDbId, + obsUnitPIO, + studyPIO, + referencedTraits + ); } } } @@ -966,10 +980,12 @@ private PendingImportObject getGidPOI(ExperimentObservation impo return null; } - private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException, MissingRequiredInfoException { + private PendingImportObject fetchOrCreateObsUnitPIO(Program program, boolean commit, String envSeqValue, ExperimentObservation importRow) throws ApiException, MissingRequiredInfoException, UnprocessableEntityException { PendingImportObject pio; String key = createObservationUnitKey(importRow); - if (this.observationUnitByNameNoScope.containsKey(key)) { + if (hasAllReferenceUnitIds) { + pio = pendingObsUnitByOUId.get(importRow.getObsUnitID()); + } else if (observationUnitByNameNoScope.containsKey(key)) { pio = observationUnitByNameNoScope.get(key); } else { String germplasmName = ""; @@ -978,7 +994,7 @@ private PendingImportObject fetchOrCreateObsUnitPIO(Progra .getBrAPIObject() .getGermplasmName(); } - PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); + PendingImportObject trialPIO = trialByNameNoScope.get(importRow.getExpTitle());; UUID trialID = trialPIO.getId(); UUID datasetId = null; if (commit) { @@ -1041,12 +1057,20 @@ private void fetchOrCreateObservationPIO(Program program, boolean commit, String seasonDbId, PendingImportObject obsUnitPIO, - List referencedTraits) throws ApiException { + PendingImportObject studyPIO, + List referencedTraits) throws ApiException, UnprocessableEntityException { PendingImportObject pio; BrAPIObservation newObservation; String variableName = column.name(); String value = column.getString(rowNum); - String key = getImportObservationHash(importRow, variableName); + String key; + if (hasAllReferenceUnitIds) { + String unitName = obsUnitPIO.getBrAPIObject().getObservationUnitName(); + String studyName = studyPIO.getBrAPIObject().getStudyName(); + key = getObservationHash(unitName, variableName, studyName); + } else { + key = getImportObservationHash(importRow, variableName); + } if (existingObsByObsHash.containsKey(key)) { if (StringUtils.isNotBlank(value) && !isObservationMatched(key, value, column, rowNum)){ @@ -1071,9 +1095,11 @@ private void fetchOrCreateObservationPIO(Program program, } else if (!this.observationByHash.containsKey(key)){ // new observation - PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); + UUID trialID = trialPIO.getId(); - PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); + //PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); UUID studyID = studyPIO.getId(); UUID id = UUID.randomUUID(); newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject(), commit, program, user, BRAPI_REFERENCE_SOURCE, trialID, studyID, obsUnitPIO.getId(), id); @@ -1083,7 +1109,7 @@ private void fetchOrCreateObservationPIO(Program program, newObservation.setObservationTimeStamp(OffsetDateTime.parse(timeStampValue)); } - newObservation.setStudyDbId(this.studyByNameNoScope.get(importRow.getEnv()).getId().toString()); //set as the BI ID to facilitate looking up studies when saving new observations + newObservation.setStudyDbId(this.studyByNameNoScope.get(pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName()).getId().toString()); //set as the BI ID to facilitate looking up studies when saving new observations pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); this.observationByHash.put(key, pio); @@ -1105,14 +1131,10 @@ private void addObsVarsToDatasetDetails(PendingImportObject pi } }); } - private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) { + private void fetchOrCreateDatasetPIO(ExperimentObservation importRow, Program program, List referencedTraits) throws UnprocessableEntityException { PendingImportObject pio; - PendingImportObject trialPIO; - if (hasAllReferenceUnitIds) { - trialPIO = trialByNameNoScope.values().iterator().next(); - } else { - trialPIO = trialByNameNoScope.get(importRow.getExpTitle()); - } + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); String name = String.format("Observation Dataset [%s-%s]", program.getKey(), trialPIO.getBrAPIObject() @@ -1144,15 +1166,26 @@ private PendingImportObject fetchOrCreateStudyPIO( boolean commit, String expSequenceValue, ExperimentObservation importRow, - Supplier envNextVal) { + Supplier envNextVal + ) throws UnprocessableEntityException { PendingImportObject pio; - if (studyByNameNoScope.containsKey(importRow.getEnv())) { + if (hasAllReferenceUnitIds) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName(), + program.getKey() + ); + pio = studyByNameNoScope.get(studyName); + if (!commit){ + addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); + } + } else if (studyByNameNoScope.containsKey(importRow.getEnv())) { pio = studyByNameNoScope.get(importRow.getEnv()); if (!commit){ addYearToStudyAdditionalInfo(program, pio.getBrAPIObject()); } } else { - PendingImportObject trialPIO = this.trialByNameNoScope.get(importRow.getExpTitle()); + PendingImportObject trialPIO = hasAllReferenceUnitIds ? + getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); UUID id = UUID.randomUUID(); BrAPIStudy newStudy = importRow.constructBrAPIStudy(program, commit, BRAPI_REFERENCE_SOURCE, expSequenceValue, trialID, id, envNextVal); @@ -1208,10 +1241,13 @@ private void addYearToStudyAdditionalInfo(Program program, BrAPIStudy study, Str private void fetchOrCreateLocationPIO(ExperimentObservation importRow) { PendingImportObject pio; - if (! locationByName.containsKey((importRow.getEnvLocation()))) { - ProgramLocation newLocation = importRow.constructProgramLocation(); + String envLocationName = hasAllReferenceUnitIds ? + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getLocationName() : importRow.getEnvLocation(); + if (!locationByName.containsKey((importRow.getEnvLocation()))) { + ProgramLocation newLocation = new ProgramLocation(); + newLocation.setName(envLocationName); pio = new PendingImportObject<>(ImportObjectState.NEW, newLocation, UUID.randomUUID()); - this.locationByName.put(importRow.getEnvLocation(), pio); + this.locationByName.put(envLocationName, pio); } } @@ -1226,10 +1262,7 @@ private PendingImportObject fetchOrCreateTrialPIO( // use the prior trial if observation unit IDs are supplied if (hasAllReferenceUnitIds) { - if (trialByNameNoScope.size() != 1) { - throw new UnprocessableEntityException(MULTIPLE_EXP_TITLES); - } - trialPio = trialByNameNoScope.values().iterator().next(); + trialPio = getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES); // otherwise create a new trial, but there can be only one allowed } else { @@ -1548,8 +1581,10 @@ private Map> initializeTrialByNameNoScop .collect(Collectors.toList()); try { String trialRefSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName()); - brapiTrialDAO.getTrialsByName(uniqueTrialNames, program) - .forEach(existingTrial -> processAndCacheTrial(existingTrial, program, trialRefSource, trialByName)); + + brapiTrialDAO.getTrialsByName(uniqueTrialNames, program).forEach(existingTrial -> + processAndCacheTrial(existingTrial, program, trialRefSource, trialByName) + ); } catch (ApiException e) { log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); @@ -2116,4 +2151,11 @@ private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { Integer yearInt = (season == null) ? null : season.getYear(); return (yearInt == null) ? "" : yearInt.toString(); } + + private V getSingleEntryValue(Map map, String message) throws UnprocessableEntityException { + if (map.size() != 1) { + throw new UnprocessableEntityException(message); + } + return map.values().iterator().next(); + } } From 87cc4bdc454fed889388d8ab5a548f8018d9685f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 4 Mar 2024 15:24:50 -0500 Subject: [PATCH 182/220] [BI-2046] initialize brapi maps when OUIds supplied --- .../processors/ExperimentProcessor.java | 111 ++++++++++++++---- 1 file changed, 88 insertions(+), 23 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index adbef9852..c38fd0774 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -24,6 +24,7 @@ import io.micronaut.http.HttpStatus; import io.micronaut.http.exceptions.HttpStatusException; import io.micronaut.http.server.exceptions.InternalServerException; +import io.reactivex.functions.Function; import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.collections4.map.CaseInsensitiveMap; @@ -46,6 +47,7 @@ import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.*; import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.base.Study; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.imports.PendingImport; @@ -197,9 +199,18 @@ public void getExistingBrapiData(List importRows, Program program) // get all prior units referenced in import pendingObsUnitByOUId = fetchReferenceObservationUnits(referenceOUIds, program); observationUnitByNameNoScope = mapPendingObservationUnitByName(pendingObsUnitByOUId, program); + initializeTrialsForExistingObservationUnits(program, trialByNameNoScope); + initializeStudiesForExistingObservationUnits(program, studyByNameNoScope); + locationByName = initializeLocationByName(program, studyByNameNoScope); + + + pendingStudyByOUId = fetchStudyByOUId(referenceOUIds, pendingObsUnitByOUId, program); } catch (ApiException e) { log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); + } catch (Exception e) { + log.error("Error processing experiment with ", e); + throw new InternalServerException(e.toString(), e); } } else if (hasNoReferenceUnitIds) { observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); @@ -1489,6 +1500,15 @@ private Map> initializeObserva } } + private Map> mapPendingTrialOUId( + Map> trialByName, + Program program + ) { + Map> trialByOUId = new HashMap<>(); + + + return trialByOUId; + } private Map> mapPendingObservationUnitByName( Map> pendingUnitById, Program program @@ -1508,9 +1528,6 @@ private Map> mapPendingObserva return pendingUnitByName; } - - - /** * Retrieves a list of pending Observation Units based on their IDs. * @@ -1534,13 +1551,13 @@ private Map> fetchReferenceObs ); // Construct the DeltaBreed observation unit source for external references - String deltabreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); + String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); if (referenceObsUnits.size() == referenceOUIds.size()) { // Iterate through reference Observation Units referenceObsUnits.forEach(unit -> { // Get external reference for the Observation Unit - BrAPIExternalReference unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltabreedOUSource) + BrAPIExternalReference unitXref = Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) .orElseThrow(() -> new IllegalStateException("External reference does not exist for Deltabreed ObservationUnit ID")); // Set pending Observation Unit by its ID @@ -1554,7 +1571,7 @@ private Map> fetchReferenceObs // Handle missing Observation Unit IDs List missingIds = new ArrayList<>(referenceOUIds); Set fetchedIds = referenceObsUnits.stream().map(unit -> - Utilities.getExternalReference(unit.getExternalReferences(), deltabreedOUSource) + Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource) .orElseThrow(() -> new InternalServerException("External reference does not exist for Deltabreed ObservationUnit ID")) .getReferenceId()) .collect(Collectors.toSet()); @@ -1580,10 +1597,8 @@ private Map> initializeTrialByNameNoScop .distinct() .collect(Collectors.toList()); try { - String trialRefSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName()); - brapiTrialDAO.getTrialsByName(uniqueTrialNames, program).forEach(existingTrial -> - processAndCacheTrial(existingTrial, program, trialRefSource, trialByName) + processAndCacheTrial(existingTrial, program, trialByName) ); } catch (ApiException e) { log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); @@ -1604,6 +1619,9 @@ private Map> initializeStudyByNameNoScop } catch (ApiException e) { log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); + } catch (Exception e) { + log.error("Error processing studies", e); + throw new InternalServerException(e.toString(), e); } List existingStudies; @@ -1615,16 +1633,24 @@ private Map> initializeStudyByNameNoScop } UUID experimentId = trial.get().getId(); existingStudies = brAPIStudyDAO.getStudiesByExperimentID(experimentId, program); - existingStudies.forEach(existingStudy -> processAndCacheStudy(existingStudy, program, studyByName)); + for (BrAPIStudy existingStudy : existingStudies) { + processAndCacheStudy(existingStudy, program, BrAPIStudy::getStudyName, studyByName); + } } catch (ApiException e) { log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); + } catch (Exception e) { + log.error("Error processing studies: ", e); + throw new InternalServerException(e.toString(), e); } return studyByName; } - private void initializeStudiesForExistingObservationUnits(Program program, Map> studyByName) throws ApiException { + private void initializeStudiesForExistingObservationUnits( + Program program, + Map> studyByName + ) throws Exception { Set studyDbIds = observationUnitByNameNoScope.values() .stream() .map(pio -> pio.getBrAPIObject() @@ -1632,7 +1658,9 @@ private void initializeStudiesForExistingObservationUnits(Program program, Map studies = fetchStudiesByDbId(studyDbIds, program); - studies.forEach(study -> processAndCacheStudy(study, program, studyByName)); + for (BrAPIStudy study : studies) { + processAndCacheStudy(study, program, BrAPIStudy::getStudyName, studyByName); + } } /** @@ -1659,6 +1687,33 @@ private List fetchStudiesByDbId(Set studyDbIds, Program prog return studies; } + Map> initializeLocationByName( + Program program, + Map> studyByName + ) { + Map> locationByName = new HashMap<>(); + + List existingLocations = new ArrayList<>(); + if(studyByName.size() > 0) { + Set locationDbIds = studyByName.values() + .stream() + .map(study -> study.getBrAPIObject() + .getLocationDbId()) + .collect(Collectors.toSet()); + try { + existingLocations.addAll(locationService.getLocationsByDbId(locationDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching locations: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + existingLocations.forEach(existingLocation -> locationByName.put( + existingLocation.getName(), + new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()) + ) + ); + return locationByName; + } private Map> initializeUniqueLocationNames(Program program, List experimentImportRows) { Map> locationByName = new HashMap<>(); @@ -1794,8 +1849,13 @@ private void processAndCacheObservationUnit(BrAPIObservationUnit brAPIObservatio brAPIObservationUnit, UUID.fromString(idRef.getReferenceId()))); } - - private void processAndCacheStudy(BrAPIStudy existingStudy, Program program, Map> studyByName) { + private PendingImportObject processAndCacheStudy( + BrAPIStudy existingStudy, + Program program, + Function getterFunction, + Map> studyMap + ) throws Exception { + PendingImportObject pendingStudy; BrAPIExternalReference xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) .orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId())); // map season dbid to year @@ -1804,9 +1864,16 @@ private void processAndCacheStudy(BrAPIStudy existingStudy, Program program, Map String seasonYear = this.seasonDbIdToYear(seasonDbId, program.getId()); existingStudy.setSeasons(Collections.singletonList(seasonYear)); } - studyByName.put( - Utilities.removeProgramKeyAndUnknownAdditionalData(existingStudy.getStudyName(), program.getKey()), - new PendingImportObject<>(ImportObjectState.EXISTING, (BrAPIStudy) Utilities.formatBrapiObjForDisplay(existingStudy, BrAPIStudy.class, program), UUID.fromString(xref.getReferenceID()))); + pendingStudy = new PendingImportObject<>( + ImportObjectState.EXISTING, + (BrAPIStudy) Utilities.formatBrapiObjForDisplay(existingStudy, BrAPIStudy.class, program), + UUID.fromString(xref.getReferenceId()) + ); + studyMap.put( + Utilities.removeProgramKeyAndUnknownAdditionalData(getterFunction.apply(existingStudy), program.getKey()), + pendingStudy + ); + return pendingStudy; } private void setPendingTrialByOUId(Program program) { @@ -1892,8 +1959,7 @@ private void initializeTrialsForExistingObservationUnits(Program program, Map processAndCacheTrial(trial, program, trialRefSource, trialByName)); + trials.forEach(trial -> processAndCacheTrial(trial, program, trialByName)); } catch (ApiException e) { log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); @@ -1915,16 +1981,15 @@ private Set fetchTrialDbidsForStudies(Set studyDbIds, Program pr } private void processAndCacheTrial( - BrAPITrial existingTrial, + BrAPITrial existingTrial, Program program, - String trialRefSource, Map> trialByNameNoScope ) { //get TrialId from existingTrial - BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), trialRefSource) + BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) .orElseThrow(() -> new InternalServerException("An Experiment ID was not found in any of the external references")); - UUID experimentId = UUID.fromString(experimentIDRef.getReferenceID()); + UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); trialByNameNoScope.put( Utilities.removeProgramKey(existingTrial.getTrialName(), program.getKey()), From 8d3c29a8e14e538196090b1fbe732199a49ef6ae Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 4 Mar 2024 17:13:33 -0500 Subject: [PATCH 183/220] [BI-2046] map trials and studies to OU Id --- .../processors/ExperimentProcessor.java | 44 +++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index c38fd0774..763c740ca 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -202,7 +202,12 @@ public void getExistingBrapiData(List importRows, Program program) initializeTrialsForExistingObservationUnits(program, trialByNameNoScope); initializeStudiesForExistingObservationUnits(program, studyByNameNoScope); locationByName = initializeLocationByName(program, studyByNameNoScope); - + for (Map.Entry> unitEntry : pendingObsUnitByOUId.entrySet()) { + String unitId = unitEntry.getKey(); + BrAPIObservationUnit unit = unitEntry.getValue().getBrAPIObject(); + mapPendingTrialByOUId(unitId, unit, trialByNameNoScope, studyByNameNoScope, pendingTrialByOUId, program); + mapPendingStudyByOUId(unitId, unit, studyByNameNoScope, pendingStudyByOUId, program); + } pendingStudyByOUId = fetchStudyByOUId(referenceOUIds, pendingObsUnitByOUId, program); } catch (ApiException e) { @@ -1500,12 +1505,43 @@ private Map> initializeObserva } } - private Map> mapPendingTrialOUId( - Map> trialByName, + private Map> mapPendingStudyByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> studyByName, + Map> studyByOUId, Program program ) { - Map> trialByOUId = new HashMap<>(); + if (unit.getStudyName() != null) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); + studyByOUId.put(unitId, studyByName.get(studyName)); + } else { + throw new IllegalStateException("Observation unit missing study name: " + unitId); + } + return studyByOUId; + } + private Map> mapPendingTrialByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> trialByName, + Map> studyByName, + Map> trialByOUId, + Program program + ) { + String trialName; + if (unit.getTrialName() != null) { + trialName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getTrialName(), program.getKey()); + } else if (unit.getStudyName() != null) { + String studyName = Utilities.removeProgramKeyAndUnknownAdditionalData(unit.getStudyName(), program.getKey()); + trialName = Utilities.removeProgramKeyAndUnknownAdditionalData( + studyByName.get(studyName).getBrAPIObject().getTrialName(), + program.getKey() + ); + } else { + throw new IllegalStateException("Observation unit missing trial name and study name: " + unitId); + } + trialByOUId.put(unitId, trialByName.get(trialName)); return trialByOUId; } From 6cb0251360e0c89ed96479ab4b4ff774a6a3e528 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 4 Mar 2024 17:42:11 -0500 Subject: [PATCH 184/220] [BI-2046] map locations by OU id --- .../processors/ExperimentProcessor.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 763c740ca..61e03c3f5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -207,6 +207,8 @@ public void getExistingBrapiData(List importRows, Program program) BrAPIObservationUnit unit = unitEntry.getValue().getBrAPIObject(); mapPendingTrialByOUId(unitId, unit, trialByNameNoScope, studyByNameNoScope, pendingTrialByOUId, program); mapPendingStudyByOUId(unitId, unit, studyByNameNoScope, pendingStudyByOUId, program); + mapPendingLocationByOUId(unitId, unit, pendingStudyByOUId, locationByName, pendingLocationByOUId); + } pendingStudyByOUId = fetchStudyByOUId(referenceOUIds, pendingObsUnitByOUId, program); @@ -1505,6 +1507,27 @@ private Map> initializeObserva } } + private Map> mapPendingLocationByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> studyByOUId, + Map> locationByName, + Map> locationByOUId + ) { + if (unit.getLocationName() != null) { + locationByOUId.put(unitId, locationByName.get(unit.getLocationName())); + } else if (studyByOUId.get(unitId) != null && studyByOUId.get(unitId).getBrAPIObject().getLocationName() != null) { + locationByOUId.put( + unitId, + locationByName.get(studyByOUId.get(unitId).getBrAPIObject().getLocationName()) + ); + } else { + throw new IllegalStateException("Observation unit missing location: " + unitId); + } + + return locationByOUId; + } + private Map> mapPendingStudyByOUId( String unitId, BrAPIObservationUnit unit, From 7b3eb166dee3b06e5619ef60a565b68bc0830434 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 5 Mar 2024 12:32:53 -0500 Subject: [PATCH 185/220] [BI-2046] initialize dataset maps for existing observation units --- .../processors/ExperimentProcessor.java | 49 ++++++++++++++++++- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 61e03c3f5..7ad90a13f 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -208,7 +208,7 @@ public void getExistingBrapiData(List importRows, Program program) mapPendingTrialByOUId(unitId, unit, trialByNameNoScope, studyByNameNoScope, pendingTrialByOUId, program); mapPendingStudyByOUId(unitId, unit, studyByNameNoScope, pendingStudyByOUId, program); mapPendingLocationByOUId(unitId, unit, pendingStudyByOUId, locationByName, pendingLocationByOUId); - + initializeObsVarDatasetForExistingObservationUnit(unitId, pendingObsDatasetByOUId, obsVarDatasetByName, pendingTrialByOUId, program); } pendingStudyByOUId = fetchStudyByOUId(referenceOUIds, pendingObsUnitByOUId, program); @@ -1808,6 +1808,43 @@ private Map> initializeUniqueLocati existingLocations.forEach(existingLocation -> locationByName.put(existingLocation.getName(), new PendingImportObject<>(ImportObjectState.EXISTING, existingLocation, existingLocation.getId()))); return locationByName; } + + private Map> initializeObsVarDatasetForExistingObservationUnit( + String unitId, + Map> obsVarDatasetByOUId, + Map> obsVarDatasetByName, + Map> pendingTrialByOUId, + Program program + ) { + if (pendingTrialByOUId.get(unitId) != null && + pendingTrialByOUId.get(unitId).getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + String datasetId = pendingTrialByOUId.get(unitId).getBrAPIObject() + .getAdditionalInfo() + .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) + .getAsString(); + + try { + List existingDatasets = brAPIListDAO + .getListByTypeAndExternalRef(BrAPIListTypes.OBSERVATIONVARIABLES, + program.getId(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName()), + UUID.fromString(datasetId)); + if (existingDatasets == null || existingDatasets.isEmpty()) { + throw new InternalServerException("existing dataset summary not returned from brapi server"); + } + BrAPIListDetails dataSetDetails = brAPIListDAO + .getListById(existingDatasets.get(0).getListDbId(), program.getId()) + .getResult(); + processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); + processAndCacheObsVarDatasetByOUId(dataSetDetails, unitId, obsVarDatasetByOUId); + } catch (ApiException e) { + log.error(Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + return obsVarDatasetByOUId; + } + private Map> initializeObsVarDatasetByName(Program program, List experimentImportRows) { Map> obsVarDatasetByName = new HashMap<>(); @@ -1854,12 +1891,20 @@ private Optional> getTrialPIO(List> obsVarDatasetByOUId) { + BrAPIExternalReference xref = Utilities.getExternalReference(existingList.getExternalReferences(), + String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for list (dbid): " + existingList.getListDbId())); + obsVarDatasetByOUId.put(unitId, + new PendingImportObject(ImportObjectState.EXISTING, existingList, UUID.fromString(xref.getReferenceId()))); + } private void processAndCacheObsVarDataset(BrAPIListDetails existingList, Map> obsVarDatasetByName) { BrAPIExternalReference xref = Utilities.getExternalReference(existingList.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) .orElseThrow(() -> new IllegalStateException("External references wasn't found for list (dbid): " + existingList.getListDbId())); obsVarDatasetByName.put(existingList.getListName(), - new PendingImportObject(ImportObjectState.EXISTING, existingList, UUID.fromString(xref.getReferenceID()))); + new PendingImportObject(ImportObjectState.EXISTING, existingList, UUID.fromString(xref.getReferenceId()))); } private Map> initializeExistingGermplasmByGID(Program program, List experimentImportRows) { Map> existingGermplasmByGID = new HashMap<>(); From 346b629bd9d7d69f08f342e829facb72664baffd Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 5 Mar 2024 13:35:26 -0500 Subject: [PATCH 186/220] [BI-2046] refactor initialization of dataset map --- .../processors/ExperimentProcessor.java | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 7ad90a13f..1ad614d5a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -202,13 +202,13 @@ public void getExistingBrapiData(List importRows, Program program) initializeTrialsForExistingObservationUnits(program, trialByNameNoScope); initializeStudiesForExistingObservationUnits(program, studyByNameNoScope); locationByName = initializeLocationByName(program, studyByNameNoScope); + initializeObsVarDatasetForExistingObservationUnits(pendingTrialByOUId, program); for (Map.Entry> unitEntry : pendingObsUnitByOUId.entrySet()) { String unitId = unitEntry.getKey(); BrAPIObservationUnit unit = unitEntry.getValue().getBrAPIObject(); mapPendingTrialByOUId(unitId, unit, trialByNameNoScope, studyByNameNoScope, pendingTrialByOUId, program); mapPendingStudyByOUId(unitId, unit, studyByNameNoScope, pendingStudyByOUId, program); mapPendingLocationByOUId(unitId, unit, pendingStudyByOUId, locationByName, pendingLocationByOUId); - initializeObsVarDatasetForExistingObservationUnit(unitId, pendingObsDatasetByOUId, obsVarDatasetByName, pendingTrialByOUId, program); } pendingStudyByOUId = fetchStudyByOUId(referenceOUIds, pendingObsUnitByOUId, program); @@ -1809,16 +1809,15 @@ private Map> initializeUniqueLocati return locationByName; } - private Map> initializeObsVarDatasetForExistingObservationUnit( - String unitId, - Map> obsVarDatasetByOUId, - Map> obsVarDatasetByName, + private Map> initializeObsVarDatasetForExistingObservationUnits( Map> pendingTrialByOUId, Program program ) { - if (pendingTrialByOUId.get(unitId) != null && - pendingTrialByOUId.get(unitId).getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { - String datasetId = pendingTrialByOUId.get(unitId).getBrAPIObject() + Map> obsVarDatasetByName = new HashMap<>(); + + if (pendingTrialByOUId.size() > 0 && + pendingTrialByOUId.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + String datasetId = pendingTrialByOUId.values().iterator().next().getBrAPIObject() .getAdditionalInfo() .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) .getAsString(); @@ -1836,13 +1835,12 @@ private Map> initializeObsVarDatas .getListById(existingDatasets.get(0).getListDbId(), program.getId()) .getResult(); processAndCacheObsVarDataset(dataSetDetails, obsVarDatasetByName); - processAndCacheObsVarDatasetByOUId(dataSetDetails, unitId, obsVarDatasetByOUId); } catch (ApiException e) { log.error(Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); } } - return obsVarDatasetByOUId; + return obsVarDatasetByName; } private Map> initializeObsVarDatasetByName(Program program, List experimentImportRows) { From 18ff6e23338855e4f02897fcd4fb35ac70603b3c Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 5 Mar 2024 14:40:20 -0500 Subject: [PATCH 187/220] [BI-2046] map obsVarDataset by OU id --- .../services/processors/ExperimentProcessor.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 1ad614d5a..f75c6aacc 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -209,6 +209,7 @@ public void getExistingBrapiData(List importRows, Program program) mapPendingTrialByOUId(unitId, unit, trialByNameNoScope, studyByNameNoScope, pendingTrialByOUId, program); mapPendingStudyByOUId(unitId, unit, studyByNameNoScope, pendingStudyByOUId, program); mapPendingLocationByOUId(unitId, unit, pendingStudyByOUId, locationByName, pendingLocationByOUId); + mapPendingObsDatasetByOUId(unitId, pendingTrialByOUId, obsVarDatasetByName, pendingObsDatasetByOUId, program); } pendingStudyByOUId = fetchStudyByOUId(referenceOUIds, pendingObsUnitByOUId, program); @@ -1809,6 +1810,20 @@ private Map> initializeUniqueLocati return locationByName; } + private Map> mapPendingObsDatasetByOUId( + String unitId, + Map> trialByOUId, + Map> obsVarDatasetByName, + Map> obsVarDatasetByOUId, + Program program + ) { + if (!trialByOUId.isEmpty() && !obsVarDatasetByName.isEmpty() && + trialByOUId.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + obsVarDatasetByOUId.put(unitId, obsVarDatasetByName.values().iterator().next()); + } + + return obsVarDatasetByOUId; + } private Map> initializeObsVarDatasetForExistingObservationUnits( Map> pendingTrialByOUId, Program program From aa397ce6353752d4cbac9e070709eef9d517035b Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 5 Mar 2024 17:57:17 -0500 Subject: [PATCH 188/220] [BI-2046] fix NPEs --- .../processors/ExperimentProcessor.java | 66 +++++++++++++++---- 1 file changed, 53 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index f75c6aacc..fc0307668 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -47,6 +47,7 @@ import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.*; import org.breedinginsight.brapps.importer.model.ImportUpload; +import org.breedinginsight.brapps.importer.model.base.ObservationUnit; import org.breedinginsight.brapps.importer.model.base.Study; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; @@ -122,23 +123,24 @@ public class ExperimentProcessor implements Processor { //These BrapiData-objects are initially populated by the getExistingBrapiData() method, // then updated by the initNewBrapiData() method. - private Map> trialByNameNoScope = null; - private Map> pendingTrialByOUId = null; + private Map> trialByNameNoScope = new HashMap<>(); + private Map> pendingTrialByOUId = new HashMap<>(); private Map> locationByName = null; - private Map> pendingLocationByOUId = null; - private Map> studyByNameNoScope = null; - private Map> pendingStudyByOUId = null; + private Map> pendingLocationByOUId = new HashMap<>(); + private Map> studyByNameNoScope = new HashMap<>(); + private Map> pendingStudyByOUId = new HashMap<>(); private Map> obsVarDatasetByName = null; - private Map> pendingObsDatasetByOUId = null; + private Map> pendingObsDatasetByOUId = new HashMap<>(); // It is assumed that there are no preexisting Observation Units for the given environment (so this will not be // initialized by getExistingBrapiData() ) private Map> observationUnitByNameNoScope = null; - private Map> pendingObsUnitByOUId = null; + private Map> pendingObsUnitByOUId = new HashMap<>(); private final Map> observationByHash = new HashMap<>(); private Map existingObsByObsHash = new HashMap<>(); // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the initNewBrapiData() method private Map> existingGermplasmByGID = null; + private Map> pendingGermplasmByOUId = null; // Associates timestamp columns to associated phenotype column name for ease of storage private final Map> timeStampColByPheno = new HashMap<>(); @@ -202,7 +204,8 @@ public void getExistingBrapiData(List importRows, Program program) initializeTrialsForExistingObservationUnits(program, trialByNameNoScope); initializeStudiesForExistingObservationUnits(program, studyByNameNoScope); locationByName = initializeLocationByName(program, studyByNameNoScope); - initializeObsVarDatasetForExistingObservationUnits(pendingTrialByOUId, program); + obsVarDatasetByName = initializeObsVarDatasetForExistingObservationUnits(trialByNameNoScope, program); + existingGermplasmByGID = initializeGermplasmByGIDForExistingObservationUnits(observationUnitByNameNoScope, program); for (Map.Entry> unitEntry : pendingObsUnitByOUId.entrySet()) { String unitId = unitEntry.getKey(); BrAPIObservationUnit unit = unitEntry.getValue().getBrAPIObject(); @@ -210,9 +213,10 @@ public void getExistingBrapiData(List importRows, Program program) mapPendingStudyByOUId(unitId, unit, studyByNameNoScope, pendingStudyByOUId, program); mapPendingLocationByOUId(unitId, unit, pendingStudyByOUId, locationByName, pendingLocationByOUId); mapPendingObsDatasetByOUId(unitId, pendingTrialByOUId, obsVarDatasetByName, pendingObsDatasetByOUId, program); + mapGermplasmByOUId(unitId, unit, existingGermplasmByGID, pendingGermplasmByOUId); } - pendingStudyByOUId = fetchStudyByOUId(referenceOUIds, pendingObsUnitByOUId, program); + //pendingStudyByOUId = fetchStudyByOUId(referenceOUIds, pendingObsUnitByOUId, program); } catch (ApiException e) { log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); @@ -1810,6 +1814,16 @@ private Map> initializeUniqueLocati return locationByName; } + private Map> mapGermplasmByOUId( + String unitId, + BrAPIObservationUnit unit, + Map> germplasmByName, + Map> germplasmByOUId + ) { + //String dbId = unit. + //germplasmByName.values().stream().filter(pio -> pio.getBrAPIObject().getGermplasmDbId()) + return germplasmByOUId; + } private Map> mapPendingObsDatasetByOUId( String unitId, Map> trialByOUId, @@ -1825,14 +1839,14 @@ private Map> mapPendingObsDatasetB return obsVarDatasetByOUId; } private Map> initializeObsVarDatasetForExistingObservationUnits( - Map> pendingTrialByOUId, + Map> trialByName, Program program ) { Map> obsVarDatasetByName = new HashMap<>(); - if (pendingTrialByOUId.size() > 0 && - pendingTrialByOUId.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { - String datasetId = pendingTrialByOUId.values().iterator().next().getBrAPIObject() + if (trialByName.size() > 0 && + trialByName.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { + String datasetId = trialByName.values().iterator().next().getBrAPIObject() .getAdditionalInfo() .get(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID) .getAsString(); @@ -1919,6 +1933,32 @@ private void processAndCacheObsVarDataset(BrAPIListDetails existingList, Map(ImportObjectState.EXISTING, existingList, UUID.fromString(xref.getReferenceId()))); } + private Map> initializeGermplasmByGIDForExistingObservationUnits( + Map> unitByName, + Program program + ) { + Map> existingGermplasmByGID = new HashMap<>(); + + List existingGermplasms = new ArrayList<>(); + if(unitByName.size() > 0) { + Set germplasmDbIds = unitByName.values().stream().map(ou -> ou.getBrAPIObject().getGermplasmDbId()).collect(Collectors.toSet()); + try { + existingGermplasms.addAll(brAPIGermplasmDAO.getGermplasmsByDBID(germplasmDbIds, program.getId())); + } catch (ApiException e) { + log.error("Error fetching germplasm: " + Utilities.generateApiExceptionLogMessage(e), e); + throw new InternalServerException(e.toString(), e); + } + } + + existingGermplasms.forEach(existingGermplasm -> { + BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) + .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceID()))); + }); + return existingGermplasmByGID; + } + + private Map> initializeExistingGermplasmByGID(Program program, List experimentImportRows) { Map> existingGermplasmByGID = new HashMap<>(); From 9c33f31eeadf052a65d16aa7b8644038f904fa1f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 5 Mar 2024 18:16:06 -0500 Subject: [PATCH 189/220] [BI-2046] map germplasm by observation unit id --- .../services/processors/ExperimentProcessor.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index fc0307668..347e894b6 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -139,8 +139,8 @@ public class ExperimentProcessor implements Processor { private final Map> observationByHash = new HashMap<>(); private Map existingObsByObsHash = new HashMap<>(); // existingGermplasmByGID is populated by getExistingBrapiData(), but not updated by the initNewBrapiData() method - private Map> existingGermplasmByGID = null; - private Map> pendingGermplasmByOUId = null; + private Map> existingGermplasmByGID = new HashMap<>(); + private Map> pendingGermplasmByOUId = new HashMap<>(); // Associates timestamp columns to associated phenotype column name for ease of storage private final Map> timeStampColByPheno = new HashMap<>(); @@ -216,7 +216,6 @@ public void getExistingBrapiData(List importRows, Program program) mapGermplasmByOUId(unitId, unit, existingGermplasmByGID, pendingGermplasmByOUId); } - //pendingStudyByOUId = fetchStudyByOUId(referenceOUIds, pendingObsUnitByOUId, program); } catch (ApiException e) { log.error("Error fetching observation units: " + Utilities.generateApiExceptionLogMessage(e), e); throw new InternalServerException(e.toString(), e); @@ -1820,8 +1819,9 @@ private Map> mapGermplasmByOUId( Map> germplasmByName, Map> germplasmByOUId ) { - //String dbId = unit. - //germplasmByName.values().stream().filter(pio -> pio.getBrAPIObject().getGermplasmDbId()) + String gid = unit.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.GID).getAsString(); + germplasmByOUId.put(unitId, germplasmByName.get(gid)); + return germplasmByOUId; } private Map> mapPendingObsDatasetByOUId( From e9ef80d3a50579a92c61563df284bfa4186d0e92 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Tue, 5 Mar 2024 18:18:37 -0500 Subject: [PATCH 190/220] [BI-2046] refaactor --- .../services/processors/ExperimentProcessor.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 347e894b6..5a4120e00 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -225,17 +225,17 @@ public void getExistingBrapiData(List importRows, Program program) } } else if (hasNoReferenceUnitIds) { observationUnitByNameNoScope = initializeObservationUnits(program, experimentImportRows); - + trialByNameNoScope = initializeTrialByNameNoScope(program, experimentImportRows); + studyByNameNoScope = initializeStudyByNameNoScope(program, experimentImportRows); + locationByName = initializeUniqueLocationNames(program, experimentImportRows); + obsVarDatasetByName = initializeObsVarDatasetByName(program, experimentImportRows); + existingGermplasmByGID = initializeExistingGermplasmByGID(program, experimentImportRows); + } else { // can't proceed if you have a mix of ObsUnitId for some but not all rows throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, MISSING_OBS_UNIT_ID_ERROR); } - trialByNameNoScope = initializeTrialByNameNoScope(program, experimentImportRows); - studyByNameNoScope = initializeStudyByNameNoScope(program, experimentImportRows); - locationByName = initializeUniqueLocationNames(program, experimentImportRows); - obsVarDatasetByName = initializeObsVarDatasetByName(program, experimentImportRows); - existingGermplasmByGID = initializeExistingGermplasmByGID(program, experimentImportRows); } /** From 74eeeb5f3bf30385bb26027a40083b1649a39438 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 6 Mar 2024 17:13:32 -0500 Subject: [PATCH 191/220] [BI-2046] prepare for validation using reference OU ids --- .../processors/ExperimentProcessor.java | 50 +++++++++++++------ 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 5a4120e00..7eaf38550 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -230,7 +230,7 @@ public void getExistingBrapiData(List importRows, Program program) locationByName = initializeUniqueLocationNames(program, experimentImportRows); obsVarDatasetByName = initializeObsVarDatasetByName(program, experimentImportRows); existingGermplasmByGID = initializeExistingGermplasmByGID(program, experimentImportRows); - + } else { // can't proceed if you have a mix of ObsUnitId for some but not all rows @@ -467,22 +467,42 @@ public void postBrapiData(Map mappedBrAPIImport, Program private void prepareDataForValidation(List importRows, List> phenotypeCols, Map mappedBrAPIImport) { for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); - PendingImport mappedImportRow = mappedBrAPIImport.getOrDefault(rowNum, new PendingImport()); - mappedImportRow.setTrial(this.trialByNameNoScope.get(importRow.getExpTitle())); - mappedImportRow.setLocation(this.locationByName.get(importRow.getEnvLocation())); - mappedImportRow.setStudy(this.studyByNameNoScope.get(importRow.getEnv())); - mappedImportRow.setObservationUnit(this.observationUnitByNameNoScope.get(createObservationUnitKey(importRow))); - - // loop over phenotype column observation data for current row List> observations = mappedImportRow.getObservations(); - for (Column column : phenotypeCols) { - // if value was blank won't be entry in map for this observation - observations.add(this.observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); - } + String observationHash; + if (hasAllReferenceUnitIds) { + mappedImportRow.setTrial(pendingTrialByOUId.get(importRow.getObsUnitID()); + mappedImportRow.setLocation(pendingLocationByOUId.get(importRow.getObsUnitID())); + mappedImportRow.setStudy(pendingStudyByOUId.get(importRow.getObsUnitID())); + mappedImportRow.setObservationUnit(pendingObsUnitByOUId.get(importRow.getObsUnitID())); + mappedImportRow.setGermplasm(pendingGermplasmByOUId.get(importRow.getObsUnitID())); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + observationHash = getObservationHash( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getObservationUnitName(), + getVariableNameFromColumn(column), + pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() + ); - PendingImportObject germplasmPIO = getGidPOI(importRow); - mappedImportRow.setGermplasm(germplasmPIO); + // if value was blank won't be entry in map for this observation + observations.add(observationByHash.get(observationHash)); + } + + } else { + mappedImportRow.setTrial(trialByNameNoScope.get(importRow.getExpTitle())); + mappedImportRow.setLocation(locationByName.get(importRow.getEnvLocation())); + mappedImportRow.setStudy(studyByNameNoScope.get(importRow.getEnv())); + mappedImportRow.setObservationUnit(observationUnitByNameNoScope.get(createObservationUnitKey(importRow))); + mappedImportRow.setGermplasm(getGidPIO(importRow)); + + // loop over phenotype column observation data for current row + for (Column column : phenotypeCols) { + + // if value was blank won't be entry in map for this observation + observations.add(observationByHash.get(getImportObservationHash(importRow, getVariableNameFromColumn(column)))); + } + } mappedBrAPIImport.put(rowNum, mappedImportRow); } @@ -994,7 +1014,7 @@ private void validateTestOrCheck(ExperimentObservation importRow, ValidationErro } } - private PendingImportObject getGidPOI(ExperimentObservation importRow) { + private PendingImportObject getGidPIO(ExperimentObservation importRow) { if (this.existingGermplasmByGID.containsKey(importRow.getGid())) { return existingGermplasmByGID.get(importRow.getGid()); } From eaeb42394fab77cfa9366f0cd8dca557e417adf8 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 7 Mar 2024 10:50:30 -0500 Subject: [PATCH 192/220] [BI-2046] validate observations for reference observation units --- .../processors/ExperimentProcessor.java | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 7eaf38550..bf616273a 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -471,7 +471,7 @@ private void prepareDataForValidation(List importRows, List> observations = mappedImportRow.getObservations(); String observationHash; if (hasAllReferenceUnitIds) { - mappedImportRow.setTrial(pendingTrialByOUId.get(importRow.getObsUnitID()); + mappedImportRow.setTrial(pendingTrialByOUId.get(importRow.getObsUnitID())); mappedImportRow.setLocation(pendingLocationByOUId.get(importRow.getObsUnitID())); mappedImportRow.setStudy(pendingStudyByOUId.get(importRow.getObsUnitID())); mappedImportRow.setObservationUnit(pendingObsUnitByOUId.get(importRow.getObsUnitID())); @@ -687,17 +687,26 @@ private void validateFields(List importRows, ValidationErrors valid for (int rowNum = 0; rowNum < importRows.size(); rowNum++) { ExperimentObservation importRow = (ExperimentObservation) importRows.get(rowNum); PendingImport mappedImportRow = mappedBrAPIImport.get(rowNum); - if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. - validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); + if (hasAllReferenceUnitIds) { + validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); + } else { + if (StringUtils.isNotBlank(importRow.getGid())) { // if GID is blank, don't bother to check if it is valid. + validateGermplasm(importRow, validationErrors, rowNum, mappedImportRow.getGermplasm()); + } + validateTestOrCheck(importRow, validationErrors, rowNum); + validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); + validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); + validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); } - validateTestOrCheck(importRow, validationErrors, rowNum); - validateConditionallyRequired(validationErrors, rowNum, importRow, program, commit); - validateObservationUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); - validateObservations(validationErrors, rowNum, importRow, phenotypeCols, colVarMap, commit, user); } } - private void validateObservationUnits(ValidationErrors validationErrors, Set uniqueStudyAndObsUnit, int rowNum, ExperimentObservation importRow) { + private void validateObservationUnits( + ValidationErrors validationErrors, + Set uniqueStudyAndObsUnit, + int rowNum, + ExperimentObservation importRow + ) { validateUniqueObsUnits(validationErrors, uniqueStudyAndObsUnit, rowNum, importRow); String key = createObservationUnitKey(importRow); @@ -759,9 +768,20 @@ private void validateObservations(ValidationErrors validationErrors, boolean commit, User user) { phenotypeCols.forEach(phenoCol -> { - String importHash = getImportObservationHash(importRow, phenoCol.name()); + String importHash; String importObsValue = phenoCol.getString(rowNum); + if (hasAllReferenceUnitIds) { + importHash = getObservationHash( + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getObservationUnitName(), + getVariableNameFromColumn(phenoCol), + pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() + ); + + } else { + importHash = getImportObservationHash(importRow, phenoCol.name()); + } + // error if import observation data already exists and user has not selected to overwrite if(commit && "false".equals(importRow.getOverwrite() == null ? "false" : importRow.getOverwrite()) && this.existingObsByObsHash.containsKey(importHash) && @@ -845,7 +865,8 @@ private void validateUniqueObsUnits( ValidationErrors validationErrors, Set uniqueStudyAndObsUnit, int rowNum, - ExperimentObservation importRow) { + ExperimentObservation importRow + ) { String envIdPlusStudyId = createObservationUnitKey(importRow); if (uniqueStudyAndObsUnit.contains(envIdPlusStudyId)) { String errorMessage = String.format("The ID (%s) is not unique within the environment(%s)", importRow.getExpUnitId(), importRow.getEnv()); From bd480f24cc4e218f70c4050f9569d25ef97bfd1c Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 7 Mar 2024 15:06:41 -0500 Subject: [PATCH 193/220] [BI-2046] add test for updating observations referenced by OU id --- .../importer/ExperimentFileImportTest.java | 105 +++++++++++++++--- 1 file changed, 91 insertions(+), 14 deletions(-) diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 59607a577..3ff0271f7 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -735,19 +735,6 @@ public void importNewObsVarByObsUnitId() { assertTrue(ouIdXref.isPresent()); Map newObsVar = new HashMap<>(); -// newObsVar.put(Columns.GERMPLASM_GID, "1"); -// newObsVar.put(Columns.TEST_CHECK, "T"); -// newObsVar.put(Columns.EXP_TITLE, "Test Exp"); -// newObsVar.put(Columns.EXP_UNIT, "Plot"); -// newObsVar.put(Columns.EXP_TYPE, "Phenotyping"); -// newObsVar.put(Columns.ENV, "New Env"); -// newObsVar.put(Columns.ENV_LOCATION, "Location A"); -// newObsVar.put(Columns.ENV_YEAR, "2023"); -// newObsVar.put(Columns.EXP_UNIT_ID, "a-1"); -// newObsVar.put(Columns.REP_NUM, "1"); -// newObsVar.put(Columns.BLOCK_NUM, "1"); -// newObsVar.put(Columns.ROW, "1"); -// newObsVar.put(Columns.COLUMN, "1"); newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); @@ -763,7 +750,58 @@ public void importNewObsVarByObsUnitId() { assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); - assertRowSaved(newObsVar, program, traits); + assertRowReferencedByOUIdSaved(newObsVar, program, traits); + } + + @Test + @SneakyThrows + public void importNewObservationDataByObsUnitId() { + List traits = importTestUtils.createTraits(1); + Program program = createProgram("New Observation Referring to OU by ID", "OUDAT", "DATAOU", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Map newExp = new HashMap<>(); + newExp.put(Columns.GERMPLASM_GID, "1"); + newExp.put(Columns.TEST_CHECK, "T"); + newExp.put(Columns.EXP_TITLE, "Test Exp"); + newExp.put(Columns.EXP_UNIT, "Plot"); + newExp.put(Columns.EXP_TYPE, "Phenotyping"); + newExp.put(Columns.ENV, "New Env"); + newExp.put(Columns.ENV_LOCATION, "Location A"); + newExp.put(Columns.ENV_YEAR, "2023"); + newExp.put(Columns.EXP_UNIT_ID, "a-1"); + newExp.put(Columns.REP_NUM, "1"); + newExp.put(Columns.BLOCK_NUM, "1"); + newExp.put(Columns.ROW, "1"); + newExp.put(Columns.COLUMN, "1"); + newExp.put(traits.get(0).getObservationVariableName(), null); + + importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); + + BrAPITrial brAPITrial = brAPITrialDAO.getTrialsByName(List.of((String)newExp.get(Columns.EXP_TITLE)), program).get(0); + Optional trialIdXref = Utilities.getExternalReference(brAPITrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())); + assertTrue(trialIdXref.isPresent()); + BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByExperimentID(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); + + BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); + Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); + assertTrue(ouIdXref.isPresent()); + + Map newObsVar = new HashMap<>(); + newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); + newObsVar.put(traits.get(0).getObservationVariableName(), "newData"); + + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); + assertTrue(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().has("observationDatasetId")); + assertTrue(importTestUtils.UUID_REGEX.matcher(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().get("observationDatasetId").getAsString()).matches()); + assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); + assertRowReferencedByOUIdSaved(newObsVar, program, traits); } @@ -1106,6 +1144,45 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { } } + private Map assertRowReferencedByOUIdSaved(Map expected, Program program, List traits) throws ApiException { + Map ret = new HashMap<>(); + + List units = ouDAO.getObservationUnitsById(List.of((String)expected.get(Columns.OBS_UNIT_ID)), program); + assertFalse(units.isEmpty()); + + List observations = null; + if(traits != null) { + observations = observationDAO.getObservationsByObservationUnitsAndVariables( + units.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList()), + traits.stream().map(Trait::getObservationVariableDbId).collect(Collectors.toList()), + program + ); + if (expected.get(traits.get(0).getObservationVariableName()) == null) { + assertTrue(observations.isEmpty()); + } else { + assertFalse(observations.isEmpty()); + List expectedVariableObservation = new ArrayList<>(); + List actualVariableObservation = new ArrayList<>(); + observations.forEach(observation -> actualVariableObservation.add(String.format("%s:%s", Utilities.removeProgramKey(observation.getObservationVariableName(), program.getKey()), observation.getValue()))); + for(Trait trait : traits) { + if (expected.get(trait.getObservationVariableName()) != null) { + expectedVariableObservation.add(String.format("%s:%s", trait.getObservationVariableName(), expected.get(trait.getObservationVariableName()))); + } + } + if (actualVariableObservation.isEmpty()) { + assertTrue(expectedVariableObservation.isEmpty()); + } else { + assertThat("Missing Variable:Observation combo", actualVariableObservation, containsInAnyOrder(expectedVariableObservation.toArray())); + } + + } + + ret.put("observations", observations); + } + + return ret; + } + private Map assertRowSaved(Map expected, Program program, List traits) throws ApiException { Map ret = new HashMap<>(); From 97047658944dd447ed3b7be2801887156bf3e9f2 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 11 Mar 2024 13:44:32 -0400 Subject: [PATCH 194/220] [BI-2046] fix studyPIO reference when creating new observation PIO --- .../importer/services/processors/ExperimentProcessor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index bf616273a..a47d42e9c 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1162,7 +1162,6 @@ private void fetchOrCreateObservationPIO(Program program, getSingleEntryValue(trialByNameNoScope, MULTIPLE_EXP_TITLES) : trialByNameNoScope.get(importRow.getExpTitle()); UUID trialID = trialPIO.getId(); - //PendingImportObject studyPIO = this.studyByNameNoScope.get(importRow.getEnv()); UUID studyID = studyPIO.getId(); UUID id = UUID.randomUUID(); newObservation = importRow.constructBrAPIObservation(value, variableName, seasonDbId, obsUnitPIO.getBrAPIObject(), commit, program, user, BRAPI_REFERENCE_SOURCE, trialID, studyID, obsUnitPIO.getId(), id); @@ -1172,7 +1171,7 @@ private void fetchOrCreateObservationPIO(Program program, newObservation.setObservationTimeStamp(OffsetDateTime.parse(timeStampValue)); } - newObservation.setStudyDbId(this.studyByNameNoScope.get(pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName()).getId().toString()); //set as the BI ID to facilitate looking up studies when saving new observations + newObservation.setStudyDbId(studyPIO.getId().toString()); //set as the BI ID to facilitate looking up studies when saving new observations pio = new PendingImportObject<>(ImportObjectState.NEW, newObservation); this.observationByHash.put(key, pio); From 885ef5325316ade47c1ed08e5058a28aaf709bf1 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Fri, 15 Mar 2024 14:27:58 -0400 Subject: [PATCH 195/220] [BI-2046] correct observation hash when OU id referenced --- .../importer/services/processors/ExperimentProcessor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index a47d42e9c..6e53b37ef 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -480,6 +480,7 @@ private void prepareDataForValidation(List importRows, List column : phenotypeCols) { observationHash = getObservationHash( + pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() + pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getObservationUnitName(), getVariableNameFromColumn(column), pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() @@ -1130,7 +1131,7 @@ private void fetchOrCreateObservationPIO(Program program, if (hasAllReferenceUnitIds) { String unitName = obsUnitPIO.getBrAPIObject().getObservationUnitName(); String studyName = studyPIO.getBrAPIObject().getStudyName(); - key = getObservationHash(unitName, variableName, studyName); + key = getObservationHash(studyName + unitName, variableName, studyName); } else { key = getImportObservationHash(importRow, variableName); } From a39c505ae100f4299aa0bbe2f6793a69e434864a Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 27 Mar 2024 11:11:19 -0400 Subject: [PATCH 196/220] [BI-2046] set preview state for existing empty datasets --- .../processors/ExperimentProcessor.java | 25 ++++++---- .../importer/ExperimentFileImportTest.java | 47 ++++++++++++++++--- 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 6e53b37ef..5230f3819 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -233,7 +233,7 @@ public void getExistingBrapiData(List importRows, Program program) } else { - // can't proceed if you have a mix of ObsUnitId for some but not all rows + // can't proceed if the import has a mix of ObsUnitId for some but not all rows throw new HttpStatusException(HttpStatus.UNPROCESSABLE_ENTITY, MISSING_OBS_UNIT_ID_ERROR); } } @@ -666,6 +666,10 @@ private String createObservationUnitKey(String studyName, String obsUnitName) { return studyName + obsUnitName; } + private String getImportObservationHash(String obsUnitName, String variableName, String studyName) { + return getObservationHash(createObservationUnitKey(studyName, obsUnitName), variableName, studyName); + } + private String getImportObservationHash(ExperimentObservation importRow, String variableName) { return getObservationHash(createObservationUnitKey(importRow), variableName, importRow.getEnv()); } @@ -773,7 +777,7 @@ private void validateObservations(ValidationErrors validationErrors, String importObsValue = phenoCol.getString(rowNum); if (hasAllReferenceUnitIds) { - importHash = getObservationHash( + importHash = getImportObservationHash( pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getObservationUnitName(), getVariableNameFromColumn(phenoCol), pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() @@ -832,14 +836,17 @@ private void validateObservations(ValidationErrors validationErrors, pendingObservation.getAdditionalInfo().get(BrAPIAdditionalInfoFields.CHANGELOG).getAsJsonArray().add(gson.toJsonTree(change).getAsJsonObject()); } - // preview case where observation has already been committed and import ObsVar data is either empty or the - // same as has been committed prior to import - } else if(existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)) || - isObservationMatched(importHash, importObsValue, phenoCol, rowNum))) { + // preview case where observation has already been committed and import ObsVar data is the + // same as has been committed prior to import + } else if(isObservationMatched(importHash, importObsValue, phenoCol, rowNum)) { BrAPIObservation existingObs = this.existingObsByObsHash.get(importHash); existingObs.setObservationVariableName(phenoCol.name()); observationByHash.get(importHash).setState(ImportObjectState.EXISTING); observationByHash.get(importHash).setBrAPIObject(existingObs); + + // preview case where observation has already been committed and import ObsVar data is empty prior to import + } else if(!existingObsByObsHash.containsKey(importHash) && (StringUtils.isBlank(phenoCol.getString(rowNum)))) { + observationByHash.get(importHash).setState(ImportObjectState.EXISTING); } else { validateObservationValue(colVarMap.get(phenoCol.name()), phenoCol.getString(rowNum), phenoCol.name(), validationErrors, rowNum); @@ -1097,7 +1104,7 @@ boolean isTimestampMatched(String observationHash, String timeStamp) { } boolean isValueMatched(String observationHash, String value) { - if (existingObsByObsHash.get(observationHash).getValue() == null) { + if (!existingObsByObsHash.containsKey(observationHash) || existingObsByObsHash.get(observationHash).getValue() == null) { return value == null; } return existingObsByObsHash.get(observationHash).getValue().equals(value); @@ -1994,7 +2001,7 @@ private Map> initializeGermplasmByGI existingGermplasms.forEach(existingGermplasm -> { BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); - existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceID()))); + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); }); return existingGermplasmByGID; } @@ -2030,7 +2037,7 @@ private Map> initializeExistingGermp existingGermplasms.forEach(existingGermplasm -> { BrAPIExternalReference xref = Utilities.getExternalReference(existingGermplasm.getExternalReferences(), String.format("%s", BRAPI_REFERENCE_SOURCE)) .orElseThrow(() -> new IllegalStateException("External references wasn't found for germplasm (dbid): " + existingGermplasm.getGermplasmDbId())); - existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceID()))); + existingGermplasmByGID.put(existingGermplasm.getAccessionNumber(), new PendingImportObject<>(ImportObjectState.EXISTING, existingGermplasm, UUID.fromString(xref.getReferenceId()))); }); return existingGermplasmByGID; } diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 3ff0271f7..33591b431 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -757,7 +757,7 @@ public void importNewObsVarByObsUnitId() { @SneakyThrows public void importNewObservationDataByObsUnitId() { List traits = importTestUtils.createTraits(1); - Program program = createProgram("New Observation Referring to OU by ID", "OUDAT", "DATAOU", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Program program = createProgram("New Observation Referring to OU by ID", "OUDAT", "DATOU", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); @@ -787,7 +787,7 @@ public void importNewObservationDataByObsUnitId() { Map newObsVar = new HashMap<>(); newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); - newObsVar.put(traits.get(0).getObservationVariableName(), "newData"); + newObsVar.put(traits.get(0).getObservationVariableName(), "1"); JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId); @@ -921,7 +921,24 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "2"); - uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName(), commit); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, true, client, program, mappingId); + + JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); + assertEquals(1, previewRows.size()); + JsonObject row = previewRows.get(0).getAsJsonObject(); + + assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); + assertTrue(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().has("observationDatasetId")); + assertTrue(importTestUtils.UUID_REGEX.matcher(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().get("observationDatasetId").getAsString()).matches()); + assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); + assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); + + if(commit) { + assertRowSaved(newObservation, program, traits); + } else { + assertValidPreviewRow(newObservation, row, program, traits); + } } /* @@ -1136,7 +1153,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); - newObservation.put(traits.get(0).getObservationVariableName(), "1"); + //newObservation.put(traits.get(0).getObservationVariableName(), "1"); if(commit) { assertRowSaved(newObservation, program, traits); } else { @@ -1152,6 +1169,7 @@ private Map assertRowReferencedByOUIdSaved(Map e List observations = null; if(traits != null) { + //observations = observationDAO.getObservationsByStudyName(List.of(units.get(0).getStudyName()), program); observations = observationDAO.getObservationsByObservationUnitsAndVariables( units.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList()), traits.stream().map(Trait::getObservationVariableDbId).collect(Collectors.toList()), @@ -1160,7 +1178,7 @@ private Map assertRowReferencedByOUIdSaved(Map e if (expected.get(traits.get(0).getObservationVariableName()) == null) { assertTrue(observations.isEmpty()); } else { - assertFalse(observations.isEmpty()); + //assertFalse(observations.isEmpty()); List expectedVariableObservation = new ArrayList<>(); List actualVariableObservation = new ArrayList<>(); observations.forEach(observation -> actualVariableObservation.add(String.format("%s:%s", Utilities.removeProgramKey(observation.getObservationVariableName(), program.getKey()), observation.getValue()))); @@ -1286,7 +1304,24 @@ private Map assertRowSaved(Map expected, Program if(traits != null) { List expectedVariableObservation = new ArrayList<>(); List actualVariableObservation = new ArrayList<>(); - observations.forEach(observation -> actualVariableObservation.add(String.format("%s:%s", Utilities.removeProgramKey(observation.getObservationVariableName(), program.getKey()), observation.getValue()))); + observations.forEach(observation -> actualVariableObservation.add( + String.format( + "%s:%s", + Utilities.removeProgramKey(observation.getObservationVariableName(), program.getKey()), + observation.getValue() + ) + )); + +// //Remove observations duplicated when re-imported with no change +// Set uniqueObservations = new HashSet<>(); +// Iterator iterator = actualVariableObservation.iterator(); +// +// while (iterator.hasNext()) { +// String item = iterator.next(); +// if (!uniqueObservations.add(item)) { +// iterator.remove(); // Remove the duplicate actual observation +// } +// } for(Trait trait : traits) { if (expected.get(trait.getObservationVariableName()) != null) { expectedVariableObservation.add(String.format("%s:%s", trait.getObservationVariableName(), expected.get(trait.getObservationVariableName()))); From 3e04828858faf59713e25fe62fe123fa4ba7571a Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 27 Mar 2024 18:53:35 -0400 Subject: [PATCH 197/220] fix test --- .../processors/ExperimentProcessor.java | 2 +- .../importer/ExperimentFileImportTest.java | 72 +++++++++---------- 2 files changed, 33 insertions(+), 41 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 5230f3819..c9d861906 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1144,7 +1144,7 @@ private void fetchOrCreateObservationPIO(Program program, } if (existingObsByObsHash.containsKey(key)) { - if (StringUtils.isNotBlank(value) && !isObservationMatched(key, value, column, rowNum)){ + if (!isObservationMatched(key, value, column, rowNum)){ // prior observation with updated value newObservation = gson.fromJson(gson.toJson(existingObsByObsHash.get(key)), BrAPIObservation.class); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 33591b431..2e28f5efc 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -753,11 +753,12 @@ public void importNewObsVarByObsUnitId() { assertRowReferencedByOUIdSaved(newObsVar, program, traits); } - @Test + @ParameterizedTest + @ValueSource(booleans = {true, false}) @SneakyThrows - public void importNewObservationDataByObsUnitId() { + public void importNewObservationDataByObsUnitId(boolean commit) { List traits = importTestUtils.createTraits(1); - Program program = createProgram("New Observation Referring to OU by ID", "OUDAT", "DATOU", BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); + Program program = createProgram("New Observation Referring to OU by ID"+(commit ? "C" : "P"), "OUDAT"+(commit ? "C" : "P"), "DATOU"+(commit ? "C" : "P"), BRAPI_REFERENCE_SOURCE, createGermplasm(1), traits); Map newExp = new HashMap<>(); newExp.put(Columns.GERMPLASM_GID, "1"); newExp.put(Columns.TEST_CHECK, "T"); @@ -772,7 +773,7 @@ public void importNewObservationDataByObsUnitId() { newExp.put(Columns.BLOCK_NUM, "1"); newExp.put(Columns.ROW, "1"); newExp.put(Columns.COLUMN, "1"); - newExp.put(traits.get(0).getObservationVariableName(), null); + newExp.put(traits.get(0).getObservationVariableName(), null); // empty dataset importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newExp), null), null, true, client, program, mappingId); @@ -786,10 +787,23 @@ public void importNewObservationDataByObsUnitId() { assertTrue(ouIdXref.isPresent()); Map newObsVar = new HashMap<>(); + newObsVar.put(Columns.GERMPLASM_GID, "1"); + newObsVar.put(Columns.TEST_CHECK, "T"); + newObsVar.put(Columns.EXP_TITLE, "Test Exp"); + newObsVar.put(Columns.EXP_UNIT, "Plot"); + newObsVar.put(Columns.EXP_TYPE, "Phenotyping"); + newObsVar.put(Columns.ENV, "New Env"); + newObsVar.put(Columns.ENV_LOCATION, "Location A"); + newObsVar.put(Columns.ENV_YEAR, "2023"); + newObsVar.put(Columns.EXP_UNIT_ID, "a-1"); + newObsVar.put(Columns.REP_NUM, "1"); + newObsVar.put(Columns.BLOCK_NUM, "1"); + newObsVar.put(Columns.ROW, "1"); + newObsVar.put(Columns.COLUMN, "1"); newObsVar.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObsVar.put(traits.get(0).getObservationVariableName(), "1"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, true, client, program, mappingId); + JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits), null, commit, client, program, mappingId); JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); assertEquals(1, previewRows.size()); @@ -801,7 +815,12 @@ public void importNewObservationDataByObsUnitId() { assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); - assertRowReferencedByOUIdSaved(newObsVar, program, traits); + if(commit) { + //assertRowReferencedByOUIdSaved(newObsVar, program, traits); + assertRowSaved(newObsVar, program, traits); + } else { + assertValidPreviewRow(newObsVar, row, program, traits); + } } @@ -921,24 +940,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { newObservation.put(Columns.OBS_UNIT_ID, ouIdXref.get().getReferenceId()); newObservation.put(traits.get(0).getObservationVariableName(), "2"); - JsonObject result = importTestUtils.uploadAndFetch(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), null, true, client, program, mappingId); - - JsonArray previewRows = result.get("preview").getAsJsonObject().get("rows").getAsJsonArray(); - assertEquals(1, previewRows.size()); - JsonObject row = previewRows.get(0).getAsJsonObject(); - - assertEquals("EXISTING", row.getAsJsonObject("trial").get("state").getAsString()); - assertTrue(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().has("observationDatasetId")); - assertTrue(importTestUtils.UUID_REGEX.matcher(row.getAsJsonObject("trial").get("brAPIObject").getAsJsonObject().get("additionalInfo").getAsJsonObject().get("observationDatasetId").getAsString()).matches()); - assertEquals("EXISTING", row.getAsJsonObject("location").get("state").getAsString()); - assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); - assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); - - if(commit) { - assertRowSaved(newObservation, program, traits); - } else { - assertValidPreviewRow(newObservation, row, program, traits); - } + uploadAndVerifyFailure(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits), traits.get(0).getObservationVariableName(), commit); } /* @@ -1169,12 +1171,12 @@ private Map assertRowReferencedByOUIdSaved(Map e List observations = null; if(traits != null) { - //observations = observationDAO.getObservationsByStudyName(List.of(units.get(0).getStudyName()), program); - observations = observationDAO.getObservationsByObservationUnitsAndVariables( - units.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList()), - traits.stream().map(Trait::getObservationVariableDbId).collect(Collectors.toList()), - program - ); + observations = observationDAO.getObservationsByStudyName(List.of(units.get(0).getStudyName()), program); +// observations = observationDAO.getObservationsByObservationUnitsAndVariables( +// units.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList()), +// traits.stream().map(Trait::getObservationVariableDbId).collect(Collectors.toList()), +// program +// ); if (expected.get(traits.get(0).getObservationVariableName()) == null) { assertTrue(observations.isEmpty()); } else { @@ -1312,16 +1314,6 @@ private Map assertRowSaved(Map expected, Program ) )); -// //Remove observations duplicated when re-imported with no change -// Set uniqueObservations = new HashSet<>(); -// Iterator iterator = actualVariableObservation.iterator(); -// -// while (iterator.hasNext()) { -// String item = iterator.next(); -// if (!uniqueObservations.add(item)) { -// iterator.remove(); // Remove the duplicate actual observation -// } -// } for(Trait trait : traits) { if (expected.get(trait.getObservationVariableName()) != null) { expectedVariableObservation.add(String.format("%s:%s", trait.getObservationVariableName(), expected.get(trait.getObservationVariableName()))); From 17b08e36dffa002c2a6b115346d71b48dda556cd Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 28 Mar 2024 10:32:32 -0400 Subject: [PATCH 198/220] [BI-2046] remove unused helper method --- .../brapps/importer/services/FileMappingUtil.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java b/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java index a3e96164a..cb93daf48 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/FileMappingUtil.java @@ -91,13 +91,4 @@ public List sortByField(List sortedFields, List unsortedItems, return unsortedItems; } - - public boolean isValidUUID(String id) { - try { - UUID.fromString(id); - return true; - } catch (IllegalArgumentException e) { - return false; - } - } } From 55a0f788fc244952a7d920bda82ff8b18e220d61 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 28 Mar 2024 12:45:30 -0400 Subject: [PATCH 199/220] [BI-2046] refactor --- .../processors/ExperimentProcessor.java | 31 ++++++++++++------- .../importer/ExperimentFileImportTest.java | 9 +----- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index c9d861906..7a5ef92ce 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -35,7 +35,6 @@ import org.brapi.v2.model.core.*; import org.brapi.v2.model.core.request.BrAPIListNewRequest; import org.brapi.v2.model.core.response.BrAPIListDetails; -import org.brapi.v2.model.geno.BrAPIReference; import org.brapi.v2.model.germ.BrAPIGermplasm; import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationUnit; @@ -47,8 +46,6 @@ import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.*; import org.breedinginsight.brapps.importer.model.ImportUpload; -import org.breedinginsight.brapps.importer.model.base.ObservationUnit; -import org.breedinginsight.brapps.importer.model.base.Study; import org.breedinginsight.brapps.importer.model.imports.BrAPIImport; import org.breedinginsight.brapps.importer.model.imports.ChangeLogEntry; import org.breedinginsight.brapps.importer.model.imports.PendingImport; @@ -471,19 +468,20 @@ private void prepareDataForValidation(List importRows, List> observations = mappedImportRow.getObservations(); String observationHash; if (hasAllReferenceUnitIds) { - mappedImportRow.setTrial(pendingTrialByOUId.get(importRow.getObsUnitID())); - mappedImportRow.setLocation(pendingLocationByOUId.get(importRow.getObsUnitID())); - mappedImportRow.setStudy(pendingStudyByOUId.get(importRow.getObsUnitID())); - mappedImportRow.setObservationUnit(pendingObsUnitByOUId.get(importRow.getObsUnitID())); - mappedImportRow.setGermplasm(pendingGermplasmByOUId.get(importRow.getObsUnitID())); + String refOUId = importRow.getObsUnitID(); + mappedImportRow.setTrial(pendingTrialByOUId.get(refOUId)); + mappedImportRow.setLocation(pendingLocationByOUId.get(refOUId)); + mappedImportRow.setStudy(pendingStudyByOUId.get(refOUId)); + mappedImportRow.setObservationUnit(pendingObsUnitByOUId.get(refOUId)); + mappedImportRow.setGermplasm(pendingGermplasmByOUId.get(refOUId)); // loop over phenotype column observation data for current row for (Column column : phenotypeCols) { observationHash = getObservationHash( - pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() + - pendingObsUnitByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getObservationUnitName(), + pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() + + pendingObsUnitByOUId.get(refOUId).getBrAPIObject().getObservationUnitName(), getVariableNameFromColumn(column), - pendingStudyByOUId.get(importRow.getObsUnitID()).getBrAPIObject().getStudyName() + pendingStudyByOUId.get(refOUId).getBrAPIObject().getStudyName() ); // if value was blank won't be entry in map for this observation @@ -666,6 +664,17 @@ private String createObservationUnitKey(String studyName, String obsUnitName) { return studyName + obsUnitName; } + /** + * This method is responsible for generating a hash based on the import observation unit information. + * It takes the observation unit name, variable name, and study name as input parameters. + * The observation unit key is created using the study name and observation unit name. + * The hash is generated based on the observation unit key, variable name, and study name. + * + * @param obsUnitName The name of the observation unit being imported. + * @param variableName The name of the variable associated with the observation unit. + * @param studyName The name of the study associated with the observation unit. + * @return A string representing the hash of the import observation unit information. + */ private String getImportObservationHash(String obsUnitName, String variableName, String studyName) { return getObservationHash(createObservationUnitKey(studyName, obsUnitName), variableName, studyName); } diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 2e28f5efc..9420f18f0 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -816,7 +816,6 @@ public void importNewObservationDataByObsUnitId(boolean commit) { assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); if(commit) { - //assertRowReferencedByOUIdSaved(newObsVar, program, traits); assertRowSaved(newObsVar, program, traits); } else { assertValidPreviewRow(newObsVar, row, program, traits); @@ -1155,7 +1154,6 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { assertEquals("EXISTING", row.getAsJsonObject("study").get("state").getAsString()); assertEquals("EXISTING", row.getAsJsonObject("observationUnit").get("state").getAsString()); - //newObservation.put(traits.get(0).getObservationVariableName(), "1"); if(commit) { assertRowSaved(newObservation, program, traits); } else { @@ -1172,15 +1170,10 @@ private Map assertRowReferencedByOUIdSaved(Map e List observations = null; if(traits != null) { observations = observationDAO.getObservationsByStudyName(List.of(units.get(0).getStudyName()), program); -// observations = observationDAO.getObservationsByObservationUnitsAndVariables( -// units.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toList()), -// traits.stream().map(Trait::getObservationVariableDbId).collect(Collectors.toList()), -// program -// ); if (expected.get(traits.get(0).getObservationVariableName()) == null) { assertTrue(observations.isEmpty()); } else { - //assertFalse(observations.isEmpty()); + assertFalse(observations.isEmpty()); List expectedVariableObservation = new ArrayList<>(); List actualVariableObservation = new ArrayList<>(); observations.forEach(observation -> actualVariableObservation.add(String.format("%s:%s", Utilities.removeProgramKey(observation.getObservationVariableName(), program.getKey()), observation.getValue()))); From cc68f285ef4433d7b554f85fad934122df1d5a8c Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 28 Mar 2024 15:10:50 -0400 Subject: [PATCH 200/220] [BI-2046] document exp processhelper method --- .../processors/ExperimentProcessor.java | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 7a5ef92ce..3ed5a0d01 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -882,8 +882,7 @@ private void validateUniqueObsUnits( ValidationErrors validationErrors, Set uniqueStudyAndObsUnit, int rowNum, - ExperimentObservation importRow - ) { + ExperimentObservation importRow) { String envIdPlusStudyId = createObservationUnitKey(importRow); if (uniqueStudyAndObsUnit.contains(envIdPlusStudyId)) { String errorMessage = String.format("The ID (%s) is not unique within the environment(%s)", importRow.getExpUnitId(), importRow.getEnv()); @@ -1568,6 +1567,24 @@ private Map> initializeObserva } } + /** + * Maps pending locations by Observation Unit (OU) Id based on given parameters. + * + * This method takes in a unitId, BrAPIObservationUnit unit, Maps of studyByOUId, locationByName, + * and locationByOUId. It then associates the location of the observation unit with the respective OU Id. + * If the locationName is not null for the unit, it is directly added to locationByOUId. + * If the locationName is null, it checks the studyByOUId map for a location related to the unit. + * If a location related to the unit is found, it maps that location with the respective OU Id. + * If no location is found, it throws an IllegalStateException. + * + * @param unitId the Observation Unit Id + * @param unit the BrAPIObservationUnit object + * @param studyByOUId a Map of Study by Observation Unit Id + * @param locationByName a Map of Location by Name + * @param locationByOUId a Map of Location by Observation Unit Id + * @return the updated locationByOUId map after mapping the pending locations + * @throws IllegalStateException if the Observation Unit is missing a location + */ private Map> mapPendingLocationByOUId( String unitId, BrAPIObservationUnit unit, From 068f046ab61c75186b6e0700891c41376e81af07 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 28 Mar 2024 15:16:17 -0400 Subject: [PATCH 201/220] [BI-2046] document exp process helper method --- .../services/processors/ExperimentProcessor.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 3ed5a0d01..658241ddb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1666,14 +1666,13 @@ private Map> mapPendingObserva } /** - * Retrieves a list of pending Observation Units based on their IDs. + * Retrieves reference Observation Units based on a set of reference Observation Unit IDs and a Program. + * Constructs DeltaBreed observation unit source for external references and sets up pending Observation Units. * - * This function retrieves Observation Units based on a list of reference Observation Unit IDs - * and the associated program. It then sets pending Observation Units for each ID. - * - * @return List of BrAPIObservationUnit: The list of reference Observation Units retrieved. - * - * @throws InternalServerException if an error occurs during the process. + * @param referenceOUIds A set of reference Observation Unit IDs to retrieve + * @param program The Program associated with the Observation Units + * @return A Map containing pending Observation Units by their ID + * @throws ApiException if an error occurs during the process */ private Map> fetchReferenceObservationUnits( Set referenceOUIds, From e39a02def0143a5fe67cb9f9e36047f06d4a48db Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 28 Mar 2024 15:20:11 -0400 Subject: [PATCH 202/220] [BI-2046] document exp process helper method --- .../processors/ExperimentProcessor.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 658241ddb..46f9bb95b 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1800,17 +1800,16 @@ private void initializeStudiesForExistingObservationUnits( } /** - * Fetches a list of BrAPI studies by studyDbIds belonging to a specific - * program. - * If not all studyDbIds are found in the database, it throws an - * IllegalStateException. + * Fetches a list of BrAPI studies by their study database IDs for a given program. * - * @param studyDbIds A set of studyDbIds to fetch studies for - * @param program The program for which the studies are associated - * @return A list of BrAPIStudy objects representing the fetched studies - * @throws ApiException if an error occurs during the database fetch - * operation - * @throws IllegalStateException if not all studyDbIds are found in the database + * This method queries the BrAPIStudyDAO to retrieve studies based on the provided study database IDs and the program. + * It ensures that all requested study database IDs are found in the result set, throwing an IllegalStateException if any are missing. + * + * @param studyDbIds a Set of Strings representing the study database IDs to fetch + * @param program the Program object representing the program context in which to fetch studies + * @return a List of BrAPIStudy objects matching the provided study database IDs + * @throws ApiException if there is an issue fetching the studies + * @throws IllegalStateException if any requested study database IDs are not found in the result set */ private List fetchStudiesByDbId(Set studyDbIds, Program program) throws ApiException { List studies = brAPIStudyDAO.getStudiesByStudyDbId(studyDbIds, program); From a42b98f3de226ad10f539965eaacae17eb752592 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 28 Mar 2024 15:23:28 -0400 Subject: [PATCH 203/220] [BI-2046] document exp process helper method --- .../services/processors/ExperimentProcessor.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 46f9bb95b..f283150a0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1822,10 +1822,21 @@ private List fetchStudiesByDbId(Set studyDbIds, Program prog return studies; } + /** + * Initializes a map of ProgramLocation objects by their names using the given Program and a map of BrAPIStudy objects by their names. + * + * This method takes a Program object and a map of BrAPIStudy objects by their names, retrieves the location database IDs from the studies, + * and fetches existing ProgramLocation objects based on the database IDs. It then creates a map of ProgramLocation objects by their names + * with PendingImportObject wrappers that indicate the state of the object as existing. + * + * @param program the Program object to associate with the locations + * @param studyByName a map of BrAPIStudy objects by their names + * @return a map of ProgramLocation objects by their names with PendingImportObject wrappers + * @throws InternalServerException if an error occurs during the location retrieval process + */ Map> initializeLocationByName( Program program, - Map> studyByName - ) { + Map> studyByName) { Map> locationByName = new HashMap<>(); List existingLocations = new ArrayList<>(); From b3f819cfcd092985a7c546bc52454e0c67da92cd Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 28 Mar 2024 15:29:28 -0400 Subject: [PATCH 204/220] [BI-2046] document exp process helper method --- .../processors/ExperimentProcessor.java | 49 ++++++++++++++++--- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index f283150a0..3137fa371 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -209,7 +209,7 @@ public void getExistingBrapiData(List importRows, Program program) mapPendingTrialByOUId(unitId, unit, trialByNameNoScope, studyByNameNoScope, pendingTrialByOUId, program); mapPendingStudyByOUId(unitId, unit, studyByNameNoScope, pendingStudyByOUId, program); mapPendingLocationByOUId(unitId, unit, pendingStudyByOUId, locationByName, pendingLocationByOUId); - mapPendingObsDatasetByOUId(unitId, pendingTrialByOUId, obsVarDatasetByName, pendingObsDatasetByOUId, program); + mapPendingObsDatasetByOUId(unitId, pendingTrialByOUId, obsVarDatasetByName, pendingObsDatasetByOUId); mapGermplasmByOUId(unitId, unit, existingGermplasmByGID, pendingGermplasmByOUId); } @@ -1896,24 +1896,49 @@ private Map> initializeUniqueLocati return locationByName; } + /** + * Maps a given germplasm to an observation unit ID in a given map of germplasm by observation unit ID. + * + * This method retrieves the Global Identifier (GID) of the provided observation unit and uses it to lookup + * the corresponding PendingImportObject in the map of germplasm by name. The found germplasm + * object is then mapped to the observation unit ID in the provided map of germplasm by observation unit ID. + * The updated map is returned after the mapping operation has been performed. + * + * @param unitId The observation unit ID to which the germplasm should be mapped. + * @param unit The BrAPIObservationUnit object representing the observation unit. + * @param germplasmByName The map of germplasm objects by name used to lookup the desired germplasm. + * @param germplasmByOUId The map of germplasm objects by observation unit ID to update with the mapping result. + * @return The updated map of germplasm objects by observation unit ID after mapping the germplasm to the provided observation unit ID. + */ private Map> mapGermplasmByOUId( String unitId, BrAPIObservationUnit unit, Map> germplasmByName, - Map> germplasmByOUId - ) { + Map> germplasmByOUId) { String gid = unit.getAdditionalInfo().getAsJsonObject().get(BrAPIAdditionalInfoFields.GID).getAsString(); germplasmByOUId.put(unitId, germplasmByName.get(gid)); return germplasmByOUId; } + + /** + * Maps the pending observation dataset by OU Id based on the given inputs. + * This function checks if the trialByOUId map is not empty, the obsVarDatasetByName map is not empty, + * and if the first entry in the trialByOUId map contains observation dataset id in its additional info. + * If the conditions are met, it adds the pending import object from the obsVarDatasetByName map to the + * obsVarDatasetByOUId map using the unitId as the key. + * + * @param unitId the unit ID based on which the mapping is done + * @param trialByOUId a map containing pending import objects with BrAPITrial as the value, mapped by OU Id + * @param obsVarDatasetByName a map containing pending import objects with BrAPIListDetails as the value, mapped by dataset name + * @param obsVarDatasetByOUId a map containing pending import objects with BrAPIListDetails as the value, mapped by OU Id + * @return the updated obsVarDatasetByOUId map after potential addition of a pending import object + */ private Map> mapPendingObsDatasetByOUId( String unitId, Map> trialByOUId, Map> obsVarDatasetByName, - Map> obsVarDatasetByOUId, - Program program - ) { + Map> obsVarDatasetByOUId) { if (!trialByOUId.isEmpty() && !obsVarDatasetByName.isEmpty() && trialByOUId.values().iterator().next().getBrAPIObject().getAdditionalInfo().has(BrAPIAdditionalInfoFields.OBSERVATION_DATASET_ID)) { obsVarDatasetByOUId.put(unitId, obsVarDatasetByName.values().iterator().next()); @@ -1921,10 +1946,18 @@ private Map> mapPendingObsDatasetB return obsVarDatasetByOUId; } + + /** + * Initializes observation variable dataset for existing observation units. This function retrieves existing datasets related to observation variables for the specified trial and program, processes the dataset details, and caches the data accordingly. + * + * @param trialByName A map containing trial information indexed by trial name. + * @param program The program to which the datasets are related. + * @return A map of observation variable dataset objects indexed by dataset name. + * @throws InternalServerException If the existing dataset summary is not retrieved from the BrAPI server, or an error occurs during API communication. + */ private Map> initializeObsVarDatasetForExistingObservationUnits( Map> trialByName, - Program program - ) { + Program program) { Map> obsVarDatasetByName = new HashMap<>(); if (trialByName.size() > 0 && From 2e6161f06c1d7ae3d06608d9957e2ea34d271751 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 28 Mar 2024 15:31:22 -0400 Subject: [PATCH 205/220] [BI-2046] delete unused helper method --- .../services/processors/ExperimentProcessor.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 3137fa371..32a6f9b78 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -2034,14 +2034,7 @@ private Optional> getTrialPIO(List> obsVarDatasetByOUId) { - BrAPIExternalReference xref = Utilities.getExternalReference(existingList.getExternalReferences(), - String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) - .orElseThrow(() -> new IllegalStateException("External references wasn't found for list (dbid): " + existingList.getListDbId())); - obsVarDatasetByOUId.put(unitId, - new PendingImportObject(ImportObjectState.EXISTING, existingList, UUID.fromString(xref.getReferenceId()))); - } + private void processAndCacheObsVarDataset(BrAPIListDetails existingList, Map> obsVarDatasetByName) { BrAPIExternalReference xref = Utilities.getExternalReference(existingList.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) From a3f16c2fd923a4372349de33fb4b5411e9582233 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 28 Mar 2024 15:35:10 -0400 Subject: [PATCH 206/220] [BI-2046] document exp process helper method --- .../processors/ExperimentProcessor.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 32a6f9b78..16529c1ca 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -2034,7 +2034,7 @@ private Optional> getTrialPIO(List> obsVarDatasetByName) { BrAPIExternalReference xref = Utilities.getExternalReference(existingList.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.DATASET.getName())) @@ -2042,10 +2042,21 @@ private void processAndCacheObsVarDataset(BrAPIListDetails existingList, Map(ImportObjectState.EXISTING, existingList, UUID.fromString(xref.getReferenceId()))); } + + /** + * Initializes a mapping of BrAPI Germplasm objects by Germplasm ID for existing BrAPI Observation Units. + * This method retrieves existing Germplasms associated with the provided Observation Units and creates a mapping + * using their Accession Number as the key and a PendingImportObject containing the Germplasm object and a reference ID. + * If no existing Germplasms are found, an empty mapping is returned. + * + * @param unitByName A mapping of Observation Units by name. + * @param program The BrAPI Program object to which the Germplasms belong. + * @return A mapping of BrAPI Germplasm objects by Germplasm ID for existing Observation Units. + * @throws InternalServerException If an error occurs while fetching Germplasms from the database. + */ private Map> initializeGermplasmByGIDForExistingObservationUnits( Map> unitByName, - Program program - ) { + Program program) { Map> existingGermplasmByGID = new HashMap<>(); List existingGermplasms = new ArrayList<>(); From aab081e193638fc00a9b69881286000c473a7f94 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 28 Mar 2024 15:39:05 -0400 Subject: [PATCH 207/220] [BI-2046] delete unused helper method --- .../processors/ExperimentProcessor.java | 49 +------------------ 1 file changed, 1 insertion(+), 48 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 16529c1ca..6239e4325 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -2130,8 +2130,7 @@ private PendingImportObject processAndCacheStudy( BrAPIStudy existingStudy, Program program, Function getterFunction, - Map> studyMap - ) throws Exception { + Map> studyMap) throws Exception { PendingImportObject pendingStudy; BrAPIExternalReference xref = Utilities.getExternalReference(existingStudy.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.STUDIES.getName())) .orElseThrow(() -> new IllegalStateException("External references wasn't found for study (dbid): " + existingStudy.getStudyDbId())); @@ -2153,52 +2152,6 @@ private PendingImportObject processAndCacheStudy( return pendingStudy; } - private void setPendingTrialByOUId(Program program) { - if(observationUnitByNameNoScope.size() > 0) { - Set trialDbIds = new HashSet<>(); - Set studyDbIds = new HashSet<>(); - - observationUnitByNameNoScope.values() - .forEach(pio -> { - BrAPIObservationUnit existingOu = pio.getBrAPIObject(); - if (StringUtils.isBlank(existingOu.getTrialDbId()) && StringUtils.isBlank(existingOu.getStudyDbId())) { - throw new IllegalStateException("TrialDbId and StudyDbId are not set for an existing ObservationUnit"); - } - - if (StringUtils.isNotBlank(existingOu.getTrialDbId())) { - trialDbIds.add(existingOu.getTrialDbId()); - } else { - studyDbIds.add(existingOu.getStudyDbId()); - } - }); - - //if the OU doesn't have the trialDbId set, then fetch the study to fetch the trialDbId - if(!studyDbIds.isEmpty()) { - try { - trialDbIds.addAll(fetchTrialDbidsForStudies(studyDbIds, program)); - } catch (ApiException e) { - log.error("Error fetching studies: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - - try { - List trials = brapiTrialDAO.getTrialsByDbIds(trialDbIds, program); - if (trials.size() != trialDbIds.size()) { - List missingIds = new ArrayList<>(trialDbIds); - missingIds.removeAll(trials.stream().map(BrAPITrial::getTrialDbId).collect(Collectors.toList())); - throw new IllegalStateException("Trial not found for trialDbId(s): " + String.join(COMMA_DELIMITER, missingIds)); - } - - String trialRefSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName()); - trials.forEach(trial -> processAndCachePendingTrial(trial, trialRefSource, "foo")); - } catch (ApiException e) { - log.error("Error fetching trials: " + Utilities.generateApiExceptionLogMessage(e), e); - throw new InternalServerException(e.toString(), e); - } - } - } - private void initializeTrialsForExistingObservationUnits(Program program, Map> trialByName) { if(observationUnitByNameNoScope.size() > 0) { Set trialDbIds = new HashSet<>(); From b3396c8a913375f1293556bae8f74cfbe71b93f7 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 28 Mar 2024 15:42:43 -0400 Subject: [PATCH 208/220] [BI-2046] delete unused helper method --- .../processors/ExperimentProcessor.java | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 6239e4325..b7c68fa50 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -2226,36 +2226,6 @@ private void processAndCacheTrial( new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); } - /** - * Cache pending trial with reference to observation unit. - * - * This method processes the given existing trial, associates it with the - * specified - * program, retrieves the experiment ID from the trial's external references, - * then caches it with the provided organizational unit reference ID. - * - * @param existingTrial The existing trial to process and cache. - * @param trialRefSource The source to retrieve the trial's reference. - * @param referenceOUId The ID of the reference observation unit. - */ - private void processAndCachePendingTrial( - BrAPITrial existingTrial, - String trialRefSource, - String referenceOUId) { - - // Retrieve experiment ID from existingTrial - BrAPIExternalReference experimentIDRef = Utilities - .getExternalReference(existingTrial.getExternalReferences(), trialRefSource) - .orElseThrow(() -> new InternalServerException( - "An Experiment ID was not found in any of the external references")); - UUID experimentId = UUID.fromString(experimentIDRef.getReferenceId()); - - // Cache pending trial by observation unit ID - pendingTrialByOUId.put( - referenceOUId, - new PendingImportObject<>(ImportObjectState.EXISTING, existingTrial, experimentId)); - } - /** * Sets the reference Observation Unit IDs based on the import rows. * Checks for references to existing observation units and populates the From ac8072b19f0096a1f0484e73c5afb61a26729c8f Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 28 Mar 2024 15:46:06 -0400 Subject: [PATCH 209/220] [BI-2046] document exp process helper method --- .../services/processors/ExperimentProcessor.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index b7c68fa50..412b34222 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -2213,8 +2213,7 @@ private Set fetchTrialDbidsForStudies(Set studyDbIds, Program pr private void processAndCacheTrial( BrAPITrial existingTrial, Program program, - Map> trialByNameNoScope - ) { + Map> trialByNameNoScope) { //get TrialId from existingTrial BrAPIExternalReference experimentIDRef = Utilities.getExternalReference(existingTrial.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.TRIALS.getName())) @@ -2227,10 +2226,12 @@ private void processAndCacheTrial( } /** - * Sets the reference Observation Unit IDs based on the import rows. - * Checks for references to existing observation units and populates the - * referenceOUIds list. - * Sets flags hasAllReferenceUnitIds and hasNoReferenceUnitIds accordingly. + * This function collates unique ObsUnitID values from a list of BrAPIImport objects. + * It iterates through the list and adds non-blank ObsUnitID values to a Set. It also checks for any repeated ObsUnitIDs. + * + * @param importRows a List of BrAPIImport objects containing ExperimentObservation data + * @return a Set of unique ObsUnitID strings + * @throws IllegalStateException if a repeated ObsUnitID is encountered */ private Set collateReferenceOUIds(List importRows) { Set referenceOUIds = new HashSet<>(); From 4a328778c060ada73065cc2b014337dc56849c87 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 28 Mar 2024 15:49:24 -0400 Subject: [PATCH 210/220] [BI-2046] document exp process helper method --- .../importer/services/processors/ExperimentProcessor.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index 412b34222..f4bc0a0d0 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -2418,6 +2418,14 @@ private String seasonDbIdToYearFromDatabase(String seasonDbId, UUID programId) { return (yearInt == null) ? "" : yearInt.toString(); } + /** + * Returns the single value from the given map, throwing an UnprocessableEntityException if the map does not contain exactly one entry. + * + * @param map The map from which to retrieve the single value. + * @param message The error message to include in the UnprocessableEntityException if the map does not contain exactly one entry. + * @return The single value from the map. + * @throws UnprocessableEntityException if the map does not contain exactly one entry. + */ private V getSingleEntryValue(Map map, String message) throws UnprocessableEntityException { if (map.size() != 1) { throw new UnprocessableEntityException(message); From b98da08c0df8b8309b70accf9ee700b1e529e97e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Thu, 28 Mar 2024 15:57:40 -0400 Subject: [PATCH 211/220] [BI-2046] clean up --- .../importer/services/processors/ExperimentProcessor.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index f4bc0a0d0..b14b935c2 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -128,8 +128,6 @@ public class ExperimentProcessor implements Processor { private Map> pendingStudyByOUId = new HashMap<>(); private Map> obsVarDatasetByName = null; private Map> pendingObsDatasetByOUId = new HashMap<>(); - // It is assumed that there are no preexisting Observation Units for the given environment (so this will not be - // initialized by getExistingBrapiData() ) private Map> observationUnitByNameNoScope = null; private Map> pendingObsUnitByOUId = new HashMap<>(); From be76c1238619748fa75fbbf179f90312fc8fc84e Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Mon, 1 Apr 2024 15:15:44 -0400 Subject: [PATCH 212/220] [BI-2046] Update expected error message in test --- .../importer/services/processors/ExperimentProcessor.java | 7 ++++--- .../brapps/importer/ExperimentFileImportTest.java | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index b14b935c2..c8757c8eb 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1345,8 +1345,9 @@ private PendingImportObject fetchOrCreateTrialPIO( if (trialByNameNoScope.containsKey(importRow.getExpTitle())) { PendingImportObject envPio; trialPio = trialByNameNoScope.get(importRow.getExpTitle()); - envPio = this.studyByNameNoScope.get(importRow.getEnv()); - if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ + envPio = studyByNameNoScope.get(importRow.getEnv()); + if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && + (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); } } else if (!trialByNameNoScope.isEmpty()) { @@ -1359,7 +1360,7 @@ private PendingImportObject fetchOrCreateTrialPIO( } BrAPITrial newTrial = importRow.constructBrAPITrial(program, user, commit, BRAPI_REFERENCE_SOURCE, id, expSeqValue); trialPio = new PendingImportObject<>(ImportObjectState.NEW, newTrial, id); - this.trialByNameNoScope.put(importRow.getExpTitle(), trialPio); + trialByNameNoScope.put(importRow.getExpTitle(), trialPio); } } return trialPio; diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 9420f18f0..6bc203d64 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -635,7 +635,7 @@ public void verifyFailureNewOuExistingEnv(boolean commit) { JsonObject result = JsonParser.parseString(upload.body()).getAsJsonObject().getAsJsonObject("result"); assertEquals(422, result.getAsJsonObject("progress").get("statuscode").getAsInt(), "Returned data: " + result); - assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experimental entities are missing ObsUnitIDs")); + assertTrue(result.getAsJsonObject("progress").get("message").getAsString().startsWith("Experiment Title already exists")); } @Test From 7ea6dc49757de1af0eea413829d9c5be627f7d08 Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 3 Apr 2024 15:44:55 -0400 Subject: [PATCH 213/220] [BI-2046] remove redundant part of error message --- .../importer/services/processors/ExperimentProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index c8757c8eb..e584f7ca4 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -84,7 +84,7 @@ public class ExperimentProcessor implements Processor { private static final String NAME = "Experiment"; - private static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs. Import cannot proceed"; + private static final String MISSING_OBS_UNIT_ID_ERROR = "Experimental entities are missing ObsUnitIDs"; private static final String PREEXISTING_EXPERIMENT_TITLE = "Experiment Title already exists"; private static final String MULTIPLE_EXP_TITLES = "File contains more than one Experiment Title"; private static final String MIDNIGHT = "T00:00:00-00:00"; From 8bc7bc56b88f1b41be99391e544bbd936a35485b Mon Sep 17 00:00:00 2001 From: dmeidlin <14339308+dmeidlin@users.noreply.github.com> Date: Wed, 3 Apr 2024 17:37:59 -0400 Subject: [PATCH 214/220] [BI-2046] add comment --- .../importer/services/processors/ExperimentProcessor.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java index e584f7ca4..cbc7299d1 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/ExperimentProcessor.java @@ -1346,6 +1346,8 @@ private PendingImportObject fetchOrCreateTrialPIO( PendingImportObject envPio; trialPio = trialByNameNoScope.get(importRow.getExpTitle()); envPio = studyByNameNoScope.get(importRow.getEnv()); + + // creating new units for existing experiments and environments is not possible if (trialPio!=null && ImportObjectState.EXISTING==trialPio.getState() && (StringUtils.isBlank( importRow.getObsUnitID() )) && (envPio!=null && ImportObjectState.EXISTING==envPio.getState() ) ){ throw new UnprocessableEntityException(PREEXISTING_EXPERIMENT_TITLE); @@ -2226,7 +2228,9 @@ private void processAndCacheTrial( /** * This function collates unique ObsUnitID values from a list of BrAPIImport objects. - * It iterates through the list and adds non-blank ObsUnitID values to a Set. It also checks for any repeated ObsUnitIDs. + * It iterates through the list and adds non-blank ObsUnitID values to a Set. + * It also checks for any repeated ObsUnitIDs. The instance variables hasAllReferenceUnitIds and + * hasNoReferenceUnitIds are updated. * * @param importRows a List of BrAPIImport objects containing ExperimentObservation data * @return a Set of unique ObsUnitID strings From bec8d86c9f4d26a9333210924d578c99909e681e Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Fri, 5 Apr 2024 20:12:50 +0000 Subject: [PATCH 215/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 55429bbd0..b24ac1f50 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+699 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/34c257df6a7e2c52672f8e10023dbd7dd5bcddf0 +version=v0.9.0+701 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/0db3d2fa3ff9f300f6434615ba7edc2dc79d03e0 From b262e94165ce5e9887d056ec2a908b239457a039 Mon Sep 17 00:00:00 2001 From: David Randolph Phillips Date: Wed, 10 Apr 2024 12:39:01 -0400 Subject: [PATCH 216/220] [BI-2017] changed 'Name' to 'Germplasm Name' in import file --- .../v2/services/BrAPIGermplasmService.java | 2 +- .../germplasm/GermplasmFileColumns.java | 2 +- .../V1.20.0__germplasm_change_name_field.sql | 21 +++++++++++++++++++ .../importer/GermplasmFileImportTest.java | 10 ++++----- .../files/fileutil/file_all_null_rows.csv | 2 +- .../files/fileutil/file_some_null_rows.csv | 2 +- .../germplasm_import/bad_breeding_methods.csv | 2 +- .../circular_parent_dependencies.csv | 2 +- .../germplasm_import/duplicate_db_names.csv | 2 +- .../duplicate_entry_numbers.csv | 2 +- .../empty_required_fields.csv | 2 +- .../empty_values_required_fields.csv | 2 +- .../female_dbid_not_exist.csv | 2 +- .../female_entry_number_not_exist.csv | 2 +- .../files/germplasm_import/full_import.csv | 2 +- .../germplasm_column_casing.csv | 2 +- .../male_dbid_no_female_success.csv | 2 +- .../germplasm_import/male_dbid_not_exist.csv | 2 +- .../male_entry_number_not_exist.csv | 2 +- .../minimal_germplasm_import.csv | 2 +- .../missing_optional_header.csv | 2 +- .../missing_required_header.csv | 2 +- .../no_female_parent_blank_pedigree.csv | 2 +- .../nonnumerical_entry_numbers.csv | 2 +- .../self_ref_parent_dependencies.csv | 2 +- .../germplasm_import/some_entry_numbers.csv | 2 +- 26 files changed, 50 insertions(+), 29 deletions(-) create mode 100644 src/main/resources/db/migration/V1.20.0__germplasm_change_name_field.sql diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java index b64a88c23..74fc8017d 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPIGermplasmService.java @@ -78,7 +78,7 @@ public List> processListData(List germplasm, for (BrAPIGermplasm germplasmEntry: germplasm) { HashMap row = new HashMap<>(); row.put("GID", Integer.valueOf(germplasmEntry.getAccessionNumber())); - row.put("Name", germplasmEntry.getGermplasmName()); + row.put("Germplasm Name", germplasmEntry.getGermplasmName()); row.put("Breeding Method", germplasmEntry.getAdditionalInfo().get(BrAPIAdditionalInfoFields.GERMPLASM_BREEDING_METHOD).getAsString()); String source = germplasmEntry.getSeedSource(); row.put("Source", source); diff --git a/src/main/java/org/breedinginsight/services/parsers/germplasm/GermplasmFileColumns.java b/src/main/java/org/breedinginsight/services/parsers/germplasm/GermplasmFileColumns.java index 632b1f42d..613cfdf6a 100644 --- a/src/main/java/org/breedinginsight/services/parsers/germplasm/GermplasmFileColumns.java +++ b/src/main/java/org/breedinginsight/services/parsers/germplasm/GermplasmFileColumns.java @@ -25,7 +25,7 @@ public enum GermplasmFileColumns { GID("GID", Column.ColumnDataType.INTEGER), - NAME("Name", Column.ColumnDataType.STRING), + NAME("Germplasm Name", Column.ColumnDataType.STRING), BREEDING_METHOD("Breeding Method", Column.ColumnDataType.STRING), SOURCE("Source", Column.ColumnDataType.STRING), FEMALE_PARENT_GID("Female Parent GID", Column.ColumnDataType.INTEGER), diff --git a/src/main/resources/db/migration/V1.20.0__germplasm_change_name_field.sql b/src/main/resources/db/migration/V1.20.0__germplasm_change_name_field.sql new file mode 100644 index 000000000..f878f934b --- /dev/null +++ b/src/main/resources/db/migration/V1.20.0__germplasm_change_name_field.sql @@ -0,0 +1,21 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +update importer_mapping +set mapping = '[{"id": "f8d43c7e-a618-4c16-8829-3085f7202a67", "mapping": [{"id": "f384837e-ad8d-4dbe-b54e-87b57070bed1", "value": {"fileFieldName": "Germplasm Name"}, "objectId": "germplasmName"}, {"id": "39628d14-458b-429b-8e66-bb48e0445a83", "value": {"fileFieldName": "Breeding Method"}, "objectId": "breedingMethod"}, {"id": "f1ba63e1-f5e4-433f-a53e-1c2f3e2fa71f", "value": {"fileFieldName": "Source"}, "objectId": "germplasmSource"}, {"id": "f5892565-f888-4596-be82-ab8eeabf37ce", "value": {"fileFieldName": "External UID"}, "objectId": "externalUID"}, {"id": "65507e5d-2d66-4595-8763-e772fe25c870", "value": {"fileFieldName": "Entry No"}, "objectId": "entryNo"}, {"id": "3eae24c1-ca4a-48a2-96d0-3cf4630acd3a", "value": {"fileFieldName": "Female Parent GID"}, "objectId": "femaleParentDBID"}, {"id": "2dbd7262-93a1-44b0-86b7-f5fca290b965", "value": {"fileFieldName": "Male Parent GID"}, "objectId": "maleParentDBID"}, {"id": "6f7f1539-6e8f-4ede-b7d3-3423cc63abec", "value": {"fileFieldName": "Female Parent Entry No"}, "objectId": "femaleParentEntryNo"}, {"id": "25fe9954-bca7-42f1-818a-5f71e242fa1f", "value": {"fileFieldName": "Male Parent Entry No"}, "objectId": "maleParentEntryNo"}, {"id": "b910adfe-a474-47a0-8410-514578898436", "value": {"fileFieldName": "Synonyms"}, "objectId": "synonyms"}, {"id": "15836d5f-8194-40a8-a771-114eaae31eb4", "objectId": "germplasmPUI"}, {"id": "675b6af8-5a17-4146-a503-2e4e1a65d5fa", "objectId": "acquisitionDate"}, {"id": "69a3bd3c-cebc-435c-acdd-0be62dda25ed", "objectId": "countryOfOrigin"}, {"id": "8ab25267-20f2-450e-89ca-21634ff8fadb", "objectId": "collection"}, {"id": "ce1701e2-2f61-4250-8595-9536e3f5ddcf", "objectId": "AdditionalInfo"}, {"id": "3470e9df-a028-45b7-943f-198bc62b6dbe", "objectId": "ExternalReference"}], "objectId": "Germplasm"}]', + file = '[{"Germplasm Name": "BITest Pinot Noir", "Source": "Unknown", "Entry No": "1", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": "", "Synonyms": "test1;test2"}, {"Name": "BITest Pixie", "Source": "Winters Nursery", "Entry No": "2", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": "1", "Synonyms": "test1;test2"}, {"Name": "BITest BI002", "Source": "Ithaca Nursery", "Entry No": "7", "External UID": "12231321", "Breeding Method": "Biparental cross", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "", "Female Parent Entry No": "2", "Synonyms": "test1;test2"}, {"Name": "BITest BI003", "Source": "Ithaca Nursery", "Entry No": "8", "External UID": "", "Breeding Method": "BPC", "Male Parent GID": "", "Female Parent GID": "", "Male Parent Entry No": "7", "Female Parent Entry No": "2", "Synonyms": "test1;test2"}, {"Name": "BITest Pinot Noir", "Source": "Unknown", "Entry No": "3", "External UID": "", "Breeding Method": "UMM", "Male Parent GID": "", "Female Parent GID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": "", "Synonyms": "test1;test2"}, {"Name": "BITest Pixie", "Source": "Winters Nursery", "Entry No": "4", "External UID": "", "Breeding Method": "CFV", "Male Parent GID": "640e8b58-1b1c-44a6-91a6-85b2b773376b", "Female Parent GID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": "", "Synonyms": "test1;test2"}, {"Name": "BITest BI002", "Source": "Ithaca Nursery", "Entry No": "5", "External UID": "12231321", "Breeding Method": "Biparental cross", "Male Parent GID": "", "Female Parent GID": "5fb01ea5-c212-4cfa-84e4-6d190379341f", "Male Parent Entry No": "", "Female Parent Entry No": "2", "Synonyms": "test1;test2"}]' +where name = 'GermplasmTemplateMap'; \ No newline at end of file diff --git a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java index 0110f0d3a..35fe42946 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/GermplasmFileImportTest.java @@ -167,7 +167,7 @@ public void fullImportPreviewSuccess() { // Check preview specific items // Germplasm name (display name) - assertEquals(fileData.getString(i, "Name"), germplasm.get("germplasmName").getAsString()); + assertEquals(fileData.getString(i, "Germplasm Name"), germplasm.get("germplasmName").getAsString()); JsonObject additionalInfo = germplasm.getAsJsonObject("additionalInfo"); // Created Date (not present) assertFalse(additionalInfo.has(BrAPIAdditionalInfoFields.CREATED_DATE), "createdDate is present, but should not be"); @@ -229,7 +229,7 @@ public void fullImportCommitSuccess() { // Check commit specific items // Germplasm name (display name) - String expectedGermplasmName = String.format("%s [%s-%s]", fileData.getString(i, "Name"), validProgram.getKey(), germplasm.get("accessionNumber").getAsString()); + String expectedGermplasmName = String.format("%s [%s-%s]", fileData.getString(i, "Germplasm Name"), validProgram.getKey(), germplasm.get("accessionNumber").getAsString()); assertEquals(expectedGermplasmName, germplasm.get("germplasmName").getAsString()); // Created Date JsonObject additionalInfo = germplasm.getAsJsonObject("additionalInfo"); @@ -531,8 +531,8 @@ public void emptyRequiredFieldsError() { assertEquals(1, errors.size(), "Not enough errors were returned"); JsonObject error = errors.get(0).getAsJsonObject(); assertEquals(422, error.get("httpStatusCode").getAsInt(), "Incorrect http status code"); - assertEquals("Name", error.get("field").getAsString(), "Incorrect field name"); - assertEquals(importService.getBlankRequiredFieldMsg("Name"), error.get("errorMessage").getAsString(), "Incorrect error message"); + assertEquals("Germplasm Name", error.get("field").getAsString(), "Incorrect field name"); + assertEquals(importService.getBlankRequiredFieldMsg("Germplasm Name"), error.get("errorMessage").getAsString(), "Incorrect error message"); JsonObject rowError2 = rowErrors.get(1).getAsJsonObject(); JsonArray errors2 = rowError2.getAsJsonArray("errors"); @@ -648,7 +648,7 @@ public void selfReferenceParentError() { public void checkBasicResponse(JsonObject germplasm, Table fileData, Integer i) { // Germplasm display name - assertEquals(fileData.getString(i, "Name"), germplasm.get("defaultDisplayName").getAsString(), "Wrong display name"); + assertEquals(fileData.getString(i, "Germplasm Name"), germplasm.get("defaultDisplayName").getAsString(), "Wrong display name"); // Entry Number gson.fromJson(germplasm.getAsJsonObject("additionalInfo").getAsJsonObject("listEntryNumbers"), Map.class).forEach((listId, entryNumber) -> assertEquals(fileData.getString(i, "Entry No"), entryNumber, "Wrong entry number")); JsonObject additionalInfo = germplasm.getAsJsonObject("additionalInfo"); diff --git a/src/test/resources/files/fileutil/file_all_null_rows.csv b/src/test/resources/files/fileutil/file_all_null_rows.csv index 5eb2cc5e6..30d94f9c3 100644 --- a/src/test/resources/files/fileutil/file_all_null_rows.csv +++ b/src/test/resources/files/fileutil/file_all_null_rows.csv @@ -1,4 +1,4 @@ -Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID +Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID Buzz-1,BPC,Middle Earth,1,16,,,, Buzz-2,CFV,Middle Earth,10,,,,, ,,,,,,,, diff --git a/src/test/resources/files/fileutil/file_some_null_rows.csv b/src/test/resources/files/fileutil/file_some_null_rows.csv index 8be693b39..a7d9a67c5 100644 --- a/src/test/resources/files/fileutil/file_some_null_rows.csv +++ b/src/test/resources/files/fileutil/file_some_null_rows.csv @@ -1,4 +1,4 @@ -Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID +Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID Buzz-1,BPC,Middle Earth,1,16,,,, Buzz-2,CFV,Middle Earth,10,,,,, yeah,,,,,,,, diff --git a/src/test/resources/files/germplasm_import/bad_breeding_methods.csv b/src/test/resources/files/germplasm_import/bad_breeding_methods.csv index 9faa60aee..2225ed58b 100644 --- a/src/test/resources/files/germplasm_import/bad_breeding_methods.csv +++ b/src/test/resources/files/germplasm_import/bad_breeding_methods.csv @@ -1,4 +1,4 @@ -GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Germplasm 1,ANE,Wild,,,,,,, ,Germplasm 2,BAD,Wild,,,,,,, ,Germplasm 3,BAD1,Wild,,,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/circular_parent_dependencies.csv b/src/test/resources/files/germplasm_import/circular_parent_dependencies.csv index eda6c2c49..144f5b045 100644 --- a/src/test/resources/files/germplasm_import/circular_parent_dependencies.csv +++ b/src/test/resources/files/germplasm_import/circular_parent_dependencies.csv @@ -1,4 +1,4 @@ -GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Germplasm 1,BCR,Wild,,,1,2,3,1234, ,Germplasm 2,BCR,Wild,,,2,1,2,5678, ,Germplasm 3,BCR,Wild,,,3,2,3,9123, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/duplicate_db_names.csv b/src/test/resources/files/germplasm_import/duplicate_db_names.csv index 0bdc5c5fe..06eba2526 100644 --- a/src/test/resources/files/germplasm_import/duplicate_db_names.csv +++ b/src/test/resources/files/germplasm_import/duplicate_db_names.csv @@ -1,4 +1,4 @@ -GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Full Germplasm 1,ANE,Wild,,,,,,, ,Full Germplasm 2,ANE,Wild,,,,,,, ,Full Germplasm 3,ANE,Wild,,,,,,, diff --git a/src/test/resources/files/germplasm_import/duplicate_entry_numbers.csv b/src/test/resources/files/germplasm_import/duplicate_entry_numbers.csv index e20f5c845..943876183 100644 --- a/src/test/resources/files/germplasm_import/duplicate_entry_numbers.csv +++ b/src/test/resources/files/germplasm_import/duplicate_entry_numbers.csv @@ -1,4 +1,4 @@ -GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Germplasm 1,BCR,Wild,,,1,,,, ,Germplasm 2,BCR,Wild,,,1,,,, ,Germplasm 3,BCR,Wild,,,3,,,, diff --git a/src/test/resources/files/germplasm_import/empty_required_fields.csv b/src/test/resources/files/germplasm_import/empty_required_fields.csv index 3072ce9b2..eb7aeca5b 100644 --- a/src/test/resources/files/germplasm_import/empty_required_fields.csv +++ b/src/test/resources/files/germplasm_import/empty_required_fields.csv @@ -1,4 +1,4 @@ -GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Germplasm 1,ANE,Wild,,,,,,, ,,ANE,Wild,,,,,,, ,Germplasm 3,ANE,,,,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/empty_values_required_fields.csv b/src/test/resources/files/germplasm_import/empty_values_required_fields.csv index 6af9e43a1..ae72d3717 100644 --- a/src/test/resources/files/germplasm_import/empty_values_required_fields.csv +++ b/src/test/resources/files/germplasm_import/empty_values_required_fields.csv @@ -1,4 +1,4 @@ -GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Germplasm 1,,Wild,,,,,,, ,,,Wild,,,,,,, ,Germplasm 3,,,,,,,,, diff --git a/src/test/resources/files/germplasm_import/female_dbid_not_exist.csv b/src/test/resources/files/germplasm_import/female_dbid_not_exist.csv index 08a931520..359b54694 100644 --- a/src/test/resources/files/germplasm_import/female_dbid_not_exist.csv +++ b/src/test/resources/files/germplasm_import/female_dbid_not_exist.csv @@ -1,4 +1,4 @@ -GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Germplasm 1,BCR,Wild,1000,,,,,, ,Germplasm 2,BCR,Cultivated,1001,,,,,, ,Germplasm 3,BCR,Kinda Wild,1002,,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/female_entry_number_not_exist.csv b/src/test/resources/files/germplasm_import/female_entry_number_not_exist.csv index e8e3f4be4..60e47b782 100644 --- a/src/test/resources/files/germplasm_import/female_entry_number_not_exist.csv +++ b/src/test/resources/files/germplasm_import/female_entry_number_not_exist.csv @@ -1,4 +1,4 @@ -GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Germplasm 1,BCR,Wild,,,,1,,, ,Germplasm 2,BCR,Cultivated,,,,2,,, ,Germplasm 3,BCR,Kinda Wild,,,,3,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/full_import.csv b/src/test/resources/files/germplasm_import/full_import.csv index 9073ac69f..dcc785ac6 100644 --- a/src/test/resources/files/germplasm_import/full_import.csv +++ b/src/test/resources/files/germplasm_import/full_import.csv @@ -1,4 +1,4 @@ -GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Full Germplasm 1,ANE,Wild,1,2,2,,,1234,test1;test2 ,Full Germplasm 2,ANE,Wild,2,,3,,,5678,test3;test2 ,Full Germplasm 3,ANE,Wild,,,4,2,3,9123,test3;test4 \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/germplasm_column_casing.csv b/src/test/resources/files/germplasm_import/germplasm_column_casing.csv index f4c2e226a..c8f42c6d7 100644 --- a/src/test/resources/files/germplasm_import/germplasm_column_casing.csv +++ b/src/test/resources/files/germplasm_import/germplasm_column_casing.csv @@ -1,2 +1,2 @@ -GID,Name,BREEDING METHOD,SOURCE,fEmAlE pArEnT gId,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Germplasm Name,BREEDING METHOD,SOURCE,fEmAlE pArEnT gId,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Germ A,ANE,the shire,,,1,,,ABC-123, diff --git a/src/test/resources/files/germplasm_import/male_dbid_no_female_success.csv b/src/test/resources/files/germplasm_import/male_dbid_no_female_success.csv index 23d9a47d8..8a1bb0020 100644 --- a/src/test/resources/files/germplasm_import/male_dbid_no_female_success.csv +++ b/src/test/resources/files/germplasm_import/male_dbid_no_female_success.csv @@ -1,4 +1,4 @@ -GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Germplasm 1,,Wild,,100,,,,, ,Germplasm 2,,Cultivated,,101,,,,, ,Germplasm 3,,Kinda Wild,,102,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/male_dbid_not_exist.csv b/src/test/resources/files/germplasm_import/male_dbid_not_exist.csv index f01b9a62d..115a3958e 100644 --- a/src/test/resources/files/germplasm_import/male_dbid_not_exist.csv +++ b/src/test/resources/files/germplasm_import/male_dbid_not_exist.csv @@ -1,4 +1,4 @@ -GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Germplasm 1,BCR,Wild,,100,,,,, ,Germplasm 2,BCR,Cultivated,,101,,,,, ,Germplasm 3,BCR,Kinda Wild,,102,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/male_entry_number_not_exist.csv b/src/test/resources/files/germplasm_import/male_entry_number_not_exist.csv index bfa7f9c0d..e1b484c71 100644 --- a/src/test/resources/files/germplasm_import/male_entry_number_not_exist.csv +++ b/src/test/resources/files/germplasm_import/male_entry_number_not_exist.csv @@ -1,4 +1,4 @@ -GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Germplasm 1,BCR,Wild,,,,,1,, ,Germplasm 2,BCR,Cultivated,,,,,2,, ,Germplasm 3,BCR,Kinda Wild,,,,,3,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/minimal_germplasm_import.csv b/src/test/resources/files/germplasm_import/minimal_germplasm_import.csv index acb35b785..7ff102940 100644 --- a/src/test/resources/files/germplasm_import/minimal_germplasm_import.csv +++ b/src/test/resources/files/germplasm_import/minimal_germplasm_import.csv @@ -1,4 +1,4 @@ -GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Germplasm 1,BCR,Wild,,,,,,, ,Germplasm 2,BCR,Cultivated,,,,,,, ,Germplasm 3,BCR,Kinda Wild,,,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/missing_optional_header.csv b/src/test/resources/files/germplasm_import/missing_optional_header.csv index 0b1844cbb..d9f0feafc 100644 --- a/src/test/resources/files/germplasm_import/missing_optional_header.csv +++ b/src/test/resources/files/germplasm_import/missing_optional_header.csv @@ -1,4 +1,4 @@ -GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Germplasm 1,ANE,Wild,,,,,, ,Germplasm 2,ANE,Wild,,,,,, ,Germplasm 3,ANE,Wild,,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/missing_required_header.csv b/src/test/resources/files/germplasm_import/missing_required_header.csv index 584bfac1a..a3a22a73f 100644 --- a/src/test/resources/files/germplasm_import/missing_required_header.csv +++ b/src/test/resources/files/germplasm_import/missing_required_header.csv @@ -1,4 +1,4 @@ -GID,Name,Breeding Method,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Germplasm Name,Breeding Method,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Germplasm 1,,,,,,,, ,Germplasm 2,,,,,,,, ,Germplasm 3,,,,,,,, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/no_female_parent_blank_pedigree.csv b/src/test/resources/files/germplasm_import/no_female_parent_blank_pedigree.csv index 8d1a4b461..a5282bd52 100644 --- a/src/test/resources/files/germplasm_import/no_female_parent_blank_pedigree.csv +++ b/src/test/resources/files/germplasm_import/no_female_parent_blank_pedigree.csv @@ -1,4 +1,4 @@ -GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Germplasm 1,BCR,Wild,0,2,,,,1234, ,Germplasm 2,BCR,Wild,0,3,,,,5678, ,Germplasm 3,BCR,Wild,0,3,,,,9123, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/nonnumerical_entry_numbers.csv b/src/test/resources/files/germplasm_import/nonnumerical_entry_numbers.csv index fbca1ed3d..a8808d479 100644 --- a/src/test/resources/files/germplasm_import/nonnumerical_entry_numbers.csv +++ b/src/test/resources/files/germplasm_import/nonnumerical_entry_numbers.csv @@ -1,4 +1,4 @@ -GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Germplasm 1,,Wild,,,a,,,, ,Germplasm 2,,Wild,,,b,,,, ,Germplasm 3,,Wild,,,3,,,, diff --git a/src/test/resources/files/germplasm_import/self_ref_parent_dependencies.csv b/src/test/resources/files/germplasm_import/self_ref_parent_dependencies.csv index 16144b020..34750b4ed 100644 --- a/src/test/resources/files/germplasm_import/self_ref_parent_dependencies.csv +++ b/src/test/resources/files/germplasm_import/self_ref_parent_dependencies.csv @@ -1,4 +1,4 @@ -GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Germplasm 1,BCR,Wild,,,1,1,1,1234, ,Germplasm 2,BCR,Wild,,,2,2,2,5678, ,Germplasm 3,BCR,Wild,,,3,3,3,9123, \ No newline at end of file diff --git a/src/test/resources/files/germplasm_import/some_entry_numbers.csv b/src/test/resources/files/germplasm_import/some_entry_numbers.csv index b86064c06..7edf4e979 100644 --- a/src/test/resources/files/germplasm_import/some_entry_numbers.csv +++ b/src/test/resources/files/germplasm_import/some_entry_numbers.csv @@ -1,4 +1,4 @@ -GID,Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms +GID,Germplasm Name,Breeding Method,Source,Female Parent GID,Male Parent GID,Entry No,Female Parent Entry No,Male Parent Entry No,External UID,Synonyms ,Germplasm 1,BCR,Wild,,,1,,,, ,Germplasm 2,BCR,Wild,,,,,,, ,Germplasm 3,BCR,Wild,,,3,,,, From 57f649efd1bb7d79b7e80df51ba15623c1b45a35 Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 10 Apr 2024 17:41:16 +0000 Subject: [PATCH 217/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index b24ac1f50..137b3c6ae 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+701 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/0db3d2fa3ff9f300f6434615ba7edc2dc79d03e0 +version=v0.9.0+705 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/202483703288ef760d1285ea90c5f66d8abfeec3 From 482c8a6979f688ddf4188ee0ac456da5af554b72 Mon Sep 17 00:00:00 2001 From: mlm483 <128052931+mlm483@users.noreply.github.com> Date: Wed, 10 Apr 2024 14:14:52 -0400 Subject: [PATCH 218/220] [BI-1680] - added migration to remove method --- .../V1.21.0__remove_breeding_method.sql | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 src/main/resources/db/migration/V1.21.0__remove_breeding_method.sql diff --git a/src/main/resources/db/migration/V1.21.0__remove_breeding_method.sql b/src/main/resources/db/migration/V1.21.0__remove_breeding_method.sql new file mode 100644 index 000000000..16b566d63 --- /dev/null +++ b/src/main/resources/db/migration/V1.21.0__remove_breeding_method.sql @@ -0,0 +1,27 @@ +/* + * See the NOTICE file distributed with this work for additional information + * regarding copyright ownership. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +DO $$ + + BEGIN + -- Delete any rows with FK. + DELETE FROM program_enabled_breeding_methods + WHERE breeding_method_id = (SELECT id FROM breeding_method WHERE abbreviation = 'SOC' AND name = 'Somoclone'); + -- Delete breeding method. + DELETE FROM breeding_method + WHERE abbreviation = 'SOC' AND name = 'Somoclone'; + END $$; \ No newline at end of file From 4f4173ead8ab0cc0de6bb9c21096f5fd6cb4f47d Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 10 Apr 2024 19:19:01 +0000 Subject: [PATCH 219/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 137b3c6ae..d1d81549b 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+705 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/202483703288ef760d1285ea90c5f66d8abfeec3 +version=v0.9.0+707 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/253cc007b211fcc889d0396711256ea39bc37cc2 From 9350f39d64ceedb38a76e8fe19843c7f71f4a8bb Mon Sep 17 00:00:00 2001 From: rob-ouser-bi Date: Wed, 10 Apr 2024 21:12:59 +0000 Subject: [PATCH 220/220] [autocommit] bumping build number --- src/main/resources/version.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/resources/version.properties b/src/main/resources/version.properties index 0b68f9692..c984b914d 100644 --- a/src/main/resources/version.properties +++ b/src/main/resources/version.properties @@ -14,5 +14,5 @@ # limitations under the License. # -version=v0.9.0+707 -versionInfo=https://github.com/Breeding-Insight/bi-api/commit/253cc007b211fcc889d0396711256ea39bc37cc2 \ No newline at end of file +version=v0.9.0+709 +versionInfo=https://github.com/Breeding-Insight/bi-api/commit/77e813a97f0866853a6275291f6100110f904cbb \ No newline at end of file