From e6ddd59276f208b9d379e58b8e88aa6961e1b4b8 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Tue, 4 Jun 2024 11:52:58 -0500 Subject: [PATCH 01/16] add additional field to indicate data type --- .../service/dfr3/controllers/FragilityController.java | 7 ++++++- .../incore/service/dfr3/controllers/MappingController.java | 5 +++++ .../incore/service/dfr3/controllers/RepairController.java | 5 +++++ .../service/dfr3/controllers/RestorationController.java | 5 +++++ .../ncsa/incore/service/dfr3/daos/MongoDBFragilityDAO.java | 1 + .../ncsa/incore/service/dfr3/daos/MongoDBMappingDAO.java | 6 ++++-- .../ncsa/incore/service/dfr3/daos/MongoDBRepairDAO.java | 1 + .../incore/service/dfr3/daos/MongoDBRestorationDAO.java | 1 + .../illinois/ncsa/incore/service/dfr3/models/DFR3Set.java | 3 +++ .../ncsa/incore/service/dfr3/models/MappingSet.java | 3 +++ .../dfr3-service/src/test/java/mocks/MockFragilityDAO.java | 1 + .../dfr3-service/src/test/resources/building_mapping.json | 1 + server/dfr3-service/src/test/resources/fragility.json | 4 ++++ .../dfr3-service/src/test/resources/fragility_request.json | 1 + 14 files changed, 41 insertions(+), 3 deletions(-) diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java index 24a613766..aa004ef24 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java @@ -114,7 +114,8 @@ public FragilityController( @Operation(tags = "Gets list of fragilities", summary = "Apply filters to get the desired set of fragilities") public List getFragilities(@Parameter(name = "demand type filter", example = "PGA") @QueryParam("demand") String demandType, @Parameter(name = "hazard type filter", example = "earthquake") @QueryParam("hazard") String hazardType, - @Parameter(name = "Inventory type", example = "building") @QueryParam("inventory") String inventoryType, + @Parameter(name = "Inventory type filter", example = "building") @QueryParam("inventory") String inventoryType, + @Parameter(name = "Data type filter", example = "ergo:buildingInventoryVer7") @QueryParam("dataType") String dataType, @Parameter(name = "not implemented", hidden = true) @QueryParam("author") String author, @Parameter(name = "Legacy fragility Id from v1") @QueryParam("legacy_id") String legacyId, @Parameter(name = "Fragility creator's username") @QueryParam("creator") String creator, @@ -139,6 +140,10 @@ public List getFragilities(@Parameter(name = "demand type filter", queryMap.put("inventoryType", inventoryType); } + if (dataType != null){ + queryMap.put("dataType", dataType); + } + if (creator != null) { queryMap.put("creator", creator); } diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java index d356abeaa..fadc2a4c2 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java @@ -87,6 +87,7 @@ public MappingController( @Operation(tags = "Gets list of all inventory mappings", summary = "Apply filters to get the desired set of mappings") public List getMappings(@Parameter(name= "hazard type filter", example = "earthquake") @QueryParam("hazard") String hazardType, @Parameter(name = "Inventory type", example = "building") @QueryParam("inventory") String inventoryType, + @Parameter(name = "Data type", example = "ergo:buildingInventoryVer7") @QueryParam("dataType") String dataType, @Parameter(name = "DFR3 Mapping type", example = "fragility, restoration, repair") @QueryParam( "mappingType") String mappingType, @Parameter(name = "Creator's username") @QueryParam("creator") String creator, @@ -103,6 +104,10 @@ public List getMappings(@Parameter(name= "hazard type filter", exam queryMap.put("inventoryType", inventoryType); } + if (dataType != null) { + queryMap.put("dataType", dataType); + } + if (creator != null) { queryMap.put("creator", creator); } diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RepairController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RepairController.java index eb7fe2988..fc17075b4 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RepairController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RepairController.java @@ -104,6 +104,7 @@ public RepairController( @Operation(tags = "Gets list of repairs", summary = "Apply filters to get the desired set of repairs") public List getRepairs(@Parameter(name = "hazard type filter", example = "earthquake") @QueryParam("hazard") String hazardType, @Parameter(name = "Inventory type", example = "building") @QueryParam("inventory") String inventoryType, + @Parameter(name = "Data type filter", example = "ergo:buildingInventoryVer7") @QueryParam("dataType") String dataType, @Parameter(name = "Repair creator's username") @QueryParam("creator") String creator, @Parameter(name = "Name of space") @DefaultValue("") @QueryParam("space") String spaceName, @Parameter(name = "Skip the first n results") @QueryParam("skip") int offset, @@ -118,6 +119,10 @@ public List getRepairs(@Parameter(name = "hazard type filter", examp queryMap.put("inventoryType", inventoryType); } + if (dataType != null){ + queryMap.put("dataType", dataType); + } + if (creator != null) { queryMap.put("creator", creator); } diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RestorationController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RestorationController.java index 7c42dbbf5..e290f3685 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RestorationController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RestorationController.java @@ -103,6 +103,7 @@ public RestorationController( @Operation(tags = "Gets list of restorations", description = "Apply filters to get the desired set of restorations") public List getRestorations(@Parameter(name = "hazard type filter", example = "earthquake") @QueryParam("hazard") String hazardType, @Parameter(name = "Inventory type", example = "building") @QueryParam("inventory") String inventoryType, + @Parameter(name = "Data type filter", example = "ergo:buildingInventoryVer7") @QueryParam("dataType") String dataType, @Parameter(name = "Restoration creator's username") @QueryParam("creator") String creator, @Parameter(name = "Name of space") @DefaultValue("") @QueryParam("space") String spaceName, @Parameter(name = "Skip the first n results") @QueryParam("skip") int offset, @@ -118,6 +119,10 @@ public List getRestorations(@Parameter(name = "hazard type filt queryMap.put("inventoryType", inventoryType); } + if (dataType != null){ + queryMap.put("dataType", dataType); + } + if (creator != null) { queryMap.put("creator", creator); } diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBFragilityDAO.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBFragilityDAO.java index 7391c74b8..f4d12eefa 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBFragilityDAO.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBFragilityDAO.java @@ -85,6 +85,7 @@ public List searchFragilities(String text) { Filters.regex("legacyId").pattern(text).caseInsensitive(), Filters.regex("hazardType").pattern(text).caseInsensitive(), Filters.regex("inventoryType").pattern(text).caseInsensitive(), + Filters.regex("dataType").pattern(text).caseInsensitive(), Filters.regex("description").pattern(text).caseInsensitive(), Filters.regex("authors").pattern(text).caseInsensitive())); List sets = query.iterator().toList(); diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBMappingDAO.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBMappingDAO.java index b67c21f62..fe2ff514c 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBMappingDAO.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBMappingDAO.java @@ -95,12 +95,14 @@ public List searchMappings(String text, String mappingType) { .filter(Filters.or( Filters.regex("name").pattern(text).caseInsensitive(), Filters.regex("hazardType").pattern(text).caseInsensitive(), - Filters.regex("inventoryType").pattern(text).caseInsensitive())); + Filters.regex("inventoryType").pattern(text).caseInsensitive(), + Filters.regex("dataType").pattern(text).caseInsensitive())); } else { query.filter(Filters.or( Filters.regex("name").pattern(text).caseInsensitive(), Filters.regex("hazardType").pattern(text).caseInsensitive(), - Filters.regex("inventoryType").pattern(text).caseInsensitive())); + Filters.regex("inventoryType").pattern(text).caseInsensitive(), + Filters.regex("dataType").pattern(text).caseInsensitive())); } List sets = query.iterator().toList(); diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBRepairDAO.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBRepairDAO.java index afc1ad819..f37a46026 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBRepairDAO.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBRepairDAO.java @@ -85,6 +85,7 @@ public List searchRepairs(String text) { Filters.regex("legacyId").pattern(text).caseInsensitive(), Filters.regex("hazardType").pattern(text).caseInsensitive(), Filters.regex("inventoryType").pattern(text).caseInsensitive(), + Filters.regex("dataType").pattern(text).caseInsensitive(), Filters.regex("description").pattern(text).caseInsensitive(), Filters.regex("authors").pattern(text).caseInsensitive())); List sets = query.iterator().toList(); diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBRestorationDAO.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBRestorationDAO.java index f6bb09f73..e738b9d7c 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBRestorationDAO.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBRestorationDAO.java @@ -86,6 +86,7 @@ public List searchRestorations(String text) { Filters.regex("legacyId").pattern(text).caseInsensitive(), Filters.regex("hazardType").pattern(text).caseInsensitive(), Filters.regex("inventoryType").pattern(text).caseInsensitive(), + Filters.regex("dataType").pattern(text).caseInsensitive(), Filters.regex("description").pattern(text).caseInsensitive(), Filters.regex("authors").pattern(text).caseInsensitive())); List sets = query.iterator().toList(); diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/models/DFR3Set.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/models/DFR3Set.java index e8a770eae..d34a04a01 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/models/DFR3Set.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/models/DFR3Set.java @@ -31,6 +31,7 @@ public abstract class DFR3Set { protected String hazardType; protected String inventoryType; + protected String dataType; protected String creator; protected String owner; @@ -79,6 +80,8 @@ public String getInventoryType() { return inventoryType; } + public String getDataType() { return dataType; } + public String getCreator() { return creator; } diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/models/MappingSet.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/models/MappingSet.java index 6ab40b35a..18c3bb7e1 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/models/MappingSet.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/models/MappingSet.java @@ -31,6 +31,7 @@ public class MappingSet { private String name; private String hazardType; private String inventoryType; + private String dataType; @JsonProperty("mappingType") private String mappingType; @@ -63,6 +64,8 @@ public String getInventoryType() { return inventoryType; } + public String getDataType() { return dataType; } + public List getMappings() { return mappings; } diff --git a/server/dfr3-service/src/test/java/mocks/MockFragilityDAO.java b/server/dfr3-service/src/test/java/mocks/MockFragilityDAO.java index 4a8197062..7890e9dab 100644 --- a/server/dfr3-service/src/test/java/mocks/MockFragilityDAO.java +++ b/server/dfr3-service/src/test/java/mocks/MockFragilityDAO.java @@ -117,6 +117,7 @@ public List searchFragilities(String text) { Filters.regex("legacyId").pattern(text).caseInsensitive(), Filters.regex("hazardType").pattern(text).caseInsensitive(), Filters.regex("inventoryType").pattern(text).caseInsensitive(), + Filters.regex("dataType").pattern(text).caseInsensitive(), Filters.regex("description").pattern(text).caseInsensitive(), Filters.regex("authors").pattern(text).caseInsensitive())); List sets = query.iterator(new FindOptions().limit(100)).toList(); diff --git a/server/dfr3-service/src/test/resources/building_mapping.json b/server/dfr3-service/src/test/resources/building_mapping.json index 67c80c648..2725a3943 100644 --- a/server/dfr3-service/src/test/resources/building_mapping.json +++ b/server/dfr3-service/src/test/resources/building_mapping.json @@ -2,6 +2,7 @@ "name": "Default Building Fragility Mapping 1.0", "hazardType": "earthquake", "inventoryType": "building", + "dataType": "ergo:buildingInventoryVer7", "mappings": [ { "legacyEntry": { diff --git a/server/dfr3-service/src/test/resources/fragility.json b/server/dfr3-service/src/test/resources/fragility.json index b03b61061..f1b180625 100644 --- a/server/dfr3-service/src/test/resources/fragility.json +++ b/server/dfr3-service/src/test/resources/fragility.json @@ -4,6 +4,7 @@ "legacyId": "NSDS_PFM_MTB_UL_475_W1_4", "hazardType": "earthquake", "inventoryType": "building", + "dataType": "ergo:buildingInventoryVer7", "authors": [ "Amr S. Elnashai", "Seong-Hoon Jeong", @@ -54,6 +55,7 @@ "legacyId": "STR_PFM_MTB_UL_475_S2_9", "hazardType": "earthquake", "inventoryType": "building", + "dataType": "ergo:buildingInventoryVer7", "authors": [ "Amr S. Elnashai", "Seong-Hoon Jeong", @@ -111,6 +113,7 @@ "resultType": "Limit State", "hazardType": "tsunami", "inventoryType": "electric_facility", + "dataType": "incore:epfVer2", "creator": "cwang138", "demandType": "Vmax", "demandUnits": "ft/s", @@ -209,6 +212,7 @@ "id": "5e99d0b3e05cb9ccbc32d6e6", "hazardType": "earthquake", "inventoryType": "bridge", + "dataType": "ergo:bridgesVer3", "authors": [ "HAZUS" ], diff --git a/server/dfr3-service/src/test/resources/fragility_request.json b/server/dfr3-service/src/test/resources/fragility_request.json index cd1fa94ea..5956686ce 100644 --- a/server/dfr3-service/src/test/resources/fragility_request.json +++ b/server/dfr3-service/src/test/resources/fragility_request.json @@ -2,6 +2,7 @@ "legacyId": "legacyId", "hazardType": "hazardType", "inventoryType": "inventoryType", + "dataType": "incore:dataType", "authors": [ "Chen Wang" ], From 25e98d7ed82229e131c34f6e1846383ae16ecc89 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Wed, 5 Jun 2024 10:22:45 -0500 Subject: [PATCH 02/16] put common parameters into constant; bypass certain fields --- server/dfr3-service/build.gradle | 7 ++- .../dfr3/controllers/FragilityController.java | 28 ++++++++++ .../incore/service/dfr3/utils/CommonUtil.java | 27 +++++++++ .../service/dfr3/utils/ServiceUtil.java | 55 +++++++++++++++++++ .../incore/common/SemanticsConstants.java | 14 +++++ 5 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/utils/ServiceUtil.java create mode 100644 server/incore-common/src/main/java/edu/illinois/ncsa/incore/common/SemanticsConstants.java diff --git a/server/dfr3-service/build.gradle b/server/dfr3-service/build.gradle index 445d8874c..51a2a6417 100644 --- a/server/dfr3-service/build.gradle +++ b/server/dfr3-service/build.gradle @@ -3,6 +3,7 @@ gretty { contextPath '/dfr3/' } -// dependencies { -// implementation "org.dom4j:dom4j:1.6.1" -// } +dependencies { + implementation group: 'org.apache.httpcomponents', name: 'httpmime', version: '4.5.5' + implementation group: 'org.apache.commons', name: 'commons-collections4', version: '4.0' +} diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java index aa004ef24..e2df99454 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java @@ -11,6 +11,7 @@ package edu.illinois.ncsa.incore.service.dfr3.controllers; import edu.illinois.ncsa.incore.common.AllocationConstants; +import edu.illinois.ncsa.incore.common.SemanticsConstants; import edu.illinois.ncsa.incore.common.auth.Authorizer; import edu.illinois.ncsa.incore.common.auth.IAuthorizer; import edu.illinois.ncsa.incore.common.auth.Privileges; @@ -28,6 +29,8 @@ import edu.illinois.ncsa.incore.service.dfr3.daos.IMappingDAO; import edu.illinois.ncsa.incore.service.dfr3.models.FragilitySet; +import edu.illinois.ncsa.incore.service.dfr3.utils.CommonUtil; +import edu.illinois.ncsa.incore.service.dfr3.utils.ServiceUtil; import io.swagger.v3.oas.annotations.OpenAPIDefinition; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -48,6 +51,8 @@ import jakarta.ws.rs.*; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; + +import java.io.IOException; import java.util.*; import java.util.stream.Collectors; @@ -84,6 +89,7 @@ public class FragilityController { private final String username; private final List groups; + private final String userGroups; @Inject IAuthorizer authorizer; @@ -106,6 +112,7 @@ public FragilityController( @Parameter(name = "User groups.", required = false) @HeaderParam("x-auth-usergroup") String userGroups ) { this.username = UserInfoUtils.getUsername(userInfo); + this.userGroups = userGroups; this.groups = UserGroupUtils.getUserGroups(userGroups); } @@ -238,6 +245,27 @@ public FragilitySet uploadFragilitySet(@Parameter(name = "json representing the } } } + String dataType = fragilitySet.getDataType(); + if (dataType == null) { + throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataType is a required field."); + } + try { + String SemanticsDefinition = ServiceUtil.getJsonFromSemanticsEndpoint(dataType, username, userGroups); + List columns = CommonUtil.getColumnNames(SemanticsDefinition); + + // check if curve parameters matches the dataType columns + fragilitySet.getCurveParameters().forEach((params) -> { + // only check curve parameter does not belong to a part of the demand type + if (!demandTypes.contains(params.fullName) && !demandUnits.contains(params.name) && !SemanticsConstants.RESERVED_COLUMNS.contains(params.name)) { + if (!columns.contains(params.name)){ + throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Curve parameter: " + params.name + " not found in the dataType: " + dataType); + } + } + }); + + } catch (IOException e) { + throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Could not check the fragility curve parameter matches the dataType columns."); + } fragilitySet.setCreator(username); fragilitySet.setOwner(username); diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/utils/CommonUtil.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/utils/CommonUtil.java index 85ef92fb6..86d22d8e7 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/utils/CommonUtil.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/utils/CommonUtil.java @@ -1,7 +1,13 @@ package edu.illinois.ncsa.incore.service.dfr3.utils; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import org.json.simple.JSONObject; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + public class CommonUtil { public static JSONObject createUserStatusJson(String creator, String keyDatabase, int numDfr) { JSONObject outJson = new JSONObject(); @@ -11,4 +17,25 @@ public static JSONObject createUserStatusJson(String creator, String keyDatabase return outJson; } + + public static List getColumnNames(String semanticDefinition) throws IOException { + + // Parse the JSON response + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode rootNode = objectMapper.readTree(semanticDefinition); + + // Navigate to "tableSchema" -> "columns" + JsonNode columnsNode = rootNode.path("tableSchema").path("columns"); + + // Extract the "name" field from each column and add to a list + List columnNames = new ArrayList<>(); + if (columnsNode.isArray()) { + for (JsonNode column : columnsNode) { + String columnName = column.path("name").asText(); + columnNames.add(columnName); + } + } + + return columnNames; + } } diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/utils/ServiceUtil.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/utils/ServiceUtil.java new file mode 100644 index 000000000..8d1443120 --- /dev/null +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/utils/ServiceUtil.java @@ -0,0 +1,55 @@ +package edu.illinois.ncsa.incore.service.dfr3.utils; + +import edu.illinois.ncsa.incore.common.SemanticsConstants; +import org.apache.http.HttpResponse; +import org.apache.http.client.HttpClient; +import org.apache.http.client.ResponseHandler; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.BasicResponseHandler; + + +import java.io.IOException; + + +public class ServiceUtil { + + private static final String SEMANTICS_ENDPOINT = "http://localhost:8080/semantics/api/types"; + + public static String getJsonFromSemanticsEndpoint(String name, String username, String userGroups) throws IOException { + + String dataEndpoint = createEndpoint(); + + HttpClientBuilder builder = HttpClientBuilder.create(); + HttpClient httpclient = builder.build(); + + String requestUrl = dataEndpoint + SemanticsConstants.SEMANTICS_ENDPOINT + "/" + name; + + HttpGet httpGet = new HttpGet(requestUrl); + httpGet.setHeader(SemanticsConstants.X_AUTH_USERINFO, "{\"preferred_username\": \"" + username + "\"}"); + httpGet.setHeader(SemanticsConstants.X_AUTH_USERGROUP, userGroups); + httpGet.setHeader("contentType", "application/json"); + httpGet.setHeader("accept", "application/json"); + + HttpResponse response = null; + + response = httpclient.execute(httpGet); + ResponseHandler responseHandler = new BasicResponseHandler(); + + return responseHandler.handleResponse(response); + } + + public static String createEndpoint() { + String endpoint = "http://localhost:8080/"; + String endpointProp = System.getenv("SERVICES_URL"); + if (endpointProp != null && !endpointProp.isEmpty()) { + endpoint = endpointProp; + if (!endpoint.endsWith("/")) { + endpoint += "/"; + } + } + + return endpoint; + } + +} diff --git a/server/incore-common/src/main/java/edu/illinois/ncsa/incore/common/SemanticsConstants.java b/server/incore-common/src/main/java/edu/illinois/ncsa/incore/common/SemanticsConstants.java new file mode 100644 index 000000000..e2e400d28 --- /dev/null +++ b/server/incore-common/src/main/java/edu/illinois/ncsa/incore/common/SemanticsConstants.java @@ -0,0 +1,14 @@ +package edu.illinois.ncsa.incore.common; + +import java.util.Arrays; +import java.util.List; + +public class SemanticsConstants { + + public static final List RESERVED_COLUMNS = Arrays.asList("num_stories", "period"); + public static final String SEMANTICS_ENDPOINT = "semantics/api/types"; + + // Auth headers + public static final String X_AUTH_USERINFO = "x-auth-userinfo"; + public static final String X_AUTH_USERGROUP = "x-auth-usergroup"; +} From e7002e5e97b8a0e6f4395395af208492ea6943e6 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Wed, 5 Jun 2024 10:34:10 -0500 Subject: [PATCH 03/16] refine the logic --- .../dfr3/controllers/FragilityController.java | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java index e2df99454..4785d8ff5 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java @@ -246,27 +246,33 @@ public FragilitySet uploadFragilitySet(@Parameter(name = "json representing the } } String dataType = fragilitySet.getDataType(); + String inventoryType = fragilitySet.getInventoryType(); if (dataType == null) { throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataType is a required field."); } try { - String SemanticsDefinition = ServiceUtil.getJsonFromSemanticsEndpoint(dataType, username, userGroups); - List columns = CommonUtil.getColumnNames(SemanticsDefinition); + String semanticsDefinition = ServiceUtil.getJsonFromSemanticsEndpoint(dataType, username, userGroups); + List columns = CommonUtil.getColumnNames(semanticsDefinition); - // check if curve parameters matches the dataType columns fragilitySet.getCurveParameters().forEach((params) -> { - // only check curve parameter does not belong to a part of the demand type - if (!demandTypes.contains(params.fullName) && !demandUnits.contains(params.name) && !SemanticsConstants.RESERVED_COLUMNS.contains(params.name)) { - if (!columns.contains(params.name)){ + // Only check curve parameter if it does not belong to a part of the demand type + if (!demandTypes.contains(params.fullName) && !demandUnits.contains(params.name)) { + // Check if inventoryType is "building" and the column is not reserved + boolean isBuildingAndNotReserved = "building".equals(inventoryType) && SemanticsConstants.RESERVED_COLUMNS.contains(params.name); + + // If it's not a building parameter that is reserved, check if it's in the columns + if (!isBuildingAndNotReserved && !columns.contains(params.name)) { throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Curve parameter: " + params.name + " not found in the dataType: " + dataType); } } }); + } catch (IOException e) { - throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Could not check the fragility curve parameter matches the dataType columns."); + throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Could not check the fragility curve parameter matches the dataType columns.", e); } + fragilitySet.setCreator(username); fragilitySet.setOwner(username); From 85528c1ba52307166c49b0f471b458afa12e537e Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Wed, 5 Jun 2024 10:42:13 -0500 Subject: [PATCH 04/16] throw exception bugfix --- .../incore/service/dfr3/controllers/FragilityController.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java index 4785d8ff5..ead784b25 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java @@ -267,9 +267,8 @@ public FragilitySet uploadFragilitySet(@Parameter(name = "json representing the } }); - } catch (IOException e) { - throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Could not check the fragility curve parameter matches the dataType columns.", e); + throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Could not check the fragility curve parameter matches the dataType columns."); } From 2bb2fe5cbb4d320212e0b5e0c865b3f8dd9d59ff Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Wed, 5 Jun 2024 10:51:21 -0500 Subject: [PATCH 05/16] add same logic to restoration and repair curves --- .../dfr3/controllers/FragilityController.java | 2 +- .../dfr3/controllers/RepairController.java | 26 +++++++++++++++++++ .../controllers/RestorationController.java | 25 ++++++++++++++++++ .../incore/service/dfr3/utils/CommonUtil.java | 4 +++ 4 files changed, 56 insertions(+), 1 deletion(-) diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java index ead784b25..5abd1909d 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java @@ -245,6 +245,7 @@ public FragilitySet uploadFragilitySet(@Parameter(name = "json representing the } } } + // check if the parameters matches the defined data type in semantics String dataType = fragilitySet.getDataType(); String inventoryType = fragilitySet.getInventoryType(); if (dataType == null) { @@ -271,7 +272,6 @@ public FragilitySet uploadFragilitySet(@Parameter(name = "json representing the throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Could not check the fragility curve parameter matches the dataType columns."); } - fragilitySet.setCreator(username); fragilitySet.setOwner(username); diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RepairController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RepairController.java index fc17075b4..343a2ecf9 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RepairController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RepairController.java @@ -11,6 +11,7 @@ package edu.illinois.ncsa.incore.service.dfr3.controllers; import edu.illinois.ncsa.incore.common.AllocationConstants; +import edu.illinois.ncsa.incore.common.SemanticsConstants; import edu.illinois.ncsa.incore.common.auth.Authorizer; import edu.illinois.ncsa.incore.common.auth.IAuthorizer; import edu.illinois.ncsa.incore.common.auth.Privileges; @@ -28,6 +29,8 @@ import edu.illinois.ncsa.incore.service.dfr3.daos.IMappingDAO; import edu.illinois.ncsa.incore.service.dfr3.daos.IRepairDAO; import edu.illinois.ncsa.incore.service.dfr3.models.RepairSet; +import edu.illinois.ncsa.incore.service.dfr3.utils.CommonUtil; +import edu.illinois.ncsa.incore.service.dfr3.utils.ServiceUtil; import io.swagger.v3.oas.annotations.OpenAPIDefinition; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -43,6 +46,8 @@ import jakarta.ws.rs.*; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; + +import java.io.IOException; import java.util.*; import java.util.stream.Collectors; @@ -76,6 +81,7 @@ public class RepairController { private final String username; private final List groups; + private final String userGroups; @Inject IAuthorizer authorizer; @@ -96,6 +102,7 @@ public RepairController( @Parameter(name = "User groups.", required = false) @HeaderParam("x-auth-usergroup") String userGroups ) { this.username = UserInfoUtils.getUsername(userInfo); + this.userGroups = userGroups; this.groups = UserGroupUtils.getUserGroups(userGroups); } @@ -185,6 +192,25 @@ public RepairSet uploadRepairSet(@Parameter(name = "json representing the repair AllocationConstants.HAZARD_DFR3_ALLOCATION_MESSAGE); } + // check if the parameters matches the defined data type in semantics + String dataType = repairSet.getDataType(); + if (dataType == null) { + throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataType is a required field."); + } + try { + String semanticsDefinition = ServiceUtil.getJsonFromSemanticsEndpoint(dataType, username, userGroups); + List columns = CommonUtil.getColumnNames(semanticsDefinition); + + repairSet.getCurveParameters().forEach((params) -> { + if(!columns.contains(params.name)) { + throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Curve parameter: " + params.name + " not found in the dataType: " + dataType); + } + }); + + } catch (IOException e) { + throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Could not check the repair curve parameter matches the dataType columns."); + } + repairSet.setCreator(username); repairSet.setOwner(username); diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RestorationController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RestorationController.java index e290f3685..860b77be3 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RestorationController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RestorationController.java @@ -26,6 +26,8 @@ import edu.illinois.ncsa.incore.service.dfr3.daos.IMappingDAO; import edu.illinois.ncsa.incore.service.dfr3.daos.IRestorationDAO; import edu.illinois.ncsa.incore.service.dfr3.models.RestorationSet; +import edu.illinois.ncsa.incore.service.dfr3.utils.CommonUtil; +import edu.illinois.ncsa.incore.service.dfr3.utils.ServiceUtil; import io.swagger.v3.oas.annotations.OpenAPIDefinition; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -41,6 +43,8 @@ import jakarta.ws.rs.*; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; + +import java.io.IOException; import java.util.*; import java.util.stream.Collectors; @@ -75,6 +79,7 @@ public class RestorationController { private final String username; private final List groups; + private final String userGroups; @Inject IAuthorizer authorizer; @@ -95,6 +100,7 @@ public RestorationController( @Parameter(name = "User groups.", required = false) @HeaderParam("x-auth-usergroup") String userGroups ) { this.username = UserInfoUtils.getUsername(userInfo); + this.userGroups = userGroups; this.groups = UserGroupUtils.getUserGroups(userGroups); } @@ -189,6 +195,25 @@ public RestorationSet uploadRestorationSet(@Parameter(name = "json representing AllocationConstants.HAZARD_DFR3_ALLOCATION_MESSAGE); } + // check if the parameters matches the defined data type in semantics + String dataType = restorationSet.getDataType(); + if (dataType == null) { + throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataType is a required field."); + } + try { + String semanticsDefinition = ServiceUtil.getJsonFromSemanticsEndpoint(dataType, username, userGroups); + List columns = CommonUtil.getColumnNames(semanticsDefinition); + + restorationSet.getCurveParameters().forEach((params) -> { + if(!columns.contains(params.name)) { + throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Curve parameter: " + params.name + " not found in the dataType: " + dataType); + } + }); + + } catch (IOException e) { + throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Could not check the restoration curve parameter matches the dataType columns."); + } + restorationSet.setCreator(username); restorationSet.setOwner(username); diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/utils/CommonUtil.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/utils/CommonUtil.java index 86d22d8e7..bc8e21ae9 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/utils/CommonUtil.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/utils/CommonUtil.java @@ -2,6 +2,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import edu.illinois.ncsa.incore.common.SemanticsConstants; +import edu.illinois.ncsa.incore.common.exceptions.IncoreHTTPException; +import edu.illinois.ncsa.incore.service.dfr3.models.DFR3Set; +import jakarta.ws.rs.core.Response; import org.json.simple.JSONObject; import java.io.IOException; From 4261ee7dadca860003bd5607b8ac7d3f3a56f88a Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Wed, 5 Jun 2024 11:20:07 -0500 Subject: [PATCH 06/16] initial code parsing mapping rules to get columns --- .../dfr3/controllers/MappingController.java | 41 +++++++++++++++++++ .../incore/service/dfr3/utils/CommonUtil.java | 32 +++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java index fadc2a4c2..3b2365642 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java @@ -29,6 +29,8 @@ import edu.illinois.ncsa.incore.service.dfr3.daos.IRestorationDAO; import edu.illinois.ncsa.incore.service.dfr3.models.Mapping; import edu.illinois.ncsa.incore.service.dfr3.models.MappingSet; +import edu.illinois.ncsa.incore.service.dfr3.utils.CommonUtil; +import edu.illinois.ncsa.incore.service.dfr3.utils.ServiceUtil; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; @@ -38,9 +40,13 @@ import jakarta.ws.rs.*; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; + +import java.io.IOException; import java.util.*; import java.util.stream.Collectors; +import static edu.illinois.ncsa.incore.service.dfr3.utils.CommonUtil.extractColumnsFromMapping; + @Tag(name = "Mapping") @Path("mappings") @@ -49,6 +55,7 @@ public class MappingController { private final String username; private final List groups; + private final String userGroups; @Inject private IMappingDAO mappingDAO; @@ -79,6 +86,7 @@ public MappingController( @Parameter(name = "User groups.", required = false) @HeaderParam("x-auth-usergroup") String userGroups ) { this.username = UserInfoUtils.getUsername(userInfo); + this.userGroups = userGroups; this.groups = UserGroupUtils.getUserGroups(userGroups); } @@ -193,6 +201,9 @@ public MappingSet uploadMapping(@Parameter(name = "json representing the fragili } List mappings = mappingSet.getMappings(); + List uniqueColumns = new ArrayList<>(); + Set columnSet = new HashSet<>(); + int idx = 0; String prevRuleClassName = ""; // This validates if the format of the "rules" being submitted is an Array or Hash. It is needed because we made "rules" attribute @@ -213,7 +224,37 @@ public MappingSet uploadMapping(@Parameter(name = "json representing the fragili prevRuleClassName = mapping.getRules().getClass().getName(); } idx++; + + // get unique column names + if (mapping.getRules() instanceof ArrayList) { + extractColumnsFromMapping((ArrayList) mapping.getRules(), columnSet); + } + else if(mapping.getRules() instanceof HashMap) { + extractColumnsFromMapping((HashMap) mapping.getRules(), columnSet); + } + uniqueColumns.addAll(columnSet); + } + + // check if the parameters matches the defined data type in semantics + String dataType = mappingSet.getDataType(); + if (dataType == null) { + throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataType is a required field."); } + try { + String semanticsDefinition = ServiceUtil.getJsonFromSemanticsEndpoint(dataType, username, userGroups); + List columns = CommonUtil.getColumnNames(semanticsDefinition); + + // parse mapping rules to find column names + uniqueColumns.forEach((uniqueColumn) -> { + if(!columns.contains(uniqueColumn)) { + throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Column: " + uniqueColumn + "in the Mapping Rules not found in the dataType: " + dataType); + } + }); + + } catch (IOException e) { + throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Could not check if the column in the mapping rules matches the dataType columns."); + } + mappingSet.setCreator(username); mappingSet.setOwner(username); diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/utils/CommonUtil.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/utils/CommonUtil.java index bc8e21ae9..032c96f1e 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/utils/CommonUtil.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/utils/CommonUtil.java @@ -11,6 +11,8 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Set; public class CommonUtil { public static JSONObject createUserStatusJson(String creator, String keyDatabase, int numDfr) { @@ -42,4 +44,34 @@ public static List getColumnNames(String semanticDefinition) throws IOEx return columnNames; } + + public static void extractColumnsFromMapping(Map rules, Set columnSet) { + rules.forEach((key, value) -> { + if (value instanceof List) { + for (Object item : (List) value) { + if (item instanceof String) { + String[] parts = ((String) item).split(" "); + if (parts.length > 1) { + columnSet.add(parts[1]); + } + } else if (item instanceof Map) { + extractColumnsFromMapping((Map) item, columnSet); + } + } + } + }); + } + + public static void extractColumnsFromMapping(ArrayList rules, Set columnSet) { + rules.forEach((rule) -> { + if (rule instanceof List) { + ((List) rule).forEach((r) -> { + String[] parts = r.toString().split(" "); + if (parts.length > 1) { + columnSet.add(parts[1]); + } + }); + } + }); + } } From f253c0caf604b9afebb46bff783be19b2cc40c86 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Wed, 5 Jun 2024 12:00:53 -0500 Subject: [PATCH 07/16] fix bugs in mapping check --- .../incore/service/dfr3/controllers/MappingController.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java index 3b2365642..49443db79 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java @@ -201,7 +201,6 @@ public MappingSet uploadMapping(@Parameter(name = "json representing the fragili } List mappings = mappingSet.getMappings(); - List uniqueColumns = new ArrayList<>(); Set columnSet = new HashSet<>(); int idx = 0; @@ -232,9 +231,10 @@ public MappingSet uploadMapping(@Parameter(name = "json representing the fragili else if(mapping.getRules() instanceof HashMap) { extractColumnsFromMapping((HashMap) mapping.getRules(), columnSet); } - uniqueColumns.addAll(columnSet); } + List uniqueColumns = new ArrayList<>(columnSet); + // check if the parameters matches the defined data type in semantics String dataType = mappingSet.getDataType(); if (dataType == null) { @@ -247,7 +247,7 @@ else if(mapping.getRules() instanceof HashMap) { // parse mapping rules to find column names uniqueColumns.forEach((uniqueColumn) -> { if(!columns.contains(uniqueColumn)) { - throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Column: " + uniqueColumn + "in the Mapping Rules not found in the dataType: " + dataType); + throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Column: " + uniqueColumn + " in the Mapping Rules not found in the dataType: " + dataType); } }); From 9f7b4758375b152976c893ee2580967712e07418 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Wed, 5 Jun 2024 12:01:59 -0500 Subject: [PATCH 08/16] changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c983364c..97e9ab950 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## [Unreleased] +- Valid DFR3 set and mapping set match semantcis definition [#295](https://github.com/IN-CORE/incore-services/issues/295) + ## [1.26.1] - 2024-04-08 ### Fixed From 0ae602b9be06bdae0e409652f8b3d2935f489503 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Thu, 11 Jul 2024 15:51:01 -0500 Subject: [PATCH 09/16] typo in changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97e9ab950..4dbad2b10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] -- Valid DFR3 set and mapping set match semantcis definition [#295](https://github.com/IN-CORE/incore-services/issues/295) +- Valid DFR3 set and mapping set match semantics definition [#295](https://github.com/IN-CORE/incore-services/issues/295) ## [1.26.1] - 2024-04-08 From 912c5b554161e29a4576bd8cfb91fdcafed22ae1 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Fri, 12 Jul 2024 13:21:09 -0500 Subject: [PATCH 10/16] remove it from fragility, repair and restoration --- .../dfr3/controllers/FragilityController.java | 31 ------------------- .../dfr3/controllers/RepairController.java | 24 -------------- .../controllers/RestorationController.java | 24 -------------- .../dfr3/daos/MongoDBFragilityDAO.java | 1 - .../service/dfr3/daos/MongoDBRepairDAO.java | 1 - .../dfr3/daos/MongoDBRestorationDAO.java | 1 - .../incore/service/dfr3/models/DFR3Set.java | 3 -- 7 files changed, 85 deletions(-) diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java index 5abd1909d..d4475a831 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java @@ -122,7 +122,6 @@ public FragilityController( public List getFragilities(@Parameter(name = "demand type filter", example = "PGA") @QueryParam("demand") String demandType, @Parameter(name = "hazard type filter", example = "earthquake") @QueryParam("hazard") String hazardType, @Parameter(name = "Inventory type filter", example = "building") @QueryParam("inventory") String inventoryType, - @Parameter(name = "Data type filter", example = "ergo:buildingInventoryVer7") @QueryParam("dataType") String dataType, @Parameter(name = "not implemented", hidden = true) @QueryParam("author") String author, @Parameter(name = "Legacy fragility Id from v1") @QueryParam("legacy_id") String legacyId, @Parameter(name = "Fragility creator's username") @QueryParam("creator") String creator, @@ -147,10 +146,6 @@ public List getFragilities(@Parameter(name = "demand type filter", queryMap.put("inventoryType", inventoryType); } - if (dataType != null){ - queryMap.put("dataType", dataType); - } - if (creator != null) { queryMap.put("creator", creator); } @@ -245,32 +240,6 @@ public FragilitySet uploadFragilitySet(@Parameter(name = "json representing the } } } - // check if the parameters matches the defined data type in semantics - String dataType = fragilitySet.getDataType(); - String inventoryType = fragilitySet.getInventoryType(); - if (dataType == null) { - throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataType is a required field."); - } - try { - String semanticsDefinition = ServiceUtil.getJsonFromSemanticsEndpoint(dataType, username, userGroups); - List columns = CommonUtil.getColumnNames(semanticsDefinition); - - fragilitySet.getCurveParameters().forEach((params) -> { - // Only check curve parameter if it does not belong to a part of the demand type - if (!demandTypes.contains(params.fullName) && !demandUnits.contains(params.name)) { - // Check if inventoryType is "building" and the column is not reserved - boolean isBuildingAndNotReserved = "building".equals(inventoryType) && SemanticsConstants.RESERVED_COLUMNS.contains(params.name); - - // If it's not a building parameter that is reserved, check if it's in the columns - if (!isBuildingAndNotReserved && !columns.contains(params.name)) { - throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Curve parameter: " + params.name + " not found in the dataType: " + dataType); - } - } - }); - - } catch (IOException e) { - throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Could not check the fragility curve parameter matches the dataType columns."); - } fragilitySet.setCreator(username); fragilitySet.setOwner(username); diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RepairController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RepairController.java index 343a2ecf9..be7289950 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RepairController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RepairController.java @@ -111,7 +111,6 @@ public RepairController( @Operation(tags = "Gets list of repairs", summary = "Apply filters to get the desired set of repairs") public List getRepairs(@Parameter(name = "hazard type filter", example = "earthquake") @QueryParam("hazard") String hazardType, @Parameter(name = "Inventory type", example = "building") @QueryParam("inventory") String inventoryType, - @Parameter(name = "Data type filter", example = "ergo:buildingInventoryVer7") @QueryParam("dataType") String dataType, @Parameter(name = "Repair creator's username") @QueryParam("creator") String creator, @Parameter(name = "Name of space") @DefaultValue("") @QueryParam("space") String spaceName, @Parameter(name = "Skip the first n results") @QueryParam("skip") int offset, @@ -126,10 +125,6 @@ public List getRepairs(@Parameter(name = "hazard type filter", examp queryMap.put("inventoryType", inventoryType); } - if (dataType != null){ - queryMap.put("dataType", dataType); - } - if (creator != null) { queryMap.put("creator", creator); } @@ -192,25 +187,6 @@ public RepairSet uploadRepairSet(@Parameter(name = "json representing the repair AllocationConstants.HAZARD_DFR3_ALLOCATION_MESSAGE); } - // check if the parameters matches the defined data type in semantics - String dataType = repairSet.getDataType(); - if (dataType == null) { - throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataType is a required field."); - } - try { - String semanticsDefinition = ServiceUtil.getJsonFromSemanticsEndpoint(dataType, username, userGroups); - List columns = CommonUtil.getColumnNames(semanticsDefinition); - - repairSet.getCurveParameters().forEach((params) -> { - if(!columns.contains(params.name)) { - throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Curve parameter: " + params.name + " not found in the dataType: " + dataType); - } - }); - - } catch (IOException e) { - throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Could not check the repair curve parameter matches the dataType columns."); - } - repairSet.setCreator(username); repairSet.setOwner(username); diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RestorationController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RestorationController.java index 860b77be3..68b6b5393 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RestorationController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RestorationController.java @@ -109,7 +109,6 @@ public RestorationController( @Operation(tags = "Gets list of restorations", description = "Apply filters to get the desired set of restorations") public List getRestorations(@Parameter(name = "hazard type filter", example = "earthquake") @QueryParam("hazard") String hazardType, @Parameter(name = "Inventory type", example = "building") @QueryParam("inventory") String inventoryType, - @Parameter(name = "Data type filter", example = "ergo:buildingInventoryVer7") @QueryParam("dataType") String dataType, @Parameter(name = "Restoration creator's username") @QueryParam("creator") String creator, @Parameter(name = "Name of space") @DefaultValue("") @QueryParam("space") String spaceName, @Parameter(name = "Skip the first n results") @QueryParam("skip") int offset, @@ -125,10 +124,6 @@ public List getRestorations(@Parameter(name = "hazard type filt queryMap.put("inventoryType", inventoryType); } - if (dataType != null){ - queryMap.put("dataType", dataType); - } - if (creator != null) { queryMap.put("creator", creator); } @@ -195,25 +190,6 @@ public RestorationSet uploadRestorationSet(@Parameter(name = "json representing AllocationConstants.HAZARD_DFR3_ALLOCATION_MESSAGE); } - // check if the parameters matches the defined data type in semantics - String dataType = restorationSet.getDataType(); - if (dataType == null) { - throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataType is a required field."); - } - try { - String semanticsDefinition = ServiceUtil.getJsonFromSemanticsEndpoint(dataType, username, userGroups); - List columns = CommonUtil.getColumnNames(semanticsDefinition); - - restorationSet.getCurveParameters().forEach((params) -> { - if(!columns.contains(params.name)) { - throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Curve parameter: " + params.name + " not found in the dataType: " + dataType); - } - }); - - } catch (IOException e) { - throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Could not check the restoration curve parameter matches the dataType columns."); - } - restorationSet.setCreator(username); restorationSet.setOwner(username); diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBFragilityDAO.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBFragilityDAO.java index f4d12eefa..7391c74b8 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBFragilityDAO.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBFragilityDAO.java @@ -85,7 +85,6 @@ public List searchFragilities(String text) { Filters.regex("legacyId").pattern(text).caseInsensitive(), Filters.regex("hazardType").pattern(text).caseInsensitive(), Filters.regex("inventoryType").pattern(text).caseInsensitive(), - Filters.regex("dataType").pattern(text).caseInsensitive(), Filters.regex("description").pattern(text).caseInsensitive(), Filters.regex("authors").pattern(text).caseInsensitive())); List sets = query.iterator().toList(); diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBRepairDAO.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBRepairDAO.java index f37a46026..afc1ad819 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBRepairDAO.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBRepairDAO.java @@ -85,7 +85,6 @@ public List searchRepairs(String text) { Filters.regex("legacyId").pattern(text).caseInsensitive(), Filters.regex("hazardType").pattern(text).caseInsensitive(), Filters.regex("inventoryType").pattern(text).caseInsensitive(), - Filters.regex("dataType").pattern(text).caseInsensitive(), Filters.regex("description").pattern(text).caseInsensitive(), Filters.regex("authors").pattern(text).caseInsensitive())); List sets = query.iterator().toList(); diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBRestorationDAO.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBRestorationDAO.java index e738b9d7c..f6bb09f73 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBRestorationDAO.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBRestorationDAO.java @@ -86,7 +86,6 @@ public List searchRestorations(String text) { Filters.regex("legacyId").pattern(text).caseInsensitive(), Filters.regex("hazardType").pattern(text).caseInsensitive(), Filters.regex("inventoryType").pattern(text).caseInsensitive(), - Filters.regex("dataType").pattern(text).caseInsensitive(), Filters.regex("description").pattern(text).caseInsensitive(), Filters.regex("authors").pattern(text).caseInsensitive())); List sets = query.iterator().toList(); diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/models/DFR3Set.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/models/DFR3Set.java index d34a04a01..e8a770eae 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/models/DFR3Set.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/models/DFR3Set.java @@ -31,7 +31,6 @@ public abstract class DFR3Set { protected String hazardType; protected String inventoryType; - protected String dataType; protected String creator; protected String owner; @@ -80,8 +79,6 @@ public String getInventoryType() { return inventoryType; } - public String getDataType() { return dataType; } - public String getCreator() { return creator; } From 61ef9318461b5095edfa276907d64841602ee93a Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Fri, 12 Jul 2024 14:30:29 -0500 Subject: [PATCH 11/16] update to a list of data types --- CHANGELOG.md | 5 ++- .../dfr3/controllers/MappingController.java | 45 ++++++++++++------- .../incore/service/dfr3/daos/IMappingDAO.java | 2 +- .../service/dfr3/daos/MongoDBMappingDAO.java | 16 ++++++- .../service/dfr3/models/MappingSet.java | 4 +- .../src/test/java/mocks/MockFragilityDAO.java | 1 - .../java/mocks/MockFragilityMappingDAO.java | 2 +- .../src/test/resources/building_mapping.json | 2 +- .../src/test/resources/fragility.json | 4 -- .../src/test/resources/fragility_request.json | 2 +- 10 files changed, 52 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dbad2b10..abe85f0bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,10 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] -- Valid DFR3 set and mapping set match semantics definition [#295](https://github.com/IN-CORE/incore-services/issues/295) + +### Added + +- Valid DFR3 mapping set match semantics definition [#295](https://github.com/IN-CORE/incore-services/issues/295) ## [1.26.1] - 2024-04-08 diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java index 49443db79..3577d3181 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java @@ -112,10 +112,6 @@ public List getMappings(@Parameter(name= "hazard type filter", exam queryMap.put("inventoryType", inventoryType); } - if (dataType != null) { - queryMap.put("dataType", dataType); - } - if (creator != null) { queryMap.put("creator", creator); } @@ -127,7 +123,7 @@ public List getMappings(@Parameter(name= "hazard type filter", exam List mappingSets; if (queryMap.isEmpty()) { - mappingSets = this.mappingDAO.getMappingSets(); + mappingSets = this.mappingDAO.getMappingSets(dataType); } else { mappingSets = this.mappingDAO.queryMappingSets(queryMap); } @@ -236,23 +232,38 @@ else if(mapping.getRules() instanceof HashMap) { List uniqueColumns = new ArrayList<>(columnSet); // check if the parameters matches the defined data type in semantics - String dataType = mappingSet.getDataType(); - if (dataType == null) { + List dataTypes = mappingSet.getDataTypes(); + if (dataTypes == null || dataTypes.isEmpty()) { throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataType is a required field."); } - try { - String semanticsDefinition = ServiceUtil.getJsonFromSemanticsEndpoint(dataType, username, userGroups); - List columns = CommonUtil.getColumnNames(semanticsDefinition); - // parse mapping rules to find column names - uniqueColumns.forEach((uniqueColumn) -> { - if(!columns.contains(uniqueColumn)) { - throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Column: " + uniqueColumn + " in the Mapping Rules not found in the dataType: " + dataType); + boolean columnFound = false; + + for (String dataType : dataTypes) { + try { + String semanticsDefinition = ServiceUtil.getJsonFromSemanticsEndpoint(dataType, username, userGroups); + List columnsDefinition = CommonUtil.getColumnNames(semanticsDefinition); + + // Check if any uniqueColumn is found in columns + for (String uniqueColumn : uniqueColumns) { + if (columnsDefinition.contains(uniqueColumn)) { + columnFound = true; + break; // Break the inner loop + } } - }); - } catch (IOException e) { - throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Could not check if the column in the mapping rules matches the dataType columns."); + // If column is found, break the outer loop + if (columnFound) break; + + } catch (IOException e) { + throw new IncoreHTTPException(Response.Status.BAD_REQUEST, + "Could not check if the column in the mapping rules matches the dataType:" + dataType); + } + } + + if (!columnFound) { + throw new IncoreHTTPException(Response.Status.BAD_REQUEST, + "The columns in the mapping rules do not match the columns in any of the listed dataType."); } diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/IMappingDAO.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/IMappingDAO.java index cfefb910e..1add3b07b 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/IMappingDAO.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/IMappingDAO.java @@ -19,7 +19,7 @@ public interface IMappingDAO { void initialize(); - List getMappingSets(); + List getMappingSets(String dataType); Optional getMappingSetById(String id); diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBMappingDAO.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBMappingDAO.java index fe2ff514c..48cb24ff7 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBMappingDAO.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBMappingDAO.java @@ -20,6 +20,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; public class MongoDBMappingDAO extends MongoDAO implements IMappingDAO { @@ -33,8 +34,19 @@ public void initialize() { } @Override - public List getMappingSets() { - return this.dataStore.find(MappingSet.class).iterator().toList(); + public List getMappingSets(String dataType) { + // Fetch all MappingSets + List allMappingSets = this.dataStore.find(MappingSet.class).iterator().toList(); + + // If dataType is null, return all MappingSets without filtering + if (dataType == null) { + return allMappingSets; + } + + // Filter the MappingSets based on the dataType parameter + return allMappingSets.stream() + .filter(mappingSet -> mappingSet.getDataTypes().contains(dataType)) + .collect(Collectors.toList()); } @Override diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/models/MappingSet.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/models/MappingSet.java index 18c3bb7e1..3fe660071 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/models/MappingSet.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/models/MappingSet.java @@ -31,7 +31,7 @@ public class MappingSet { private String name; private String hazardType; private String inventoryType; - private String dataType; + private List dataTypes; @JsonProperty("mappingType") private String mappingType; @@ -64,7 +64,7 @@ public String getInventoryType() { return inventoryType; } - public String getDataType() { return dataType; } + public List getDataTypes() { return dataTypes; } public List getMappings() { return mappings; diff --git a/server/dfr3-service/src/test/java/mocks/MockFragilityDAO.java b/server/dfr3-service/src/test/java/mocks/MockFragilityDAO.java index 7890e9dab..4a8197062 100644 --- a/server/dfr3-service/src/test/java/mocks/MockFragilityDAO.java +++ b/server/dfr3-service/src/test/java/mocks/MockFragilityDAO.java @@ -117,7 +117,6 @@ public List searchFragilities(String text) { Filters.regex("legacyId").pattern(text).caseInsensitive(), Filters.regex("hazardType").pattern(text).caseInsensitive(), Filters.regex("inventoryType").pattern(text).caseInsensitive(), - Filters.regex("dataType").pattern(text).caseInsensitive(), Filters.regex("description").pattern(text).caseInsensitive(), Filters.regex("authors").pattern(text).caseInsensitive())); List sets = query.iterator(new FindOptions().limit(100)).toList(); diff --git a/server/dfr3-service/src/test/java/mocks/MockFragilityMappingDAO.java b/server/dfr3-service/src/test/java/mocks/MockFragilityMappingDAO.java index 32f26f478..4038a2b87 100644 --- a/server/dfr3-service/src/test/java/mocks/MockFragilityMappingDAO.java +++ b/server/dfr3-service/src/test/java/mocks/MockFragilityMappingDAO.java @@ -36,7 +36,7 @@ public void initialize() { } @Override - public List getMappingSets() { + public List getMappingSets(String dataType) { return mappingSets; } diff --git a/server/dfr3-service/src/test/resources/building_mapping.json b/server/dfr3-service/src/test/resources/building_mapping.json index 2725a3943..59f69e2e6 100644 --- a/server/dfr3-service/src/test/resources/building_mapping.json +++ b/server/dfr3-service/src/test/resources/building_mapping.json @@ -2,7 +2,7 @@ "name": "Default Building Fragility Mapping 1.0", "hazardType": "earthquake", "inventoryType": "building", - "dataType": "ergo:buildingInventoryVer7", + "dataTypes": ["ergo:buildingInventoryVer6","ergo:buildingInventoryVer7"], "mappings": [ { "legacyEntry": { diff --git a/server/dfr3-service/src/test/resources/fragility.json b/server/dfr3-service/src/test/resources/fragility.json index f1b180625..b03b61061 100644 --- a/server/dfr3-service/src/test/resources/fragility.json +++ b/server/dfr3-service/src/test/resources/fragility.json @@ -4,7 +4,6 @@ "legacyId": "NSDS_PFM_MTB_UL_475_W1_4", "hazardType": "earthquake", "inventoryType": "building", - "dataType": "ergo:buildingInventoryVer7", "authors": [ "Amr S. Elnashai", "Seong-Hoon Jeong", @@ -55,7 +54,6 @@ "legacyId": "STR_PFM_MTB_UL_475_S2_9", "hazardType": "earthquake", "inventoryType": "building", - "dataType": "ergo:buildingInventoryVer7", "authors": [ "Amr S. Elnashai", "Seong-Hoon Jeong", @@ -113,7 +111,6 @@ "resultType": "Limit State", "hazardType": "tsunami", "inventoryType": "electric_facility", - "dataType": "incore:epfVer2", "creator": "cwang138", "demandType": "Vmax", "demandUnits": "ft/s", @@ -212,7 +209,6 @@ "id": "5e99d0b3e05cb9ccbc32d6e6", "hazardType": "earthquake", "inventoryType": "bridge", - "dataType": "ergo:bridgesVer3", "authors": [ "HAZUS" ], diff --git a/server/dfr3-service/src/test/resources/fragility_request.json b/server/dfr3-service/src/test/resources/fragility_request.json index 5956686ce..22165554a 100644 --- a/server/dfr3-service/src/test/resources/fragility_request.json +++ b/server/dfr3-service/src/test/resources/fragility_request.json @@ -2,7 +2,7 @@ "legacyId": "legacyId", "hazardType": "hazardType", "inventoryType": "inventoryType", - "dataType": "incore:dataType", + "dataTypes": ["incore:dataType"], "authors": [ "Chen Wang" ], From 8f9eadee0fdb77a708e332d19e08e853475bd6e2 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Fri, 12 Jul 2024 15:05:35 -0500 Subject: [PATCH 12/16] add logic to check data type empty --- .../service/dfr3/controllers/MappingController.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java index 3577d3181..2ae9d6812 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java @@ -233,8 +233,11 @@ else if(mapping.getRules() instanceof HashMap) { // check if the parameters matches the defined data type in semantics List dataTypes = mappingSet.getDataTypes(); - if (dataTypes == null || dataTypes.isEmpty()) { - throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataType is a required field."); + if (dataTypes == null) { + throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataTypes is a required field."); + } + else if (dataTypes.isEmpty()) { + throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataTypes cannot be empty."); } boolean columnFound = false; @@ -263,10 +266,9 @@ else if(mapping.getRules() instanceof HashMap) { if (!columnFound) { throw new IncoreHTTPException(Response.Status.BAD_REQUEST, - "The columns in the mapping rules do not match the columns in any of the listed dataType."); + "The columns in the mapping rules do not match the columns in any of the listed dataTypes."); } - mappingSet.setCreator(username); mappingSet.setOwner(username); From e4c07d539793976e51ac7819e01f121cb4ff1c15 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Fri, 12 Jul 2024 15:37:35 -0500 Subject: [PATCH 13/16] loop through each fraigility curve to check required parameters --- .../dfr3/controllers/MappingController.java | 49 +++++++++++++------ 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java index 2ae9d6812..2bf1920af 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java @@ -27,8 +27,7 @@ import edu.illinois.ncsa.incore.service.dfr3.daos.IMappingDAO; import edu.illinois.ncsa.incore.service.dfr3.daos.IRepairDAO; import edu.illinois.ncsa.incore.service.dfr3.daos.IRestorationDAO; -import edu.illinois.ncsa.incore.service.dfr3.models.Mapping; -import edu.illinois.ncsa.incore.service.dfr3.models.MappingSet; +import edu.illinois.ncsa.incore.service.dfr3.models.*; import edu.illinois.ncsa.incore.service.dfr3.utils.CommonUtil; import edu.illinois.ncsa.incore.service.dfr3.utils.ServiceUtil; import io.swagger.v3.oas.annotations.Parameter; @@ -198,6 +197,7 @@ public MappingSet uploadMapping(@Parameter(name = "json representing the fragili List mappings = mappingSet.getMappings(); Set columnSet = new HashSet<>(); + Set dfr3CurveSets = new HashSet<>(); int idx = 0; String prevRuleClassName = ""; @@ -227,6 +227,25 @@ public MappingSet uploadMapping(@Parameter(name = "json representing the fragili else if(mapping.getRules() instanceof HashMap) { extractColumnsFromMapping((HashMap) mapping.getRules(), columnSet); } + + // get unique dfr3 curves + Optional.ofNullable(mapping.getEntry()) + .ifPresent(entry -> { + if ("fragility".equals(mappingSet.getMappingType())) { + entry.values().forEach(id -> + this.fragilityDAO.getFragilitySetById(id).ifPresent(dfr3CurveSets::add) + ); + } else if ("restoration".equals(mappingSet.getMappingType())) { + entry.values().forEach(id -> + this.restorationDAO.getRestorationSetById(id).ifPresent(dfr3CurveSets::add) + ); + } else if ("repair".equals(mappingSet.getMappingType())) { + entry.values().forEach(id -> + this.repairDAO.getRepairSetById(id).ifPresent(dfr3CurveSets::add) + ); + } + }); + } List uniqueColumns = new ArrayList<>(columnSet); @@ -241,32 +260,34 @@ else if (dataTypes.isEmpty()) { } boolean columnFound = false; - for (String dataType : dataTypes) { try { String semanticsDefinition = ServiceUtil.getJsonFromSemanticsEndpoint(dataType, username, userGroups); List columnsDefinition = CommonUtil.getColumnNames(semanticsDefinition); - // Check if any uniqueColumn is found in columns - for (String uniqueColumn : uniqueColumns) { - if (columnsDefinition.contains(uniqueColumn)) { - columnFound = true; - break; // Break the inner loop - } - } + // Check if all uniqueColumns are found in columns + boolean allUniqueColumnsFound = columnsDefinition.containsAll(uniqueColumns); - // If column is found, break the outer loop - if (columnFound) break; + // Check if all curveParameters are found in columns for every curve set + boolean allCurveParametersFound = dfr3CurveSets.stream().allMatch(dfr3CurveSet -> { + List curveParameters = dfr3CurveSet.getCurveParameters(); + return curveParameters != null && curveParameters.stream().allMatch(param -> columnsDefinition.contains(param.name)); + }); + // If both conditions are met, set columnFound to true + if (allUniqueColumnsFound && allCurveParametersFound) { + columnFound = true; + break; // Break the outer loop if both conditions are met + } } catch (IOException e) { throw new IncoreHTTPException(Response.Status.BAD_REQUEST, - "Could not check if the column in the mapping rules matches the dataType:" + dataType); + "Could not check if the column in the mapping rules matches the dataType: " + dataType); } } if (!columnFound) { throw new IncoreHTTPException(Response.Status.BAD_REQUEST, - "The columns in the mapping rules do not match the columns in any of the listed dataTypes."); + "The columns in the mapping rules and/or fragility parameters do not match the columns in any of the listed dataTypes."); } mappingSet.setCreator(username); From 129f6ef43534e7f3fd93a9b3650916d0a40abf87 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Mon, 15 Jul 2024 10:59:23 -0500 Subject: [PATCH 14/16] add check to each fragility curve --- .../dfr3/controllers/FragilityController.java | 10 +++------ .../dfr3/controllers/MappingController.java | 21 +++++++++++++++++-- .../dfr3/controllers/RepairController.java | 11 ++-------- .../controllers/RestorationController.java | 2 -- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java index d4475a831..f5cd7af31 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java @@ -21,7 +21,6 @@ import edu.illinois.ncsa.incore.common.dao.IUserFinalQuotaRepository; import edu.illinois.ncsa.incore.common.exceptions.IncoreHTTPException; import edu.illinois.ncsa.incore.common.models.Space; -import edu.illinois.ncsa.incore.common.models.UserAllocations; import edu.illinois.ncsa.incore.common.utils.AllocationUtils; import edu.illinois.ncsa.incore.common.utils.UserGroupUtils; import edu.illinois.ncsa.incore.common.utils.UserInfoUtils; @@ -29,8 +28,6 @@ import edu.illinois.ncsa.incore.service.dfr3.daos.IMappingDAO; import edu.illinois.ncsa.incore.service.dfr3.models.FragilitySet; -import edu.illinois.ncsa.incore.service.dfr3.utils.CommonUtil; -import edu.illinois.ncsa.incore.service.dfr3.utils.ServiceUtil; import io.swagger.v3.oas.annotations.OpenAPIDefinition; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -52,7 +49,6 @@ import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; -import java.io.IOException; import java.util.*; import java.util.stream.Collectors; @@ -110,7 +106,7 @@ public class FragilityController { public FragilityController( @Parameter(name = "User credentials.", required = true) @HeaderParam("x-auth-userinfo") String userInfo, @Parameter(name = "User groups.", required = false) @HeaderParam("x-auth-usergroup") String userGroups - ) { + ) { this.username = UserInfoUtils.getUsername(userInfo); this.userGroups = userGroups; this.groups = UserGroupUtils.getUserGroups(userGroups); @@ -244,7 +240,7 @@ public FragilitySet uploadFragilitySet(@Parameter(name = "json representing the fragilitySet.setCreator(username); fragilitySet.setOwner(username); - if (fragilitySet.getFragilityCurves().size() == 0){ + if (fragilitySet.getFragilityCurves().size() == 0) { throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "No fragility curves are included in the json. " + "Please provide at least one."); } @@ -313,7 +309,7 @@ public FragilitySet deleteFragilityById(@Parameter(name = "fragility id", exampl } // remove dfr3 in the usage - AllocationUtils.decreaseUsage(allocationsRepository, username, "dfr3"); + AllocationUtils.decreaseUsage(allocationsRepository, username, "dfr3"); return this.fragilityDAO.deleteFragilitySetById(id); } diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java index 2bf1920af..43da4d3fb 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java @@ -11,6 +11,7 @@ package edu.illinois.ncsa.incore.service.dfr3.controllers; import edu.illinois.ncsa.incore.common.AllocationConstants; +import edu.illinois.ncsa.incore.common.SemanticsConstants; import edu.illinois.ncsa.incore.common.auth.Authorizer; import edu.illinois.ncsa.incore.common.auth.IAuthorizer; import edu.illinois.ncsa.incore.common.auth.Privileges; @@ -19,7 +20,6 @@ import edu.illinois.ncsa.incore.common.dao.IUserFinalQuotaRepository; import edu.illinois.ncsa.incore.common.exceptions.IncoreHTTPException; import edu.illinois.ncsa.incore.common.models.Space; -import edu.illinois.ncsa.incore.common.models.UserAllocations; import edu.illinois.ncsa.incore.common.utils.AllocationUtils; import edu.illinois.ncsa.incore.common.utils.UserGroupUtils; import edu.illinois.ncsa.incore.common.utils.UserInfoUtils; @@ -271,7 +271,24 @@ else if (dataTypes.isEmpty()) { // Check if all curveParameters are found in columns for every curve set boolean allCurveParametersFound = dfr3CurveSets.stream().allMatch(dfr3CurveSet -> { List curveParameters = dfr3CurveSet.getCurveParameters(); - return curveParameters != null && curveParameters.stream().allMatch(param -> columnsDefinition.contains(param.name)); + if (dfr3CurveSet instanceof FragilitySet) { + return curveParameters != null && curveParameters.stream().allMatch(param -> { + // Only check curve parameter if it does not belong to a part of the demand type + if (!((FragilitySet) dfr3CurveSet).getDemandTypes().contains(param.fullName) + && !((FragilitySet) dfr3CurveSet).getDemandUnits().contains(param.name)) { + // Check if inventoryType is "building" and the column is not reserved + boolean isBuildingAndNotReserved = "building".equals(((FragilitySet) dfr3CurveSet).getInventoryType()) + && SemanticsConstants.RESERVED_COLUMNS.contains(param.name); + + // If it's not a building parameter that is reserved, check if it's in the columns + return isBuildingAndNotReserved || columnsDefinition.contains(param.name); + } + return true; + }); + } else { + // For RestorationSet or other types, just check if all curveParameters are in columnsDefinition + return curveParameters != null && curveParameters.stream().allMatch(param -> columnsDefinition.contains(param.name)); + } }); // If both conditions are met, set columnFound to true diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RepairController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RepairController.java index be7289950..75ceef520 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RepairController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RepairController.java @@ -11,26 +11,20 @@ package edu.illinois.ncsa.incore.service.dfr3.controllers; import edu.illinois.ncsa.incore.common.AllocationConstants; -import edu.illinois.ncsa.incore.common.SemanticsConstants; import edu.illinois.ncsa.incore.common.auth.Authorizer; import edu.illinois.ncsa.incore.common.auth.IAuthorizer; import edu.illinois.ncsa.incore.common.auth.Privileges; import edu.illinois.ncsa.incore.common.dao.ISpaceRepository; import edu.illinois.ncsa.incore.common.dao.IUserAllocationsRepository; import edu.illinois.ncsa.incore.common.dao.IUserFinalQuotaRepository; -import edu.illinois.ncsa.incore.common.dao.IUserAllocationsRepository; -import edu.illinois.ncsa.incore.common.dao.IUserFinalQuotaRepository; import edu.illinois.ncsa.incore.common.exceptions.IncoreHTTPException; import edu.illinois.ncsa.incore.common.models.Space; -import edu.illinois.ncsa.incore.common.models.UserAllocations; import edu.illinois.ncsa.incore.common.utils.AllocationUtils; import edu.illinois.ncsa.incore.common.utils.UserGroupUtils; import edu.illinois.ncsa.incore.common.utils.UserInfoUtils; import edu.illinois.ncsa.incore.service.dfr3.daos.IMappingDAO; import edu.illinois.ncsa.incore.service.dfr3.daos.IRepairDAO; import edu.illinois.ncsa.incore.service.dfr3.models.RepairSet; -import edu.illinois.ncsa.incore.service.dfr3.utils.CommonUtil; -import edu.illinois.ncsa.incore.service.dfr3.utils.ServiceUtil; import io.swagger.v3.oas.annotations.OpenAPIDefinition; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; @@ -47,7 +41,6 @@ import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response; -import java.io.IOException; import java.util.*; import java.util.stream.Collectors; @@ -99,7 +92,7 @@ public class RepairController { @Inject public RepairController( @Parameter(name = "User credentials.", required = true) @HeaderParam("x-auth-userinfo") String userInfo, - @Parameter(name = "User groups.", required = false) @HeaderParam("x-auth-usergroup") String userGroups + @Parameter(name = "User groups.", required = false) @HeaderParam("x-auth-usergroup") String userGroups ) { this.username = UserInfoUtils.getUsername(userInfo); this.userGroups = userGroups; @@ -190,7 +183,7 @@ public RepairSet uploadRepairSet(@Parameter(name = "json representing the repair repairSet.setCreator(username); repairSet.setOwner(username); - if (repairSet.getRepairCurves().size() == 0){ + if (repairSet.getRepairCurves().size() == 0) { throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "No repair curves are included in the json. " + "Please provide at least one."); } diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RestorationController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RestorationController.java index 68b6b5393..eaf546876 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RestorationController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RestorationController.java @@ -26,8 +26,6 @@ import edu.illinois.ncsa.incore.service.dfr3.daos.IMappingDAO; import edu.illinois.ncsa.incore.service.dfr3.daos.IRestorationDAO; import edu.illinois.ncsa.incore.service.dfr3.models.RestorationSet; -import edu.illinois.ncsa.incore.service.dfr3.utils.CommonUtil; -import edu.illinois.ncsa.incore.service.dfr3.utils.ServiceUtil; import io.swagger.v3.oas.annotations.OpenAPIDefinition; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; From 4b00907acdc9d0eefde07bc3526ee49269683a50 Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Mon, 15 Jul 2024 12:23:40 -0500 Subject: [PATCH 15/16] fix bug and finalize the logic --- .../dfr3/controllers/MappingController.java | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java index 43da4d3fb..2fed61744 100644 --- a/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java +++ b/server/dfr3-service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java @@ -266,33 +266,35 @@ else if (dataTypes.isEmpty()) { List columnsDefinition = CommonUtil.getColumnNames(semanticsDefinition); // Check if all uniqueColumns are found in columns - boolean allUniqueColumnsFound = columnsDefinition.containsAll(uniqueColumns); - + boolean allMappingRuleColumnsFound = columnsDefinition.containsAll(uniqueColumns); // Check if all curveParameters are found in columns for every curve set - boolean allCurveParametersFound = dfr3CurveSets.stream().allMatch(dfr3CurveSet -> { - List curveParameters = dfr3CurveSet.getCurveParameters(); - if (dfr3CurveSet instanceof FragilitySet) { - return curveParameters != null && curveParameters.stream().allMatch(param -> { - // Only check curve parameter if it does not belong to a part of the demand type - if (!((FragilitySet) dfr3CurveSet).getDemandTypes().contains(param.fullName) - && !((FragilitySet) dfr3CurveSet).getDemandUnits().contains(param.name)) { - // Check if inventoryType is "building" and the column is not reserved - boolean isBuildingAndNotReserved = "building".equals(((FragilitySet) dfr3CurveSet).getInventoryType()) - && SemanticsConstants.RESERVED_COLUMNS.contains(param.name); - - // If it's not a building parameter that is reserved, check if it's in the columns - return isBuildingAndNotReserved || columnsDefinition.contains(param.name); - } - return true; - }); - } else { - // For RestorationSet or other types, just check if all curveParameters are in columnsDefinition - return curveParameters != null && curveParameters.stream().allMatch(param -> columnsDefinition.contains(param.name)); - } - }); + boolean allDFR3CurveParameterColumnsFound = false; + if (allMappingRuleColumnsFound) { + allDFR3CurveParameterColumnsFound = dfr3CurveSets.stream().allMatch(dfr3CurveSet -> { + List curveParameters = dfr3CurveSet.getCurveParameters(); + if (dfr3CurveSet instanceof FragilitySet) { + return curveParameters != null && curveParameters.stream().allMatch(param -> { + // Only check curve parameter if it does not belong to a part of the demand type + if (!((FragilitySet) dfr3CurveSet).getDemandTypes().contains(param.fullName) + && !((FragilitySet) dfr3CurveSet).getDemandTypes().contains(param.name)) { + // Check if inventoryType is "building" and the column is not reserved + boolean isBuildingAndNotReserved = "building".equals(((FragilitySet) dfr3CurveSet).getInventoryType()) + && SemanticsConstants.RESERVED_COLUMNS.contains(param.name); + + // If it's not a building parameter that is reserved, check if it's in the columns + return isBuildingAndNotReserved || columnsDefinition.contains(param.name); + } + return true; + }); + } else { + // For RestorationSet or other types, just check if all curveParameters are in columnsDefinition + return curveParameters != null && curveParameters.stream().allMatch(param -> columnsDefinition.contains(param.name)); + } + }); + } // If both conditions are met, set columnFound to true - if (allUniqueColumnsFound && allCurveParametersFound) { + if (allMappingRuleColumnsFound && allDFR3CurveParameterColumnsFound) { columnFound = true; break; // Break the outer loop if both conditions are met } From 83438f12c7ecfb296f00ba850d650df508c7a9df Mon Sep 17 00:00:00 2001 From: Chen Wang Date: Wed, 24 Jul 2024 09:51:28 -0500 Subject: [PATCH 16/16] redundant data types --- server/dfr3-service/src/test/resources/fragility_request.json | 1 - 1 file changed, 1 deletion(-) diff --git a/server/dfr3-service/src/test/resources/fragility_request.json b/server/dfr3-service/src/test/resources/fragility_request.json index 22165554a..cd1fa94ea 100644 --- a/server/dfr3-service/src/test/resources/fragility_request.json +++ b/server/dfr3-service/src/test/resources/fragility_request.json @@ -2,7 +2,6 @@ "legacyId": "legacyId", "hazardType": "hazardType", "inventoryType": "inventoryType", - "dataTypes": ["incore:dataType"], "authors": [ "Chen Wang" ],