Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

295 semantics with dfr3 curves and mapping #297

Open
wants to merge 17 commits into
base: develop
Choose a base branch
from
Open

295 semantics with dfr3 curves and mapping #297

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
e6ddd59
add additional field to indicate data type
longshuicy Jun 4, 2024
25e98d7
put common parameters into constant; bypass certain fields
longshuicy Jun 5, 2024
e7002e5
refine the logic
longshuicy Jun 5, 2024
85528c1
throw exception bugfix
longshuicy Jun 5, 2024
2bb2fe5
add same logic to restoration and repair curves
longshuicy Jun 5, 2024
4261ee7
initial code parsing mapping rules to get columns
longshuicy Jun 5, 2024
f253c0c
fix bugs in mapping check
longshuicy Jun 5, 2024
9f7b475
changelog
longshuicy Jun 5, 2024
0ae602b
typo in changelog entry
longshuicy Jul 11, 2024
912c5b5
remove it from fragility, repair and restoration
longshuicy Jul 12, 2024
61ef931
update to a list of data types
longshuicy Jul 12, 2024
8f9eade
add logic to check data type empty
longshuicy Jul 12, 2024
e4c07d5
loop through each fraigility curve to check required parameters
longshuicy Jul 12, 2024
129f6ef
add check to each fragility curve
longshuicy Jul 15, 2024
4b00907
fix bug and finalize the logic
longshuicy Jul 15, 2024
2b61df5
Merge branch 'develop' into 295-semantics-with-dfr3-curves-and-mapping
longshuicy Jul 15, 2024
83438f1
redundant data types
longshuicy Jul 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).

## [Unreleased]
- Valid DFR3 set and mapping set match semantcis definition [#295](https://github.com/IN-CORE/incore-services/issues/295)

### Added
- Valid DFR3 mapping set match semantics definition [#295](https://github.com/IN-CORE/incore-services/issues/295)

### Changed
- Jersey to version 3.1.7 [#250](https://github.com/IN-CORE/incore-services/issues/250)

## [1.26.1] - 2024-04-08

Expand Down
41 changes: 3 additions & 38 deletions .../src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,13 @@
import edu.illinois.ncsa.incore.common.dao.IUserFinalQuotaRepository;
import edu.illinois.ncsa.incore.common.exceptions.IncoreHTTPException;
import edu.illinois.ncsa.incore.common.models.Space;
import edu.illinois.ncsa.incore.common.models.UserAllocations;
import edu.illinois.ncsa.incore.common.utils.AllocationUtils;
import edu.illinois.ncsa.incore.common.utils.UserGroupUtils;
import edu.illinois.ncsa.incore.common.utils.UserInfoUtils;
import edu.illinois.ncsa.incore.service.dfr3.daos.IFragilityDAO;
import edu.illinois.ncsa.incore.service.dfr3.daos.IMappingDAO;
import edu.illinois.ncsa.incore.service.dfr3.models.FragilitySet;

import edu.illinois.ncsa.incore.service.dfr3.utils.CommonUtil;
import edu.illinois.ncsa.incore.service.dfr3.utils.ServiceUtil;
import io.swagger.v3.oas.annotations.OpenAPIDefinition;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
Expand All @@ -52,7 +49,6 @@
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;

import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -110,7 +106,7 @@ public class FragilityController {
public FragilityController(
@Parameter(name = "User credentials.", required = true) @HeaderParam("x-auth-userinfo") String userInfo,
@Parameter(name = "User groups.", required = false) @HeaderParam("x-auth-usergroup") String userGroups
) {
) {
this.username = UserInfoUtils.getUsername(userInfo);
this.userGroups = userGroups;
this.groups = UserGroupUtils.getUserGroups(userGroups);
Expand All @@ -122,7 +118,6 @@ public FragilityController(
public List<FragilitySet> getFragilities(@Parameter(name = "demand type filter", example = "PGA") @QueryParam("demand") String demandType,
@Parameter(name = "hazard type filter", example = "earthquake") @QueryParam("hazard") String hazardType,
@Parameter(name = "Inventory type filter", example = "building") @QueryParam("inventory") String inventoryType,
@Parameter(name = "Data type filter", example = "ergo:buildingInventoryVer7") @QueryParam("dataType") String dataType,
@Parameter(name = "not implemented", hidden = true) @QueryParam("author") String author,
@Parameter(name = "Legacy fragility Id from v1") @QueryParam("legacy_id") String legacyId,
@Parameter(name = "Fragility creator's username") @QueryParam("creator") String creator,
Expand All @@ -147,10 +142,6 @@ public List<FragilitySet> getFragilities(@Parameter(name = "demand type filter",
queryMap.put("inventoryType", inventoryType);
}

if (dataType != null){
queryMap.put("dataType", dataType);
}

if (creator != null) {
queryMap.put("creator", creator);
}
Expand Down Expand Up @@ -245,37 +236,11 @@ public FragilitySet uploadFragilitySet(@Parameter(name = "json representing the
}
}
}
// check if the parameters matches the defined data type in semantics
String dataType = fragilitySet.getDataType();
String inventoryType = fragilitySet.getInventoryType();
if (dataType == null) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataType is a required field.");
}
try {
String semanticsDefinition = ServiceUtil.getJsonFromSemanticsEndpoint(dataType, username, userGroups);
List<String> columns = CommonUtil.getColumnNames(semanticsDefinition);

fragilitySet.getCurveParameters().forEach((params) -> {
// Only check curve parameter if it does not belong to a part of the demand type
if (!demandTypes.contains(params.fullName) && !demandUnits.contains(params.name)) {
// Check if inventoryType is "building" and the column is not reserved
boolean isBuildingAndNotReserved = "building".equals(inventoryType) && SemanticsConstants.RESERVED_COLUMNS.contains(params.name);

// If it's not a building parameter that is reserved, check if it's in the columns
if (!isBuildingAndNotReserved && !columns.contains(params.name)) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Curve parameter: " + params.name + " not found in the dataType: " + dataType);
}
}
});

} catch (IOException e) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Could not check the fragility curve parameter matches the dataType columns.");
}

fragilitySet.setCreator(username);
fragilitySet.setOwner(username);

if (fragilitySet.getFragilityCurves().size() == 0){
if (fragilitySet.getFragilityCurves().size() == 0) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "No fragility curves are included in the json. " +
"Please provide at least one.");
}
Expand Down Expand Up @@ -344,7 +309,7 @@ public FragilitySet deleteFragilityById(@Parameter(name = "fragility id", exampl
}

// remove dfr3 in the usage
AllocationUtils.decreaseUsage(allocationsRepository, username, "dfr3");
AllocationUtils.decreaseUsage(allocationsRepository, username, "dfr3");

return this.fragilityDAO.deleteFragilitySetById(id);
}
Expand Down
95 changes: 74 additions & 21 deletions ...ce/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
package edu.illinois.ncsa.incore.service.dfr3.controllers;

import edu.illinois.ncsa.incore.common.AllocationConstants;
import edu.illinois.ncsa.incore.common.SemanticsConstants;
import edu.illinois.ncsa.incore.common.auth.Authorizer;
import edu.illinois.ncsa.incore.common.auth.IAuthorizer;
import edu.illinois.ncsa.incore.common.auth.Privileges;
Expand All @@ -19,16 +20,14 @@
import edu.illinois.ncsa.incore.common.dao.IUserFinalQuotaRepository;
import edu.illinois.ncsa.incore.common.exceptions.IncoreHTTPException;
import edu.illinois.ncsa.incore.common.models.Space;
import edu.illinois.ncsa.incore.common.models.UserAllocations;
import edu.illinois.ncsa.incore.common.utils.AllocationUtils;
import edu.illinois.ncsa.incore.common.utils.UserGroupUtils;
import edu.illinois.ncsa.incore.common.utils.UserInfoUtils;
import edu.illinois.ncsa.incore.service.dfr3.daos.IFragilityDAO;
import edu.illinois.ncsa.incore.service.dfr3.daos.IMappingDAO;
import edu.illinois.ncsa.incore.service.dfr3.daos.IRepairDAO;
import edu.illinois.ncsa.incore.service.dfr3.daos.IRestorationDAO;
import edu.illinois.ncsa.incore.service.dfr3.models.Mapping;
import edu.illinois.ncsa.incore.service.dfr3.models.MappingSet;
import edu.illinois.ncsa.incore.service.dfr3.models.*;
import edu.illinois.ncsa.incore.service.dfr3.utils.CommonUtil;
import edu.illinois.ncsa.incore.service.dfr3.utils.ServiceUtil;
import io.swagger.v3.oas.annotations.Parameter;
Expand Down Expand Up @@ -112,10 +111,6 @@ public List<MappingSet> getMappings(@Parameter(name= "hazard type filter", exam
queryMap.put("inventoryType", inventoryType);
}

if (dataType != null) {
queryMap.put("dataType", dataType);
}

if (creator != null) {
queryMap.put("creator", creator);
}
Expand All @@ -127,7 +122,7 @@ public List<MappingSet> getMappings(@Parameter(name= "hazard type filter", exam
List<MappingSet> mappingSets;

if (queryMap.isEmpty()) {
mappingSets = this.mappingDAO.getMappingSets();
mappingSets = this.mappingDAO.getMappingSets(dataType);
} else {
mappingSets = this.mappingDAO.queryMappingSets(queryMap);
}
Expand Down Expand Up @@ -202,6 +197,7 @@ public MappingSet uploadMapping(@Parameter(name = "json representing the fragili

List<Mapping> mappings = mappingSet.getMappings();
Set<String> columnSet = new HashSet<>();
Set<DFR3Set> dfr3CurveSets = new HashSet<>();

int idx = 0;
String prevRuleClassName = "";
Expand Down Expand Up @@ -231,30 +227,87 @@ public MappingSet uploadMapping(@Parameter(name = "json representing the fragili
else if(mapping.getRules() instanceof HashMap) {
extractColumnsFromMapping((HashMap<?, ?>) mapping.getRules(), columnSet);
}

// get unique dfr3 curves
Optional.ofNullable(mapping.getEntry())
.ifPresent(entry -> {
if ("fragility".equals(mappingSet.getMappingType())) {
entry.values().forEach(id ->
this.fragilityDAO.getFragilitySetById(id).ifPresent(dfr3CurveSets::add)
);
} else if ("restoration".equals(mappingSet.getMappingType())) {
entry.values().forEach(id ->
this.restorationDAO.getRestorationSetById(id).ifPresent(dfr3CurveSets::add)
);
} else if ("repair".equals(mappingSet.getMappingType())) {
entry.values().forEach(id ->
this.repairDAO.getRepairSetById(id).ifPresent(dfr3CurveSets::add)
);
}
});

}

List<String> uniqueColumns = new ArrayList<>(columnSet);

// check if the parameters matches the defined data type in semantics
String dataType = mappingSet.getDataType();
if (dataType == null) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataType is a required field.");
List<String> dataTypes = mappingSet.getDataTypes();
if (dataTypes == null) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataTypes is a required field.");
}
else if (dataTypes.isEmpty()) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataTypes cannot be empty.");
}
try {
String semanticsDefinition = ServiceUtil.getJsonFromSemanticsEndpoint(dataType, username, userGroups);
List<String> columns = CommonUtil.getColumnNames(semanticsDefinition);

// parse mapping rules to find column names
uniqueColumns.forEach((uniqueColumn) -> {
if(!columns.contains(uniqueColumn)) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Column: " + uniqueColumn + " in the Mapping Rules not found in the dataType: " + dataType);
boolean columnFound = false;
for (String dataType : dataTypes) {
try {
String semanticsDefinition = ServiceUtil.getJsonFromSemanticsEndpoint(dataType, username, userGroups);
List<String> columnsDefinition = CommonUtil.getColumnNames(semanticsDefinition);

// Check if all uniqueColumns are found in columns
boolean allMappingRuleColumnsFound = columnsDefinition.containsAll(uniqueColumns);
// Check if all curveParameters are found in columns for every curve set
boolean allDFR3CurveParameterColumnsFound = false;
if (allMappingRuleColumnsFound) {
allDFR3CurveParameterColumnsFound = dfr3CurveSets.stream().allMatch(dfr3CurveSet -> {
List<CurveParameter> curveParameters = dfr3CurveSet.getCurveParameters();
if (dfr3CurveSet instanceof FragilitySet) {
return curveParameters != null && curveParameters.stream().allMatch(param -> {
// Only check curve parameter if it does not belong to a part of the demand type
if (!((FragilitySet) dfr3CurveSet).getDemandTypes().contains(param.fullName)
&& !((FragilitySet) dfr3CurveSet).getDemandTypes().contains(param.name)) {
// Check if inventoryType is "building" and the column is not reserved
boolean isBuildingAndNotReserved = "building".equals(((FragilitySet) dfr3CurveSet).getInventoryType())
&& SemanticsConstants.RESERVED_COLUMNS.contains(param.name);

// If it's not a building parameter that is reserved, check if it's in the columns
return isBuildingAndNotReserved || columnsDefinition.contains(param.name);
}
return true;
});
} else {
// For RestorationSet or other types, just check if all curveParameters are in columnsDefinition
return curveParameters != null && curveParameters.stream().allMatch(param -> columnsDefinition.contains(param.name));
}
});
}
});

} catch (IOException e) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Could not check if the column in the mapping rules matches the dataType columns.");
// If both conditions are met, set columnFound to true
if (allMappingRuleColumnsFound && allDFR3CurveParameterColumnsFound) {
columnFound = true;
break; // Break the outer loop if both conditions are met
}
} catch (IOException e) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST,
"Could not check if the column in the mapping rules matches the dataType: " + dataType);
}
}

if (!columnFound) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST,
"The columns in the mapping rules and/or fragility parameters do not match the columns in any of the listed dataTypes.");
}

mappingSet.setCreator(username);
mappingSet.setOwner(username);
Expand Down
35 changes: 2 additions & 33 deletions ...ice/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RepairController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,26 +11,20 @@
package edu.illinois.ncsa.incore.service.dfr3.controllers;

import edu.illinois.ncsa.incore.common.AllocationConstants;
import edu.illinois.ncsa.incore.common.SemanticsConstants;
import edu.illinois.ncsa.incore.common.auth.Authorizer;
import edu.illinois.ncsa.incore.common.auth.IAuthorizer;
import edu.illinois.ncsa.incore.common.auth.Privileges;
import edu.illinois.ncsa.incore.common.dao.ISpaceRepository;
import edu.illinois.ncsa.incore.common.dao.IUserAllocationsRepository;
import edu.illinois.ncsa.incore.common.dao.IUserFinalQuotaRepository;
import edu.illinois.ncsa.incore.common.dao.IUserAllocationsRepository;
import edu.illinois.ncsa.incore.common.dao.IUserFinalQuotaRepository;
import edu.illinois.ncsa.incore.common.exceptions.IncoreHTTPException;
import edu.illinois.ncsa.incore.common.models.Space;
import edu.illinois.ncsa.incore.common.models.UserAllocations;
import edu.illinois.ncsa.incore.common.utils.AllocationUtils;
import edu.illinois.ncsa.incore.common.utils.UserGroupUtils;
import edu.illinois.ncsa.incore.common.utils.UserInfoUtils;
import edu.illinois.ncsa.incore.service.dfr3.daos.IMappingDAO;
import edu.illinois.ncsa.incore.service.dfr3.daos.IRepairDAO;
import edu.illinois.ncsa.incore.service.dfr3.models.RepairSet;
import edu.illinois.ncsa.incore.service.dfr3.utils.CommonUtil;
import edu.illinois.ncsa.incore.service.dfr3.utils.ServiceUtil;
import io.swagger.v3.oas.annotations.OpenAPIDefinition;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
Expand All @@ -47,7 +41,6 @@
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;

import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -99,7 +92,7 @@ public class RepairController {
@Inject
public RepairController(
@Parameter(name = "User credentials.", required = true) @HeaderParam("x-auth-userinfo") String userInfo,
@Parameter(name = "User groups.", required = false) @HeaderParam("x-auth-usergroup") String userGroups
@Parameter(name = "User groups.", required = false) @HeaderParam("x-auth-usergroup") String userGroups
) {
this.username = UserInfoUtils.getUsername(userInfo);
this.userGroups = userGroups;
Expand All @@ -111,7 +104,6 @@ public RepairController(
@Operation(tags = "Gets list of repairs", summary = "Apply filters to get the desired set of repairs")
public List<RepairSet> getRepairs(@Parameter(name = "hazard type filter", example = "earthquake") @QueryParam("hazard") String hazardType,
@Parameter(name = "Inventory type", example = "building") @QueryParam("inventory") String inventoryType,
@Parameter(name = "Data type filter", example = "ergo:buildingInventoryVer7") @QueryParam("dataType") String dataType,
@Parameter(name = "Repair creator's username") @QueryParam("creator") String creator,
@Parameter(name = "Name of space") @DefaultValue("") @QueryParam("space") String spaceName,
@Parameter(name = "Skip the first n results") @QueryParam("skip") int offset,
Expand All @@ -126,10 +118,6 @@ public List<RepairSet> getRepairs(@Parameter(name = "hazard type filter", examp
queryMap.put("inventoryType", inventoryType);
}

if (dataType != null){
queryMap.put("dataType", dataType);
}

if (creator != null) {
queryMap.put("creator", creator);
}
Expand Down Expand Up @@ -192,29 +180,10 @@ public RepairSet uploadRepairSet(@Parameter(name = "json representing the repair
AllocationConstants.HAZARD_DFR3_ALLOCATION_MESSAGE);
}

// check if the parameters matches the defined data type in semantics
String dataType = repairSet.getDataType();
if (dataType == null) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataType is a required field.");
}
try {
String semanticsDefinition = ServiceUtil.getJsonFromSemanticsEndpoint(dataType, username, userGroups);
List<String> columns = CommonUtil.getColumnNames(semanticsDefinition);

repairSet.getCurveParameters().forEach((params) -> {
if(!columns.contains(params.name)) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Curve parameter: " + params.name + " not found in the dataType: " + dataType);
}
});

} catch (IOException e) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Could not check the repair curve parameter matches the dataType columns.");
}

repairSet.setCreator(username);
repairSet.setOwner(username);

if (repairSet.getRepairCurves().size() == 0){
if (repairSet.getRepairCurves().size() == 0) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "No repair curves are included in the json. " +
"Please provide at least one.");
}
Expand Down
Loading
Loading