Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

295 semantics with dfr3 curves and mapping #297

Open
wants to merge 17 commits into
base: develop
Choose a base branch
from
Open

295 semantics with dfr3 curves and mapping #297

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
e6ddd59
add additional field to indicate data type
longshuicy Jun 4, 2024
25e98d7
put common parameters into constant; bypass certain fields
longshuicy Jun 5, 2024
e7002e5
refine the logic
longshuicy Jun 5, 2024
85528c1
throw exception bugfix
longshuicy Jun 5, 2024
2bb2fe5
add same logic to restoration and repair curves
longshuicy Jun 5, 2024
4261ee7
initial code parsing mapping rules to get columns
longshuicy Jun 5, 2024
f253c0c
fix bugs in mapping check
longshuicy Jun 5, 2024
9f7b475
changelog
longshuicy Jun 5, 2024
0ae602b
typo in changelog entry
longshuicy Jul 11, 2024
912c5b5
remove it from fragility, repair and restoration
longshuicy Jul 12, 2024
61ef931
update to a list of data types
longshuicy Jul 12, 2024
8f9eade
add logic to check data type empty
longshuicy Jul 12, 2024
e4c07d5
loop through each fraigility curve to check required parameters
longshuicy Jul 12, 2024
129f6ef
add check to each fragility curve
longshuicy Jul 15, 2024
4b00907
fix bug and finalize the logic
longshuicy Jul 15, 2024
2b61df5
Merge branch 'develop' into 295-semantics-with-dfr3-curves-and-mapping
longshuicy Jul 15, 2024
83438f1
redundant data types
longshuicy Jul 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/)
and this project adheres to [Semantic Versioning](http://semver.org/).

## [Unreleased]
- Valid DFR3 set and mapping set match semantcis definition [#295](https://github.com/IN-CORE/incore-services/issues/295)

## [1.26.1] - 2024-04-08

### Fixed
Expand Down
7 changes: 4 additions & 3 deletions server/dfr3-service/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ gretty {
contextPath '/dfr3/'
}

// dependencies {
// implementation "org.dom4j:dom4j:1.6.1"
// }
dependencies {
implementation group: 'org.apache.httpcomponents', name: 'httpmime', version: '4.5.5'
implementation group: 'org.apache.commons', name: 'commons-collections4', version: '4.0'
}
40 changes: 39 additions & 1 deletion .../src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/FragilityController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
package edu.illinois.ncsa.incore.service.dfr3.controllers;

import edu.illinois.ncsa.incore.common.AllocationConstants;
import edu.illinois.ncsa.incore.common.SemanticsConstants;
import edu.illinois.ncsa.incore.common.auth.Authorizer;
import edu.illinois.ncsa.incore.common.auth.IAuthorizer;
import edu.illinois.ncsa.incore.common.auth.Privileges;
Expand All @@ -28,6 +29,8 @@
import edu.illinois.ncsa.incore.service.dfr3.daos.IMappingDAO;
import edu.illinois.ncsa.incore.service.dfr3.models.FragilitySet;

import edu.illinois.ncsa.incore.service.dfr3.utils.CommonUtil;
import edu.illinois.ncsa.incore.service.dfr3.utils.ServiceUtil;
import io.swagger.v3.oas.annotations.OpenAPIDefinition;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
Expand All @@ -48,6 +51,8 @@
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;

import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -84,6 +89,7 @@ public class FragilityController {

private final String username;
private final List<String> groups;
private final String userGroups;

@Inject
IAuthorizer authorizer;
Expand All @@ -106,6 +112,7 @@ public FragilityController(
@Parameter(name = "User groups.", required = false) @HeaderParam("x-auth-usergroup") String userGroups
) {
this.username = UserInfoUtils.getUsername(userInfo);
this.userGroups = userGroups;
this.groups = UserGroupUtils.getUserGroups(userGroups);
}

Expand All @@ -114,7 +121,8 @@ public FragilityController(
@Operation(tags = "Gets list of fragilities", summary = "Apply filters to get the desired set of fragilities")
public List<FragilitySet> getFragilities(@Parameter(name = "demand type filter", example = "PGA") @QueryParam("demand") String demandType,
@Parameter(name = "hazard type filter", example = "earthquake") @QueryParam("hazard") String hazardType,
@Parameter(name = "Inventory type", example = "building") @QueryParam("inventory") String inventoryType,
@Parameter(name = "Inventory type filter", example = "building") @QueryParam("inventory") String inventoryType,
@Parameter(name = "Data type filter", example = "ergo:buildingInventoryVer7") @QueryParam("dataType") String dataType,
@Parameter(name = "not implemented", hidden = true) @QueryParam("author") String author,
@Parameter(name = "Legacy fragility Id from v1") @QueryParam("legacy_id") String legacyId,
@Parameter(name = "Fragility creator's username") @QueryParam("creator") String creator,
Expand All @@ -139,6 +147,10 @@ public List<FragilitySet> getFragilities(@Parameter(name = "demand type filter",
queryMap.put("inventoryType", inventoryType);
}

if (dataType != null){
queryMap.put("dataType", dataType);
}

if (creator != null) {
queryMap.put("creator", creator);
}
Expand Down Expand Up @@ -233,6 +245,32 @@ public FragilitySet uploadFragilitySet(@Parameter(name = "json representing the
}
}
}
// check if the parameters matches the defined data type in semantics
String dataType = fragilitySet.getDataType();
String inventoryType = fragilitySet.getInventoryType();
if (dataType == null) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataType is a required field.");
}
try {
String semanticsDefinition = ServiceUtil.getJsonFromSemanticsEndpoint(dataType, username, userGroups);
List<String> columns = CommonUtil.getColumnNames(semanticsDefinition);

fragilitySet.getCurveParameters().forEach((params) -> {
// Only check curve parameter if it does not belong to a part of the demand type
if (!demandTypes.contains(params.fullName) && !demandUnits.contains(params.name)) {
// Check if inventoryType is "building" and the column is not reserved
boolean isBuildingAndNotReserved = "building".equals(inventoryType) && SemanticsConstants.RESERVED_COLUMNS.contains(params.name);

// If it's not a building parameter that is reserved, check if it's in the columns
if (!isBuildingAndNotReserved && !columns.contains(params.name)) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Curve parameter: " + params.name + " not found in the dataType: " + dataType);
}
}
});

} catch (IOException e) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Could not check the fragility curve parameter matches the dataType columns.");
}

fragilitySet.setCreator(username);
fragilitySet.setOwner(username);
Expand Down
46 changes: 46 additions & 0 deletions ...ce/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/MappingController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@
import edu.illinois.ncsa.incore.service.dfr3.daos.IRestorationDAO;
import edu.illinois.ncsa.incore.service.dfr3.models.Mapping;
import edu.illinois.ncsa.incore.service.dfr3.models.MappingSet;
import edu.illinois.ncsa.incore.service.dfr3.utils.CommonUtil;
import edu.illinois.ncsa.incore.service.dfr3.utils.ServiceUtil;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
Expand All @@ -38,9 +40,13 @@
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;

import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;

import static edu.illinois.ncsa.incore.service.dfr3.utils.CommonUtil.extractColumnsFromMapping;


@Tag(name = "Mapping")
@Path("mappings")
Expand All @@ -49,6 +55,7 @@ public class MappingController {

private final String username;
private final List<String> groups;
private final String userGroups;

@Inject
private IMappingDAO mappingDAO;
Expand Down Expand Up @@ -79,6 +86,7 @@ public MappingController(
@Parameter(name = "User groups.", required = false) @HeaderParam("x-auth-usergroup") String userGroups
) {
this.username = UserInfoUtils.getUsername(userInfo);
this.userGroups = userGroups;
this.groups = UserGroupUtils.getUserGroups(userGroups);
}

Expand All @@ -87,6 +95,7 @@ public MappingController(
@Operation(tags = "Gets list of all inventory mappings", summary = "Apply filters to get the desired set of mappings")
public List<MappingSet> getMappings(@Parameter(name= "hazard type filter", example = "earthquake") @QueryParam("hazard") String hazardType,
@Parameter(name = "Inventory type", example = "building") @QueryParam("inventory") String inventoryType,
@Parameter(name = "Data type", example = "ergo:buildingInventoryVer7") @QueryParam("dataType") String dataType,
@Parameter(name = "DFR3 Mapping type", example = "fragility, restoration, repair") @QueryParam(
"mappingType") String mappingType,
@Parameter(name = "Creator's username") @QueryParam("creator") String creator,
Expand All @@ -103,6 +112,10 @@ public List<MappingSet> getMappings(@Parameter(name= "hazard type filter", exam
queryMap.put("inventoryType", inventoryType);
}

if (dataType != null) {
queryMap.put("dataType", dataType);
}

if (creator != null) {
queryMap.put("creator", creator);
}
Expand Down Expand Up @@ -188,6 +201,8 @@ public MappingSet uploadMapping(@Parameter(name = "json representing the fragili
}

List<Mapping> mappings = mappingSet.getMappings();
Set<String> columnSet = new HashSet<>();

int idx = 0;
String prevRuleClassName = "";
// This validates if the format of the "rules" being submitted is an Array or Hash. It is needed because we made "rules" attribute
Expand All @@ -208,8 +223,39 @@ public MappingSet uploadMapping(@Parameter(name = "json representing the fragili
prevRuleClassName = mapping.getRules().getClass().getName();
}
idx++;

// get unique column names
if (mapping.getRules() instanceof ArrayList) {
extractColumnsFromMapping((ArrayList<?>) mapping.getRules(), columnSet);
}
else if(mapping.getRules() instanceof HashMap) {
extractColumnsFromMapping((HashMap<?, ?>) mapping.getRules(), columnSet);
}
}

List<String> uniqueColumns = new ArrayList<>(columnSet);

// check if the parameters matches the defined data type in semantics
String dataType = mappingSet.getDataType();
if (dataType == null) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataType is a required field.");
}
try {
String semanticsDefinition = ServiceUtil.getJsonFromSemanticsEndpoint(dataType, username, userGroups);
List<String> columns = CommonUtil.getColumnNames(semanticsDefinition);

// parse mapping rules to find column names
uniqueColumns.forEach((uniqueColumn) -> {
if(!columns.contains(uniqueColumn)) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Column: " + uniqueColumn + " in the Mapping Rules not found in the dataType: " + dataType);
}
});

} catch (IOException e) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Could not check if the column in the mapping rules matches the dataType columns.");
}


mappingSet.setCreator(username);
mappingSet.setOwner(username);

Expand Down
31 changes: 31 additions & 0 deletions ...ice/src/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RepairController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
package edu.illinois.ncsa.incore.service.dfr3.controllers;

import edu.illinois.ncsa.incore.common.AllocationConstants;
import edu.illinois.ncsa.incore.common.SemanticsConstants;
import edu.illinois.ncsa.incore.common.auth.Authorizer;
import edu.illinois.ncsa.incore.common.auth.IAuthorizer;
import edu.illinois.ncsa.incore.common.auth.Privileges;
Expand All @@ -28,6 +29,8 @@
import edu.illinois.ncsa.incore.service.dfr3.daos.IMappingDAO;
import edu.illinois.ncsa.incore.service.dfr3.daos.IRepairDAO;
import edu.illinois.ncsa.incore.service.dfr3.models.RepairSet;
import edu.illinois.ncsa.incore.service.dfr3.utils.CommonUtil;
import edu.illinois.ncsa.incore.service.dfr3.utils.ServiceUtil;
import io.swagger.v3.oas.annotations.OpenAPIDefinition;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
Expand All @@ -43,6 +46,8 @@
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;

import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -76,6 +81,7 @@ public class RepairController {

private final String username;
private final List<String> groups;
private final String userGroups;

@Inject
IAuthorizer authorizer;
Expand All @@ -96,6 +102,7 @@ public RepairController(
@Parameter(name = "User groups.", required = false) @HeaderParam("x-auth-usergroup") String userGroups
) {
this.username = UserInfoUtils.getUsername(userInfo);
this.userGroups = userGroups;
this.groups = UserGroupUtils.getUserGroups(userGroups);
}

Expand All @@ -104,6 +111,7 @@ public RepairController(
@Operation(tags = "Gets list of repairs", summary = "Apply filters to get the desired set of repairs")
public List<RepairSet> getRepairs(@Parameter(name = "hazard type filter", example = "earthquake") @QueryParam("hazard") String hazardType,
@Parameter(name = "Inventory type", example = "building") @QueryParam("inventory") String inventoryType,
@Parameter(name = "Data type filter", example = "ergo:buildingInventoryVer7") @QueryParam("dataType") String dataType,
@Parameter(name = "Repair creator's username") @QueryParam("creator") String creator,
@Parameter(name = "Name of space") @DefaultValue("") @QueryParam("space") String spaceName,
@Parameter(name = "Skip the first n results") @QueryParam("skip") int offset,
Expand All @@ -118,6 +126,10 @@ public List<RepairSet> getRepairs(@Parameter(name = "hazard type filter", examp
queryMap.put("inventoryType", inventoryType);
}

if (dataType != null){
queryMap.put("dataType", dataType);
}

if (creator != null) {
queryMap.put("creator", creator);
}
Expand Down Expand Up @@ -180,6 +192,25 @@ public RepairSet uploadRepairSet(@Parameter(name = "json representing the repair
AllocationConstants.HAZARD_DFR3_ALLOCATION_MESSAGE);
}

// check if the parameters matches the defined data type in semantics
String dataType = repairSet.getDataType();
if (dataType == null) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataType is a required field.");
}
try {
String semanticsDefinition = ServiceUtil.getJsonFromSemanticsEndpoint(dataType, username, userGroups);
List<String> columns = CommonUtil.getColumnNames(semanticsDefinition);

repairSet.getCurveParameters().forEach((params) -> {
if(!columns.contains(params.name)) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Curve parameter: " + params.name + " not found in the dataType: " + dataType);
}
});

} catch (IOException e) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Could not check the repair curve parameter matches the dataType columns.");
}

repairSet.setCreator(username);
repairSet.setOwner(username);

Expand Down
30 changes: 30 additions & 0 deletions ...rc/main/java/edu/illinois/ncsa/incore/service/dfr3/controllers/RestorationController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import edu.illinois.ncsa.incore.service.dfr3.daos.IMappingDAO;
import edu.illinois.ncsa.incore.service.dfr3.daos.IRestorationDAO;
import edu.illinois.ncsa.incore.service.dfr3.models.RestorationSet;
import edu.illinois.ncsa.incore.service.dfr3.utils.CommonUtil;
import edu.illinois.ncsa.incore.service.dfr3.utils.ServiceUtil;
import io.swagger.v3.oas.annotations.OpenAPIDefinition;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
Expand All @@ -41,6 +43,8 @@
import jakarta.ws.rs.*;
import jakarta.ws.rs.core.MediaType;
import jakarta.ws.rs.core.Response;

import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -75,6 +79,7 @@ public class RestorationController {

private final String username;
private final List<String> groups;
private final String userGroups;

@Inject
IAuthorizer authorizer;
Expand All @@ -95,6 +100,7 @@ public RestorationController(
@Parameter(name = "User groups.", required = false) @HeaderParam("x-auth-usergroup") String userGroups
) {
this.username = UserInfoUtils.getUsername(userInfo);
this.userGroups = userGroups;
this.groups = UserGroupUtils.getUserGroups(userGroups);
}

Expand All @@ -103,6 +109,7 @@ public RestorationController(
@Operation(tags = "Gets list of restorations", description = "Apply filters to get the desired set of restorations")
public List<RestorationSet> getRestorations(@Parameter(name = "hazard type filter", example = "earthquake") @QueryParam("hazard") String hazardType,
@Parameter(name = "Inventory type", example = "building") @QueryParam("inventory") String inventoryType,
@Parameter(name = "Data type filter", example = "ergo:buildingInventoryVer7") @QueryParam("dataType") String dataType,
@Parameter(name = "Restoration creator's username") @QueryParam("creator") String creator,
@Parameter(name = "Name of space") @DefaultValue("") @QueryParam("space") String spaceName,
@Parameter(name = "Skip the first n results") @QueryParam("skip") int offset,
Expand All @@ -118,6 +125,10 @@ public List<RestorationSet> getRestorations(@Parameter(name = "hazard type filt
queryMap.put("inventoryType", inventoryType);
}

if (dataType != null){
queryMap.put("dataType", dataType);
}

if (creator != null) {
queryMap.put("creator", creator);
}
Expand Down Expand Up @@ -184,6 +195,25 @@ public RestorationSet uploadRestorationSet(@Parameter(name = "json representing
AllocationConstants.HAZARD_DFR3_ALLOCATION_MESSAGE);
}

// check if the parameters matches the defined data type in semantics
String dataType = restorationSet.getDataType();
if (dataType == null) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "dataType is a required field.");
}
try {
String semanticsDefinition = ServiceUtil.getJsonFromSemanticsEndpoint(dataType, username, userGroups);
List<String> columns = CommonUtil.getColumnNames(semanticsDefinition);

restorationSet.getCurveParameters().forEach((params) -> {
if(!columns.contains(params.name)) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Curve parameter: " + params.name + " not found in the dataType: " + dataType);
}
});

} catch (IOException e) {
throw new IncoreHTTPException(Response.Status.BAD_REQUEST, "Could not check the restoration curve parameter matches the dataType columns.");
}

restorationSet.setCreator(username);
restorationSet.setOwner(username);

Expand Down
1 change: 1 addition & 0 deletions ...service/src/main/java/edu/illinois/ncsa/incore/service/dfr3/daos/MongoDBFragilityDAO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ public List<FragilitySet> searchFragilities(String text) {
Filters.regex("legacyId").pattern(text).caseInsensitive(),
Filters.regex("hazardType").pattern(text).caseInsensitive(),
Filters.regex("inventoryType").pattern(text).caseInsensitive(),
Filters.regex("dataType").pattern(text).caseInsensitive(),
Filters.regex("description").pattern(text).caseInsensitive(),
Filters.regex("authors").pattern(text).caseInsensitive()));
List<FragilitySet> sets = query.iterator().toList();
Expand Down
Loading
Loading