Skip to content

Commit

Permalink
Implement the new Gazelle validation API
Browse files Browse the repository at this point in the history 
Fixes #141
  • Loading branch information
@qligier
qligier committed Feb 28, 2024
1 parent 467864e commit cf9395f
Show file tree
Hide file tree
Showing 28 changed files with 2,512 additions and 153 deletions.
1 change: 0 additions & 1 deletion .idea/misc.xml
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions docs/changelog.md
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
2024/XX/YY Release 3.6.1

- Implemented the new Gazelle validation API [#141](https://github.com/ahdis/matchbox/issues/141)

2024/02/27 Release 3.6.0

- `docker pull europe-west6-docker.pkg.dev/ahdis-ch/ahdis/matchbox:v3.6.0`
Expand Down
2 changes: 1 addition & 1 deletion matchbox-engine/src/test/java/ch/ahdis/matchbox/engine/tests/CdaToFhirTransformTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ void TestObservation() throws FHIRException, IOException {
assertNotNull(resource);
}

@Test
@Test
void TestObservationSt() throws FHIRException, IOException {
InputStream in = getResourceAsStream("cda-it-observation-st.xml");

Expand Down
8 changes: 5 additions & 3 deletions matchbox-server/src/main/java/ca/uhn/fhir/jpa/starter/Application.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package ca.uhn.fhir.jpa.starter;

import ch.ahdis.matchbox.spring.MatchboxEventListener;
import ch.ahdis.matchbox.gazelle.GazelleValidationWs;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.config.AutowireCapableBeanFactory;
import org.springframework.boot.SpringApplication;
Expand All @@ -26,7 +27,8 @@
MdmConfig.class,
MatchboxJpaConfig.class,
FhirServerConfigR4.class,
MatchboxEventListener.class})
MatchboxEventListener.class,
GazelleValidationWs.class})
public class Application extends SpringBootServletInitializer {

public static void main(String[] args) {
Expand All @@ -48,8 +50,8 @@ protected SpringApplicationBuilder configure(

@Bean
@Conditional(OnEitherVersion.class)
public ServletRegistrationBean hapiServletRegistration(RestfulServer restfulServer) {
ServletRegistrationBean servletRegistrationBean = new ServletRegistrationBean();
public ServletRegistrationBean<RestfulServer> hapiServletRegistration(RestfulServer restfulServer) {
ServletRegistrationBean<RestfulServer> servletRegistrationBean = new ServletRegistrationBean<>();
beanFactory.autowireBean(restfulServer);
servletRegistrationBean.setServlet(restfulServer);
servletRegistrationBean.addUrlMappings("/fhir/*");
Expand Down
84 changes: 40 additions & 44 deletions matchbox-server/src/main/java/ch/ahdis/fhir/hapi/jpa/validation/ValidationProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,32 +21,27 @@
*/

import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.jpa.dao.data.INpmPackageVersionDao;
import ca.uhn.fhir.rest.annotation.Operation;
import ca.uhn.fhir.rest.annotation.OperationParam;
import ca.uhn.fhir.rest.api.EncodingEnum;
import ca.uhn.fhir.util.StopWatch;
import ca.uhn.fhir.validation.ResultSeverityEnum;
import ca.uhn.fhir.validation.SingleValidationMessage;
import ch.ahdis.matchbox.CliContext;
import ch.ahdis.matchbox.MatchboxEngineSupport;
import ch.ahdis.matchbox.engine.MatchboxEngine;
import ch.ahdis.matchbox.engine.cli.VersionUtil;
import org.apache.commons.beanutils.BeanUtils;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.IOUtils;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.hl7.fhir.convertors.factory.VersionConvertorFactory_40_50;
import org.hl7.fhir.instance.model.api.IBase;
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.r4.model.StructureDefinition;
import org.hl7.fhir.r5.elementmodel.Manager.FhirFormat;
import org.hl7.fhir.r5.model.DateTimeType;
import org.hl7.fhir.r5.model.DateType;
import org.hl7.fhir.r5.model.Duration;
import org.hl7.fhir.r5.model.OperationOutcome;
import org.hl7.fhir.r5.model.StringType;
import org.hl7.fhir.r5.model.TimeType;
import org.hl7.fhir.r5.utils.EOperationOutcome;
import org.hl7.fhir.r5.model.UriType;
import org.hl7.fhir.r5.utils.OperationOutcomeUtilities;
import org.hl7.fhir.r5.utils.ToolingExtensions;
Expand Down Expand Up @@ -104,7 +99,7 @@ public IBaseResource validate(final HttpServletRequest theRequest) {
log.debug("$validate");
final ArrayList<SingleValidationMessage> addedValidationMessages = new ArrayList<>();

final StopWatch sw = new StopWatch();
final var sw = new StopWatch();
sw.startTask("Total");

// we extract here all config
Expand Down Expand Up @@ -142,7 +137,13 @@ public IBaseResource validate(final HttpServletRequest theRequest) {
reload = theRequest.getParameter("reload").equals("true");
}

final String contentString = this.getContentString(theRequest, addedValidationMessages);
String contentString = "";
try {
contentString = new String(theRequest.getInputStream().readAllBytes(), StandardCharsets.UTF_8);
} catch (final Exception e) {
log.error(e.getMessage(), e);
}

if (contentString.isEmpty()) {
return this.getOoForError("No content provided in HTTP body");
} else {
Expand Down Expand Up @@ -174,10 +175,7 @@ public IBaseResource validate(final HttpServletRequest theRequest) {

final List<ValidationMessage> messages;
try {
final var format = encoding == EncodingEnum.XML ? FhirFormat.XML : FhirFormat.JSON;
final var stream = new ByteArrayInputStream(contentString.getBytes(StandardCharsets.UTF_8));
messages = engine.validate(format, stream, profile);

messages = doValidate(engine, contentString, encoding, profile);
} catch (final Exception e) {
sw.endCurrentTask();
log.debug("Validation time: {}", sw);
Expand All @@ -187,36 +185,10 @@ public IBaseResource validate(final HttpServletRequest theRequest) {

long millis = sw.getMillis();
log.debug("Validation time: {}", sw);

return this.getOperationOutcome(sha3Hex, messages, profile, engine, millis, cliContext);
}

private String getContentString(final HttpServletRequest theRequest,
final List<SingleValidationMessage> addedValidationMessages) {
byte[] bytes = null;
String contentString = "";
try {
bytes = IOUtils.toByteArray(theRequest.getInputStream());
if (bytes.length > 2 && bytes[0] == -17 && bytes[1] == -69 && bytes[2] == -65) {
byte[] dest = new byte[bytes.length - 3];
System.arraycopy(bytes, 3, dest, 0, bytes.length - 3);
bytes = dest;
if (addedValidationMessages != null) {
final var m = new SingleValidationMessage();
m.setSeverity(ResultSeverityEnum.WARNING);
m.setMessage(
"Resource content has a UTF-8 BOM marking, skipping BOM, see https://en.wikipedia.org/wiki/Byte_order_mark");
m.setLocationCol(0);
m.setLocationLine(0);
addedValidationMessages.add(m);
}
}
contentString = new String(bytes);
} catch (final IOException e) {
log.error(e.getMessage(), e);
}
return contentString;
}

private IBaseResource getOperationOutcome(final String id,
final List<ValidationMessage> messages,
Expand All @@ -234,7 +206,6 @@ private IBaseResource getOperationOutcome(final String id,
issue.setCode(OperationOutcome.IssueType.INFORMATIONAL);

final StructureDefinition structDef = engine.getStructureDefinition(profile);

final org.hl7.fhir.r5.model.StructureDefinition structDefR5 = (org.hl7.fhir.r5.model.StructureDefinition) VersionConvertorFactory_40_50.convertResource(structDef);

final var profileDate = (structDef.getDateElement() != null)
Expand All @@ -247,7 +218,7 @@ private IBaseResource getOperationOutcome(final String id,
structDef.getVersion(),
profileDate,
String.join(", ", engine.getContext().getLoadedPackages()),
"" + ms/1000.0+ "s",
ms/1000.0+ "s",
VersionUtil.getPoweredBy(),
cliContext.toString()
));
Expand All @@ -258,9 +229,7 @@ private IBaseResource getOperationOutcome(final String id,
ext.addExtension("profileDate", structDefR5.getDateElement());

ext.addExtension("total", new Duration().setUnit("ms").setValue(ms) );
if (matchboxEngineSupport.getSessionId(engine) != null) {
ext.addExtension("validatorVersion", new StringType(VersionUtil.getPoweredBy()));
}
ext.addExtension("validatorVersion", new StringType(VersionUtil.getPoweredBy()));
cliContext.addContextToExtension(ext);
if (matchboxEngineSupport.getSessionId(engine) != null) {
ext.addExtension("sessionId", new StringType(matchboxEngineSupport.getSessionId(engine)));
Expand All @@ -273,6 +242,10 @@ private IBaseResource getOperationOutcome(final String id,

// Map the SingleValidationMessages to OperationOutcomeIssue
for (final ValidationMessage message : messages) {
if (message.getType() == null) {
// TODO: this did not happen with other core versions
message.setType(ValidationMessage.IssueType.UNKNOWN);
}
final var issue = OperationOutcomeUtilities.convertToIssue(message, oo);

// Note: the message is mapped to details.text by HAPI, but we still need it in diagnostics for the EVSClient,
Expand Down Expand Up @@ -319,4 +292,27 @@ private IBaseResource getOoForError(final @NonNull String message) {
issue.addExtension().setUrl(ToolingExtensions.EXT_ISSUE_SOURCE).setValue(new StringType("ValidationProvider"));
return VersionConvertorFactory_40_50.convertResource(oo);
}

public static List<ValidationMessage> doValidate(final MatchboxEngine engine,
String content,
final EncodingEnum encoding,
final String profile) throws EOperationOutcome, IOException {
final List<ValidationMessage> messages = new ArrayList<>();

if (content.startsWith("\uFEFF")) {
content = content.replace("\uFEFF", "");
final var m = new ValidationMessage();
m.setLevel(ValidationMessage.IssueSeverity.WARNING);
m.setMessage(
"Resource content has a UTF-8 BOM marking, skipping BOM, see https://en.wikipedia.org/wiki/Byte_order_mark");
m.setCol(0);
m.setLine(0);
messages.add(m);
}

final var format = encoding == EncodingEnum.XML ? FhirFormat.XML : FhirFormat.JSON;
final var stream = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
messages.addAll(engine.validate(format, stream, profile));
return messages;
}
}
88 changes: 43 additions & 45 deletions matchbox-server/src/main/java/ch/ahdis/matchbox/ConformancePackageResourceProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,10 @@

import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;
import java.util.stream.Collectors;

import ch.ahdis.matchbox.util.MatchboxServerUtils;
import jakarta.servlet.http.HttpServletRequest;

import org.hl7.fhir.convertors.factory.VersionConvertorFactory_40_50;
Expand All @@ -17,10 +14,7 @@
import org.hl7.fhir.instance.model.api.IBaseResource;
import org.hl7.fhir.instance.model.api.IDomainResource;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.CanonicalType;
import org.hl7.fhir.r4.model.MetadataResource;
import org.hl7.fhir.r4.model.OperationOutcome;
import org.hl7.fhir.r4.model.Resource;
import org.hl7.fhir.r4.model.*;
import org.quartz.DisallowConcurrentExecution;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
Expand Down Expand Up @@ -204,43 +198,47 @@ public ca.uhn.fhir.rest.api.server.IBundleProvider search(jakarta.servlet.http.H

public List<CanonicalType> getCanonicals() {
return new TransactionTemplate(myTxManager).execute(tx -> {
Slice<NpmPackageVersionResourceEntity> outcome = myPackageVersionResourceDao
.findByResourceType(PageRequest.of(0, 2147483646), resourceType);
List<String> versioned = outcome.stream().filter(t -> !t.getPackageVersion().isCurrentVersion())
.map(t -> (t.getCanonicalUrl() + "|" + t.getCanonicalVersion())).collect(Collectors.toList());
Slice<NpmPackageVersionResourceEntity> current = myPackageVersionResourceDao
.findCurrentByResourceType(PageRequest.of(0, 2147483646), resourceType);
versioned.addAll(current.stream().map(t -> (t.getCanonicalUrl())).collect(Collectors.toList()));
return versioned.stream().sorted().map(t -> new CanonicalType(t)).collect(Collectors.toList());
final var page = PageRequest.of(0, 2147483646);
final var currentEntityIds =
this.myPackageVersionResourceDao.findCurrentByResourceType(page, this.resourceType)
.stream()
.map(NpmPackageVersionResourceEntity::getId)
.collect(Collectors.toUnmodifiableSet());

return this.myPackageVersionResourceDao.findByResourceType(page, this.resourceType)
.stream()
.sorted(Comparator
.comparing(NpmPackageVersionResourceEntity::getCanonicalUrl)
.thenComparing(NpmPackageVersionResourceEntity::getCanonicalVersion))
.map(entity -> {
final var canonical = new CanonicalType(entity.getCanonicalUrl());
canonical.addExtension().setUrl("ig-id").setValue(new StringType(entity.getPackageVersion().getPackageId()));
canonical.addExtension().setUrl("ig-version").setValue(new StringType(entity.getCanonicalVersion()));
canonical.addExtension().setUrl("ig-current").setValue(new BooleanType(currentEntityIds.contains(entity.getId())));
canonical.addExtension().setUrl("sd-canonical").setValue(new StringType(entity.getCanonicalUrl()));
if (entity.getFilename() != null && !entity.getFilename().isBlank()) {
canonical.addExtension().setUrl("sd-title").setValue(new StringType(entity.getFilename()));
} else {
canonical.addExtension().setUrl("sd-title").setValue(new StringType(entity.getCanonicalUrl()));
}
return canonical;
})
.toList();
});
}

/**
* Helper method which will attempt to use the IBinaryStorageSvc to resolve the
* binary blob if available. If the bean is unavailable, fallback to assuming we
* are using an embedded base64 in the data element.
*
* @param theBinary the Binary who's `data` blob you want to retrieve
* @return a byte array containing the blob.
*
* @throws IOException
*/
private byte[] fetchBlobFromBinary(IBaseBinary theBinary) throws IOException {
if (myBinaryStorageSvc != null && !(myBinaryStorageSvc instanceof NullBinaryStorageSvcImpl)) {
return myBinaryStorageSvc.fetchDataBlobFromBinary(theBinary);
} else {
byte[] value = BinaryUtil.getOrCreateData(myCtx, theBinary).getValue();
if (value == null) {
throw new InternalErrorException(
Msg.code(1296) + "Failed to fetch blob from Binary/" + theBinary.getIdElement());
}
return value;
}
public List<NpmPackageVersionResourceEntity> getPackageResources() {
return new TransactionTemplate(this.myTxManager).execute(tx -> {
return myPackageVersionResourceDao
.findByResourceType(PageRequest.of(0, 2147483646), resourceType).stream().toList();
});
}

@SuppressWarnings("unchecked")
private IFhirResourceDao<IBaseBinary> getBinaryDao() {
return myDaoRegistry.getResourceDao("Binary");
public List<NpmPackageVersionResourceEntity> getCurrentPackageResources() {
return new TransactionTemplate(this.myTxManager).execute(tx -> {
return myPackageVersionResourceDao
.findCurrentByResourceType(PageRequest.of(0, 2147483646), resourceType).stream().toList();
});
}

private IBaseResource loadPackageEntityAdjustId(NpmPackageVersionResourceEntity contents) {
Expand Down Expand Up @@ -270,10 +268,10 @@ public org.hl7.fhir.r5.model.CanonicalResource getCanonical(IBaseResource theRes

private IBaseResource loadPackageEntity(NpmPackageVersionResourceEntity contents) {
try {
JpaPid binaryPid = JpaPid.fromId(contents.getResourceBinary().getId());
IBaseBinary binary = getBinaryDao().readByPid(binaryPid);
byte[] resourceContentsBytes = fetchBlobFromBinary(binary);
String resourceContents = new String(resourceContentsBytes, StandardCharsets.UTF_8);
final var binary = MatchboxServerUtils.getBinaryFromId(contents.getResourceBinary().getId(), myDaoRegistry);
final byte[] resourceContentsBytes = MatchboxServerUtils.fetchBlobFromBinary(binary, myBinaryStorageSvc,
myCtx);
final String resourceContents = new String(resourceContentsBytes, StandardCharsets.UTF_8);
switch (contents.getFhirVersion()) {
case R4:
return new org.hl7.fhir.r4.formats.JsonParser().parse(resourceContents);
Expand Down
Loading

0 comments on commit cf9395f

Please sign in to comment.