Skip to content

Commit

Permalink
Engagement/jessica/15542 evaluate filters (#15865)
Browse files Browse the repository at this point in the history 
* Adding in validation for filters as well as applying those filters to the bundle when applicable
  • Loading branch information
@JessicaWNava
JessicaWNava authored Sep 19, 2024
1 parent 694f677 commit 672fa54
Show file tree
Hide file tree
Showing 3 changed files with 111 additions and 38 deletions.
145 changes: 109 additions & 36 deletions prime-router/src/main/kotlin/cli/ProcessFhirCommands.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,19 @@ import gov.cdc.prime.router.Hl7Configuration
import gov.cdc.prime.router.Metadata
import gov.cdc.prime.router.MimeFormat
import gov.cdc.prime.router.Receiver
import gov.cdc.prime.router.ReportStreamFilter
import gov.cdc.prime.router.azure.BlobAccess
import gov.cdc.prime.router.azure.ConditionStamper
import gov.cdc.prime.router.azure.LookupTableConditionMapper
import gov.cdc.prime.router.cli.CommandUtilities.Companion.abort
import gov.cdc.prime.router.cli.helpers.HL7DiffHelper
import gov.cdc.prime.router.common.Environment
import gov.cdc.prime.router.common.JacksonMapperUtilities
import gov.cdc.prime.router.config.validation.OrganizationValidation
import gov.cdc.prime.router.fhirengine.config.HL7TranslationConfig
import gov.cdc.prime.router.fhirengine.engine.FHIRConverter
import gov.cdc.prime.router.fhirengine.engine.FHIRReceiverFilter
import gov.cdc.prime.router.fhirengine.engine.FHIRReceiverFilter.ReceiverFilterEvaluationResult
import gov.cdc.prime.router.fhirengine.engine.encodePreserveEncodingChars
import gov.cdc.prime.router.fhirengine.translation.HL7toFhirTranslator
import gov.cdc.prime.router.fhirengine.translation.hl7.FhirToHl7Context
Expand All @@ -44,6 +49,7 @@ import org.hl7.fhir.r4.model.Base
import org.hl7.fhir.r4.model.Bundle
import org.hl7.fhir.r4.model.Extension
import org.hl7.fhir.r4.model.Reference
import java.util.UUID

/**
* Process data into/from FHIR.
Expand Down Expand Up @@ -132,13 +138,61 @@ class ProcessFhirCommands : CliktCommand(
val inputFileType = inputFile.extension.uppercase()
val receiver = getReceiver()

// If there is a receiver, check the filters
var bundle = FhirTranscoder.decode(contents)
if (receiver != null) {
val reportStreamFilters = mutableListOf<ReportStreamFilter>()
reportStreamFilters.add(receiver.jurisdictionalFilter)
reportStreamFilters.add(receiver.qualityFilter)
reportStreamFilters.add(receiver.routingFilter)
reportStreamFilters.add(receiver.processingModeFilter)

val validationErrors = mutableListOf<String>()
reportStreamFilters.forEach { reportStreamFilter ->
reportStreamFilter.forEach { filter ->
val validation = OrganizationValidation.validateFilter(filter)
if (!validation) {
validationErrors.add("Filter '$filter' is not valid.")
} else {
val result = FhirPathUtils.evaluate(
CustomContext(
bundle,
bundle,
FHIRConverter().loadFhirPathShorthandLookupTable(),
CustomFhirPathFunctions()
),
bundle,
bundle,
filter
)
if (result.isEmpty() ||
(result[0].isBooleanPrimitive && result[0].primitiveValue() == "false")
) {
throw CliktError("Filter '$filter' filtered out everything, nothing to return.")
}
}
}
}

if (validationErrors.isNotEmpty()) {
throw CliktError(validationErrors.joinToString("\n"))
}

receiver.conditionFilter.forEach { conditionFilter ->
val validation = OrganizationValidation.validateFilter(conditionFilter)
if (!validation) {
throw CliktError("Condition filter '$conditionFilter' is not valid.")
}
}
}

when {
// HL7 to FHIR conversion
inputFileType == "HL7" && (
outputFormat == MimeFormat.FHIR.toString() ||
(receiver != null && receiver.format == MimeFormat.FHIR)
) -> {
var fhirMessage = convertHl7ToFhir(contents).first
var fhirMessage = convertHl7ToFhir(contents, receiver).first
fhirMessage = applyEnrichmentSchemas(fhirMessage)
if (receiver != null && receiver.enrichmentSchemaNames.isNotEmpty()) {
receiver.enrichmentSchemaNames.forEach { currentSchema ->
Expand All @@ -153,24 +207,24 @@ class ProcessFhirCommands : CliktCommand(
// FHIR to HL7 conversion
(inputFileType == "FHIR" || inputFileType == "JSON") && (
outputFormat == MimeFormat.HL7.toString() ||
(receiver != null && receiver.format == MimeFormat.HL7)
(receiver != null && (receiver.format == MimeFormat.HL7 || receiver.format == MimeFormat.HL7_BATCH))
) -> {
if (receiver == null) {
return outputResult(convertFhirToHl7(contents))
}

var bundle = FhirTranscoder.decode(contents)
bundle = FhirTranscoder.decode(contents)
if (receiver.enrichmentSchemaNames.isNotEmpty()) {
receiver.enrichmentSchemaNames.forEach { currentSchema ->
bundle = FhirTransformer(currentSchema).process(bundle)
}
}
outputResult(
convertFhirToHl7(
FhirTranscoder.encode(bundle),
receiver.translation as Hl7Configuration,
receiver
)
FhirTranscoder.encode(bundle),
receiver.translation as Hl7Configuration,
receiver
)
)
}

Expand All @@ -179,33 +233,16 @@ class ProcessFhirCommands : CliktCommand(
outputFormat == MimeFormat.FHIR.toString() ||
(receiver != null && receiver.format == MimeFormat.FHIR)
) -> {
var bundle = FhirTranscoder.decode(contents)
if (receiver != null) {
if (receiver.enrichmentSchemaNames.isNotEmpty()) {
receiver.enrichmentSchemaNames.forEach { currentSchema ->
bundle = FhirTransformer(currentSchema).process(bundle)
}
}
}
outputResult(convertFhirToFhir(FhirTranscoder.encode(bundle)), actionLogger)
outputResult(convertFhirToFhir(FhirTranscoder.encode(bundle), receiver), actionLogger)
}

// HL7 to FHIR to HL7 conversion
inputFileType == "HL7" && (
outputFormat == MimeFormat.HL7.toString() ||
(receiver != null && receiver.format == MimeFormat.HL7)
(receiver != null && (receiver.format == MimeFormat.HL7 || receiver.format == MimeFormat.HL7_BATCH))
) -> {
var (bundle, inputMessage) = convertHl7ToFhir(contents)

if (receiver != null) {
if (receiver.enrichmentSchemaNames.isNotEmpty()) {
receiver.enrichmentSchemaNames.forEach { currentSchema ->
bundle = FhirTransformer(currentSchema).process(bundle)
}
}
}

val output = convertFhirToHl7(FhirTranscoder.encode(bundle))
val (bundle2, inputMessage) = convertHl7ToFhir(contents, receiver)
val output = convertFhirToHl7(FhirTranscoder.encode(bundle2))
outputResult(output)
if (diffHl7Output != null) {
val differences = hl7DiffHelper.diffHl7(output, inputMessage)
Expand All @@ -219,7 +256,27 @@ class ProcessFhirCommands : CliktCommand(
}
}

fun getReceiver(): Receiver? {
private fun applyConditionFilter(receiver: Receiver, bundle: Bundle): Bundle {
val trackingId = if (bundle.id != null) {
bundle.id
} else {
// this is just for logging so it is fine to just make it up
UUID.randomUUID().toString()
}
val result = FHIRReceiverFilter().evaluateObservationConditionFilters(
receiver,
bundle,
ActionLogger(),
trackingId
)
if (result is ReceiverFilterEvaluationResult.Success) {
return result.bundle
} else {
throw CliktError("Condition filter failed.")
}
}

private fun getReceiver(): Receiver? {
if (!environment.isNullOrBlank() && !receiverName.isNullOrBlank() && !orgName.isNullOrBlank()) {
if (!outputFormat.isNullOrBlank()) {
throw CliktError(
Expand Down Expand Up @@ -279,12 +336,15 @@ class ProcessFhirCommands : CliktCommand(
// for that from a receiver schema.
throw CliktError("You must specify a receiver schema using --receiver-schema.")

receiverSchema != null -> {
val bundle = applySenderTransforms(fhirMessage)
receiverSchema != null -> {
var bundle = applySenderTransforms(fhirMessage)
val stamper = ConditionStamper(LookupTableConditionMapper(Metadata.getInstance()))
fhirMessage.getObservations().forEach { observation ->
bundle.getObservations().forEach { observation ->
stamper.stampObservation(observation)
}
if (receiver != null) {
bundle = applyConditionFilter(receiver, bundle)
}
FhirToHl7Converter(
receiverSchema!!,
BlobAccess.BlobContainerMetadata.build("metadata", Environment.get().storageEnvVar),
Expand All @@ -299,7 +359,8 @@ class ProcessFhirCommands : CliktCommand(
).process(bundle)
}
receiver != null && receiver.schemaName.isNotBlank() -> {
val bundle = applySenderTransforms(fhirMessage)
var bundle = applySenderTransforms(fhirMessage)
bundle = applyConditionFilter(receiver, bundle)
FhirToHl7Converter(
receiver.schemaName,
BlobAccess.BlobContainerMetadata.build("metadata", Environment.get().storageEnvVar),
Expand All @@ -322,12 +383,20 @@ class ProcessFhirCommands : CliktCommand(
/**
* convert an FHIR message to FHIR message
*/
private fun convertFhirToFhir(jsonString: String): Bundle {
private fun convertFhirToFhir(jsonString: String, receiver: Receiver?): Bundle {
var fhirMessage = FhirTranscoder.decode(jsonString)
val stamper = ConditionStamper(LookupTableConditionMapper(Metadata.getInstance()))
fhirMessage.getObservations().forEach { observation ->
stamper.stampObservation(observation)
}
if (receiver != null) {
fhirMessage = applyConditionFilter(receiver, fhirMessage)
if (receiver.enrichmentSchemaNames.isNotEmpty()) {
receiver.enrichmentSchemaNames.forEach { currentSchema ->
fhirMessage = FhirTransformer(currentSchema).process(fhirMessage)
}
}
}
fhirMessage = applyEnrichmentSchemas(fhirMessage)
if (receiverSchema == null && senderSchema == null) {
// Must have at least one schema or else why are you doing this
Expand All @@ -345,7 +414,7 @@ class ProcessFhirCommands : CliktCommand(
* look like.
* @return a FHIR bundle and the parsed HL7 input that represents the data in the one HL7 message
*/
private fun convertHl7ToFhir(hl7String: String): Pair<Bundle, Message> {
private fun convertHl7ToFhir(hl7String: String, receiver: Receiver?): Pair<Bundle, Message> {
val hasFiveEncodingChars = hl7MessageHasFiveEncodingChars(hl7String)
// Some HL7 2.5.1 implementations have adopted the truncation character # that was added in 2.7
// However, the library used to encode the HL7 message throws an error it there are more than 4 encoding
Expand All @@ -365,7 +434,7 @@ class ProcessFhirCommands : CliktCommand(
}
val hl7profile = HL7Reader.getMessageProfile(hl7message.toString())
// search hl7 profile map and create translator with config path if found
val fhirMessage = when (val configPath = HL7Reader.profileDirectoryMap[hl7profile]) {
var fhirMessage = when (val configPath = HL7Reader.profileDirectoryMap[hl7profile]) {
null -> HL7toFhirTranslator(inputSchema).translate(hl7message)
else -> HL7toFhirTranslator(configPath).translate(hl7message)
}
Expand All @@ -375,6 +444,10 @@ class ProcessFhirCommands : CliktCommand(
stamper.stampObservation(observation)
}

if (receiver != null) {
fhirMessage = applyConditionFilter(receiver, fhirMessage)
}

return Pair(fhirMessage, hl7message)
}

Expand Down
2 changes: 1 addition & 1 deletion prime-router/src/main/kotlin/config/validation/Validations.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ object OrganizationValidation : KonformValidation<List<DeepOrganization>>() {
}

// Check to see if we have a valid filter structure
private fun validateFilter(filter: String): Boolean {
fun validateFilter(filter: String): Boolean {
val isReportStreamFormat = allowedFilters.any { filter.startsWith(it) }
return isReportStreamFormat || validFhirPath(filter)
}
Expand Down
2 changes: 1 addition & 1 deletion prime-router/src/main/kotlin/fhirengine/engine/FHIRReceiverFilter.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ class FHIRReceiverFilter(
*
* [actionLogger] and [trackingId] facilitate logging
*/
private fun evaluateObservationConditionFilters(
fun evaluateObservationConditionFilters(
receiver: Receiver,
bundle: Bundle,
actionLogger: ActionLogger,
Expand Down

0 comments on commit 672fa54

Please sign in to comment.