diff --git a/.github/workflows/deploy-rc.yml b/.github/workflows/deploy-rc.yml new file mode 100644 index 00000000..dd542a87 --- /dev/null +++ b/.github/workflows/deploy-rc.yml @@ -0,0 +1,15 @@ +name: "TST/STG - Deploy release candidate" + +on: + workflow_dispatch: + push: + tags: + - '*' + +jobs: + staging: + uses: ./.github/workflows/remote-cd-trigger-template.yml + with: + WORKFLOW: aks-deployment-pstatus-graphql-stg.yml + REF: '${{ github.ref_name }}' # Resolves to the tag that is pushed + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/graphql-service-ci.yml b/.github/workflows/graphql-service-ci.yml new file mode 100644 index 00000000..76e0975a --- /dev/null +++ b/.github/workflows/graphql-service-ci.yml @@ -0,0 +1,21 @@ +name: "CI - GraphQL Service" + +on: + workflow_dispatch: + pull_request: + paths: + - pstatus-graphql-ktor/** + +defaults: + run: + working-directory: pstatus-graphql-ktor/ + +jobs: + unit-test: + name: Unit Test + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Run Gradle Test + run: ./gradlew test \ No newline at end of file diff --git a/.github/workflows/graphql-service.yml b/.github/workflows/graphql-service.yml new file mode 100644 index 00000000..37669bff --- /dev/null +++ b/.github/workflows/graphql-service.yml @@ -0,0 +1,23 @@ +name: DEV - GraphQL Service + +on: + workflow_dispatch: + inputs: + REF: + description: Branch from CDCgov/data-exchange-processing-status that you want to deploy to the dev environment. + default: main + required: true + type: string + push: + branches: + - main + paths: + - pstatus-graphql-ktor/** + +jobs: + remote-trigger: + uses: ./.github/workflows/remote-cd-trigger-template.yml + with: + WORKFLOW: aks-deployment-pstatus-graphql-dev-shared.yml + REF: ${{ inputs.REF }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/remote-cd-trigger-template.yml b/.github/workflows/remote-cd-trigger-template.yml new file mode 100644 index 00000000..ba2df354 --- /dev/null +++ b/.github/workflows/remote-cd-trigger-template.yml @@ -0,0 +1,45 @@ +name: Template - Remote trigger to CDCent +on: + workflow_call: + inputs: + WORKFLOW: + type: string + description: "Workflow yml file that should be triggered." + required: true + REF: + type: string + description: "Git tag or branch the workflow is running on." + required: false + default: 'main' + +jobs: + invoke-cd-trigger: + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ github.token }} + steps: + - name: Checkout Gen GitHub App Access Token + uses: actions/checkout@v3 + with: + repository: kave/github-app-token + - name: Generate Token + run: | + sudo gem install jwt + echo "${{ secrets.CDC_COE_BOTFREY_PEM }}" > app-private-key.pem + chmod +x ./get-github-app-access-token.sh; + . ./get-github-app-access-token.sh; + echo "access_token=${TOKEN}" >> "$GITHUB_ENV" + - name: Manually Dispatch Remote CICD Trigger Event + uses: actions/github-script@v6 + with: + github-token: ${{ env.access_token }} + script: | + await github.rest.actions.createWorkflowDispatch({ + owner: 'cdcent', + repo: 'data-exchange-pstatus-devops', + workflow_id: '${{ inputs.WORKFLOW }}', + ref: 'main', + inputs: { + REF: '${{ inputs.REF }}' + } + }) \ No newline at end of file diff --git a/processing-status-api-function-app/README.md b/processing-status-api-function-app/README.md index bb4cf73e..3b5a1918 100644 --- a/processing-status-api-function-app/README.md +++ b/processing-status-api-function-app/README.md @@ -11,9 +11,6 @@ The following is needed in order to build and deploy this function app: ## Required Application Settings In addition to the standard required application settings, the processing status API function app requires the following. -- `JAEGER_OTEL_COLLECTOR_END_POINT` - URL of the Jaeger trace collector, listening on port 4317 -- `JAEGER_TRACE_ENDPOINT` - URL of the Jaeger web UI, listening on port 16686 -- `JAEGER_HEALTH_END_POINT` - URL of the Jaeger health end point, listening on port 14269 - `CosmosDbEndpoint` - URL of the cosmos database - `CosmosDbKey` - shared key used to connect to the cosmos database - `ServiceBusQueueName` - service bus queue name, which should always be `processing-status-cosmos-db-queue` diff --git a/processing-status-api-function-app/build.gradle b/processing-status-api-function-app/build.gradle index 165f983f..55146c95 100644 --- a/processing-status-api-function-app/build.gradle +++ b/processing-status-api-function-app/build.gradle @@ -33,11 +33,6 @@ dependencies { implementation 'com.microsoft.azure:applicationinsights-core:3.4.19' implementation 'com.azure:azure-cosmos:4.55.0' implementation "com.fasterxml.jackson.module:jackson-module-kotlin:2.14.2" - implementation 'io.opentelemetry:opentelemetry-api:1.29.0' - implementation 'io.opentelemetry:opentelemetry-sdk:1.29.0' - implementation 'io.opentelemetry:opentelemetry-exporter-logging:1.29.0' - implementation 'io.opentelemetry:opentelemetry-exporter-otlp:1.29.0' - implementation 'io.opentelemetry:opentelemetry-semconv:1.29.0-alpha' implementation 'com.google.code.gson:gson:2.10.1' implementation group: 'io.github.microutils', name: 'kotlin-logging-jvm', version: '3.0.5' implementation group: 'org.slf4j', name: 'slf4j-api', version: '1.7.36' @@ -55,8 +50,6 @@ dependencies { implementation 'org.owasp.encoder:encoder:1.2.3' implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.3.1' - agent "io.opentelemetry.javaagent:opentelemetry-javaagent:1.29.0" - testImplementation("org.mockito.kotlin:mockito-kotlin:4.0.0") testImplementation "org.testng:testng:7.4.0" implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version" diff --git a/processing-status-api-function-app/gradle/wrapper/gradle-wrapper.properties b/processing-status-api-function-app/gradle/wrapper/gradle-wrapper.properties index bdc9a83b..fbcb72cf 100644 --- a/processing-status-api-function-app/gradle/wrapper/gradle-wrapper.properties +++ b/processing-status-api-function-app/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ +#Fri Jun 07 10:39:15 EDT 2024 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.0.2-bin.zip -networkTimeout=10000 +distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/processing-status-api-function-app/src/main/java/gov/cdc/ocio/processingstatusapi/FunctionJavaWrappers.java b/processing-status-api-function-app/src/main/java/gov/cdc/ocio/processingstatusapi/FunctionJavaWrappers.java index cd284403..1b437022 100644 --- a/processing-status-api-function-app/src/main/java/gov/cdc/ocio/processingstatusapi/FunctionJavaWrappers.java +++ b/processing-status-api-function-app/src/main/java/gov/cdc/ocio/processingstatusapi/FunctionJavaWrappers.java @@ -13,10 +13,6 @@ import gov.cdc.ocio.processingstatusapi.functions.status.GetUploadStatusFunction; import gov.cdc.ocio.processingstatusapi.functions.reports.ServiceBusProcessor; import gov.cdc.ocio.processingstatusapi.functions.status.GetStatusFunction; -import gov.cdc.ocio.processingstatusapi.functions.traces.AddSpanToTraceFunction; -import gov.cdc.ocio.processingstatusapi.functions.traces.CreateTraceFunction; -import gov.cdc.ocio.processingstatusapi.functions.traces.GetSpanFunction; -import gov.cdc.ocio.processingstatusapi.functions.traces.GetTraceFunction; import gov.cdc.ocio.processingstatusapi.model.DispositionType; import org.owasp.encoder.Encode; @@ -35,72 +31,6 @@ public HttpResponseMessage healthCheck( return new HealthCheckFunction(request).run(); } - @FunctionName("CreateTrace") - public HttpResponseMessage createTrace( - @HttpTrigger( - name = "req", - methods = {HttpMethod.POST}, - route = "trace", - authLevel = AuthorizationLevel.ANONYMOUS) HttpRequestMessage> request) { - return new CreateTraceFunction(request).create(); - } - - @FunctionName("TraceStartSpan") - public HttpResponseMessage traceStartSpan( - @HttpTrigger( - name = "req", - methods = {HttpMethod.PUT}, - route = "trace/startSpan/{traceId}/{parentSpanId}", - authLevel = AuthorizationLevel.ANONYMOUS) HttpRequestMessage> request, - @BindingName("traceId") String traceId, - @BindingName("parentSpanId") String parentSpanId) { - return new AddSpanToTraceFunction(request).startSpan(traceId, parentSpanId); - } - - @FunctionName("TraceStopSpan") - public HttpResponseMessage traceStopSpan( - @HttpTrigger( - name = "req", - methods = {HttpMethod.PUT}, - route = "trace/stopSpan/{traceId}/{spanId}", - authLevel = AuthorizationLevel.ANONYMOUS) HttpRequestMessage> request, - @BindingName("traceId") String traceId, - @BindingName("spanId") String spanId) { - return new AddSpanToTraceFunction(request).stopSpan(traceId, spanId); - } - - @FunctionName("GetTraceByTraceId") - public HttpResponseMessage getTraceByTraceId( - @HttpTrigger( - name = "req", - methods = {HttpMethod.GET}, - route = "trace/traceId/{traceId}", - authLevel = AuthorizationLevel.ANONYMOUS) HttpRequestMessage> request, - @BindingName("traceId") String traceId) { - return new GetTraceFunction(request).withTraceId(traceId); - } - - @FunctionName("GetTraceByUploadId") - public HttpResponseMessage getTraceByUploadId( - @HttpTrigger( - name = "req", - methods = {HttpMethod.GET}, - route = "trace/uploadId/{uploadId}", - authLevel = AuthorizationLevel.ANONYMOUS) HttpRequestMessage> request, - @BindingName("uploadId") String uploadId) { - return new GetTraceFunction(request).withUploadId(uploadId); - } - - @FunctionName("GetTraceSpan") - public HttpResponseMessage getTraceSpanByUploadIdStageName( - @HttpTrigger( - name = "req", - methods = {HttpMethod.GET}, - route = "trace/span", - authLevel = AuthorizationLevel.ANONYMOUS) HttpRequestMessage> request) { - return new GetSpanFunction(request).withQueryParams(); - } - /*** * Process a message from the service bus queue. * diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/HealthCheckFunction.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/HealthCheckFunction.kt index 090effc8..b1899707 100644 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/HealthCheckFunction.kt +++ b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/HealthCheckFunction.kt @@ -11,7 +11,6 @@ import com.microsoft.azure.servicebus.primitives.ServiceBusException import gov.cdc.ocio.processingstatusapi.cosmos.CosmosClientManager import gov.cdc.ocio.processingstatusapi.model.CosmosDb import gov.cdc.ocio.processingstatusapi.model.HealthCheck -import gov.cdc.ocio.processingstatusapi.model.Jaeger import gov.cdc.ocio.processingstatusapi.model.ServiceBus import gov.cdc.ocio.processingstatusapi.model.reports.Report import mu.KotlinLogging @@ -33,10 +32,8 @@ class HealthCheckFunction( fun run(): HttpResponseMessage { var cosmosDBHealthy = false var serviceBusHealthy = false - var jaegerHealthy = false val cosmosDBHealth = CosmosDb() val serviceBusHealth = ServiceBus() - val jaegerHealth = Jaeger() val time = measureTimeMillis { try { cosmosDBHealthy = isCosmosDBHealthy() @@ -54,23 +51,13 @@ class HealthCheckFunction( logger.error("Azure Service Bus is not healthy: ${ex.message}") } - try { - jaegerHealthy = isJaegerHealthy() - if(jaegerHealthy){ - jaegerHealth.status = "UP" - } - } catch (ex: Exception) { - jaegerHealth.healthIssues = ex.message - logger.error("Jaeger is not healthy: ${ex.message}") - } } val result = HealthCheck().apply { - status = if (cosmosDBHealthy && serviceBusHealthy && jaegerHealthy) "UP" else "DOWN" + status = if (cosmosDBHealthy && serviceBusHealthy) "UP" else "DOWN" totalChecksDuration = formatMillisToHMS(time) dependencyHealthChecks.add(cosmosDBHealth) dependencyHealthChecks.add(serviceBusHealth) - dependencyHealthChecks.add(jaegerHealth) } if(result.status == "DOWN"){ @@ -122,14 +109,6 @@ class HealthCheckFunction( return true } - private fun isJaegerHealthy(): Boolean { - val healthEndPoint = System.getenv("JAEGER_HEALTH_END_POINT") - logger.info("healthEndPoint: $healthEndPoint") - val response = khttp.get(healthEndPoint) - logger.info("response.statusCode:" + response.statusCode) - return response.statusCode == HttpStatus.OK.value(); - } - /** * Format the time in milliseconds to 00:00:00.000 format. * diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/reports/ReportManager.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/reports/ReportManager.kt index e3494c5d..a96d8711 100644 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/reports/ReportManager.kt +++ b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/reports/ReportManager.kt @@ -213,7 +213,6 @@ class ReportManager { } HttpStatus.TOO_MANY_REQUESTS.value() -> { - // See: https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/performance-tips?tabs=trace-net-core#429 // https://learn.microsoft.com/en-us/rest/api/cosmos-db/common-cosmosdb-rest-response-headers // https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/troubleshoot-request-rate-too-large?tabs=resource-specific val recommendedDuration = response.responseHeaders["x-ms-retry-after-ms"] diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/status/GetReportCountsFunction.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/status/GetReportCountsFunction.kt index 1ae09f1e..1cc2bf54 100644 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/status/GetReportCountsFunction.kt +++ b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/status/GetReportCountsFunction.kt @@ -13,7 +13,6 @@ import gov.cdc.ocio.processingstatusapi.model.reports.* import gov.cdc.ocio.processingstatusapi.model.reports.stagereports.HL7Debatch import gov.cdc.ocio.processingstatusapi.model.reports.stagereports.HL7Redactor import gov.cdc.ocio.processingstatusapi.model.reports.stagereports.HL7Validation -import gov.cdc.ocio.processingstatusapi.model.traces.* import gov.cdc.ocio.processingstatusapi.utils.DateUtils import gov.cdc.ocio.processingstatusapi.utils.JsonUtils import gov.cdc.ocio.processingstatusapi.utils.PageUtils diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/status/GetStatusFunction.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/status/GetStatusFunction.kt index 69cee096..02432d15 100644 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/status/GetStatusFunction.kt +++ b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/status/GetStatusFunction.kt @@ -6,12 +6,10 @@ import com.microsoft.azure.functions.HttpRequestMessage import com.microsoft.azure.functions.HttpResponseMessage import com.microsoft.azure.functions.HttpStatus import gov.cdc.ocio.processingstatusapi.cosmos.CosmosContainerManager -import gov.cdc.ocio.processingstatusapi.functions.traces.TraceUtils import gov.cdc.ocio.processingstatusapi.model.* import gov.cdc.ocio.processingstatusapi.model.reports.Report import gov.cdc.ocio.processingstatusapi.model.reports.ReportDao import gov.cdc.ocio.processingstatusapi.model.reports.ReportSerializer -import gov.cdc.ocio.processingstatusapi.model.traces.* import gov.cdc.ocio.processingstatusapi.utils.JsonUtils import mu.KotlinLogging import java.util.* @@ -49,29 +47,17 @@ class GetStatusFunction( .create() /** - * Retrieve a complete status (traces + reports) for the provided upload ID. + * Retrieve a complete status (reports) for the provided upload ID. * * @param uploadId String * @return HttpResponseMessage */ fun withUploadId(uploadId: String): HttpResponseMessage { - val traces = TraceUtils.getTraces(uploadId) - var traceResult: TraceResult? = null - if (traces != null) { - if (traces.size != 1) { - return request - .createResponseBuilder(HttpStatus.INTERNAL_SERVER_ERROR) - .body("Trace inconsistency found, expected exactly one trace for uploadId = $uploadId, but got ${traces.size}") - .build() - } - traceResult = TraceResult.buildFromTrace(traces[0]) - } - val reportResult = getReport(uploadId) - // Need at least one or the other (report or trace) - if (traceResult == null && reportResult == null) { + // Need at least one (report) + if (reportResult == null) { return request .createResponseBuilder(HttpStatus.BAD_REQUEST) .body("Invalid uploadId provided") @@ -79,10 +65,6 @@ class GetStatusFunction( } val status = StatusResult().apply { - this.uploadId = traceResult?.uploadId - this.dataStreamId = traceResult?.dataStreamId - this.dataStreamRoute = traceResult?.dataStreamRoute - trace = traceResult?.let { TraceDao.buildFromTraceResult(it) } reports = reportResult?.reports } @@ -93,80 +75,6 @@ class GetStatusFunction( .build() } - companion object { - val excludedSpanTags = listOf("spanMark", "span.kind", "internal.span.format", "otel.library.name") - } - -// private fun getTraces(uploadId: String): TraceResult? { -// val queryService: QueryServiceBlockingStub = jaeger.createBlockingQueryService() -// WaitUtils.untilQueryHasTag(queryService, SERVICE_NAME, "foo", "bar") -// -// val operations: GetOperationsResponse = queryService -// .getOperations(GetOperationsRequest.newBuilder().setService(SERVICE_NAME).build()) -// Assert.assertEquals(1, operations.getOperationNamesList().size()) -// Assert.assertEquals(OPERATION_NAME, operations.getOperationNamesList().get(0)) -// -// return null -// } - -// fun createBlockingQueryService(): QueryServiceBlockingStub { -// val channel = ManagedChannelBuilder.forTarget( -// java.lang.String.format( -// "localhost:%d", -// getQueryPort() -// ) -// ).usePlaintext().build() -// return QueryServiceGrpc.newBlockingStub(channel) -// } -// -// fun runFindTraces() { -// val queryHostPort = "172.17.0.1:16686" -// val channel = ManagedChannelBuilder.forTarget(queryHostPort).usePlaintext().build() -// val queryService: QueryServiceBlockingStub = QueryServiceGrpc.newBlockingStub(channel) -// val query = TraceQueryParameters.newBuilder().setServiceName("frontend") -// .build() -// val traceProto: Iterator = queryService.findTraces( -// FindTracesRequest.newBuilder().setQuery(query).build() -// ) -// val protoSpans: MutableList = ArrayList() -// while (traceProto.hasNext()) { -// protoSpans.addAll(traceProto.next().getSpansList()) -// } -// val traces: Trace = Converter.toModel(protoSpans) -// val graph: Graph = GraphCreator.create(traces) -// val errorSpans: Set = NumberOfErrors.calculate(graph) -// val result: MutableMap> = LinkedHashMap() -// for (errorSpan in errorSpans) { -// for (log in errorSpan.logs) { -// val err: String = log.fields.get(Tags.ERROR.getKey()) -// if (err != null) { -// var traceIdCount = result[err] -// if (traceIdCount == null) { -// traceIdCount = LinkedHashMap() -// result[err] = traceIdCount -// } -// var count = traceIdCount[errorSpan.traceId] -// if (count == null) { -// count = 0 -// } -// traceIdCount[errorSpan.traceId] = ++count -// } -// } -// } -// println("Error type, traceID and error count:") -// for ((key, value) in result) { -// System.out.printf("error type: %s\n", key) -// for ((key1, value1) in value) { -// System.out -// .printf("\tTraceID: %s, count %d\n", key1, value1) -// } -// } -// val height: Int = TraceHeight.calculate(graph) -// val networkLatencies: Map> = NetworkLatency.calculate(graph) -// System.out.printf("Trace height = %d\n", height) -// System.out.printf("Network latencies = %s\n", networkLatencies) -// } - private fun getReport(uploadId: String): ReportDao? { // Get the reports diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/traces/AddSpanToTraceFunction.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/traces/AddSpanToTraceFunction.kt deleted file mode 100644 index 7f510d89..00000000 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/traces/AddSpanToTraceFunction.kt +++ /dev/null @@ -1,152 +0,0 @@ -package gov.cdc.ocio.processingstatusapi.functions.traces - -import com.google.gson.Gson -import com.microsoft.azure.functions.HttpRequestMessage -import com.microsoft.azure.functions.HttpResponseMessage -import com.microsoft.azure.functions.HttpStatus -import gov.cdc.ocio.processingstatusapi.exceptions.BadRequestException -import gov.cdc.ocio.processingstatusapi.exceptions.BadStateException -import gov.cdc.ocio.processingstatusapi.model.traces.Tags -import gov.cdc.ocio.processingstatusapi.model.traces.TraceResult -import gov.cdc.ocio.processingstatusapi.opentelemetry.OpenTelemetryConfig -import io.opentelemetry.api.trace.* -import io.opentelemetry.context.Context -import mu.KotlinLogging -import java.util.* - - -/** - * Create a processing status span for a given trace. - */ -class AddSpanToTraceFunction( - private val request: HttpRequestMessage> -) { - private val logger = KotlinLogging.logger {} - - private val openTelemetry by lazy { - OpenTelemetryConfig.initOpenTelemetry() - } - - private val tracer = openTelemetry?.getTracer(AddSpanToTraceFunction::class.java.name) - - private val requestBody = request.body.orElse("") - - /** - * For a given HTTP request, this method creates a processing status span for a given trace. - * In order to process, the HTTP request must contain stageName and spanMark. - * - * @param traceId String - * @param parentSpanId String parent span ID - * @return HttpResponseMessage - resultant HTTP response message for the given request - */ - fun startSpan( - traceId: String, - parentSpanId: String - ): HttpResponseMessage { - - val stageName = request.queryParameters["stageName"] - ?: return request - .createResponseBuilder(HttpStatus.BAD_REQUEST) - .body("stageName is required") - .build() - - logger.info("stageName: $stageName") - - val result: TraceResult - if (tracer !=null) { - try { - // See if we were given any optional tags - var tags: Array? = null - try { - if (requestBody.isNotBlank()) { - tags = Gson().fromJson(requestBody, Array::class.java) - } - } catch (e: Exception) { - throw BadRequestException("Failed to parse the optional metadata tags in the request body") - } - - val spanContext = SpanContext.createFromRemoteParent( - traceId, - parentSpanId, - TraceFlags.getSampled(), - TraceState.getDefault() - ) - - val span = tracer.spanBuilder(stageName) - .setParent(Context.current().with(Span.wrap(spanContext))) - .startSpan() - - tags?.forEach { - if (it.key != null && it.value != null) - span.setAttribute(it.key!!, it.value!!) - } - span.end() - - result = TraceResult().apply { - this.traceId = span.spanContext.traceId - this.spanId = span.spanContext.spanId - } - - } catch (ex: BadRequestException) { - return request - .createResponseBuilder(HttpStatus.BAD_REQUEST) - .body(ex.localizedMessage) - .build() - } - } else { - result = TraceResult().apply { - this.traceId = TraceUtils.TRACING_DISABLED - this.spanId = TraceUtils.TRACING_DISABLED - } - } - - return request - .createResponseBuilder(HttpStatus.OK) - .header("Content-Type", "application/json") - .body(result) - .build() - } - - /** - * For a given HTTP request, this method creates a processing status span for a given trace. - * In order to process, the HTTP request must contain stageName and spanMark. - * - * @param traceId String - * @param spanId String - * @return HttpResponseMessage - resultant HTTP response message for the given request - * @throws BadRequestException - * @throws BadStateException - */ - fun stopSpan( - traceId: String, - spanId: String - ): HttpResponseMessage { - - if (tracer != null) { - try { - val spanContext = SpanContext.createFromRemoteParent( - traceId, - spanId, - TraceFlags.getSampled(), - TraceState.getDefault() - ) - val span = tracer.spanBuilder(spanId) - .setParent(Context.current().with(Span.wrap(spanContext))) - .startSpan() - span.end() - - } catch (ex: BadRequestException) { - return request - .createResponseBuilder(HttpStatus.BAD_REQUEST) - .body(ex.localizedMessage) - .build() - } - } - - return request - .createResponseBuilder(HttpStatus.OK) - .header("Content-Type", "application/json") - .build() - } - -} \ No newline at end of file diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/traces/CreateTraceFunction.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/traces/CreateTraceFunction.kt deleted file mode 100644 index 1ddf0312..00000000 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/traces/CreateTraceFunction.kt +++ /dev/null @@ -1,111 +0,0 @@ -package gov.cdc.ocio.processingstatusapi.functions.traces - -import com.microsoft.azure.functions.HttpRequestMessage -import com.microsoft.azure.functions.HttpResponseMessage -import com.microsoft.azure.functions.HttpStatus -import gov.cdc.ocio.processingstatusapi.opentelemetry.OpenTelemetryConfig -import gov.cdc.ocio.processingstatusapi.model.traces.TraceResult -import mu.KotlinLogging -import java.util.* - - -/** - * Creates a new distributed tracing trace for the given HTTP request - */ -class CreateTraceFunction( - private val request: HttpRequestMessage> -) { - private val logger = KotlinLogging.logger {} - - private val openTelemetry by lazy { - OpenTelemetryConfig.initOpenTelemetry() - } - - private val tracer = openTelemetry?.getTracer(CreateTraceFunction::class.java.name) - - private val uploadId = request.queryParameters["uploadId"] - - private var destinationId = request.queryParameters["destinationId"] - - private var dataStreamId = request.queryParameters["dataStreamId"] - - private var eventType = request.queryParameters["eventType"] - - private var dataStreamRoute = request.queryParameters["dataStreamRoute"] - - /** - * Creates a new distributed tracing trace for the given HTTP request. - * In order to process, the HTTP request must contain uploadId - * - * @return HttpResponseMessage - resultant HTTP response message for the given request - */ - fun create(): HttpResponseMessage { - dataStreamId = dataStreamId ?: destinationId - dataStreamRoute = dataStreamRoute ?: eventType - // Verify the request is complete and properly formatted - checkRequired()?.let { return it } - - logger.info("uploadId: $uploadId") - - val result: TraceResult - if (tracer != null) { - val span = tracer.spanBuilder(PARENT_SPAN).startSpan() - span.setAttribute("uploadId", uploadId!!) - span.setAttribute("dataStreamId", dataStreamId!!) - span.setAttribute("dataStreamRoute", dataStreamRoute!!) - span.end() - - result = TraceResult().apply { - traceId = span.spanContext.traceId - spanId = span.spanContext.spanId - } - } else { - result = TraceResult().apply { - traceId = TraceUtils.TRACING_DISABLED - spanId = TraceUtils.TRACING_DISABLED - } - } - - return request - .createResponseBuilder(HttpStatus.OK) - .header("Content-Type", "application/json") - .body(result) - .build() - } - - /** - * Checks that all the required query parameters are present in order to process the request. If not, - * an appropriate HTTP response message is generated with the details. - * - * @return HttpResponseMessage? - */ - private fun checkRequired(): HttpResponseMessage? { - - if (uploadId == null) { - return request - .createResponseBuilder(HttpStatus.BAD_REQUEST) - .body("uploadId is required") - .build() - } - - if (dataStreamId == null) { - return request - .createResponseBuilder(HttpStatus.BAD_REQUEST) - .body("destinationId or dataStreamId is required") - .build() - } - - if (dataStreamRoute == null) { - return request - .createResponseBuilder(HttpStatus.BAD_REQUEST) - .body("eventType or dataStreamRoute is required") - .build() - } - - return null - } - - companion object { - private const val PARENT_SPAN = "PARENT_SPAN" - } -} \ No newline at end of file diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/traces/GetSpanFunction.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/traces/GetSpanFunction.kt deleted file mode 100644 index 7c3f7b89..00000000 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/traces/GetSpanFunction.kt +++ /dev/null @@ -1,111 +0,0 @@ -package gov.cdc.ocio.processingstatusapi.functions.traces - -import com.microsoft.azure.functions.HttpRequestMessage -import com.microsoft.azure.functions.HttpResponseMessage -import com.microsoft.azure.functions.HttpStatus -import gov.cdc.ocio.processingstatusapi.model.traces.* -import gov.cdc.ocio.processingstatusapi.utils.JsonUtils -import mu.KotlinLogging -import java.util.* - -/** - * Collection of functions to get traces. - */ -class GetSpanFunction( - private val request: HttpRequestMessage> -) { - private val logger = KotlinLogging.logger {} - - private val gson = JsonUtils.getGsonBuilderWithUTC() - - fun withQueryParams(): HttpResponseMessage { - - val uploadId = request.queryParameters["uploadId"] - - val stageName = request.queryParameters["stageName"] - - if (!uploadId.isNullOrBlank() && !stageName.isNullOrBlank()) { - - var latestMatchingSpan: SpanResult? - if (TraceUtils.tracingEnabled) { - var attempts = 0 - var traces: List - do { - // Attempt to locate the trace by uploadId - traces = TraceUtils.getTraces(uploadId) - ?: return request - .createResponseBuilder(HttpStatus.BAD_REQUEST) - .body("The uploadId provided was not found.") - .build() - - latestMatchingSpan = checkForStageNameInTraces(traces, stageName) - if (latestMatchingSpan != null) { - break - } - Thread.sleep(500) - } while (attempts++ < 20) // try for up to 10 seconds - - if (traces.size != 1) { - return request - .createResponseBuilder(HttpStatus.INTERNAL_SERVER_ERROR) - .body("Trace inconsistency found, expected exactly one trace for uploadId = $uploadId, but got ${traces.size}") - .build() - } - } else { - latestMatchingSpan = SpanResult().apply { - this.spanId = TraceUtils.TRACING_DISABLED - this.traceId = TraceUtils.TRACING_DISABLED - this.stageName = stageName - this.timestamp = Date() - } - } - - return if (latestMatchingSpan != null) { - // Found a match, return it - request - .createResponseBuilder(HttpStatus.OK) - .header("Content-Type", "application/json") - .body(gson.toJson(latestMatchingSpan)) - .build() - } else { - request - .createResponseBuilder(HttpStatus.BAD_REQUEST) - .body("Found the uploadId provided, but not the stageName") - .build() - } - } - - return request - .createResponseBuilder(HttpStatus.BAD_REQUEST) - .body("Unrecognized combination of query parameters provided") - .build() - } - - /** - * Check for a span in the given trace with stage name provide. - * - * @param traces List - * @param stageName String - * @return SpanResult? - */ - private fun checkForStageNameInTraces(traces: List, stageName: String): SpanResult? { - if (traces.size != 1) - return null - - val traceResult = TraceResult.buildFromTrace(traces[0]) - - // Found the trace by uploadId, now try to see if we can find at least one stage with a matching name. - val latestMatchingSpan = traceResult.spans - ?.filter { span -> span.stageName == stageName } - ?.sortedBy { spanResult -> spanResult.timestamp } // natural sort order - oldest first to newest - ?.lastOrNull() // get the last one, which will be the most recent timestamp - - if (latestMatchingSpan != null) { - // Found a match, return it - return latestMatchingSpan - } - - return null - } - -} \ No newline at end of file diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/traces/GetTraceFunction.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/traces/GetTraceFunction.kt deleted file mode 100644 index 91aba519..00000000 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/traces/GetTraceFunction.kt +++ /dev/null @@ -1,122 +0,0 @@ -package gov.cdc.ocio.processingstatusapi.functions.traces - -import com.google.gson.* -import com.microsoft.azure.functions.HttpRequestMessage -import com.microsoft.azure.functions.HttpResponseMessage -import com.microsoft.azure.functions.HttpStatus -import gov.cdc.ocio.processingstatusapi.model.traces.* -import gov.cdc.ocio.processingstatusapi.utils.JsonUtils -import mu.KotlinLogging -import java.util.* - -/** - * Collection of functions to get traces. - */ -class GetTraceFunction( - private val request: HttpRequestMessage> -) { - private val logger = KotlinLogging.logger {} - - private val gson = JsonUtils.getGsonBuilderWithUTC() - - /** - * For a given HTTP request, this method fetches trace information for a given traceId. - * - * @param traceId String - * @return HttpResponseMessage - resultant HTTP response message for the given request - */ - fun withTraceId(traceId: String): HttpResponseMessage { - - logger.info("Trace Id = $traceId") - - var traceResult: TraceResult? = null - if (TraceUtils.tracingEnabled) { - val traceEndPoint = System.getenv("JAEGER_TRACE_END_POINT") + "api/traces/$traceId" - logger.info("traceEndPoint: $traceEndPoint") - var attempts = 0 - do { - val response = khttp.get(traceEndPoint) - val obj = response.jsonObject - logger.info("$obj") - - if (response.statusCode == HttpStatus.OK.value()) { - val traceModel = Gson().fromJson(obj.toString(), Base::class.java) - traceResult = TraceResult.buildFromTrace(traceModel.data[0]) - break - } - Thread.sleep(500) - } while (attempts++ < 20) // try for up to 10 seconds - } else { - traceResult = disabledTraceResult - } - - if (traceResult != null) { - return request - .createResponseBuilder(HttpStatus.OK) - .header("Content-Type", "application/json") - .body(gson.toJson(traceResult)) - .build() - } - - return request - .createResponseBuilder(HttpStatus.BAD_REQUEST) - .body("The trace identifier provided was not found.") - .build() - } - - /** - * For a given HTTP request, this method fetches trace information for a given uploadId. - * - * @param uploadId String - * @return HttpResponseMessage - resultant HTTP response message for the given request - */ - fun withUploadId(uploadId: String): HttpResponseMessage { - - logger.info("Upload Id = $uploadId") - - var traceResult: TraceResult? = null - if (TraceUtils.tracingEnabled) { - var attempts = 0 - do { - val traces = TraceUtils.getTraces(uploadId) - ?: return request - .createResponseBuilder(HttpStatus.BAD_REQUEST) - .body("The uploadId provided was not found.") - .build() - - if (traces.size == 1) { - traceResult = TraceResult.buildFromTrace(traces[0]) - break - } - - Thread.sleep(500) - } while (attempts++ < 20) // try for up to 10 seconds - } else { - traceResult = disabledTraceResult - } - - if (traceResult != null) { - return request - .createResponseBuilder(HttpStatus.OK) - .header("Content-Type", "application/json") - .body(gson.toJson(traceResult)) - .build() - } - - return request - .createResponseBuilder(HttpStatus.BAD_REQUEST) - .body("The upload identifier provided was not found.") - .build() - } - - companion object { - private val disabledTraceResult = TraceResult().apply { - this.traceId = TraceUtils.TRACING_DISABLED - this.dataStreamId = TraceUtils.TRACING_DISABLED - this.spanId = TraceUtils.TRACING_DISABLED - this.dataStreamRoute = TraceUtils.TRACING_DISABLED - this.uploadId = TraceUtils.TRACING_DISABLED - } - } - -} \ No newline at end of file diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/traces/TraceUtils.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/traces/TraceUtils.kt deleted file mode 100644 index 4b8706c2..00000000 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/traces/TraceUtils.kt +++ /dev/null @@ -1,69 +0,0 @@ -package gov.cdc.ocio.processingstatusapi.functions.traces - -import com.google.gson.Gson -import com.microsoft.azure.functions.HttpStatus -import gov.cdc.ocio.processingstatusapi.model.traces.Base -import gov.cdc.ocio.processingstatusapi.model.traces.Data -import gov.cdc.ocio.processingstatusapi.model.traces.TraceResult - -class TraceUtils { - - companion object { - - const val TRACING_DISABLED = "TracingDisabled" - - val tracingEnabled = System.getenv("EnableTracing").equals("True", ignoreCase = true) - - /** - * Locate all traces for the given uploadId. - * - * @param uploadId String - * @return List? - */ - fun getTraces(uploadId: String): List? { - // TODO: This is very temporary! - // This is very inefficient and won't scale at all. Once a better understanding - // of querying, and if it can be used to search by a tag then we'll use that. Otherwise, we may need to - // query the storage directly; i.e. elasticsearch, once it is available - - val traceEndPoint = System.getenv("JAEGER_TRACE_END_POINT") + "api/traces?limit=20000&service=dex-processing-status" - val response = khttp.get(traceEndPoint) - - if (response.statusCode != HttpStatus.OK.value()) { - return null - } - - val obj = response.jsonObject - val traces = Gson().fromJson(obj.toString(), Base::class.java) - val traceMatches = traces.data.filter { data -> - data.spans.any { span -> - span.tags.any { tag -> - tag.key.equals("uploadId") && tag.value.equals(uploadId) - } - } - } - - return traceMatches - } - - /** - * Get a trace for the given trace id and span id. - * - * @param traceId String - * @return TraceResult? - */ - fun getTrace(traceId: String): TraceResult? { - val traceEndPoint = System.getenv("JAEGER_TRACE_END_POINT")+"api/traces/$traceId" - - val response = khttp.get(traceEndPoint) - val obj = response.jsonObject - - if (response.statusCode != HttpStatus.OK.value()) { - return null - } - - val traceModel = Gson().fromJson(obj.toString(), Base::class.java) - return TraceResult.buildFromTrace(traceModel.data[0]) - } - } -} \ No newline at end of file diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/HealthCheck.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/HealthCheck.kt index d635fae4..2d4672dd 100644 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/HealthCheck.kt +++ b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/HealthCheck.kt @@ -19,10 +19,6 @@ class ServiceBus: HealthCheckSystem() { var service: String = "Azure Service Bus" } -class Jaeger: HealthCheckSystem() { - var service: String = "Jaeger" -} - class HealthCheck { var status : String? = "DOWN" diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/StatusResult.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/StatusResult.kt index a17e5c96..ba2574f2 100644 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/StatusResult.kt +++ b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/StatusResult.kt @@ -1,22 +1,7 @@ package gov.cdc.ocio.processingstatusapi.model -import com.google.gson.annotations.SerializedName import gov.cdc.ocio.processingstatusapi.model.reports.Report -import gov.cdc.ocio.processingstatusapi.model.traces.TraceDao data class StatusResult( - - @SerializedName("upload_id") - var uploadId: String? = null, - - @SerializedName("data_stream_id") - var dataStreamId: String? = null, - - @SerializedName("data_stream_route") - var dataStreamRoute: String? = null, - - @SerializedName("trace") - var trace: TraceDao? = null, - var reports: List? = null ) \ No newline at end of file diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/Base.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/Base.kt deleted file mode 100644 index 3404fb43..00000000 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/Base.kt +++ /dev/null @@ -1,11 +0,0 @@ -package gov.cdc.ocio.processingstatusapi.model.traces -import com.google.gson.annotations.SerializedName - -class Base { - @SerializedName("data" ) var data : ArrayList = arrayListOf() - @SerializedName("total" ) var total : Int? = null - @SerializedName("limit" ) var limit : Int? = null - @SerializedName("offset" ) var offset : Int? = null - @SerializedName("errors" ) var errors : String? = null - -} \ No newline at end of file diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/Data.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/Data.kt deleted file mode 100644 index 8d0b4f0b..00000000 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/Data.kt +++ /dev/null @@ -1,10 +0,0 @@ -package gov.cdc.ocio.processingstatusapi.model.traces -import com.google.gson.annotations.SerializedName - -class Data { - - @SerializedName("traceID" ) var traceID : String? = null - @SerializedName("spans" ) var spans : ArrayList = arrayListOf() - @SerializedName("processes" ) var processes : Processes? = Processes() - @SerializedName("warnings" ) var warnings : String? = null -} \ No newline at end of file diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/P1.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/P1.kt deleted file mode 100644 index 1758da25..00000000 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/P1.kt +++ /dev/null @@ -1,8 +0,0 @@ -package gov.cdc.ocio.processingstatusapi.model.traces - -import com.google.gson.annotations.SerializedName - -class P1 { - @SerializedName("serviceName" ) var serviceName : String? = null - @SerializedName("tags" ) var tags : ArrayList = arrayListOf() -} \ No newline at end of file diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/Processes.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/Processes.kt deleted file mode 100644 index 983914ba..00000000 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/Processes.kt +++ /dev/null @@ -1,8 +0,0 @@ -package gov.cdc.ocio.processingstatusapi.model.traces - -import com.google.gson.annotations.SerializedName -import gov.cdc.ocio.processingstatusapi.model.traces.P1 - -class Processes { - @SerializedName("p1" ) var p1 : P1? = P1() -} \ No newline at end of file diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/Reference.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/Reference.kt deleted file mode 100644 index 3f762647..00000000 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/Reference.kt +++ /dev/null @@ -1,9 +0,0 @@ -package gov.cdc.ocio.processingstatusapi.model.traces - -import com.google.gson.annotations.SerializedName - -class Reference { - @SerializedName("refType" ) var refType : String? = null - @SerializedName("traceID" ) var traceID : String? = null - @SerializedName("spanID" ) var spanID : String? = null -} \ No newline at end of file diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/SpanResult.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/SpanResult.kt deleted file mode 100644 index eba19a39..00000000 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/SpanResult.kt +++ /dev/null @@ -1,24 +0,0 @@ -package gov.cdc.ocio.processingstatusapi.model.traces - -import com.google.gson.annotations.SerializedName -import java.util.* - -data class SpanResult( - - @SerializedName("stage_name") - var stageName: String? = null, - - @SerializedName("trace_id") - var traceId: String? = null, - - @SerializedName("span_id") - var spanId: String? = null, - - var timestamp: Date? = null, - var status: String? = null, - - @SerializedName("elapsed_millis") - var elapsedMillis: Long? = null, - - var metadata : List? = null -) \ No newline at end of file diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/Spans.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/Spans.kt deleted file mode 100644 index f9a6c73e..00000000 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/Spans.kt +++ /dev/null @@ -1,16 +0,0 @@ -package gov.cdc.ocio.processingstatusapi.model.traces - -import com.google.gson.annotations.SerializedName - -class Spans { - - @SerializedName("traceID" ) var traceID : String? = null - @SerializedName("spanID" ) var spanID : String? = null - @SerializedName("operationName" ) var operationName : String? = null - @SerializedName("references" ) var references : ArrayList = arrayListOf() - @SerializedName("startTime" ) var startTime : Long? = null - @SerializedName("duration" ) var duration : Long? = null - @SerializedName("tags" ) var tags : ArrayList = arrayListOf() - @SerializedName("logs" ) var logs : ArrayList = arrayListOf() - @SerializedName("warnings" ) var warnings : ArrayList = arrayListOf() -} \ No newline at end of file diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/Tags.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/Tags.kt deleted file mode 100644 index 1801003a..00000000 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/Tags.kt +++ /dev/null @@ -1,10 +0,0 @@ -package gov.cdc.ocio.processingstatusapi.model.traces - -import com.google.gson.annotations.SerializedName - -class Tags { - - @SerializedName("key" ) var key : String? = null - //@SerializedName("type" ) var type : String? = null - @SerializedName("value" ) var value : String? = null -} \ No newline at end of file diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/TraceDao.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/TraceDao.kt deleted file mode 100644 index 87c35dc6..00000000 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/TraceDao.kt +++ /dev/null @@ -1,31 +0,0 @@ -package gov.cdc.ocio.processingstatusapi.model.traces - -import com.google.gson.annotations.SerializedName - -data class TraceDao( - - @SerializedName("trace_id") - var traceId: String? = null, - - @SerializedName("span_id") - var spanId: String? = null, - - var spans: List? = null -) { - companion object { - - /** - * Provide the trace DAO from the trace result. - * - * @param traceResult TraceResult - * @return TraceResult - */ - fun buildFromTraceResult(traceResult: TraceResult): TraceDao { - return TraceDao().apply { - this.traceId = traceResult.traceId - this.spanId = traceResult.spanId - this.spans = traceResult.spans - } - } - } -} \ No newline at end of file diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/TraceResult.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/TraceResult.kt deleted file mode 100644 index 48763460..00000000 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/traces/TraceResult.kt +++ /dev/null @@ -1,142 +0,0 @@ -package gov.cdc.ocio.processingstatusapi.model.traces - -import com.google.gson.annotations.SerializedName -import gov.cdc.ocio.processingstatusapi.functions.status.GetStatusFunction -import java.time.Instant -import java.util.* - -data class TraceResult( - - @SerializedName("trace_id") - var traceId: String? = null, - - @SerializedName("span_id") - var spanId: String? = null, - - @SerializedName("upload_id") - var uploadId: String? = null, - - @SerializedName("data_stream_id") - var dataStreamId: String? = null, - - @SerializedName("data_stream_route") - var dataStreamRoute: String? = null, - - var spans: List? = null -) { - companion object { - - /** - * Provide the trace result from the trace model. - * - * @param traceModel Base - * @return TraceResult - */ - fun buildFromTrace(traceModel: Data): TraceResult { - - // Parent span will always be the first element in the array - val uploadIdTag = traceModel.spans[0].tags.firstOrNull { it.key.equals("uploadId") } - val dataStreamIdTag = traceModel.spans[0].tags.firstOrNull { it.key.equals("dataStreamId") } - val dataStreamRouteTag = traceModel.spans[0].tags.firstOrNull { it.key.equals("dataStreamRoute") } - val parentSpanId = traceModel.spans[0].spanID - - // Remove the parent span since we'll be doing a foreach on all remaining spans - traceModel.spans.removeAt(0) - - // Iterate through all the remaining spans and map them by operationName aka stageName - val spanMap = mutableMapOf>() - val parentSpans = traceModel.spans.filter { span -> span.references.any { ref -> ref.spanID == parentSpanId } } - parentSpans.forEach { span -> - span.operationName?.let { stageName -> - var spansList = spanMap[stageName]?.toMutableList() - if (spansList == null) - spansList = mutableListOf() - val tag = Tags().apply { - key = "spanMark" - value = "start" - } - span.tags.add(tag) - spansList.add(span) - spanMap[stageName] = spansList - } - } - val childSpans = traceModel.spans.filter { span -> span.references.any { ref -> ref.refType == "CHILD_OF" } } - childSpans.forEach { span -> - // Find the parent span - val matchParentSpanId = span.references.firstOrNull { ref -> ref.refType == "CHILD_OF" }?.spanID - val match = spanMap.values.firstOrNull { spanList -> spanList.any { span -> span.spanID == matchParentSpanId } } - val stageName = match?.get(0)?.operationName - stageName?.let { - var stageSpansList = spanMap[stageName]?.toMutableList() - if (stageSpansList == null) - stageSpansList = mutableListOf() - val tag = Tags().apply { - key = "spanMark" - value = "stop" - } - span.tags.add(tag) - stageSpansList.add(span) - spanMap[stageName] = stageSpansList - } - } - - // Build the span results associated with this trace - val spanResults = mutableListOf() - spanMap.entries.forEach { entry -> - val stageName = entry.key - val spansList = entry.value - - // Determine whether the stage is running or completed. If the start mark is found, but not the stop mark - // then it is still running. - var startTimestamp: Long? = null - var stopTimestamp: Long? = null - var spanID: String? = null - val tags = mutableListOf() - spansList.forEach { span -> - val spanMarkTag = span.tags.firstOrNull { it.key.equals("spanMark") } - when (spanMarkTag?.value) { - "start" -> { - startTimestamp = span.startTime?.div(1000) // microseconds to milliseconds - spanID = span.spanID - } - "stop" -> { - stopTimestamp = span.startTime?.div(1000) // microseconds to milliseconds - } - } - tags.addAll(span.tags.filterNot { GetStatusFunction.excludedSpanTags.contains(it.key) }) - } - val isSpanComplete = (startTimestamp != null && stopTimestamp != null) - - val elapsedMillis = if (isSpanComplete) { - stopTimestamp!! - startTimestamp!! - } else if (startTimestamp != null) { - Instant.now().toEpochMilli() - startTimestamp!! - } else { - null - } - - val spanResult = SpanResult().apply { - this.stageName = stageName - this.traceId = traceModel.traceID - this.spanId = spanID - timestamp = startTimestamp?.let { Date(it) } - status = if (isSpanComplete) "complete" else "running" - this.elapsedMillis = elapsedMillis - metadata = tags - } - spanResults.add(spanResult) - } - - val result = TraceResult().apply { - this.traceId = traceModel.traceID - spanId = parentSpanId - uploadId = uploadIdTag?.value - dataStreamId = dataStreamIdTag?.value - dataStreamRoute = dataStreamRouteTag?.value - spans = spanResults - } - - return result - } - } -} \ No newline at end of file diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/opentelemetry/OpenTelemetryConfig.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/opentelemetry/OpenTelemetryConfig.kt deleted file mode 100644 index e520a248..00000000 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/opentelemetry/OpenTelemetryConfig.kt +++ /dev/null @@ -1,72 +0,0 @@ -package gov.cdc.ocio.processingstatusapi.opentelemetry - -import gov.cdc.ocio.processingstatusapi.functions.traces.TraceUtils -import io.opentelemetry.api.OpenTelemetry -import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator -import io.opentelemetry.context.propagation.ContextPropagators -import io.opentelemetry.exporter.logging.LoggingSpanExporter -import io.opentelemetry.exporter.otlp.trace.OtlpGrpcSpanExporter -import io.opentelemetry.sdk.OpenTelemetrySdk -import io.opentelemetry.sdk.resources.Resource -import io.opentelemetry.sdk.trace.SdkTracerProvider -import io.opentelemetry.sdk.trace.export.BatchSpanProcessor -import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor -import io.opentelemetry.semconv.resource.attributes.ResourceAttributes -import java.util.concurrent.TimeUnit -import io.opentelemetry.api.common.Attributes - -/** - * All SDK management takes place here, away from the instrumentation code, which should only access - * the OpenTelemetry APIs. - */ -internal object OpenTelemetryConfig1 { - /** - * Initializes the OpenTelemetry SDK with a logging span exporter and the W3C Trace Context - * propagator. - * - * @return A ready-to-use [OpenTelemetry] instance. - */ - fun initOpenTelemetry(): OpenTelemetry { - val sdkTracerProvider: SdkTracerProvider = SdkTracerProvider.builder() - .addSpanProcessor(SimpleSpanProcessor.create(LoggingSpanExporter())) - .build() - val sdk: OpenTelemetrySdk = OpenTelemetrySdk.builder() - .setTracerProvider(sdkTracerProvider) - .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) - .build() - Runtime.getRuntime().addShutdownHook(Thread(sdkTracerProvider::close)) - return sdk - } -} - -internal object OpenTelemetryConfig { - /** - * Initialize an OpenTelemetry SDK with a [OtlpGrpcSpanExporter] and a [ ]. - * - * @param jaegerEndpoint The endpoint of your Jaeger instance. - * @return A ready-to-use [OpenTelemetry] instance. - */ - fun initOpenTelemetry(): OpenTelemetry? { - if (!TraceUtils.tracingEnabled) - return null - - // Export traces to Jaeger over OTLP - val jaegerOtlpExporter: OtlpGrpcSpanExporter = OtlpGrpcSpanExporter.builder() - .setEndpoint(System.getenv("JAEGER_OTEL_COLLECTOR_END_POINT")) - .setTimeout(30, TimeUnit.SECONDS) - .build() - val serviceNameResource: Resource = - Resource.create(Attributes.of(ResourceAttributes.SERVICE_NAME, "dex-processing-status")) - - // Set to process the spans by the Jaeger Exporter - val tracerProvider = SdkTracerProvider.builder() - .addSpanProcessor(BatchSpanProcessor.builder(jaegerOtlpExporter).build()) - .setResource(Resource.getDefault().merge(serviceNameResource)) - .build() - val openTelemetry = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build() - - // it's always a good idea to shut down the SDK cleanly at JVM exit. - Runtime.getRuntime().addShutdownHook(Thread { tracerProvider.close() }) - return openTelemetry - } -} \ No newline at end of file diff --git a/processing-status-api-function-app/src/test/kotlin/FunctionWrapperTest.kt b/processing-status-api-function-app/src/test/kotlin/FunctionWrapperTest.kt index 035d7d37..8c419bd4 100644 --- a/processing-status-api-function-app/src/test/kotlin/FunctionWrapperTest.kt +++ b/processing-status-api-function-app/src/test/kotlin/FunctionWrapperTest.kt @@ -2,13 +2,9 @@ import com.microsoft.azure.functions.ExecutionContext import com.microsoft.azure.functions.HttpRequestMessage import com.microsoft.azure.functions.HttpStatus import gov.cdc.ocio.processingstatusapi.FunctionJavaWrappers -import gov.cdc.ocio.processingstatusapi.functions.traces.CreateTraceFunction -import gov.cdc.ocio.processingstatusapi.opentelemetry.OpenTelemetryConfig import io.mockk.every import io.mockk.mockk import io.mockk.mockkObject -import io.opentelemetry.api.OpenTelemetry -import io.opentelemetry.api.trace.Tracer import org.mockito.Mockito import org.mockito.Mockito.mock import org.testng.Assert @@ -23,8 +19,6 @@ class FunctionWrapperTest { private lateinit var request: HttpRequestMessage> private lateinit var context: ExecutionContext - private val mockOpenTelemetry = mockk() - private val mockTracer = mockk() private val testMessage = File("./src/test/kotlin/data/reports/createReport_badrequest.json").readText() @BeforeMethod @@ -33,9 +27,6 @@ class FunctionWrapperTest { request = mock(HttpRequestMessage::class.java) as HttpRequestMessage> context = mock(ExecutionContext::class.java) Mockito.`when`(request.body).thenReturn(Optional.of(testMessage)) - mockkObject(OpenTelemetryConfig) - every { OpenTelemetryConfig.initOpenTelemetry()} returns mockOpenTelemetry - every {mockOpenTelemetry.getTracer(CreateTraceFunction::class.java.name)} returns mockTracer // Setup method invocation interception when createResponseBuilder is called to avoid null pointer on real method call. Mockito.doAnswer { invocation -> @@ -50,60 +41,6 @@ class FunctionWrapperTest { assert(response.status == HttpStatus.INTERNAL_SERVER_ERROR) } - @Test - fun testCreateTrace() { - val response = FunctionJavaWrappers().createTrace(request) - } - - @Test - fun testTraceStartSpan() { - var exceptionThrown = false - try { - FunctionJavaWrappers().traceStartSpan(request, "4dad617cd7de019066a0f49cad309948", "1") - } catch(ex: Exception) { - exceptionThrown = true - } - Assert.assertTrue(exceptionThrown) - } - - @Test - fun testTraceStopSpan() { - var exceptionThrown = false - try { - FunctionJavaWrappers().traceStopSpan(request, "4dad617cd7de019066a0f49cad309948", "1") - } catch(ex: Exception) { - exceptionThrown = true - } - Assert.assertTrue(exceptionThrown) - } - - @Test - fun testGetTraceByTraceId() { - var exceptionThrown = false - try { - FunctionJavaWrappers().getTraceByTraceId(request, "4dad617cd7de019066a0f49cad309948") - } catch(ex: Exception) { - exceptionThrown = true - } - Assert.assertFalse(exceptionThrown) - } - - @Test - fun testGetTraceByUploadId() { - var exceptionThrown = false - try { - FunctionJavaWrappers().getTraceByUploadId(request, "4dad617cd7de019066a0f49cad309948") - } catch(ex: Exception) { - exceptionThrown = true - } - Assert.assertFalse(exceptionThrown) - } - - @Test - fun testGetTraceSpanByUploadIdStageName() { - val response = FunctionJavaWrappers().getTraceSpanByUploadIdStageName(request) - } - @Test fun tesCreateReportByUploadId() { val response = FunctionJavaWrappers().createReportByUploadId(request, "4dad617cd7de019066a0f49cad309948") diff --git a/processing-status-api-function-app/src/test/kotlin/data/trace/get_trace.json b/processing-status-api-function-app/src/test/kotlin/data/trace/get_trace.json deleted file mode 100644 index 8274e439..00000000 --- a/processing-status-api-function-app/src/test/kotlin/data/trace/get_trace.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "data": [ - { - "traceID": "89356b6ce9a5fea5cab099a7b88a144e", - "spans": [ - { - "traceID": "89356b6ce9a5fea5cab099a7b88a144e", - "spanID": "ba14b27a972a03ee", - "operationName": "PARENT_SPAN", - "references": [ - { - "refType": "CHILD_OF", - "traceID": "89356b6ce9a5fea5cab099a7b88a144e", - "spanID": "9375e3177815242e" - } - ], - "startTime": 1706206691876878, - "duration": 9, - "tags": [ - { - "key": "otel.library.name", - "type": "string", - "value": "gov.cdc.ocio.processingstatusapi.functions.traces.CreateTraceFunction" - }, - { - "key": "dataStreamRoute", - "type": "string", - "value": "upload" - }, - { - "key": "dataStreamId", - "type": "string", - "value": "1" - }, - { - "key": "uploadId", - "type": "string", - "value": "1" - }, - { - "key": "span.kind", - "type": "string", - "value": "internal" - }, - { - "key": "internal.span.format", - "type": "string", - "value": "proto" - } - ], - "logs": [], - "processID": "p1", - "warnings": [ - "invalid parent span IDs=9375e3177815242e; skipping clock skew adjustment" - ] - } - ], - "processes": { - "p1": { - "serviceName": "dex-processing-status", - "tags": [ - { - "key": "telemetry.sdk.language", - "type": "string", - "value": "java" - }, - { - "key": "telemetry.sdk.name", - "type": "string", - "value": "opentelemetry" - }, - { - "key": "telemetry.sdk.version", - "type": "string", - "value": "1.29.0" - } - ] - } - }, - "warnings": null - } - ], - "total": 0, - "limit": 0, - "offset": 0, - "errors": null -} \ No newline at end of file diff --git a/processing-status-api-function-app/src/test/kotlin/test/status/GetStatusFunctionTests.kt b/processing-status-api-function-app/src/test/kotlin/test/status/GetStatusFunctionTests.kt index aeb2e522..58d5662a 100644 --- a/processing-status-api-function-app/src/test/kotlin/test/status/GetStatusFunctionTests.kt +++ b/processing-status-api-function-app/src/test/kotlin/test/status/GetStatusFunctionTests.kt @@ -10,22 +10,17 @@ import com.microsoft.azure.functions.HttpStatus import gov.cdc.ocio.processingstatusapi.cosmos.CosmosContainerManager import gov.cdc.ocio.processingstatusapi.functions.status.GetStatusFunction import gov.cdc.ocio.processingstatusapi.model.reports.Report -import gov.cdc.ocio.processingstatusapi.opentelemetry.OpenTelemetryConfig import io.mockk.every import io.mockk.mockk import io.mockk.mockkObject import io.mockk.mockkStatic -import io.opentelemetry.api.OpenTelemetry -import io.opentelemetry.api.trace.Tracer import khttp.responses.Response -import org.json.JSONObject import org.mockito.Mockito.doAnswer import org.mockito.Mockito.mock import org.mockito.kotlin.any import org.testng.annotations.BeforeMethod import org.testng.annotations.Test import utils.HttpResponseMessageMock -import java.io.File import java.util.* @@ -33,12 +28,8 @@ class GetStatusFunctionTests { private lateinit var request: HttpRequestMessage> private lateinit var context: ExecutionContext - private val mockOpenTelemetry = mockk() - private val mockTracer = mockk() - private val testBytes = File("./src/test/kotlin/data/trace/get_trace.json").readText() + private val mockResponse = mockk() - // Convert the string to a JSONObject - private val jsonObject = JSONObject(testBytes) private val items = mockk>() @BeforeMethod @@ -55,10 +46,6 @@ class GetStatusFunctionTests { request = mock(HttpRequestMessage::class.java) as HttpRequestMessage> mockkStatic("khttp.KHttp") every {khttp.get(any())} returns mockResponse - every {mockResponse.jsonObject} returns jsonObject - mockkObject(OpenTelemetryConfig) - every { OpenTelemetryConfig.initOpenTelemetry()} returns mockOpenTelemetry - every {mockOpenTelemetry.getTracer(GetStatusFunction::class.java.name)} returns mockTracer every { mockCosmosContainer.queryItems(any(), any(), Report::class.java) } returns items // Setup method invocation interception when createResponseBuilder is called to avoid null pointer on real method call. @@ -69,14 +56,13 @@ class GetStatusFunctionTests { } @Test - fun testCreateSuccess() { + fun testWithUploadIdFailure() { every { items.count() > 0} returns false every {mockResponse.statusCode} returns HttpStatus.OK.value() - val response = GetStatusFunction(request).withUploadId("1"); - assert(response.status == HttpStatus.OK) + val response = GetStatusFunction(request).withUploadId("1") + assert(response.status == HttpStatus.BAD_REQUEST) } - } diff --git a/processing-status-api-function-app/src/test/kotlin/test/traces/AddSpanToTraceFunctionTests.kt b/processing-status-api-function-app/src/test/kotlin/test/traces/AddSpanToTraceFunctionTests.kt deleted file mode 100644 index 27c17705..00000000 --- a/processing-status-api-function-app/src/test/kotlin/test/traces/AddSpanToTraceFunctionTests.kt +++ /dev/null @@ -1,71 +0,0 @@ -package test.traces - -import com.microsoft.azure.functions.ExecutionContext -import com.microsoft.azure.functions.HttpRequestMessage -import com.microsoft.azure.functions.HttpStatus -import gov.cdc.ocio.processingstatusapi.functions.traces.AddSpanToTraceFunction -import gov.cdc.ocio.processingstatusapi.opentelemetry.OpenTelemetryConfig -import io.mockk.every -import io.mockk.mockk -import io.mockk.mockkObject -import io.opentelemetry.api.OpenTelemetry -import io.opentelemetry.api.trace.Tracer -import khttp.responses.Response -import org.mockito.Mockito -import org.mockito.Mockito.doAnswer -import org.mockito.kotlin.any -import org.testng.Assert.assertEquals -import org.testng.annotations.BeforeMethod -import org.testng.annotations.Test -import utils.HttpResponseMessageMock -import java.io.File -import java.util.* - - -class AddSpanToTraceFunctionTests { - - private lateinit var context: ExecutionContext - private lateinit var request: HttpRequestMessage> - val testMessage = File("./src/test/kotlin/data/reports/createReport_badrequest.json").readText() - private val mockResponse = mockk() - private val mockOpenTelemetry = mockk() - private val mockTracer = mockk() - - @BeforeMethod - fun setUp() { - context = Mockito.mock(ExecutionContext::class.java) - request = Mockito.mock(HttpRequestMessage::class.java) as HttpRequestMessage> - mockkObject(OpenTelemetryConfig) - every { OpenTelemetryConfig.initOpenTelemetry()} returns mockOpenTelemetry - every {mockOpenTelemetry.getTracer(AddSpanToTraceFunction::class.java.name)} returns mockTracer - Mockito.`when`(request.body).thenReturn(Optional.of(testMessage)) - // Setup method invocation interception when createResponseBuilder is called to avoid null pointer on real method call. - doAnswer { invocation -> - val status = invocation.arguments[0] as HttpStatus - HttpResponseMessageMock.HttpResponseMessageBuilderMock().status(status) - }.`when`(request).createResponseBuilder(any()) - } - - @Test - fun testWithStartSpanOk() { - val addSpanToTraceFunction = AddSpanToTraceFunction(request) - every {mockResponse.statusCode} returns HttpStatus.OK.value() - - val result = addSpanToTraceFunction.startSpan("1", "1") - assertEquals(400, result.statusCode) - } - - //@Test - fun testWithStopSpan_ok() { - val addSpanToTraceFunction = AddSpanToTraceFunction(request) - every {mockResponse.statusCode} returns HttpStatus.OK.value() - - val result = addSpanToTraceFunction.stopSpan("1", "1") - assertEquals(200, result.statusCode) - } - - - - -} - diff --git a/processing-status-api-function-app/src/test/kotlin/test/traces/CreateTraceFunctionTests.kt b/processing-status-api-function-app/src/test/kotlin/test/traces/CreateTraceFunctionTests.kt deleted file mode 100644 index 482ca0da..00000000 --- a/processing-status-api-function-app/src/test/kotlin/test/traces/CreateTraceFunctionTests.kt +++ /dev/null @@ -1,102 +0,0 @@ -package test.traces - -import com.microsoft.azure.functions.ExecutionContext -import com.microsoft.azure.functions.HttpRequestMessage -import com.microsoft.azure.functions.HttpStatus -import gov.cdc.ocio.processingstatusapi.functions.traces.CreateTraceFunction -import gov.cdc.ocio.processingstatusapi.opentelemetry.OpenTelemetryConfig -import io.mockk.every -import io.mockk.mockk -import io.mockk.mockkObject -import io.opentelemetry.api.OpenTelemetry -import io.opentelemetry.api.trace.Tracer -import org.mockito.Mockito -import org.mockito.Mockito.doAnswer -import org.mockito.Mockito.mock -import org.mockito.kotlin.any -import org.testng.annotations.BeforeMethod -import org.testng.annotations.Test -import utils.HttpResponseMessageMock -import java.io.File -import java.util.* - - -class CreateTraceFunctionTests { - - private lateinit var request: HttpRequestMessage> - private lateinit var context: ExecutionContext - private val mockOpenTelemetry = mockk() - private val mockTracer = mockk() - private val testMessage = File("./src/test/kotlin/data/reports/createReport_badrequest.json").readText() - - @BeforeMethod - fun setUp() { - context = mock(ExecutionContext::class.java) - request = mock(HttpRequestMessage::class.java) as HttpRequestMessage> - mockkObject(OpenTelemetryConfig) - every { OpenTelemetryConfig.initOpenTelemetry()} returns mockOpenTelemetry - every {mockOpenTelemetry.getTracer(CreateTraceFunction::class.java.name)} returns mockTracer - - // Setup method invocation interception when createResponseBuilder is called to avoid null pointer on real method call. - doAnswer { invocation -> - val status = invocation.arguments[0] as HttpStatus - HttpResponseMessageMock.HttpResponseMessageBuilderMock().status(status) - }.`when`(request).createResponseBuilder(any()) - } - - @Test - fun testCreateBadrequest() { - val response = CreateTraceFunction(request).create(); - assert(response.status == HttpStatus.BAD_REQUEST) - } - - @Test - fun testCreateUploadIdMissing() { - val queryParameters = mutableMapOf() - queryParameters["destinationId"] = "1" - queryParameters["eventType"] = "1" - Mockito.`when`(request.body).thenReturn(Optional.of(testMessage)) - Mockito.`when`(request.queryParameters).thenReturn(queryParameters) - val response = CreateTraceFunction(request).create(); - assert(response.status == HttpStatus.BAD_REQUEST) - } - - @Test - fun testCreateDestinationIdMissing() { - val queryParameters = mutableMapOf() - val testMessage = File("./src/test/kotlin/data/reports/createReport_badrequest.json").readText() - queryParameters["eventType"] = "1" - queryParameters["uploadId"] = "1" - Mockito.`when`(request.body).thenReturn(Optional.of(testMessage)) - Mockito.`when`(request.queryParameters).thenReturn(queryParameters) - val response = CreateTraceFunction(request).create(); - assert(response.status == HttpStatus.BAD_REQUEST) - } - - @Test - fun testCreateEventTypeMissing() { - val queryParameters = mutableMapOf() - val testMessage = File("./src/test/kotlin/data/reports/createReport_badrequest.json").readText() - queryParameters["destinationId"] = "1" - queryParameters["uploadId"] = "1" - Mockito.`when`(request.body).thenReturn(Optional.of(testMessage)) - Mockito.`when`(request.queryParameters).thenReturn(queryParameters) - val response = CreateTraceFunction(request).create(); - assert(response.status == HttpStatus.BAD_REQUEST) - } - - //@Test - fun testCreate_uploadId_good_request() { - val queryParameters = mutableMapOf() - val testMessage = File("./src/test/kotlin/data/reports/createReport_badrequest.json").readText() - queryParameters["uploadId"] = "1" - queryParameters["destinationId"] = "1" - queryParameters["eventType"] = "1" - Mockito.`when`(request.body).thenReturn(Optional.of(testMessage)) - Mockito.`when`(request.queryParameters).thenReturn(queryParameters) - val response = CreateTraceFunction(request).create(); - assert(response.status == HttpStatus.BAD_REQUEST) - } - -} - diff --git a/processing-status-api-function-app/src/test/kotlin/test/traces/GetSpanFunctionTests.kt b/processing-status-api-function-app/src/test/kotlin/test/traces/GetSpanFunctionTests.kt deleted file mode 100644 index b4a76cd4..00000000 --- a/processing-status-api-function-app/src/test/kotlin/test/traces/GetSpanFunctionTests.kt +++ /dev/null @@ -1,72 +0,0 @@ -package test.traces - -import com.azure.cosmos.CosmosClient -import com.azure.cosmos.CosmosContainer -import com.azure.cosmos.CosmosDatabase -import com.azure.cosmos.util.CosmosPagedIterable -import com.microsoft.azure.functions.ExecutionContext -import com.microsoft.azure.functions.HttpRequestMessage -import com.microsoft.azure.functions.HttpStatus -import gov.cdc.ocio.processingstatusapi.cosmos.CosmosContainerManager -import gov.cdc.ocio.processingstatusapi.functions.traces.GetSpanFunction -import gov.cdc.ocio.processingstatusapi.model.reports.Report -import io.mockk.every -import io.mockk.mockk -import io.mockk.mockkObject -import io.mockk.mockkStatic -import khttp.responses.Response -import org.json.JSONObject -import org.mockito.Mockito -import org.mockito.Mockito.doAnswer -import org.mockito.kotlin.any -import org.testng.Assert.assertEquals -import org.testng.annotations.BeforeMethod -import org.testng.annotations.Test -import utils.HttpResponseMessageMock -import java.io.File -import java.util.* - - -class GetSpanFunctionTests { - - private lateinit var context: ExecutionContext - private lateinit var request: HttpRequestMessage> - private val testBytes = File("./src/test/kotlin/data/trace/get_trace.json").readText() - private val mockResponse = mockk() - // Convert the string to a JSONObject - private val jsonObject = JSONObject(testBytes) - private val items = mockk>() - - @BeforeMethod - fun setUp() { - context = Mockito.mock(ExecutionContext::class.java) - request = Mockito.mock(HttpRequestMessage::class.java) as HttpRequestMessage> - val mockCosmosClient = mockk() - val mockCosmosDb = mockk() - val mockCosmosContainer = mockk() - mockkObject(CosmosContainerManager) - every { CosmosContainerManager.initDatabaseContainer(any(), any()) } returns mockCosmosContainer - every { mockCosmosClient.getDatabase(any()) } returns mockCosmosDb - every { mockCosmosDb.getContainer(any()) } returns mockCosmosContainer - every { mockCosmosContainer.queryItems(any(), any(), Report::class.java) } returns items - - mockkStatic("khttp.KHttp") - every {khttp.get(any())} returns mockResponse - every {mockResponse.jsonObject} returns jsonObject - // Setup method invocation interception when createResponseBuilder is called to avoid null pointer on real method call. - doAnswer { invocation -> - val status = invocation.arguments[0] as HttpStatus - HttpResponseMessageMock.HttpResponseMessageBuilderMock().status(status) - }.`when`(request).createResponseBuilder(any()) - } - - @Test - fun testWithQueryParamsOk() { - val getSpanFunction = GetSpanFunction(request) - every {mockResponse.statusCode} returns HttpStatus.OK.value() - - val result = getSpanFunction.withQueryParams() - assertEquals(400, result.statusCode) - } -} - diff --git a/processing-status-api-function-app/src/test/kotlin/test/traces/GetTraceFunctionTests.kt b/processing-status-api-function-app/src/test/kotlin/test/traces/GetTraceFunctionTests.kt deleted file mode 100644 index 8ba853e9..00000000 --- a/processing-status-api-function-app/src/test/kotlin/test/traces/GetTraceFunctionTests.kt +++ /dev/null @@ -1,87 +0,0 @@ -package test.traces - -import com.microsoft.azure.functions.ExecutionContext -import com.microsoft.azure.functions.HttpRequestMessage -import com.microsoft.azure.functions.HttpStatus -import gov.cdc.ocio.processingstatusapi.cosmos.CosmosContainerManager -import gov.cdc.ocio.processingstatusapi.functions.traces.GetTraceFunction -import gov.cdc.ocio.processingstatusapi.functions.traces.TraceUtils -import io.mockk.every -import io.mockk.mockk -import io.mockk.mockkObject -import io.mockk.mockkStatic -import khttp.responses.Response -import org.json.JSONObject -import org.mockito.Mockito -import org.mockito.Mockito.doAnswer -import org.mockito.kotlin.any -import org.testng.Assert.assertEquals -import org.testng.annotations.BeforeMethod -import org.testng.annotations.Test -import utils.HttpResponseMessageMock -import java.io.File -import java.util.* - - -class GetTraceFunctionTests { - - private lateinit var context: ExecutionContext - private lateinit var request: HttpRequestMessage> - private val testBytes = File("./src/test/kotlin/data/trace/get_trace.json").readText() - private val mockResponse = mockk() - // Convert the string to a JSONObject - private val jsonObject = JSONObject(testBytes) - - @BeforeMethod - fun setUp() { - context = Mockito.mock(ExecutionContext::class.java) - request = Mockito.mock(HttpRequestMessage::class.java) as HttpRequestMessage> - mockkStatic("khttp.KHttp") - mockkObject(TraceUtils) - every { TraceUtils.tracingEnabled } returns true - every {khttp.get(any())} returns mockResponse - every {mockResponse.jsonObject} returns jsonObject - // Setup method invocation interception when createResponseBuilder is called to avoid null pointer on real method call. - doAnswer { invocation -> - val status = invocation.arguments[0] as HttpStatus - HttpResponseMessageMock.HttpResponseMessageBuilderMock().status(status) - }.`when`(request).createResponseBuilder(any()) - } - - @Test - fun testWithTraceIdOk() { - val getTraceFunction = GetTraceFunction(request) - every {mockResponse.statusCode} returns HttpStatus.OK.value() - - val result = getTraceFunction.withTraceId("1") - assertEquals(200, result.statusCode) - } - - @Test - fun testWithTraceIdBadRequest() { - val getTraceFunction = GetTraceFunction(request) - every {mockResponse.statusCode} returns HttpStatus.BAD_REQUEST.value() - - val result = getTraceFunction.withTraceId("1") - assertEquals(400, result.statusCode) - } - - @Test - fun testWithUploadIdOk() { - val getTraceFunction = GetTraceFunction(request) - every {mockResponse.statusCode} returns HttpStatus.OK.value() - - val result = getTraceFunction.withUploadId("1") - assertEquals(200, result.statusCode) - } - - @Test - fun testWithUploadIdBadRequest() { - val getTraceFunction = GetTraceFunction(request) - every {mockResponse.statusCode} returns HttpStatus.BAD_REQUEST.value() - - val result = getTraceFunction.withUploadId("1") - assertEquals(400, result.statusCode) - } -} - diff --git a/processing-status-api-function-app/test/IntegrationTest/src/test/resources/hl7RedactedReportv2.json b/processing-status-api-function-app/test/IntegrationTest/src/test/resources/hl7RedactedReportv2.json index 9bf3aa57..b45538e7 100644 --- a/processing-status-api-function-app/test/IntegrationTest/src/test/resources/hl7RedactedReportv2.json +++ b/processing-status-api-function-app/test/IntegrationTest/src/test/resources/hl7RedactedReportv2.json @@ -30,8 +30,6 @@ "upload_id": "mcq1-20240216-01-d849c4b7-a447-44c3-a3b1-a0d50acec6ee", "data_stream_id": "celr", "data_stream_route": "hl7", - "trace_id": "UNKNOWN", - "span_id": "UNKNOWN", "supporting_metadata": { "message_type": "CASE" } diff --git a/processing-status-api-function-app/test/IntegrationTest/src/test/resources/hl7Validationv2.json b/processing-status-api-function-app/test/IntegrationTest/src/test/resources/hl7Validationv2.json index 37530317..b220c546 100644 --- a/processing-status-api-function-app/test/IntegrationTest/src/test/resources/hl7Validationv2.json +++ b/processing-status-api-function-app/test/IntegrationTest/src/test/resources/hl7Validationv2.json @@ -30,8 +30,6 @@ "upload_id": "$uploadId", "data_stream_id": "daart", "data_stream_route": "hl7", - "trace_id": "UNKNOWN", - "span_id": "UNKNOWN", "supporting_metadata": { "route": "daart", "system_provider": "DEX-ROUTING", diff --git a/processing-status-api-function-app/test/postman/Processing Status API.postman_collection.json b/processing-status-api-function-app/test/postman/Processing Status API.postman_collection.json index 55ae2a11..63d766a6 100644 --- a/processing-status-api-function-app/test/postman/Processing Status API.postman_collection.json +++ b/processing-status-api-function-app/test/postman/Processing Status API.postman_collection.json @@ -1,392 +1,12 @@ { "info": { - "_postman_id": "111429b9-2820-43fe-be73-f0e0991d7684", + "_postman_id": "96e8a269-204e-48c4-b789-e3156d5894cf", "name": "Processing Status API", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", - "_exporter_id": "608910" + "_exporter_id": "7274570", + "_collection_link": "https://speeding-star-507143.postman.co/workspace/Quest~68573676-b355-4f2a-b5d0-95cb61a584e1/collection/7274570-96e8a269-204e-48c4-b789-e3156d5894cf?action=share&source=collection_link&creator=7274570" }, "item": [ - { - "name": "Trace", - "item": [ - { - "name": "Create \"dex-testing\" trace", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "var jsonData = JSON.parse(responseBody);", - "pm.environment.set(\"traceId\", jsonData.trace_id);", - "pm.environment.set(\"parentSpanId\", jsonData.span_id);" - ], - "type": "text/javascript" - } - }, - { - "listen": "prerequest", - "script": { - "exec": [ - "var uuid = require('uuid');", - "var uploadId = uuid.v4();", - "console.log(uploadId);", - "", - "pm.environment.set(\"uploadId\", uploadId);" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "POST", - "header": [], - "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/trace?uploadId={{uploadId}}&destinationId=dex-testing&eventType=test-event1", - "host": [ - "{{PROCESSING_STATUS_BASE_URL}}" - ], - "path": [ - "api", - "trace" - ], - "query": [ - { - "key": "uploadId", - "value": "{{uploadId}}" - }, - { - "key": "destinationId", - "value": "dex-testing" - }, - { - "key": "eventType", - "value": "test-event1" - } - ] - } - }, - "response": [] - }, - { - "name": "Start upload span", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "var jsonData = JSON.parse(responseBody);", - "pm.environment.set(\"spanId\", jsonData.span_id);" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\n \"key\": \"extra_field1\",\n \"value\": \"value1\"\n },\n {\n \"key\": \"extra_field2\",\n \"value\": \"value2\"\n }\n]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/trace/startSpan/{{traceId}}/{{parentSpanId}}?stageName=dex-upload", - "host": [ - "{{PROCESSING_STATUS_BASE_URL}}" - ], - "path": [ - "api", - "trace", - "startSpan", - "{{traceId}}", - "{{parentSpanId}}" - ], - "query": [ - { - "key": "stageName", - "value": "dex-upload" - } - ] - } - }, - "response": [] - }, - { - "name": "Stop upload span", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "PUT", - "header": [], - "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/trace/stopSpan/{{traceId}}/{{spanId}}", - "host": [ - "{{PROCESSING_STATUS_BASE_URL}}" - ], - "path": [ - "api", - "trace", - "stopSpan", - "{{traceId}}", - "{{spanId}}" - ] - } - }, - "response": [] - }, - { - "name": "Start routing span", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "var jsonData = JSON.parse(responseBody);", - "pm.environment.set(\"spanId\", jsonData.span_id);" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "PUT", - "header": [], - "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/trace/startSpan/{{traceId}}/{{parentSpanId}}?stageName=dex-routing", - "host": [ - "{{PROCESSING_STATUS_BASE_URL}}" - ], - "path": [ - "api", - "trace", - "startSpan", - "{{traceId}}", - "{{parentSpanId}}" - ], - "query": [ - { - "key": "stageName", - "value": "dex-routing" - } - ] - } - }, - "response": [] - }, - { - "name": "Stop routing span", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "PUT", - "header": [], - "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/trace/stopSpan/{{traceId}}/{{spanId}}", - "host": [ - "{{PROCESSING_STATUS_BASE_URL}}" - ], - "path": [ - "api", - "trace", - "stopSpan", - "{{traceId}}", - "{{spanId}}" - ] - } - }, - "response": [] - }, - { - "name": "Get trace (ref = traceId)", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/trace/traceId/{{traceId}}", - "host": [ - "{{PROCESSING_STATUS_BASE_URL}}" - ], - "path": [ - "api", - "trace", - "traceId", - "{{traceId}}" - ] - } - }, - "response": [] - }, - { - "name": "Get trace (ref = uploadId)", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/trace/uploadId/{{uploadId}}", - "host": [ - "{{PROCESSING_STATUS_BASE_URL}}" - ], - "path": [ - "api", - "trace", - "uploadId", - "{{uploadId}}" - ] - } - }, - "response": [] - }, - { - "name": "Get span (ref = uploadId, stageName)", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "" - ], - "type": "text/javascript" - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/trace/span?uploadId={{uploadId}}&stageName=dex-upload", - "host": [ - "{{PROCESSING_STATUS_BASE_URL}}" - ], - "path": [ - "api", - "trace", - "span" - ], - "query": [ - { - "key": "uploadId", - "value": "{{uploadId}}" - }, - { - "key": "stageName", - "value": "dex-upload" - } - ] - } - }, - "response": [] - }, - { - "name": "Get trace directly", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{JAEGER_TRACE_END_POINT}}/api/traces/{{traceId}}", - "host": [ - "{{JAEGER_TRACE_END_POINT}}" - ], - "path": [ - "api", - "traces", - "{{traceId}}" - ] - } - }, - "response": [] - }, - { - "name": "Get list of services", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{JAEGER_TRACE_END_POINT}}/api/services", - "host": [ - "{{JAEGER_TRACE_END_POINT}}" - ], - "path": [ - "api", - "services" - ] - } - }, - "response": [] - }, - { - "name": "Get all \"dex-processing\" traces", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{JAEGER_TRACE_END_POINT}}/api/traces?limit=20000&service=dex-processing-status", - "host": [ - "{{JAEGER_TRACE_END_POINT}}" - ], - "path": [ - "api", - "traces" - ], - "query": [ - { - "key": "limit", - "value": "20000" - }, - { - "key": "service", - "value": "dex-processing-status" - } - ] - } - }, - "response": [] - } - ] - }, { "name": "Report", "item": [ @@ -406,7 +26,8 @@ "", "pm.environment.set(\"uploadId\", uploadId);" ], - "type": "text/javascript" + "type": "text/javascript", + "packages": {} } }, { @@ -416,7 +37,8 @@ "var jsonData = JSON.parse(responseBody);", "pm.environment.set(\"reportId\", jsonData.report_id);" ], - "type": "text/javascript" + "type": "text/javascript", + "packages": {} } } ], @@ -425,15 +47,10 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"schema_version\":\"0.0.1\",\n \"schema_name\":\"dex-metadata-verify\",\n \"filename\":\"some_upload1.csv\",\n \"metadata\":{\n \"meta_field2\":\"value3\"\n },\n \"issues\":[\n \"Missing required metadata field, 'meta_field1'.\",\n \"Metadata field, 'meta_field2' is set to 'value3' and does not contain one of the allowed values: [ 'value1', value2']\"\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } + "raw": "{\n \"schema_version\":\"0.0.1\",\n \"schema_name\":\"dex-metadata-verify\",\n \"filename\":\"some_upload1.csv\",\n \"metadata\":{\n \"meta_field2\":\"value3\"\n },\n \"issues\":[\n \"Missing required metadata field, 'meta_field1'.\",\n \"Metadata field, 'meta_field2' is set to 'value3' and does not contain one of the allowed values: [ 'value1', value2']\"\n ]\n}" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-metadata-verify&destinationId=dex-testing&eventType=test-event1", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-metadata-verify&dataStreamId=dex-testing&dataStreamRoute=test-event1", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -450,11 +67,11 @@ "value": "dex-metadata-verify" }, { - "key": "destinationId", + "key": "dataStreamId", "value": "dex-testing" }, { - "key": "eventType", + "key": "dataStreamRoute", "value": "test-event1" } ] @@ -475,7 +92,8 @@ "", "pm.environment.set(\"uploadId\", uploadId);" ], - "type": "text/javascript" + "type": "text/javascript", + "packages": {} } }, { @@ -485,7 +103,8 @@ "var jsonData = JSON.parse(responseBody);", "pm.environment.set(\"reportId\", jsonData.report_id);" ], - "type": "text/javascript" + "type": "text/javascript", + "packages": {} } } ], @@ -494,15 +113,10 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"schema_version\": \"0.0.1\",\n \"schema_name\": \"dex-metadata-verify\",\n \"filename\": \"some_upload1.csv\",\n \"metadata\":{\n \"meta_field1\":\"value1\"\n },\n \"issues\": null\n}", - "options": { - "raw": { - "language": "json" - } - } + "raw": "{\n \"schema_version\": \"0.0.1\",\n \"schema_name\": \"dex-metadata-verify\",\n \"filename\": \"some_upload1.csv\",\n \"metadata\":{\n \"meta_field1\":\"value1\"\n },\n \"issues\": null\n}" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-metadata-verify&destinationId=dex-testing&eventType=test-event1", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-metadata-verify&dataStreamId=dex-testing&dataStreamRoute=test-event1", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -519,11 +133,11 @@ "value": "dex-metadata-verify" }, { - "key": "destinationId", + "key": "dataStreamId", "value": "dex-testing" }, { - "key": "eventType", + "key": "dataStreamRoute", "value": "test-event1" } ] @@ -540,7 +154,8 @@ "exec": [ "" ], - "type": "text/javascript" + "type": "text/javascript", + "packages": {} } }, { @@ -550,7 +165,8 @@ "var jsonData = JSON.parse(responseBody);", "pm.environment.set(\"reportId\", jsonData.report_id);" ], - "type": "text/javascript" + "type": "text/javascript", + "packages": {} } } ], @@ -559,15 +175,10 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"schema_name\": \"upload\",\n \"schema_version\": \"1.0\",\n \"tguid\": \"{{uploadId}}\",\n \"offset\": 0,\n \"size\": 27472691,\n \"filename\": \"some_upload1.csv\",\n \"meta_destination_id\": \"ndlp\",\n \"meta_ext_event\": \"routineImmunization\",\n \"end_time_epoch_millis\": 1700009141546,\n \"start_time_epoch_millis\": 1700009137234,\n \"metadata\": {\n \"filename\": \"10MB-test-file\",\n \"filetype\": \"text/plain\",\n \"meta_destination_id\": \"ndlp\",\n \"meta_ext_event\": \"routineImmunization\",\n \"meta_ext_source\": \"IZGW\",\n \"meta_ext_sourceversion\": \"V2022-12-31\",\n \"meta_ext_entity\": \"DD2\",\n \"meta_username\": \"ygj6@cdc.gov\",\n \"meta_ext_objectkey\": \"2b18d70c-8559-11ee-b9d1-0242ac120002\",\n \"meta_ext_filename\": \"10MB-test-file\",\n \"meta_ext_submissionperiod\": \"1\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } + "raw": "{\n \"schema_name\": \"upload\",\n \"schema_version\": \"1.0\",\n \"tguid\": \"{{uploadId}}\",\n \"offset\": 0,\n \"size\": 27472691,\n \"filename\": \"some_upload1.csv\",\n \"meta_destination_id\": \"ndlp\",\n \"meta_ext_event\": \"routineImmunization\",\n \"end_time_epoch_millis\": 1700009141546,\n \"start_time_epoch_millis\": 1700009137234,\n \"metadata\": {\n \"filename\": \"10MB-test-file\",\n \"filetype\": \"text/plain\",\n \"meta_destination_id\": \"ndlp\",\n \"meta_ext_event\": \"routineImmunization\",\n \"meta_ext_source\": \"IZGW\",\n \"meta_ext_sourceversion\": \"V2022-12-31\",\n \"meta_ext_entity\": \"DD2\",\n \"meta_username\": \"ygj6@cdc.gov\",\n \"meta_ext_objectkey\": \"2b18d70c-8559-11ee-b9d1-0242ac120002\",\n \"meta_ext_filename\": \"10MB-test-file\",\n \"meta_ext_submissionperiod\": \"1\"\n }\n}" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-upload&destinationId=dex-testing&eventType=test-event1", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-upload&dataStreamId=dex-testing&dataStreamRoute=test-event1", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -584,11 +195,11 @@ "value": "dex-upload" }, { - "key": "destinationId", + "key": "dataStreamId", "value": "dex-testing" }, { - "key": "eventType", + "key": "dataStreamRoute", "value": "test-event1" } ] @@ -606,7 +217,8 @@ "var jsonData = JSON.parse(responseBody);", "pm.environment.set(\"reportId\", jsonData.report_id);" ], - "type": "text/javascript" + "type": "text/javascript", + "packages": {} } } ], @@ -615,15 +227,10 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"schema_name\": \"upload\",\n \"schema_version\": \"1.0\",\n \"tguid\": \"{{uploadId}}\",\n \"offset\": 27472691,\n \"size\": 27472691,\n \"filename\": \"some_upload1.csv\",\n \"meta_destination_id\": \"ndlp\",\n \"meta_ext_event\": \"routineImmunization\",\n \"end_time_epoch_millis\": 1700009141546,\n \"start_time_epoch_millis\": 1700009137234,\n \"metadata\": {\n \"filename\": \"10MB-test-file\",\n \"filetype\": \"text/plain\",\n \"meta_destination_id\": \"ndlp\",\n \"meta_ext_event\": \"routineImmunization\",\n \"meta_ext_source\": \"IZGW\",\n \"meta_ext_sourceversion\": \"V2022-12-31\",\n \"meta_ext_entity\": \"DD2\",\n \"meta_username\": \"ygj6@cdc.gov\",\n \"meta_ext_objectkey\": \"2b18d70c-8559-11ee-b9d1-0242ac120002\",\n \"meta_ext_filename\": \"10MB-test-file\",\n \"meta_ext_submissionperiod\": \"1\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } + "raw": "{\n \"schema_name\": \"upload\",\n \"schema_version\": \"1.0\",\n \"tguid\": \"{{uploadId}}\",\n \"offset\": 27472691,\n \"size\": 27472691,\n \"filename\": \"some_upload1.csv\",\n \"meta_destination_id\": \"ndlp\",\n \"meta_ext_event\": \"routineImmunization\",\n \"end_time_epoch_millis\": 1700009141546,\n \"start_time_epoch_millis\": 1700009137234,\n \"metadata\": {\n \"filename\": \"10MB-test-file\",\n \"filetype\": \"text/plain\",\n \"meta_destination_id\": \"ndlp\",\n \"meta_ext_event\": \"routineImmunization\",\n \"meta_ext_source\": \"IZGW\",\n \"meta_ext_sourceversion\": \"V2022-12-31\",\n \"meta_ext_entity\": \"DD2\",\n \"meta_username\": \"ygj6@cdc.gov\",\n \"meta_ext_objectkey\": \"2b18d70c-8559-11ee-b9d1-0242ac120002\",\n \"meta_ext_filename\": \"10MB-test-file\",\n \"meta_ext_submissionperiod\": \"1\"\n }\n}" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-upload&destinationId=dex-testing&eventType=test-event1", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-upload&dataStreamId=dex-testing&dataStreamRoute=test-event1", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -640,11 +247,11 @@ "value": "dex-upload" }, { - "key": "destinationId", + "key": "dataStreamId", "value": "dex-testing" }, { - "key": "eventType", + "key": "dataStreamRoute", "value": "test-event1" } ] @@ -662,7 +269,8 @@ "var jsonData = JSON.parse(responseBody);", "pm.environment.set(\"reportId\", jsonData.report_id);" ], - "type": "text/javascript" + "type": "text/javascript", + "packages": {} } } ], @@ -671,15 +279,10 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"schema_name\": \"dex-file-copy\",\n \"schema_version\": \"1.0\",\n \"file_source_blob_url\": \"../csv_file1.csv\",\n \"file_destination_blob_url\": \".../edav/csv_file1_87847487844.csv\",\n \"timestamp\": \"\",\n \"result\": \"success|failed\"\n}", - "options": { - "raw": { - "language": "json" - } - } + "raw": "{\n \"schema_name\": \"dex-file-copy\",\n \"schema_version\": \"1.0\",\n \"file_source_blob_url\": \"../csv_file1.csv\",\n \"file_destination_blob_url\": \".../edav/csv_file1_87847487844.csv\",\n \"timestamp\": \"\",\n \"result\": \"success|failed\"\n}" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-routing&destinationId=dex-testing&eventType=test-event1", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-routing&dataStreamId=dex-testing&dataStreamRoute=test-event1", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -696,11 +299,11 @@ "value": "dex-routing" }, { - "key": "destinationId", + "key": "dataStreamId", "value": "dex-testing" }, { - "key": "eventType", + "key": "dataStreamRoute", "value": "test-event1" } ] @@ -718,7 +321,8 @@ "var jsonData = JSON.parse(responseBody);", "pm.environment.set(\"reportId\", jsonData.report_id);" ], - "type": "text/javascript" + "type": "text/javascript", + "packages": {} } } ], @@ -727,15 +331,10 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"file_name\": \"postman-dtx022-19fcf009-3c83-48d0-be62-d33c2c68c45e.txt\",\n \"file_uuid\": \"{{$randomUUID}}\",\n \"routing_metadata\": {\n \"upload_id\": \"{{uploadId}}\"\n },\n \"message_batch\": \"SINGLE\",\n \"number_of_messages\": 100,\n \"number_of_messages_not_propagated\": 0,\n \"error_messages\": [],\n \"metadata\": {\n \"processes\": [\n {\n \"process_name\": \"RECEIVER\"\n }\n ]\n },\n \"schema_version\": \"0.0.1\",\n \"schema_name\": \"hl7_debatch_report\"\n}", - "options": { - "raw": { - "language": "json" - } - } + "raw": "{\n \"file_name\": \"postman-dtx022-19fcf009-3c83-48d0-be62-d33c2c68c45e.txt\",\n \"file_uuid\": \"{{$randomUUID}}\",\n \"routing_metadata\": {\n \"upload_id\": \"{{uploadId}}\"\n },\n \"message_batch\": \"SINGLE\",\n \"number_of_messages\": 100,\n \"number_of_messages_not_propagated\": 0,\n \"error_messages\": [],\n \"metadata\": {\n \"processes\": [\n {\n \"process_name\": \"RECEIVER\"\n }\n ]\n },\n \"schema_version\": \"0.0.1\",\n \"schema_name\": \"hl7_debatch_report\"\n}" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=receiver&destinationId=dex-testing&eventType=test-event1", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=receiver&dataStreamId=dex-testing&dataStreamRoute=test-event1", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -752,11 +351,11 @@ "value": "receiver" }, { - "key": "destinationId", + "key": "dataStreamId", "value": "dex-testing" }, { - "key": "eventType", + "key": "dataStreamRoute", "value": "test-event1" } ] @@ -774,7 +373,8 @@ "var jsonData = JSON.parse(responseBody);", "pm.environment.set(\"reportId\", jsonData.report_id);" ], - "type": "text/javascript" + "type": "text/javascript", + "packages": {} } } ], @@ -783,15 +383,10 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"c5d9a1dd-f8e5-44fe-be7c-30cda29f6833\",\n \"message_uuid\": \"c5d9a1dd-f8e5-44fe-be7c-30cda29f6833\",\n \"message_info\": {\n \"event_code\": \"10110\",\n \"route\": \"hepatitis\",\n \"reporting_jurisdiction\": \"01\",\n \"type\": \"CASE\",\n \"local_record_id\": \"CAS13728088AL01\"\n },\n \"routing_metadata\": {\n \"upload_id\": \"UNKNOWN\"\n },\n \"metadata\": {\n \"provenance\": {\n \"event_id\": \"876b0c8b-601e-006d-38c5-5d346d06d59d\",\n \"event_timestamp\": \"2024-02-12T15:06:05.3884264Z\",\n \"file_uuid\": \"c933b793-65c5-4a5b-b5a1-ae15a8c02eba\",\n \"file_path\": \"https://ocioedemessagesatst.blob.core.windows.net/hl7ingress/postman-dtx022-e4a79d2c-6654-420b-8d79-e4f2045fae0a.txt\",\n \"file_timestamp\": \"2024-02-12T15:06:05+00:00\",\n \"file_size\": 5513,\n \"single_or_batch\": \"SINGLE\",\n \"message_hash\": \"7e080a0ef4a2b0b7a1240304602b6197\",\n \"ext_system_provider\": \"POSTMAN\",\n \"ext_original_file_name\": \"postman-dtx022-e4a79d2c-6654-420b-8d79-e4f2045fae0a.txt\",\n \"message_index\": 1,\n \"ext_original_file_timestamp\": \"dtx0-test\",\n \"source_metadata\": {\n \"orginal_file_name\": \"test-da2e4021-4b56-48d0-add1-d3ad9f3d1c53.txt\"\n }\n },\n \"processes\": [\n {\n \"status\": \"SUCCESS\",\n \"process_name\": \"RECEIVER\",\n \"process_version\": \"1.0.0\",\n \"eventhub_queued_time\": \"2024-02-12T15:06:06.579\",\n \"eventhub_offset\": 240518168576,\n \"eventhub_sequence_number\": 115413,\n \"configs\": [],\n \"start_processing_time\": \"2024-02-12T15:06:07.435+00:00\",\n \"end_processing_time\": \"2024-02-12T15:06:07.53+00:00\"\n },\n {\n \"status\": \"SUCCESS\",\n \"report\": {\n \"entries\": [],\n \"status\": \"SUCCESS\"\n },\n \"process_name\": \"REDACTOR\",\n \"process_version\": \"1.0.0\",\n \"eventhub_queued_time\": \"2024-02-12T15:06:08.595\",\n \"eventhub_offset\": 124554051584,\n \"eventhub_sequence_number\": 5658,\n \"configs\": [\n \"case_config.txt\"\n ],\n \"start_processing_time\": \"2024-02-12T15:06:09.46+00:00\",\n \"end_processing_time\": \"2024-02-12T15:06:09.479+00:00\"\n },\n {\n \"status\": \"SUCCESS\",\n \"report\": {\n \"entries\": {\n \"structure\": [],\n \"content\": [],\n \"value-set\": []\n },\n \"error-count\": {\n \"structure\": 0,\n \"value-set\": 0,\n \"content\": 0\n },\n \"warning-count\": {\n \"structure\": 0,\n \"value-set\": 0,\n \"content\": 0\n },\n \"status\": \"VALID_MESSAGE\"\n },\n \"process_name\": \"STRUCTURE-VALIDATOR\",\n \"process_version\": \"1.0.0\",\n \"eventhub_queued_time\": \"2024-02-12T15:06:10.47\",\n \"eventhub_offset\": 485331304448,\n \"eventhub_sequence_number\": 188792,\n \"configs\": [\n \"NOTF_ORU_V3.0\"\n ],\n \"start_processing_time\": \"2024-02-12T15:06:11.454+00:00\",\n \"end_processing_time\": \"2024-02-12T15:06:11.475+00:00\"\n }\n ]\n },\n \"summary\": {\n \"current_status\": \"VALID_MESSAGE\"\n },\n \"schema_version\": \"0.0.1\",\n \"schema_name\": \"DEX HL7v2\"\n}", - "options": { - "raw": { - "language": "json" - } - } + "raw": "{\n \"id\": \"c5d9a1dd-f8e5-44fe-be7c-30cda29f6833\",\n \"message_uuid\": \"c5d9a1dd-f8e5-44fe-be7c-30cda29f6833\",\n \"message_info\": {\n \"event_code\": \"10110\",\n \"route\": \"hepatitis\",\n \"reporting_jurisdiction\": \"01\",\n \"type\": \"CASE\",\n \"local_record_id\": \"CAS13728088AL01\"\n },\n \"routing_metadata\": {\n \"upload_id\": \"UNKNOWN\"\n },\n \"metadata\": {\n \"provenance\": {\n \"event_id\": \"876b0c8b-601e-006d-38c5-5d346d06d59d\",\n \"event_timestamp\": \"2024-02-12T15:06:05.3884264Z\",\n \"file_uuid\": \"c933b793-65c5-4a5b-b5a1-ae15a8c02eba\",\n \"file_path\": \"https://ocioedemessagesatst.blob.core.windows.net/hl7ingress/postman-dtx022-e4a79d2c-6654-420b-8d79-e4f2045fae0a.txt\",\n \"file_timestamp\": \"2024-02-12T15:06:05+00:00\",\n \"file_size\": 5513,\n \"single_or_batch\": \"SINGLE\",\n \"message_hash\": \"7e080a0ef4a2b0b7a1240304602b6197\",\n \"ext_system_provider\": \"POSTMAN\",\n \"ext_original_file_name\": \"postman-dtx022-e4a79d2c-6654-420b-8d79-e4f2045fae0a.txt\",\n \"message_index\": 1,\n \"ext_original_file_timestamp\": \"dtx0-test\",\n \"source_metadata\": {\n \"orginal_file_name\": \"test-da2e4021-4b56-48d0-add1-d3ad9f3d1c53.txt\"\n }\n },\n \"processes\": [\n {\n \"status\": \"SUCCESS\",\n \"process_name\": \"RECEIVER\",\n \"process_version\": \"1.0.0\",\n \"eventhub_queued_time\": \"2024-02-12T15:06:06.579\",\n \"eventhub_offset\": 240518168576,\n \"eventhub_sequence_number\": 115413,\n \"configs\": [],\n \"start_processing_time\": \"2024-02-12T15:06:07.435+00:00\",\n \"end_processing_time\": \"2024-02-12T15:06:07.53+00:00\"\n },\n {\n \"status\": \"SUCCESS\",\n \"report\": {\n \"entries\": [],\n \"status\": \"SUCCESS\"\n },\n \"process_name\": \"REDACTOR\",\n \"process_version\": \"1.0.0\",\n \"eventhub_queued_time\": \"2024-02-12T15:06:08.595\",\n \"eventhub_offset\": 124554051584,\n \"eventhub_sequence_number\": 5658,\n \"configs\": [\n \"case_config.txt\"\n ],\n \"start_processing_time\": \"2024-02-12T15:06:09.46+00:00\",\n \"end_processing_time\": \"2024-02-12T15:06:09.479+00:00\"\n },\n {\n \"status\": \"SUCCESS\",\n \"report\": {\n \"entries\": {\n \"structure\": [],\n \"content\": [],\n \"value-set\": []\n },\n \"error-count\": {\n \"structure\": 0,\n \"value-set\": 0,\n \"content\": 0\n },\n \"warning-count\": {\n \"structure\": 0,\n \"value-set\": 0,\n \"content\": 0\n },\n \"status\": \"VALID_MESSAGE\"\n },\n \"process_name\": \"STRUCTURE-VALIDATOR\",\n \"process_version\": \"1.0.0\",\n \"eventhub_queued_time\": \"2024-02-12T15:06:10.47\",\n \"eventhub_offset\": 485331304448,\n \"eventhub_sequence_number\": 188792,\n \"configs\": [\n \"NOTF_ORU_V3.0\"\n ],\n \"start_processing_time\": \"2024-02-12T15:06:11.454+00:00\",\n \"end_processing_time\": \"2024-02-12T15:06:11.475+00:00\"\n }\n ]\n },\n \"summary\": {\n \"current_status\": \"VALID_MESSAGE\"\n },\n \"schema_version\": \"0.0.1\",\n \"schema_name\": \"DEX HL7v2\"\n}" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-hl7-structure&destinationId=dex-testing&eventType=test-event1", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-hl7-structure&dataStreamId=dex-testing&dataStreamRoute=test-event1", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -808,11 +403,11 @@ "value": "dex-hl7-structure" }, { - "key": "destinationId", + "key": "dataStreamId", "value": "dex-testing" }, { - "key": "eventType", + "key": "dataStreamRoute", "value": "test-event1" } ] @@ -830,12 +425,7 @@ "header": [], "body": { "mode": "raw", - "raw": "", - "options": { - "raw": { - "language": "json" - } - } + "raw": "" }, "url": { "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/uploadId/{{uploadId}}", @@ -862,12 +452,7 @@ "header": [], "body": { "mode": "raw", - "raw": "", - "options": { - "raw": { - "language": "json" - } - } + "raw": "" }, "url": { "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/reportId/{{reportId}}", @@ -894,15 +479,10 @@ "header": [], "body": { "mode": "raw", - "raw": "", - "options": { - "raw": { - "language": "json" - } - } + "raw": "" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/uploadId/49f78a80-bf57-4858-bbfb-5b6c7f27ae22", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/uploadId/75ab40c4-b595-4700-b967-7ea00d013f54", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -910,7 +490,7 @@ "api", "report", "uploadId", - "49f78a80-bf57-4858-bbfb-5b6c7f27ae22" + "75ab40c4-b595-4700-b967-7ea00d013f54" ] } }, @@ -926,12 +506,7 @@ "header": [], "body": { "mode": "raw", - "raw": "", - "options": { - "raw": { - "language": "json" - } - } + "raw": "" }, "url": { "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/dex-testing/dex-upload", @@ -946,7 +521,7 @@ ], "query": [ { - "key": "eventType", + "key": "dataStreamRoute", "value": "routineImmunization", "disabled": true } @@ -965,15 +540,10 @@ "header": [], "body": { "mode": "raw", - "raw": "", - "options": { - "raw": { - "language": "json" - } - } + "raw": "" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/dex-testing/stage2?eventType=test-event1", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/dex-testing/stage2?dataStreamRoute=test-event1", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -985,7 +555,7 @@ ], "query": [ { - "key": "eventType", + "key": "dataStreamRoute", "value": "test-event1" } ] @@ -1003,15 +573,10 @@ "header": [], "body": { "mode": "raw", - "raw": "", - "options": { - "raw": { - "language": "json" - } - } + "raw": "" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/dex-testing/dex-hl7-validation?eventType=test-event1", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/dex-testing/dex-hl7-validation?dataStreamRoute=test-event1", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -1023,7 +588,7 @@ ], "query": [ { - "key": "eventType", + "key": "dataStreamRoute", "value": "test-event1" } ] @@ -1043,15 +608,10 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"schema_version\": \"1.0\"\n}", - "options": { - "raw": { - "language": "json" - } - } + "raw": "{\n \"schema_version\": \"1.0\"\n}" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-upload&destinationId=dex-testing&eventType=test-event1", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-upload&dataStreamId=dex-testing&dataStreamRoute=test-event1", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -1068,11 +628,11 @@ "value": "dex-upload" }, { - "key": "destinationId", + "key": "dataStreamId", "value": "dex-testing" }, { - "key": "eventType", + "key": "dataStreamRoute", "value": "test-event1" } ] @@ -1087,15 +647,10 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"schema_name\": \"upload\"\n}", - "options": { - "raw": { - "language": "json" - } - } + "raw": "{\n \"schema_name\": \"upload\"\n}" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-upload&destinationId=dex-testing&eventType=test-event1", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-upload&dataStreamId=dex-testing&dataStreamRoute=test-event1", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -1112,11 +667,11 @@ "value": "dex-upload" }, { - "key": "destinationId", + "key": "dataStreamId", "value": "dex-testing" }, { - "key": "eventType", + "key": "dataStreamRoute", "value": "test-event1" } ] @@ -1131,15 +686,10 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"schema_name\": \"upload\",\n \"schema_version\": \"1.0\"\n}", - "options": { - "raw": { - "language": "json" - } - } + "raw": "{\n \"schema_name\": \"upload\",\n \"schema_version\": \"1.0\"\n}" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-upload&eventType=test-event1", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-upload&dataStreamRoute=test-event1", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -1156,7 +706,7 @@ "value": "dex-upload" }, { - "key": "eventType", + "key": "dataStreamRoute", "value": "test-event1" } ] @@ -1171,15 +721,10 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"schema_name\": \"upload\",\n \"schema_version\": \"1.0\"\n}", - "options": { - "raw": { - "language": "json" - } - } + "raw": "{\n \"schema_name\": \"upload\",\n \"schema_version\": \"1.0\"\n}" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-upload&destinationId=dex-testing", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-upload&dataStreamId=dex-testing", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -1196,7 +741,7 @@ "value": "dex-upload" }, { - "key": "destinationId", + "key": "dataStreamId", "value": "dex-testing" } ] @@ -1211,15 +756,10 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"schema_version\": \"1.0\"\n}", - "options": { - "raw": { - "language": "json" - } - } + "raw": "{\n \"schema_version\": \"1.0\"\n}" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-upload&destinationId=dex-testing&eventType=test-event1", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-upload&dataStreamId=dex-testing&dataStreamRouting=test-event1", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -1236,11 +776,11 @@ "value": "dex-upload" }, { - "key": "destinationId", + "key": "dataStreamId", "value": "dex-testing" }, { - "key": "eventType", + "key": "dataStreamRouting", "value": "test-event1" } ] @@ -1255,15 +795,10 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"schema_name\": \"upload\"\n}", - "options": { - "raw": { - "language": "json" - } - } + "raw": "{\n \"schema_name\": \"upload\"\n}" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-upload&destinationId=dex-testing&eventType=test-event1", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-upload&dataStreamId=dex-testing&dataStreamRoute=test-event1", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -1280,11 +815,11 @@ "value": "dex-upload" }, { - "key": "destinationId", + "key": "dataStreamId", "value": "dex-testing" }, { - "key": "eventType", + "key": "dataStreamRoute", "value": "test-event1" } ] @@ -1299,15 +834,10 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"schema_name\": \"upload\",\n \"schema_version\": \"1.0\"\n}", - "options": { - "raw": { - "language": "json" - } - } + "raw": "{\n \"schema_name\": \"upload\",\n \"schema_version\": \"1.0\"\n}" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-upload&eventType=test-event1", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-upload&dataStreamRoute=test-event1", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -1324,7 +854,7 @@ "value": "dex-upload" }, { - "key": "eventType", + "key": "dataStreamRoute", "value": "test-event1" } ] @@ -1339,15 +869,10 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"schema_name\": \"upload\",\n \"schema_version\": \"1.0\"\n}", - "options": { - "raw": { - "language": "json" - } - } + "raw": "{\n \"schema_name\": \"upload\",\n \"schema_version\": \"1.0\"\n}" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-upload&destinationId=dex-testing", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/json/uploadId/{{uploadId}}?stageName=dex-upload&dataStreamRoute=dex-testing", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -1364,7 +889,7 @@ "value": "dex-upload" }, { - "key": "destinationId", + "key": "dataStreamRoute", "value": "dex-testing" } ] @@ -1400,12 +925,7 @@ "header": [], "body": { "mode": "raw", - "raw": "", - "options": { - "raw": { - "language": "json" - } - } + "raw": "" }, "url": { "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/status/{{uploadId}}", @@ -1442,12 +962,7 @@ "header": [], "body": { "mode": "raw", - "raw": "", - "options": { - "raw": { - "language": "json" - } - } + "raw": "" }, "url": { "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/counts/{{uploadId}}", @@ -1473,7 +988,8 @@ "exec": [ "" ], - "type": "text/javascript" + "type": "text/javascript", + "packages": {} } } ], @@ -1485,15 +1001,10 @@ "header": [], "body": { "mode": "raw", - "raw": "", - "options": { - "raw": { - "language": "json" - } - } + "raw": "" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/counts/9de71b83-54ed-4231-a426-ff5f208d75ff", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/counts/75ab40c4-b595-4700-b967-7ea00d013f54", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -1501,7 +1012,7 @@ "api", "report", "counts", - "9de71b83-54ed-4231-a426-ff5f208d75ff" + "75ab40c4-b595-4700-b967-7ea00d013f54" ] } }, @@ -1516,7 +1027,8 @@ "exec": [ "" ], - "type": "text/javascript" + "type": "text/javascript", + "packages": {} } } ], @@ -1528,15 +1040,10 @@ "header": [], "body": { "mode": "raw", - "raw": "", - "options": { - "raw": { - "language": "json" - } - } + "raw": "" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/counts?destination_id=dex-testing&ext_event=test-event1&date_start=20240207T190000Z", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/report/counts?data_stream_id=dex-testing&data_stream_route=test-event1&date_start=20240207T190000Z", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -1547,11 +1054,11 @@ ], "query": [ { - "key": "destination_id", + "key": "data_stream_id", "value": "dex-testing" }, { - "key": "ext_event", + "key": "data_stream_route", "value": "test-event1" }, { @@ -1572,7 +1079,8 @@ "exec": [ "" ], - "type": "text/javascript" + "type": "text/javascript", + "packages": {} } } ], @@ -1584,22 +1092,17 @@ "header": [], "body": { "mode": "raw", - "raw": "", - "options": { - "raw": { - "language": "json" - } - } + "raw": "" }, "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/status/9de71b83-54ed-4231-a426-ff5f208d75ff", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/status/75ab40c4-b595-4700-b967-7ea00d013f54", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], "path": [ "api", "status", - "9de71b83-54ed-4231-a426-ff5f208d75ff" + "75ab40c4-b595-4700-b967-7ea00d013f54" ] } }, @@ -1611,7 +1114,7 @@ "method": "GET", "header": [], "url": { - "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/upload/dex-testing?page_number=1&page_size=20&sort_by=date&sort_order=descending&ext_event=test-event1&date_start=20240207T190000Z", + "raw": "{{PROCESSING_STATUS_BASE_URL}}/api/upload/dex-testing?page_number=1&page_size=20&sort_by=date&sort_order=descending&ext_event=test-event1&date_start=20240701T190000Z", "host": [ "{{PROCESSING_STATUS_BASE_URL}}" ], @@ -1643,7 +1146,7 @@ }, { "key": "date_start", - "value": "20240207T190000Z" + "value": "20240701T190000Z" } ] } @@ -1669,6 +1172,17 @@ } }, "response": [] + }, + { + "name": "Processing Status API", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "" + } + }, + "response": [] } ] } \ No newline at end of file diff --git a/processing-status-api-function-app/test/scripts/load-testing/pstatus_load_test.py b/processing-status-api-function-app/test/scripts/load-testing/pstatus_load_test.py index 470aebaa..4f810db0 100644 --- a/processing-status-api-function-app/test/scripts/load-testing/pstatus_load_test.py +++ b/processing-status-api-function-app/test/scripts/load-testing/pstatus_load_test.py @@ -32,16 +32,6 @@ async def send_single_message(sender, message): message = ServiceBusMessage(message) await sender.send_messages(message) -def send_start_trace(trace_id, parent_span_id, stage_name): - url = f'{pstatus_base_url}/api/trace/startSpan/{trace_id}/{parent_span_id}?stageName={stage_name}' - r = requests.put(url) - json_response = r.json() - return json_response['span_id'] - -def send_stop_trace(trace_id, span_id): - url = f'{pstatus_base_url}/api/trace/stopSpan/{trace_id}/{span_id}' - requests.put(url) - async def run(): # Generate a unqiue upload ID upload_id = str(uuid.uuid4()) @@ -56,16 +46,6 @@ async def run(): # Get a Queue Sender object to send messages to the queue sender = servicebus_client.get_queue_sender(queue_name=QUEUE_NAME) async with sender: - # Create the trace for this upload - url = f'{pstatus_base_url}/api/trace?uploadId={upload_id}&destinationId=dex-testing&eventType=test-event1' - response = requests.post(url) - response_json = response.json() - trace_id = response_json["trace_id"] - parent_span_id = response_json["span_id"] - - # Send the start trace - span_id = send_start_trace(trace_id, parent_span_id, "dex-upload") - # Send upload messages print("Sending simulated UPLOAD reports...") num_chunks = 1 @@ -76,22 +56,13 @@ async def run(): #print(f"Sending: {message}") await send_single_message(sender, message) time.sleep(1) - # Send the stop trace - send_stop_trace(trace_id, span_id) - - # Send the start trace - span_id = send_start_trace(trace_id, parent_span_id, "dex-routing") # Send routing message print("Sending simulated ROUTING report...") message = reports.create_routing(upload_id) #print(f"Sending: {message}") await send_single_message(sender, message) - # Send the stop trace - send_stop_trace(trace_id, span_id) - # Send the start trace - span_id = send_start_trace(trace_id, parent_span_id, "dex-hl7-validation") # Send hl7 validation messages print("Sending simulated HL7-VALIDATION reports...") batch_message = await sender.create_message_batch() @@ -101,9 +72,6 @@ async def run(): message = reports.create_hl7_validation(upload_id, line) #print(f"Sending: {message}") batch_message.add_message(ServiceBusMessage(message)) - # Send the stop trace - await sender.send_messages(batch_message) - send_stop_trace(trace_id, span_id) asyncio.run(run()) print("Done sending messages") \ No newline at end of file diff --git a/processing-status-api-function-app/test/scripts/race-conditions/pstatus_span_race_condition_test.py b/processing-status-api-function-app/test/scripts/race-conditions/pstatus_span_race_condition_test.py deleted file mode 100644 index 9505b3f5..00000000 --- a/processing-status-api-function-app/test/scripts/race-conditions/pstatus_span_race_condition_test.py +++ /dev/null @@ -1,68 +0,0 @@ -import asyncio -import uuid -import requests -import time -import json - -############################################################################################ -# -- INSTRUCTIONS -- -# 1. Open the file named ".env" in the same folder as this Python script. -# 2. In the .env file, set the variables according to your environment and desired settings. -############################################################################################ - -env = {} -with open("../.env") as envfile: - for line in envfile: - name, var = line.partition("=")[::2] - var = var.strip() - if var.startswith('"') and var.endswith('"'): - var = var[1:-1] - env[name.strip()] = var - -pstatus_base_url=env["pstatus_api_base_url"] - -def send_start_trace(trace_id, parent_span_id, stage_name): - url = f'{pstatus_base_url}/api/trace/startSpan/{trace_id}/{parent_span_id}?stageName={stage_name}' - r = requests.put(url) - json_response = r.json() - return json_response['span_id'] - -def send_stop_trace(trace_id, span_id): - url = f'{pstatus_base_url}/api/trace/stopSpan/{trace_id}/{span_id}' - requests.put(url) - -async def run(): - # Generate a unqiue upload ID - upload_id = str(uuid.uuid4()) - print("Upload ID = " + upload_id) - - # Create the trace for this upload - url = f'{pstatus_base_url}/api/trace?uploadId={upload_id}&destinationId=dex-testing&eventType=test-event1' - response = requests.post(url) - response_json = response.json() - trace_id = response_json["trace_id"] - parent_span_id = response_json["span_id"] - - # Send the start trace - span_id = send_start_trace(trace_id, parent_span_id, "dex-upload") - - # Immediately try to get the span we just created - print('Attempting to get the span just created') - start_time = time.time() - - url = f'{pstatus_base_url}/api/trace/span?uploadId={upload_id}&stageName=dex-upload' - r = requests.get(url) - print(f'status code = {r.status_code}') - if r.status_code != 200: - print(f'bad response = {r.content}') - exit(1) - json_object = json.loads(r.content) - json_formatted_str = json.dumps(json_object, indent=2) - print(f'JSON response: {json_formatted_str}') - print('Waited %s seconds' % (time.time() - start_time)) - - # Send the stop trace - send_stop_trace(trace_id, span_id) - -asyncio.run(run()) -print("Done sending messages") \ No newline at end of file diff --git a/processing-status-api-function-app/test/scripts/race-conditions/pstatus_trace_race_condition_test.py b/processing-status-api-function-app/test/scripts/race-conditions/pstatus_trace_race_condition_test.py deleted file mode 100644 index 90d78e78..00000000 --- a/processing-status-api-function-app/test/scripts/race-conditions/pstatus_trace_race_condition_test.py +++ /dev/null @@ -1,76 +0,0 @@ -import asyncio -import uuid -import requests -import time -import json - -############################################################################################ -# -- INSTRUCTIONS -- -# 1. Open the file named ".env" in the same folder as this Python script. -# 2. In the .env file, set the variables according to your environment and desired settings. -############################################################################################ - -env = {} -with open("../.env") as envfile: - for line in envfile: - name, var = line.partition("=")[::2] - var = var.strip() - if var.startswith('"') and var.endswith('"'): - var = var[1:-1] - env[name.strip()] = var - -pstatus_base_url=env["pstatus_api_base_url"] - -async def run(): - ### Test 1 - print('TEST 1: Attempting to a trace just created using trace_id') - - # Generate a unqiue upload ID - upload_id = str(uuid.uuid4()) - print("Upload ID = " + upload_id) - - # Create the trace for this upload - url = f'{pstatus_base_url}/api/trace?uploadId={upload_id}&destinationId=dex-testing&eventType=test-event1' - response = requests.post(url) - response_json = response.json() - trace_id = response_json["trace_id"] - print(f'trace_id = {trace_id}') - - # Immediately try to get the trace we just created - start_time = time.time() - url = f'{pstatus_base_url}/api/trace/traceId/{trace_id}' - r = requests.get(url) - print(f'status code = {r.status_code}') - if r.status_code != 200: - print(f'FAILED: bad response = {r.content}') - else: - json_object = json.loads(r.content) - json_formatted_str = json.dumps(json_object, indent=2) - print(f'SUCCESS: JSON response: {json_formatted_str}') - print('Waited %s seconds' % (time.time() - start_time)) - - ### Test 2 - print('TEST 2: Attempting to get a trace just created using upload_id') - - # Generate a unqiue upload ID - upload_id = str(uuid.uuid4()) - print("Upload ID = " + upload_id) - - # Create the trace for this upload - url = f'{pstatus_base_url}/api/trace?uploadId={upload_id}&destinationId=dex-testing&eventType=test-event1' - requests.post(url) - - # Immediately try to get the trace we just created - start_time = time.time() - url = f'{pstatus_base_url}/api/trace/uploadId/{upload_id}' - r = requests.get(url) - print(f'status code = {r.status_code}') - if r.status_code != 200: - print(f'FAILED: bad response = {r.content}') - else: - json_object = json.loads(r.content) - json_formatted_str = json.dumps(json_object, indent=2) - print(f'SUCCESS: JSON response: {json_formatted_str}') - print('Waited %s seconds' % (time.time() - start_time)) - -asyncio.run(run()) \ No newline at end of file diff --git a/pstatus-graphql-ktor/build.gradle b/pstatus-graphql-ktor/build.gradle index 0baa39e7..4ee5d88a 100644 --- a/pstatus-graphql-ktor/build.gradle +++ b/pstatus-graphql-ktor/build.gradle @@ -9,6 +9,8 @@ buildscript { plugins { id 'org.jetbrains.kotlin.jvm' version '1.9.24' id 'io.ktor.plugin' version '2.3.11' + id "org.jetbrains.kotlin.plugin.serialization" version "1.8.20" +// id ("kotlinx-serialization") } apply plugin: "java" apply plugin: "kotlin" @@ -33,6 +35,10 @@ configurations { agent } +repositories{ + mavenLocal() + mavenCentral() +} dependencies { implementation "io.ktor:ktor-server-core-jvm:$ktor_version" implementation "io.ktor:ktor-server-netty-jvm:$ktor_version" @@ -40,7 +46,15 @@ dependencies { implementation "io.ktor:ktor-server-status-pages:$ktor_version" implementation "io.ktor:ktor-server-auth:$ktor_version" implementation "io.ktor:ktor-server-auth-jwt:$ktor_version" + implementation "io.ktor:ktor-server-content-negotiation:$ktor_version" + implementation "io.ktor:ktor-serialization-jackson:$ktor_version" implementation "ch.qos.logback:logback-classic:$logback_version" + implementation("io.ktor:ktor-server-content-negotiation:2.1.0") + implementation("io.ktor:ktor-serialization-kotlinx-json:2.1.0") + implementation("io.ktor:ktor-client-core:2.1.0") + implementation("io.ktor:ktor-client-cio:2.1.0") + implementation("io.ktor:ktor-client-content-negotiation:2.1.0") + implementation("io.ktor:ktor-client-serialization:2.1.0") implementation 'com.expediagroup:graphql-kotlin-ktor-server:7.1.1' implementation 'com.azure:azure-cosmos:4.55.0' implementation 'io.github.microutils:kotlin-logging-jvm:3.0.5' @@ -50,6 +64,12 @@ dependencies { implementation 'com.graphql-java:graphql-java-extended-scalars:22.0' implementation 'joda-time:joda-time:2.12.7' implementation 'org.apache.commons:commons-lang3:3.3.1' + implementation "com.expediagroup:graphql-kotlin-server:6.0.0" + implementation "com.expediagroup:graphql-kotlin-schema-generator:6.0.0" + implementation "io.ktor:ktor-server-netty:2.1.0" + implementation "io.ktor:ktor-client-content-negotiation:2.1.0" + implementation "io.ktor:ktor-client-logging:2.1.0" + implementation "org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.0" testImplementation "io.ktor:ktor-server-tests-jvm:$ktor_version" testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlin_version" } @@ -70,6 +90,4 @@ jib { } } -repositories { - mavenCentral() -} + diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/Application.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/Application.kt index 34858d55..6d8ee153 100644 --- a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/Application.kt +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/Application.kt @@ -1,12 +1,17 @@ package gov.cdc.ocio.processingstatusapi +import gov.cdc.ocio.processingstatusapi.cosmos.CosmosConfiguration +import gov.cdc.ocio.processingstatusapi.cosmos.CosmosDeadLetterRepository import gov.cdc.ocio.processingstatusapi.cosmos.CosmosRepository +import gov.cdc.ocio.processingstatusapi.plugins.configureRouting import gov.cdc.ocio.processingstatusapi.plugins.graphQLModule import graphql.scalars.ExtendedScalars import graphql.schema.idl.RuntimeWiring +import io.ktor.serialization.jackson.* import io.ktor.server.application.* import io.ktor.server.engine.* import io.ktor.server.netty.* +import io.ktor.server.plugins.contentnegotiation.* import org.koin.core.KoinApplication import org.koin.dsl.module import org.koin.ktor.plugin.Koin @@ -16,6 +21,10 @@ fun KoinApplication.loadKoinModules(environment: ApplicationEnvironment): KoinAp val uri = environment.config.property("azure.cosmos_db.client.endpoint").getString() val authKey = environment.config.property("azure.cosmos_db.client.key").getString() single(createdAtStart = true) { CosmosRepository(uri, authKey, "Reports", "/uploadId") } + single(createdAtStart = true) { CosmosDeadLetterRepository(uri, authKey, "Reports-DeadLetter", "/uploadId") } + + // Create a CosmosDB config that can be dependency injected (for health checks) + single(createdAtStart = true) { CosmosConfiguration(uri, authKey) } } return modules(listOf(cosmosModule)) } @@ -26,9 +35,13 @@ fun main(args: Array) { fun Application.module() { graphQLModule() + configureRouting() install(Koin) { loadKoinModules(environment) } + install(ContentNegotiation) { + jackson() + } // See https://opensource.expediagroup.com/graphql-kotlin/docs/schema-generator/writing-schemas/scalars RuntimeWiring.newRuntimeWiring().scalar(ExtendedScalars.Date) diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/CustomSchemaGeneratorHooks.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/CustomSchemaGeneratorHooks.kt deleted file mode 100644 index f0aba826..00000000 --- a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/CustomSchemaGeneratorHooks.kt +++ /dev/null @@ -1,35 +0,0 @@ -package gov.cdc.ocio.processingstatusapi - -import com.expediagroup.graphql.generator.hooks.SchemaGeneratorHooks -import graphql.language.StringValue -import graphql.schema.Coercing -import graphql.schema.GraphQLScalarType -import graphql.schema.GraphQLType -import java.util.* -import kotlin.reflect.KClass -import kotlin.reflect.KType - -//class CustomSchemaGeneratorHooks : SchemaGeneratorHooks { -// -// override fun willGenerateGraphQLType(type: KType): GraphQLType? = when (type.classifier as? KClass<*>) { -// Date::class -> graphqlDateType -// else -> null -// } -//} -// -//val graphqlDateType = GraphQLScalarType.newScalar() -// .name("Date") -// .description("A type representing a formatted java.util.UUID") -// .coercing(DateCoercing) -// .build() -// -//object DateCoercing : Coercing { -// override fun parseValue(input: Any?): Date = Date.fromString(serialize(input)) -// -// override fun parseLiteral(input: Any?): UUID? { -// val dateString = (input as? StringValue)?.value -// return Date.fromString(dateString) -// } -// -// override fun serialize(dataFetcherResult: Any?): String = dataFetcherResult.toString() -//} diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosClientManager.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosClientManager.kt index 7690f004..4b3e7ba1 100644 --- a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosClientManager.kt +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosClientManager.kt @@ -3,22 +3,48 @@ package gov.cdc.ocio.processingstatusapi.cosmos import com.azure.cosmos.ConsistencyLevel import com.azure.cosmos.CosmosClient import com.azure.cosmos.CosmosClientBuilder +import kotlinx.coroutines.* +import java.time.Duration class CosmosClientManager { companion object { private var client: CosmosClient? = null + /** + * Establishes a connection to the CosmosDB and returns a client + * + * @param uri String + * @param authKey String + * @return CosmosClient? + */ + @OptIn(DelicateCoroutinesApi::class) fun getCosmosClient(uri: String, authKey: String): CosmosClient? { // Initialize a connection to cosmos that will persist across HTTP triggers if (client == null) { - client = CosmosClientBuilder() - .endpoint(uri) - .key(authKey) - .consistencyLevel(ConsistencyLevel.EVENTUAL) - .contentResponseOnWriteEnabled(true) - .clientTelemetryEnabled(false) - .buildClient() + return try { + var d: Deferred? = null + GlobalScope.launch { + d = async { + CosmosClientBuilder() + .endpoint(uri) + .key(authKey) + .consistencyLevel(ConsistencyLevel.EVENTUAL) + .gatewayMode() + .contentResponseOnWriteEnabled(true) + .clientTelemetryEnabled(false) + .buildClient() + } + } + runBlocking { + withTimeout(Duration.ofSeconds(10).toMillis()) { + client = d?.await() + } // wait with timeout + } + client + } catch (ex: TimeoutCancellationException) { + null + } } return client } diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosConfiguration.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosConfiguration.kt new file mode 100644 index 00000000..e4e62d30 --- /dev/null +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosConfiguration.kt @@ -0,0 +1,10 @@ +package gov.cdc.ocio.processingstatusapi.cosmos + +/** + * CosmosDB client configuration + * + * @property uri String + * @property authKey String + * @constructor + */ +data class CosmosConfiguration(val uri: String, val authKey: String) \ No newline at end of file diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosContainerManager.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosContainerManager.kt index 51b7cd51..e0ba3ea0 100644 --- a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosContainerManager.kt +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosContainerManager.kt @@ -21,26 +21,35 @@ class CosmosContainerManager { return cosmosClient.getDatabase(databaseResponse.properties.id) } + /** + * The function which creates the cosmos container instance + * @param uri String + * @param authKey String + * @param containerName String + * @param partitionKey String + */ fun initDatabaseContainer(uri: String, authKey: String, containerName: String, partitionKey: String): CosmosContainer? { val logger = KotlinLogging.logger {} try { logger.info("calling getCosmosClient...") - val cosmosClient = CosmosClientManager.getCosmosClient(uri, authKey) ?: return null + val cosmosClient = CosmosClientManager.getCosmosClient(uri, authKey) - // setup database - logger.info("calling createDatabaseIfNotExists...") - val db = createDatabaseIfNotExists(cosmosClient, "ProcessingStatus")!! + cosmosClient?.run { + // setup database + logger.info("calling createDatabaseIfNotExists...") + val db = createDatabaseIfNotExists(cosmosClient, "ProcessingStatus") - val containerProperties = CosmosContainerProperties(containerName, partitionKey) + val containerProperties = CosmosContainerProperties(containerName, partitionKey) - // Provision throughput - val throughputProperties = ThroughputProperties.createAutoscaledThroughput(1000) + // Provision throughput + val throughputProperties = ThroughputProperties.createAutoscaledThroughput(1000) - // Create container with 1000 RU/s - logger.info("calling createContainerIfNotExists...") - val databaseResponse = db.createContainerIfNotExists(containerProperties, throughputProperties) + // Create container with 1000 RU/s + logger.info("calling createContainerIfNotExists...") + val databaseResponse = db?.createContainerIfNotExists(containerProperties, throughputProperties) - return db.getContainer(databaseResponse.properties.id) + return db?.getContainer(databaseResponse?.properties?.id) + } } catch (ex: CosmosException) { logger.error("exception: ${ex.localizedMessage}") diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosRepository.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosRepository.kt index 869c7791..52993768 100644 --- a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosRepository.kt +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosRepository.kt @@ -1,9 +1,12 @@ package gov.cdc.ocio.processingstatusapi.cosmos -import org.koin.core.component.KoinComponent - -class CosmosRepository(uri: String, authKey: String, reportsContainerName: String, partitionKey: String): KoinComponent { +class CosmosRepository(uri: String, authKey: String, reportsContainerName: String, partitionKey: String) { val reportsContainer = CosmosContainerManager.initDatabaseContainer(uri, authKey, reportsContainerName, partitionKey) -} \ No newline at end of file +} + +class CosmosDeadLetterRepository(uri: String, authKey: String, reportsContainerName: String, partitionKey: String) { + val reportsDeadLetterContainer = + CosmosContainerManager.initDatabaseContainer(uri, authKey, reportsContainerName, partitionKey) +} diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/dataloaders/ReportDataLoader.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/dataloaders/ReportDataLoader.kt index 4c3a1af7..cb129b1f 100644 --- a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/dataloaders/ReportDataLoader.kt +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/dataloaders/ReportDataLoader.kt @@ -1,20 +1,33 @@ package gov.cdc.ocio.processingstatusapi.dataloaders import com.expediagroup.graphql.dataloader.KotlinDataLoader +import gov.cdc.ocio.processingstatusapi.loaders.ReportDeadLetterLoader import gov.cdc.ocio.processingstatusapi.models.Report import gov.cdc.ocio.processingstatusapi.loaders.ReportLoader +import gov.cdc.ocio.processingstatusapi.models.ReportDeadLetter import kotlinx.coroutines.runBlocking import graphql.GraphQLContext import org.dataloader.DataLoader import org.dataloader.DataLoaderFactory -import java.util.concurrent.CompletableFuture +import java.util.concurrent.CompletableFuture.* val ReportDataLoader = object : KotlinDataLoader { override val dataLoaderName = "REPORTS_LOADER" override fun getDataLoader(graphQLContext: GraphQLContext): DataLoader = DataLoaderFactory.newDataLoader { ids -> - CompletableFuture.supplyAsync { + supplyAsync { runBlocking { ReportLoader().search(ids).toMutableList() } } } } + + +val ReportDeadLetterDataLoader = object : KotlinDataLoader { + override val dataLoaderName = "REPORTS_DEAD_LETTER_LOADER" + override fun getDataLoader(graphQLContext: GraphQLContext): DataLoader = + DataLoaderFactory.newDataLoader { ids -> + supplyAsync { + runBlocking { ReportDeadLetterLoader().search(ids).toMutableList() } + } + } +} \ No newline at end of file diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/CosmosDeadLetterLoader.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/CosmosDeadLetterLoader.kt new file mode 100644 index 00000000..9fd67c18 --- /dev/null +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/CosmosDeadLetterLoader.kt @@ -0,0 +1,15 @@ +package gov.cdc.ocio.processingstatusapi.loaders + +import gov.cdc.ocio.processingstatusapi.cosmos.CosmosDeadLetterRepository +import mu.KotlinLogging +import org.koin.core.component.KoinComponent +import org.koin.core.component.inject + +open class CosmosDeadLetterLoader: KoinComponent { + + private val cosmosRepository by inject() + + protected val reportsDeadLetterContainer = cosmosRepository.reportsDeadLetterContainer + + protected val logger = KotlinLogging.logger {} +} \ No newline at end of file diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/CosmosLoader.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/CosmosLoader.kt index a9c64357..6683f674 100644 --- a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/CosmosLoader.kt +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/CosmosLoader.kt @@ -9,8 +9,6 @@ open class CosmosLoader: KoinComponent { private val cosmosRepository by inject() - protected val reportsContainerName = "Reports" - protected val reportsContainer = cosmosRepository.reportsContainer protected val logger = KotlinLogging.logger {} diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/ReportCountsLoader.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/ReportCountsLoader.kt index a0ae2320..b61e8ebf 100644 --- a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/ReportCountsLoader.kt +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/ReportCountsLoader.kt @@ -28,7 +28,7 @@ class ReportCountsLoader: CosmosLoader() { val reportsSqlQuery = ( "select " + "count(1) as counts, r.stageName, r.content.schema_name, r.content.schema_version " - + "from $reportsContainerName r where r.uploadId = '$uploadId' " + + "from r where r.uploadId = '$uploadId' " + "group by r.stageName, r.content.schema_name, r.content.schema_version" ) val reportItems = reportsContainer?.queryItems( @@ -39,7 +39,7 @@ class ReportCountsLoader: CosmosLoader() { // Order by timestamp (ascending) and grab the first one found, which will give us the earliest timestamp. val firstReportSqlQuery = ( - "select * from $reportsContainerName r where r.uploadId = '$uploadId' " + "select * from r where r.uploadId = '$uploadId' " + "order by r.timestamp asc offset 0 limit 1" ) @@ -92,7 +92,7 @@ class ReportCountsLoader: CosmosLoader() { "select " + "r.uploadId, r.stageName, SUM(r.content.stage.report.number_of_messages) as numberOfMessages, " + "SUM(r.content.stage.report.number_of_messages_not_propagated) as numberOfMessagesNotPropagated " - + "from $reportsContainerName r " + + "from r " + "where r.content.schema_name = '$hl7DebatchSchemaName' and r.uploadId in ($quotedUploadIds) " + "group by r.uploadId, r.stageName" ) @@ -114,7 +114,7 @@ class ReportCountsLoader: CosmosLoader() { + "r.uploadId, r.stageName, " + "count(contains(upper(r.content.summary.current_status), 'VALID_') ? 1 : undefined) as valid, " + "count(not contains(upper(r.content.summary.current_status), 'VALID_') ? 1 : undefined) as invalid " - + "from $reportsContainerName r " + + "from r " + "where r.content.schema_name = '$hl7ValidationSchemaName' and r.uploadId in ($quotedUploadIds) " + "group by r.uploadId, r.stageName" ) @@ -202,7 +202,7 @@ class ReportCountsLoader: CosmosLoader() { uploadIdCountSqlQuery.append( "select " + "value count(1) " - + "from (select distinct r.uploadId from $reportsContainerName r " + + "from (select distinct r.uploadId from r " + "where r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and $timeRangeWhereClause)" ) @@ -224,7 +224,7 @@ class ReportCountsLoader: CosmosLoader() { uploadIdsSqlQuery.append( "select " + "distinct value r.uploadId " - + "from $reportsContainerName r " + + "from r " + "where r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and " + "$timeRangeWhereClause offset $offset limit $pageSizeAsInt" ) @@ -241,7 +241,7 @@ class ReportCountsLoader: CosmosLoader() { val reportsSqlQuery = ( "select " + "r.uploadId, r.content.schema_name, r.content.schema_version, MIN(r.timestamp) as timestamp, count(r.stageName) as counts, r.stageName " - + "from $reportsContainerName r where r.uploadId in ($quotedUploadIds) " + + "from r where r.uploadId in ($quotedUploadIds) " + "group by r.uploadId, r.stageName, r.content.schema_name, r.content.schema_version" ) val reportItems = reportsContainer?.queryItems( @@ -328,7 +328,7 @@ class ReportCountsLoader: CosmosLoader() { val reportsSqlQuery = ( "select " + "value count(not contains(upper(r.content.summary.current_status), 'VALID_') ? 1 : undefined) " - + "from $reportsContainerName r " + + "from r " + "where r.content.schema_name = '${HL7Validation.schemaDefinition.schemaName}' and " + "r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and $timeRangeWhereClause" ) @@ -366,7 +366,7 @@ class ReportCountsLoader: CosmosLoader() { val numCompletedUploadingSqlQuery = ( "select " + "value count(1) " - + "from $reportsContainerName r " + + "from r " + "where r.content.schema_name = 'upload' and r.content['offset'] = r.content.size and " + "r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and $timeRangeWhereClause" ) @@ -380,7 +380,7 @@ class ReportCountsLoader: CosmosLoader() { val numUploadingSqlQuery = ( "select " + "value count(1) " - + "from $reportsContainerName r " + + "from r " + "where r.content.schema_name = 'upload' and r.content['offset'] != r.content.size and " + "r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and $timeRangeWhereClause" ) @@ -394,7 +394,7 @@ class ReportCountsLoader: CosmosLoader() { val numFailedSqlQuery = ( "select " + "value count(1) " - + "from $reportsContainerName r " + + "from r " + "where r.content.schema_name = 'dex-metadata-verify' and r.content.issues != null and " + "r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and $timeRangeWhereClause" ) @@ -435,14 +435,14 @@ class ReportCountsLoader: CosmosLoader() { val directMessageQuery = ( "select value SUM(directCounts) " - + "from (select value SUM(r.content.stage.report.number_of_messages) from $reportsContainerName r " + + "from (select value SUM(r.content.stage.report.number_of_messages) from r " + "where r.content.schema_name = '${HL7Debatch.schemaDefinition.schemaName}' and " + "r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and $timeRangeWhereClause) as directCounts" ) val indirectMessageQuery = ( "select value count(redactedCount) from ( " - + "select * from $reportsContainerName r where r.content.schema_name = '${HL7Redactor.schemaDefinition.schemaName}' and " + + "select * from r where r.content.schema_name = '${HL7Redactor.schemaDefinition.schemaName}' and " + "r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and $timeRangeWhereClause) as redactedCount" ) @@ -488,7 +488,7 @@ class ReportCountsLoader: CosmosLoader() { val directInvalidMessageQuery = ( "select value SUM(directCounts) " - + " FROM (select value SUM(r.content.stage.report.number_of_messages) from $reportsContainerName r " + + " FROM (select value SUM(r.content.stage.report.number_of_messages) from r " + " where r.content.schema_name = '${HL7Redactor.schemaDefinition.schemaName}' and " + " r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and $timeRangeWhereClause) as directCounts" ) @@ -496,14 +496,14 @@ class ReportCountsLoader: CosmosLoader() { val directStructureInvalidMessageQuery = ( "select " + "value count(not contains(upper(r.content.summary.current_status), 'VALID_') ? 1 : undefined) " - + "from $reportsContainerName r " + + "from r " + "where r.content.schema_name = '${HL7Validation.schemaDefinition.schemaName}' and " + "r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and $timeRangeWhereClause" ) val indirectInvalidMessageQuery = ( "select value count(invalidCounts) from (" - + "select * from $reportsContainerName r where r.content.schema_name != 'HL7-JSON-LAKE-TRANSFORMER' and " + + "select * from r where r.content.schema_name != 'HL7-JSON-LAKE-TRANSFORMER' and " + "r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and $timeRangeWhereClause) as invalidCounts" ) @@ -562,7 +562,7 @@ class ReportCountsLoader: CosmosLoader() { val rollupCountsQuery = ( "select " + "r.content.schema_name, r.content.schema_version, count(r.stageName) as counts, r.stageName " - + "from $reportsContainerName r where r.dataStreamId = '$dataStreamId' and " + + "from r where r.dataStreamId = '$dataStreamId' and " + "r.dataStreamRoute = '$dataStreamRoute' and $timeRangeWhereClause " + "group by r.stageName, r.content.schema_name, r.content.schema_version" ) diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/ReportDeadLetterLoader.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/ReportDeadLetterLoader.kt new file mode 100644 index 00000000..f1ffc340 --- /dev/null +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/ReportDeadLetterLoader.kt @@ -0,0 +1,143 @@ +package gov.cdc.ocio.processingstatusapi.loaders + +import com.azure.cosmos.models.CosmosQueryRequestOptions +import gov.cdc.ocio.processingstatusapi.models.ReportDeadLetter +import gov.cdc.ocio.processingstatusapi.models.dao.ReportDeadLetterDao +import gov.cdc.ocio.processingstatusapi.utils.SqlClauseBuilder +import java.text.SimpleDateFormat +import java.util.* +import mu.KotlinLogging + + +/** + * Class for generating reporting queries from cosmos db container which is then wrapped in a graphQl query service + */ +class ReportDeadLetterLoader : CosmosDeadLetterLoader() { + + /** + * Function that returns a list of DeadLetterReports based on uploadId + * + * @param uploadId String + */ + fun getByUploadId(uploadId: String): List { + val reportsSqlQuery = "select * from r where r.uploadId = '$uploadId'" + + val reportItems = reportsDeadLetterContainer?.queryItems( + reportsSqlQuery, CosmosQueryRequestOptions(), + ReportDeadLetterDao::class.java + ) + + val deadLetterReports = mutableListOf() + reportItems?.forEach { deadLetterReports.add(it.toReportDeadLetter()) } + + return deadLetterReports + } + + /** + * Function which returns list of ReportDeadLetter based on the specified parameters + * + * @param dataStreamId String + * @param dataStreamRoute String + * @param startDate String + * @param endDate String + */ + fun getByDataStreamByDateRange( + dataStreamId: String, + dataStreamRoute: String, + startDate: String?, + endDate: String?, + daysInterval: Int? + ): List { + + val formatter = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.getDefault()) + formatter.timeZone = TimeZone.getTimeZone("UTC") // Set time zone if needed + val timeRangeWhereClause = SqlClauseBuilder().buildSqlClauseForDateRange( + daysInterval, + getFormattedDateAsString(startDate), + getFormattedDateAsString(endDate) + ) + + val reportsSqlQuery = "select * from r where r.dataStreamId = '$dataStreamId' " + + "and r.dataStreamRoute= '$dataStreamRoute' " + + "and $timeRangeWhereClause" + + val reportItems = reportsDeadLetterContainer?.queryItems( + reportsSqlQuery, CosmosQueryRequestOptions(), + ReportDeadLetterDao::class.java + ) + + val deadLetterReports = mutableListOf() + reportItems?.forEach { deadLetterReports.add(it.toReportDeadLetter()) } + + return deadLetterReports + } + + /** + * Function which returns count of ReportDeadLetter items based on the specified parameters. + * + * @param dataStreamId String + * @param dataStreamRoute String? + * @param startDate String + * @param endDate String + */ + fun getCountByDataStreamByDateRange( + dataStreamId: String, + dataStreamRoute: String?, + startDate: String?, + endDate: String?, + daysInterval: Int? + ): Int { + + val logger = KotlinLogging.logger {} + val formatter = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.getDefault()) + formatter.timeZone = TimeZone.getTimeZone("UTC") // Set time zone if needed + + val timeRangeWhereClause = SqlClauseBuilder().buildSqlClauseForDateRange(daysInterval, startDate, endDate) + + val reportsSqlQuery = "select value count(1) from r where r.dataStreamId = '$dataStreamId' " + + "and $timeRangeWhereClause " + if (dataStreamRoute != null) " and r.dataStreamRoute= '$dataStreamRoute'" else "" + + val reportItems = reportsDeadLetterContainer?.queryItems( + reportsSqlQuery, CosmosQueryRequestOptions(), + Int::class.java + ) + val count = reportItems?.count() ?: 0 + logger.info("Count of records: $count") + return count + } + + /** + * Search the report deadletters by report id. + * + * @param ids List + * @return List + */ + fun search(ids: List): List { + val quotedIds = ids.joinToString("\",\"", "\"", "\"") + + val reportsSqlQuery = "select * from r where r.id in ($quotedIds)" + + val reportItems = reportsDeadLetterContainer?.queryItems( + reportsSqlQuery, CosmosQueryRequestOptions(), + ReportDeadLetterDao::class.java + ) + val deadLetterReports = mutableListOf() + reportItems?.forEach { deadLetterReports.add(it.toReportDeadLetter()) } + + return deadLetterReports + } + + /** + * Function which converts the inputted date to expected date format + * + * @param inputDate String + */ + private fun getFormattedDateAsString(inputDate: String?): String? { + if (inputDate == null) return null + val inputDateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss") + val outputDateFormat = SimpleDateFormat("yyyyMMdd'T'HHmmss'Z'") + val date: Date = inputDateFormat.parse(inputDate) + val outputDateString = outputDateFormat.format(date) + return outputDateString + } +} \ No newline at end of file diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/ReportLoader.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/ReportLoader.kt index b15452bb..be4f7d94 100644 --- a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/ReportLoader.kt +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/ReportLoader.kt @@ -1,44 +1,93 @@ package gov.cdc.ocio.processingstatusapi.loaders import com.azure.cosmos.models.CosmosQueryRequestOptions +import gov.cdc.ocio.processingstatusapi.exceptions.BadRequestException import gov.cdc.ocio.processingstatusapi.models.Report import gov.cdc.ocio.processingstatusapi.models.dao.ReportDao -import java.time.ZoneOffset +import gov.cdc.ocio.processingstatusapi.models.DataStream +import gov.cdc.ocio.processingstatusapi.models.SortOrder +import graphql.schema.DataFetchingEnvironment +import io.ktor.server.auth.* +import io.ktor.server.auth.jwt.* -class ReportLoader: CosmosLoader() { +class ForbiddenException(message: String) : RuntimeException(message) - fun getAnyReport(): Report? { - val reportsSqlQuery = "select * from $reportsContainerName r offset 0 limit 1" - val reportItems = reportsContainer?.queryItems( - reportsSqlQuery, CosmosQueryRequestOptions(), - ReportDao::class.java - ) - return if (reportItems == null || reportItems.count() == 0) - null - else { - daoToReport(reportItems.first()) +/** + * Report loader for graphql + */ +class ReportLoader: CosmosLoader() { + + /** + * Get all reports associated with the provided upload id. + * + * @param dataFetchingEnvironment DataFetchingEnvironment + * @param uploadId String + * @param reportsSortedBy String? + * @param sortOrder SortOrder? + * @return List + */ + fun getByUploadId(dataFetchingEnvironment: DataFetchingEnvironment, + uploadId: String, + reportsSortedBy: String?, + sortOrder: SortOrder? + ): List { + // Obtain the data streams available to the user from the data fetching env. + val authContext = dataFetchingEnvironment.graphQlContext.get("AuthContext") + var dataStreams: List? = null + authContext?.run { + val principal = authContext.principal() + dataStreams = principal?.payload?.getClaim("dataStreams")?.asList(DataStream::class.java) } - } - fun getByUploadId(uploadId: String): List { - val reportsSqlQuery = "select * from $reportsContainerName r where r.uploadId = '$uploadId'" + val reportsSqlQuery = StringBuilder() + reportsSqlQuery.append("select * from r where r.uploadId = '$uploadId'") + when (reportsSortedBy) { + "timestamp" -> { + val sortOrderVal = when (sortOrder) { + SortOrder.Ascending -> "asc" + SortOrder.Descending -> "desc" + else -> "asc" // default + } + reportsSqlQuery.append(" order by r.timestamp $sortOrderVal") + } + null -> { + // nothing to sort by + } + else -> { + throw BadRequestException("Reports can not be sorted by '$reportsSortedBy'") + } + } val reportItems = reportsContainer?.queryItems( - reportsSqlQuery, CosmosQueryRequestOptions(), + reportsSqlQuery.toString(), CosmosQueryRequestOptions(), ReportDao::class.java ) + // Convert the report DAOs to reports and ensure the user has access to them. val reports = mutableListOf() - reportItems?.forEach { reports.add(daoToReport(it)) } + reportItems?.forEach { reportItem -> + val report = reportItem.toReport() + dataStreams?.run { + if (dataStreams?.firstOrNull { ds -> ds.name == report.dataStreamId && ds.route == report.dataStreamRoute } == null) + throw ForbiddenException("You are not allowed to access this resource.") + } + reports.add(report) + } return reports } + /** + * Search for reports with the provided ids. + * + * @param ids List + * @return List + */ fun search(ids: List): List { val quotedIds = ids.joinToString("\",\"", "\"", "\"") - val reportsSqlQuery = "select * from $reportsContainerName r where r.id in ($quotedIds)" + val reportsSqlQuery = "select * from r where r.id in ($quotedIds)" val reportItems = reportsContainer?.queryItems( reportsSqlQuery, CosmosQueryRequestOptions(), @@ -46,24 +95,8 @@ class ReportLoader: CosmosLoader() { ) val reports = mutableListOf() - reportItems?.forEach { reports.add(daoToReport(it)) } + reportItems?.forEach { reports.add(it.toReport()) } return reports } - - private fun daoToReport(reportDao: ReportDao): Report { - return Report().apply { - this.id = reportDao.id - this.uploadId = reportDao.uploadId - this.reportId = reportDao.reportId - this.dataStreamId = reportDao.dataStreamId - this.dataStreamRoute = reportDao.dataStreamRoute - this.stageName = reportDao.stageName - this.messageId = reportDao.messageId - this.status = reportDao.status - this.timestamp = reportDao.timestamp?.toInstant()?.atOffset(ZoneOffset.UTC) - this.contentType = reportDao.contentType - this.content = reportDao.contentAsType - } - } } diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/UploadStatsLoader.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/UploadStatsLoader.kt index 1b44bd87..d9bdec12 100644 --- a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/UploadStatsLoader.kt +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/UploadStatsLoader.kt @@ -24,14 +24,14 @@ class UploadStatsLoader: CosmosLoader() { ) val numUniqueUploadsQuery = ( - "select r.uploadId from $reportsContainerName r " + "select r.uploadId from r " + "where r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and " + "$timeRangeWhereClause group by r.uploadId" ) val numUploadsWithStatusQuery = ( "select value count(1) " - + "from $reportsContainerName r " + + "from r " + "where r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and " + "r.content.schema_name = 'upload' and " + timeRangeWhereClause @@ -39,7 +39,7 @@ class UploadStatsLoader: CosmosLoader() { val badMetadataCountQuery = ( "select value count(1) " - + "from $reportsContainerName r " + + "from r " + "where r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and " + "r.content.schema_name = 'dex-metadata-verify' and " + "ARRAY_LENGTH(r.content.issues) > 0 and $timeRangeWhereClause" @@ -47,7 +47,7 @@ class UploadStatsLoader: CosmosLoader() { val inProgressUploadsCountQuery = ( "select value count(1) " - + "from $reportsContainerName r " + + "from r " + "where r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = 'dataStreamRoute' and " + "r.content.schema_name = 'upload' and " + "r.content['offset'] < r.content['size'] and $timeRangeWhereClause" @@ -55,7 +55,7 @@ class UploadStatsLoader: CosmosLoader() { val completedUploadsCountQuery = ( "select value count(1) " - + "from $reportsContainerName r " + + "from r " + "where r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and " + "r.content.schema_name = 'upload' and " + "r.content['offset'] = r.content['size'] and $timeRangeWhereClause" @@ -64,7 +64,7 @@ class UploadStatsLoader: CosmosLoader() { val duplicateFilenameCountQuery = ( "select * from " + "(select r.content.metadata.filename, count(1) as totalCount " - + "from $reportsContainerName r " + + "from r " + "where r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = 'dataStreamRoute' and " + "r.content.schema_name = 'dex-metadata-verify' and " + "$timeRangeWhereClause " diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/UploadStatusLoader.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/UploadStatusLoader.kt index d0c25b63..deb78c93 100644 --- a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/UploadStatusLoader.kt +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/UploadStatusLoader.kt @@ -51,25 +51,25 @@ class UploadStatusLoader: CosmosLoader() { val pageSizeAsInt = pageUtils.getPageSize(pageSize) val sqlQuery = StringBuilder() - sqlQuery.append("from $reportsContainerName t where t.dataStreamId = '$dataStreamId'") + sqlQuery.append("from r where r.dataStreamId = '$dataStreamId'") dataStreamRoute?.run { - sqlQuery.append(" and t.dataStreamRoute = '$dataStreamRoute'") + sqlQuery.append(" and r.dataStreamRoute = '$dataStreamRoute'") } dateStart?.run { val dateStartEpochSecs = DateUtils.getEpochFromDateString(dateStart, "date_start") - sqlQuery.append(" and t._ts >= $dateStartEpochSecs") + sqlQuery.append(" and r._ts >= $dateStartEpochSecs") } dateEnd?.run { val dateEndEpochSecs = DateUtils.getEpochFromDateString(dateEnd, "date_end") - sqlQuery.append(" and t._ts < $dateEndEpochSecs") + sqlQuery.append(" and r._ts < $dateEndEpochSecs") } - sqlQuery.append(" group by t.uploadId") + sqlQuery.append(" group by r.uploadId") // Query for getting counts in the structure of UploadCounts object. Note the MAX aggregate which is used to - // get the latest timestamp from t._ts. - val countQuery = "select count(1) as reportCounts, t.uploadId, MAX(t._ts) as latestTimestamp $sqlQuery" + // get the latest timestamp from r._ts. + val countQuery = "select count(1) as reportCounts, r.uploadId, MAX(r._ts) as latestTimestamp $sqlQuery" logger.info("upload status count query = $countQuery") var totalItems = 0 @@ -117,10 +117,10 @@ class UploadStatusLoader: CosmosLoader() { // } // } // } -// sqlQuery.append(" order by t.$sortField $sortOrderVal") +// sqlQuery.append(" order by r.$sortField $sortOrderVal") // } val offset = (pageNumberAsInt - 1) * pageSize - val dataSqlQuery = "select count(1) as reportCounts, t.uploadId, MAX(t._ts) as latestTimestamp $sqlQuery offset $offset limit $pageSizeAsInt" + val dataSqlQuery = "select count(1) as reportCounts, r.uploadId, MAX(r._ts) as latestTimestamp $sqlQuery offset $offset limit $pageSizeAsInt" logger.info("upload status data query = $dataSqlQuery") val results = reportsContainer?.queryItems( dataSqlQuery, CosmosQueryRequestOptions(), @@ -130,7 +130,7 @@ class UploadStatusLoader: CosmosLoader() { results?.forEach { report -> val uploadId = report.uploadId ?: throw BadStateException("Upload ID unexpectedly null") - val reportsSqlQuery = "select * from $reportsContainerName t where t.uploadId = '$uploadId'" + val reportsSqlQuery = "select * from r where r.uploadId = '$uploadId'" logger.info("get reports for upload query = $reportsSqlQuery") val reportsForUploadId = reportsContainer?.queryItems( reportsSqlQuery, CosmosQueryRequestOptions(), diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/DataStream.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/DataStream.kt new file mode 100644 index 00000000..31b180f5 --- /dev/null +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/DataStream.kt @@ -0,0 +1,20 @@ +package gov.cdc.ocio.processingstatusapi.models + +import java.util.ArrayList + +/** + * Defines the data stream which is used in determining access permissions. + * + * @property id Int + * @property code String? + * @property name String? + * @property route String? + * @property jurisdictions ArrayList? + */ +class DataStream { + var id: Int = 0 + var code: String? = null + var name: String? = null + var route: String? = null + var jurisdictions: ArrayList? = null +} \ No newline at end of file diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/Report.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/Report.kt index 9adddc09..617df49b 100644 --- a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/Report.kt +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/Report.kt @@ -1,7 +1,6 @@ package gov.cdc.ocio.processingstatusapi.models import com.expediagroup.graphql.generator.annotations.GraphQLDescription -import gov.cdc.ocio.processingstatusapi.models.reports.SchemaDefinition import java.time.OffsetDateTime /** @@ -49,7 +48,7 @@ data class Report( var status : String? = null, @GraphQLDescription("Content of the report. If the report is JSON then the content will be shown as JSON. Otherwise, the content is a base64 encoded string.") - var content: SchemaDefinition? = null, + var content : Map<*, *>? = null, @GraphQLDescription("Datestamp the report was recorded in the database") var timestamp: OffsetDateTime? = null diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/ReportDeadLetter.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/ReportDeadLetter.kt new file mode 100644 index 00000000..9ccb21f8 --- /dev/null +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/ReportDeadLetter.kt @@ -0,0 +1,70 @@ +package gov.cdc.ocio.processingstatusapi.models + +import com.expediagroup.graphql.generator.annotations.GraphQLDescription +import java.time.OffsetDateTime + + +/** + * DeadLetter report definition. + * + * @property id String? + * @property uploadId String? + * @property reportId String? + * @property dataStreamId String? + * @property dataStreamRoute String? + * @property stageName String? + * @property contentType String? + * @property messageId String? + * @property status String? + * @property content Map<*, *>? + * @property timestamp OffsetDateTime? + * @property dispositionType String? + * @property deadLetterReasons List? + * @property validationSchemas List? + * @constructor + */ +@GraphQLDescription("Contains Report DeadLetter content.") +data class ReportDeadLetter( + + @GraphQLDescription("Identifier of the report recorded by the database") + var id : String? = null, + + @GraphQLDescription("Upload identifier this report belongs to") + var uploadId: String? = null, + + @GraphQLDescription("Unique report identifier") + var reportId: String? = null, + + @GraphQLDescription("Data stream ID") + var dataStreamId: String? = null, + + @GraphQLDescription("Data stream route") + var dataStreamRoute: String? = null, + + @GraphQLDescription("Stage name this report is associated with") + var stageName: String? = null, + + @GraphQLDescription("Indicates the content type of the content; e.g. JSON, XML") + var contentType : String? = null, + + @GraphQLDescription("Message id this report belongs to; set to null if not applicable") + var messageId: String? = null, + + @GraphQLDescription("Status this report is indicating, such as success or failure") + var status : String? = null, + + @GraphQLDescription("Content of the report. If the report is JSON then the content will be shown as JSON. Otherwise, the content is a base64 encoded string.") + var content : Map<*, *>? = null, + + @GraphQLDescription("Datestamp the report was recorded in the database") + var timestamp: OffsetDateTime? = null, + + @GraphQLDescription("Disposition type of the report") + var dispositionType: String? = null, + + @GraphQLDescription("List of reasons the report was sent to dead-letter") + var deadLetterReasons: List? = null, + + @GraphQLDescription("Schemas used to validate the report") + var validationSchemas: List? = null +) \ No newline at end of file diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/SortOrder.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/SortOrder.kt new file mode 100644 index 00000000..a8a96687 --- /dev/null +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/SortOrder.kt @@ -0,0 +1,9 @@ +package gov.cdc.ocio.processingstatusapi.models + +/** + * Enumeration of the possible sort orders for queries. + */ +enum class SortOrder { + Ascending, + Descending +} \ No newline at end of file diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/dao/ReportDao.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/dao/ReportDao.kt index a758f5f6..738592b0 100644 --- a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/dao/ReportDao.kt +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/dao/ReportDao.kt @@ -1,64 +1,51 @@ package gov.cdc.ocio.processingstatusapi.models.dao import com.google.gson.Gson -import com.google.gson.annotations.SerializedName -import gov.cdc.ocio.processingstatusapi.models.reports.SchemaDefinition -import gov.cdc.ocio.processingstatusapi.models.reports.MetadataVerifyContent -import gov.cdc.ocio.processingstatusapi.models.reports.UploadStatusContent +import gov.cdc.ocio.processingstatusapi.models.Report +import java.time.ZoneOffset import java.util.* -data class ReportDao( +/** + * Data access object for reports, which is the structure returned from CosmosDB queries. + * + * @property id String? + * @property uploadId String? + * @property reportId String? + * @property dataStreamId String? + * @property dataStreamRoute String? + * @property stageName String? + * @property contentType String? + * @property messageId String? + * @property status String? + * @property timestamp Date? + * @property content Any? + * @property contentAsString String? + * @constructor + */ +open class ReportDao( var id : String? = null, - @SerializedName("upload_id") var uploadId: String? = null, - @SerializedName("report_id") var reportId: String? = null, - @SerializedName("data_stream_id") var dataStreamId: String? = null, - @SerializedName("data_stream_route") var dataStreamRoute: String? = null, - @SerializedName("stage_name") var stageName: String? = null, - @SerializedName("content_type") var contentType : String? = null, - @SerializedName("message_id") var messageId: String? = null, - @SerializedName("status") var status : String? = null, var timestamp: Date? = null, var content: Any? = null ) { - val contentAsType: SchemaDefinition? - get() { - if (content == null) return null - - return when (contentType?.lowercase(Locale.getDefault())) { - "json" -> { - if (content is LinkedHashMap<*, *>) { - val json = Gson().toJson(content, MutableMap::class.java).toString() - when ((content as LinkedHashMap<*, *>)["schema_name"]) { - "dex-metadata-verify" -> return Gson().fromJson(json, MetadataVerifyContent::class.java) - "upload" -> return Gson().fromJson(json, UploadStatusContent::class.java) - else -> return null - } - } else - null // unable to determine a JSON schema - } - - else -> null // unable to determine a JSON schema - } - } val contentAsString: String? get() { @@ -74,4 +61,20 @@ data class ReportDao( else -> content.toString() } } + + /** + * Convenience function to convert a cosmos data object to a Report object + */ + fun toReport() = Report().apply { + this.id = this@ReportDao.id + this.uploadId = this@ReportDao.uploadId + this.reportId = this@ReportDao.reportId + this.dataStreamId = this@ReportDao.dataStreamId + this.dataStreamRoute = this@ReportDao.dataStreamRoute + this.messageId = this@ReportDao.messageId + this.status = this@ReportDao.status + this.timestamp = this@ReportDao.timestamp?.toInstant()?.atOffset(ZoneOffset.UTC) + this.contentType = this@ReportDao.contentType + this.content = this@ReportDao.content as? Map<*, *> + } } \ No newline at end of file diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/dao/ReportDeadLetterDao.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/dao/ReportDeadLetterDao.kt new file mode 100644 index 00000000..f1d6d8d5 --- /dev/null +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/dao/ReportDeadLetterDao.kt @@ -0,0 +1,38 @@ +package gov.cdc.ocio.processingstatusapi.models.dao + +import gov.cdc.ocio.processingstatusapi.models.ReportDeadLetter +import java.time.ZoneOffset + + +/** + * Data access object for dead-letter reports, which is the structure returned from CosmosDB queries. + */ +data class ReportDeadLetterDao( + + var dispositionType: String? = null, + + var deadLetterReasons: List? = null, + + var validationSchemas: List? = null, + +) : ReportDao() { + /** + * Convenience function to convert a cosmos data object to a ReportDeadLetter object + */ + fun toReportDeadLetter() = ReportDeadLetter().apply { + this.id = this@ReportDeadLetterDao.id + this.uploadId = this@ReportDeadLetterDao.uploadId + this.reportId = this@ReportDeadLetterDao.reportId + this.dataStreamId = this@ReportDeadLetterDao.dataStreamId + this.dataStreamRoute = this@ReportDeadLetterDao.dataStreamRoute + this.messageId = this@ReportDeadLetterDao.messageId + this.status = this@ReportDeadLetterDao.status + this.timestamp = this@ReportDeadLetterDao.timestamp?.toInstant()?.atOffset(ZoneOffset.UTC) + this.contentType = this@ReportDeadLetterDao.contentType + this.content = this@ReportDeadLetterDao.content as? Map<*, *> + this.dispositionType = this@ReportDeadLetterDao.dispositionType + this.deadLetterReasons = this@ReportDeadLetterDao.deadLetterReasons + this.validationSchemas = this@ReportDeadLetterDao.validationSchemas + } + +} \ No newline at end of file diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/mutations/NotificationsMutationService.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/mutations/NotificationsMutationService.kt new file mode 100644 index 00000000..01b71fea --- /dev/null +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/mutations/NotificationsMutationService.kt @@ -0,0 +1,226 @@ +@file:Suppress("PLUGIN_IS_NOT_ENABLED") + +package gov.cdc.ocio.processingstatusapi.mutations + +import com.expediagroup.graphql.generator.annotations.GraphQLDescription +import com.expediagroup.graphql.server.operations.Mutation +import io.ktor.client.* +import io.ktor.client.call.* +import io.ktor.client.plugins.* +import io.ktor.client.request.* +import io.ktor.client.plugins.contentnegotiation.* +import io.ktor.client.plugins.logging.* +import io.ktor.client.statement.* +import io.ktor.serialization.kotlinx.json.* +import io.ktor.http.* +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.Serializable + + +/** + * Email Subscription data class which is serialized back and forth which is in turn subscribed in to the MemCache + * @param dataStreamId String + * @param dataStreamRoute String + * @param email String + * @param stageName String + * @param statusType String + */ +@Serializable +data class EmailSubscription(val dataStreamId:String, + val dataStreamRoute:String, + val email: String, + val stageName: String, + val statusType:String) + +/** + * UnSubscription data class which is serialized back and forth which is in turn used for unsubscribing from the cache for emails and webhooks using the given subscriberId + * @param subscriptionId + */ +@Serializable +data class UnSubscription(val subscriptionId:String) + +/** + * SubscriptionResult is the response class which is serialized back and forth which is in turn used for getting the response which contains the subscriberId , message and the status of subscribe/unsubscribe operations + * @param subscription_id + * @param timestamp + * @param status + * @param message + */ +@Serializable +data class SubscriptionResult( + var subscription_id: String? = null, + var timestamp: Long? = null, + var status: Boolean? = false, + var message: String? = "" +) + +/** + * The graphQL mutation class for notifications + */ + +class NotificationsMutationService : Mutation { + private val notificationsRouteBaseUrl: String =System.getenv("PSTATUS_NOTIFICATIONS_BASE_URL") + private val serviceUnavailable ="Notification service is unavailable and no connection has been established. Make sure the service is running" + private val client = HttpClient { + install(ContentNegotiation) { + json() + } + install(Logging) { + logger = Logger.DEFAULT + level = LogLevel.INFO + } + install(HttpTimeout) { + requestTimeoutMillis = 10000 + connectTimeoutMillis = 10000 + socketTimeoutMillis = 10000 + } + } + + /** + * SubscribeEmail function which inturn uses the http client to invoke the notifications ktor microservice route to subscribe + * @param dataStreamId String + * @param dataStreamRoute String + * @param email String + * @param stageName String + * @param statusType String + */ + + @GraphQLDescription("Subscribe Email Notifications") + @Suppress("unused") + fun subscribeEmail(dataStreamId:String, dataStreamRoute:String,email: String, stageName: String, statusType:String):SubscriptionResult { + val url = "$notificationsRouteBaseUrl/subscribe/email" + + return runBlocking { + try { + val response= client.post(url) { + contentType(ContentType.Application.Json) + setBody(EmailSubscription(dataStreamId, dataStreamRoute, email, stageName, statusType)) + } + return@runBlocking ProcessResponse(response) + } + catch (e:Exception){ + if(e.message!!.contains("Status:")){ + ProcessErrorCodes(url,e, null) + } + throw Exception(serviceUnavailable) + } + } + } + + /** + * UnSubscribeEmail function which in turn uses the http client to invoke the notifications ktor microservice route to unsubscribe email notifications using the subscriberId + * @param subscriptionId String + */ + + @GraphQLDescription("Unsubscribe Email Notifications") + @Suppress("unused") + fun unsubscribeEmail(subscriptionId:String):SubscriptionResult { + val url = "$notificationsRouteBaseUrl/unsubscribe/email" + + return runBlocking { + try { + val response = client.post(url) { + contentType(ContentType.Application.Json) + setBody(UnSubscription(subscriptionId)) + } + return@runBlocking ProcessResponse(response) + } + catch (e:Exception){ + if(e.message!!.contains("Status:")){ + ProcessErrorCodes(url,e, subscriptionId) + } + throw Exception(serviceUnavailable) + } + } + } + + /** + * SubscribeWebhook function which in turn uses the http client to invoke the notifications ktor microservice route to subscribe webhook notifications + * @param dataStreamId String + * @param dataStreamRoute String + * @param email String + * @param stageName String + * @param statusType String + */ + + @GraphQLDescription("Subscribe Webhook Notifications") + @Suppress("unused") + fun subscribeWebhook(dataStreamId:String, dataStreamRoute:String,email: String, stageName: String, statusType:String):SubscriptionResult { + val url = "$notificationsRouteBaseUrl/subscribe/webhook" + return runBlocking { + try{ + val response= client.post(url) { + contentType(ContentType.Application.Json) + setBody(EmailSubscription(dataStreamId, dataStreamRoute,email,stageName,statusType)) + } + return@runBlocking ProcessResponse(response) + } + catch (e:Exception){ + if(e.message!!.contains("Status:")){ + ProcessErrorCodes(url,e,null) + } + throw Exception(serviceUnavailable) + } + } + } + + /** + * UnSubscribeWebhook function which in turn uses the http client to invoke the notifications ktor microservice route to unsubscribe webhook notifications + * @param subscriptionId String + */ + + @GraphQLDescription("Unsubscribe Webhook Notifications") + @Suppress("unused") + fun unsubscribeWebhook(subscriptionId:String):SubscriptionResult { + val url = "$notificationsRouteBaseUrl/unsubscribe/webhook" + return runBlocking { + try { + val response = client.post(url) { + contentType(ContentType.Application.Json) + setBody(UnSubscription(subscriptionId)) + } + return@runBlocking ProcessResponse(response) + } + catch (e:Exception){ + if(e.message!!.contains("Status:")){ + ProcessErrorCodes(url,e, subscriptionId) + } + throw Exception(serviceUnavailable) + } + } + } + + companion object { + /** + * Function to process the http response coming from notifications service + * @param response HttpResponse + */ + private suspend fun ProcessResponse(response:HttpResponse):SubscriptionResult { + if (response.status == HttpStatusCode.OK) { + return response.body() + } else { + throw Exception("Notification service is unavailable. Status:${response.status}") + } + } + } + +} + +@Throws(Exception::class) +/** + * Function to process the http response codes and throw exception accordingly + * @param url String + * @param e Exception + * @param subscriptionId String? + */ +internal fun ProcessErrorCodes(url:String, e:Exception, subscriptionId:String?) { + val error = e.message!!.substringAfter("Status:").substringBefore(" ") + when (error) { + "500" -> throw Exception("Subscription with subscriptionId = ${subscriptionId} does not exist in the cache") + "400" -> throw Exception("Bad Request: Please check the request and retry") + "401" -> throw Exception("Unauthorized access to notifications service") + "403" -> throw Exception("Access to notifications service is forbidden") + "404" -> throw Exception("${url} not found") + else -> throw Exception(e.message) + } +} \ No newline at end of file diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/CustomGraphQLContextFactory.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/CustomGraphQLContextFactory.kt new file mode 100644 index 00000000..936b74b4 --- /dev/null +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/CustomGraphQLContextFactory.kt @@ -0,0 +1,25 @@ +package gov.cdc.ocio.processingstatusapi.plugins + +import com.expediagroup.graphql.generator.extensions.plus +import com.expediagroup.graphql.server.ktor.DefaultKtorGraphQLContextFactory +import graphql.GraphQLContext +import io.ktor.server.request.* +import io.ktor.util.* + +/** + * Custom graphql context so that we have include additional context to the default graphql context. + */ +class CustomGraphQLContextFactory : DefaultKtorGraphQLContextFactory() { + + /** + * Override the method to generate context so we can add to it. + * + * @param request ApplicationRequest + * @return GraphQLContext + */ + override suspend fun generateContext(request: ApplicationRequest): GraphQLContext = + super.generateContext(request).plus( + // Add the AuthContext to the graphql context + mapOf("AuthContext" to (request.call.attributes.getOrNull(AttributeKey("AuthContext")))) + ) +} \ No newline at end of file diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/CustomGraphQLStatusPages.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/CustomGraphQLStatusPages.kt new file mode 100644 index 00000000..4cb40816 --- /dev/null +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/CustomGraphQLStatusPages.kt @@ -0,0 +1,17 @@ +package gov.cdc.ocio.processingstatusapi.plugins + +import gov.cdc.ocio.processingstatusapi.loaders.ForbiddenException +import io.ktor.http.* +import io.ktor.server.plugins.statuspages.* +import io.ktor.server.response.* + +fun StatusPagesConfig.customGraphQLStatusPages(): StatusPagesConfig { + exception { call, cause -> + when (cause) { + is UnsupportedOperationException -> call.respond(HttpStatusCode.MethodNotAllowed) + is ForbiddenException -> call.respond(HttpStatusCode.Forbidden, cause.message ?: "Forbidden") + else -> call.respond(HttpStatusCode.BadRequest) + } + } + return this +} \ No newline at end of file diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/CustomSchemaGeneratorHooks.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/CustomSchemaGeneratorHooks.kt index 995bd713..78a16a4f 100644 --- a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/CustomSchemaGeneratorHooks.kt +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/CustomSchemaGeneratorHooks.kt @@ -10,12 +10,25 @@ import java.time.OffsetDateTime import kotlin.reflect.KClass import kotlin.reflect.KType +/** + * Define a GraphQL scalar for the Long type. + */ val graphqlLongClassType: GraphQLScalarType = GraphQLScalarType.newScalar() .name("Long") .coercing(ExtendedScalars.GraphQLLong.coercing) .build() +/** + * Custom schema generator hooks for the PS API reports. + */ class CustomSchemaGeneratorHooks : SchemaGeneratorHooks { + + /** + * Override the graphql schema generator types to include the new scalars for Long, DateTime, Maps, etc. + * + * @param type KType + * @return GraphQLType? + */ override fun willGenerateGraphQLType(type: KType): GraphQLType? = when (type.classifier as? KClass<*>) { OffsetDateTime::class -> DateTimeScalar.INSTANCE Long::class ->graphqlLongClassType diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/GraphQL.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/GraphQL.kt index 02325e50..b8279707 100644 --- a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/GraphQL.kt +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/GraphQL.kt @@ -5,56 +5,77 @@ import com.auth0.jwt.algorithms.Algorithm import com.expediagroup.graphql.dataloader.KotlinDataLoaderRegistryFactory import com.expediagroup.graphql.server.ktor.* import gov.cdc.ocio.processingstatusapi.dataloaders.ReportDataLoader -import gov.cdc.ocio.processingstatusapi.queries.HealthQueryService -import gov.cdc.ocio.processingstatusapi.queries.ReportCountsQueryService -import gov.cdc.ocio.processingstatusapi.queries.ReportQueryService -import gov.cdc.ocio.processingstatusapi.queries.UploadQueryService +import gov.cdc.ocio.processingstatusapi.dataloaders.ReportDeadLetterDataLoader +import gov.cdc.ocio.processingstatusapi.mutations.NotificationsMutationService +import gov.cdc.ocio.processingstatusapi.queries.* import io.ktor.http.* import io.ktor.serialization.jackson.* import io.ktor.server.application.* import io.ktor.server.auth.* import io.ktor.server.auth.jwt.* +import io.ktor.server.config.* import io.ktor.server.plugins.statuspages.* import io.ktor.server.response.* import io.ktor.server.routing.* import io.ktor.server.websocket.* +import mu.KotlinLogging import java.time.Duration +import java.util.* + +/** + * Implementation of the GraphQL module for PS API. + * + * @receiver Application + */ fun Application.graphQLModule() { + val logger = KotlinLogging.logger {} + install(WebSockets) { // needed for subscriptions pingPeriod = Duration.ofSeconds(1) contentConverter = JacksonWebsocketContentConverter() } // see https://ktor.io/docs/server-jwt.html#configure-verifier + // Get security settings and default to enabled if missing + val securityEnabled = environment.config.tryGetString("jwt.enabled")?.lowercase() != "false" val secret = environment.config.property("jwt.secret").getString() val issuer = environment.config.property("jwt.issuer").getString() val audience = environment.config.property("jwt.audience").getString() val myRealm = environment.config.property("jwt.realm").getString() - install(Authentication) { - jwt { - jwt("auth-jwt") { - realm = myRealm - verifier(JWT - .require(Algorithm.HMAC256(secret)) - .withAudience(audience) - .withIssuer(issuer) - .build()) - validate { credential -> - if (credential.payload.getClaim("username").asString() != "") { - JWTPrincipal(credential.payload) - } else { - null + val graphQLPath = environment.config.tryGetString("graphql.path") + if (securityEnabled) { + install(Authentication) { + jwt { + jwt("auth-jwt") { + realm = myRealm + verifier( + JWT + .require(Algorithm.HMAC256(Base64.getDecoder().decode(secret))) + .withAudience(audience) + .withIssuer(issuer) + .build() + ) + validate { credential -> + if (credential.payload.getClaim("username").asString() != "") { + JWTPrincipal(credential.payload) + } else { + logger.error("username missing from JWT claims, denying the token") + null + } + } + challenge { defaultScheme, realm -> + call.respond( + HttpStatusCode.Unauthorized, + "Token is not valid or has expired; defaultScheme = $defaultScheme, realm = $realm" + ) } - } - challenge { _, _ -> - call.respond(HttpStatusCode.Unauthorized, "Token is not valid or has expired") } } } } install(StatusPages) { - defaultGraphQLStatusPages() + customGraphQLStatusPages() } // install(CORS) { // anyHost() @@ -66,7 +87,12 @@ fun Application.graphQLModule() { HealthQueryService(), ReportQueryService(), ReportCountsQueryService(), + ReportDeadLetterQueryService(), UploadQueryService() + + ) + mutations= listOf( + NotificationsMutationService() ) // subscriptions = listOf( // ErrorSubscriptionService() @@ -75,22 +101,34 @@ fun Application.graphQLModule() { } engine { dataLoaderRegistryFactory = KotlinDataLoaderRegistryFactory( - ReportDataLoader + ReportDataLoader, + ReportDeadLetterDataLoader ) } + if (securityEnabled) { + server { + contextFactory = CustomGraphQLContextFactory() + } + } } install(Routing) { - authenticate("auth-jwt") { -// post("/graphql") { -// val principal = call.principal() -// val username = principal!!.payload.getClaim("username").asString() -// val expiresAt = principal.expiresAt?.time?.minus(System.currentTimeMillis()) -// call.respondText("Hello, $username! Token is expired at $expiresAt ms.") -// } + if (securityEnabled) { + authenticate("auth-jwt") { + graphQLPostRoute() + } + } else { + graphQLPostRoute() } - graphQLPostRoute() graphQLSubscriptionsRoute() graphQLSDLRoute() - graphiQLRoute() // Go to http://localhost:8080/graphiql for the GraphQL playground + // Go to http://localhost:8080/graphiql for the GraphQL playground + if (graphQLPath.isNullOrEmpty()) { + graphiQLRoute() + } else { + graphiQLRoute( + graphQLEndpoint = "$graphQLPath/graphql", + subscriptionsEndpoint = "$graphQLPath/subscriptions" + ) + } } } \ No newline at end of file diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/Routing.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/Routing.kt new file mode 100644 index 00000000..1b4a8d57 --- /dev/null +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/Routing.kt @@ -0,0 +1,18 @@ +package gov.cdc.ocio.processingstatusapi.plugins + +import gov.cdc.ocio.processingstatusapi.queries.HealthQueryService +import io.ktor.server.application.* +import io.ktor.server.response.* +import io.ktor.server.routing.* + +fun Application.configureRouting() { + val version = environment.config.propertyOrNull("ktor.version")?.getString() ?: "unknown" + routing { + get("/health") { + call.respond(HealthQueryService().getHealth()) + } + get("/version") { + call.respondText(version) + } + } +} \ No newline at end of file diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/queries/HealthQueryService.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/queries/HealthQueryService.kt index bb25be2d..119bfe08 100644 --- a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/queries/HealthQueryService.kt +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/queries/HealthQueryService.kt @@ -2,10 +2,20 @@ package gov.cdc.ocio.processingstatusapi.queries import com.expediagroup.graphql.generator.annotations.GraphQLDescription import com.expediagroup.graphql.server.operations.Query -import gov.cdc.ocio.processingstatusapi.loaders.ReportLoader +import gov.cdc.ocio.processingstatusapi.cosmos.CosmosClientManager +import gov.cdc.ocio.processingstatusapi.cosmos.CosmosConfiguration import mu.KotlinLogging +import org.koin.core.component.KoinComponent +import org.koin.core.component.inject import kotlin.system.measureTimeMillis +/** + * Abstract class used for modeling the health issues of an individual service. + * + * @property status String + * @property healthIssues String? + * @property service String + */ abstract class HealthCheckSystem { @GraphQLDescription("Status of the dependency") @@ -18,10 +28,22 @@ abstract class HealthCheckSystem { open val service: String = "" } +/** + * Concrete implementation of the cosmosdb service health check. + * + * @property service String + */ class CosmosDb: HealthCheckSystem() { override val service = "Cosmos DB" } +/** + * Run health checks for the service. + * + * @property status String? + * @property totalChecksDuration String? + * @property dependencyHealthChecks MutableList + */ class HealthCheck { @GraphQLDescription("Overall status of the service") @@ -31,21 +53,36 @@ class HealthCheck { var totalChecksDuration : String? = null @GraphQLDescription("Status of the service dependencies") - var dependencyHealthChecks = arrayListOf() + var dependencyHealthChecks = mutableListOf() } +/** + * GraphQL query service for getting health status. + */ class HealthQueryService : Query { + fun getHealth() = HealthCheckService().getHealth() +} + +/** + * Health checks for the service and any dependencies. + * + * @property logger KLogger + * @property cosmosConfiguration CosmosConfiguration + */ +class HealthCheckService : KoinComponent { private val logger = KotlinLogging.logger {} - @GraphQLDescription("Return a single report from the provided uploadId") + private val cosmosConfiguration by inject() + + @GraphQLDescription("Performs a service health check of the processing status API and it's dependencies.") @Suppress("unused") fun getHealth(): HealthCheck { var cosmosDBHealthy = false val cosmosDBHealth = CosmosDb() val time = measureTimeMillis { try { - cosmosDBHealthy = isCosmosDBHealthy() + cosmosDBHealthy = isCosmosDBHealthy(config = cosmosConfiguration) cosmosDBHealth.status = "UP" } catch (ex: Exception) { cosmosDBHealth.healthIssues = ex.message @@ -63,10 +100,14 @@ class HealthQueryService : Query { /** * Check whether CosmosDB is healthy. * + * @param config CosmosConfiguration * @return Boolean */ - private fun isCosmosDBHealthy(): Boolean { - return ReportLoader().getAnyReport() != null + private fun isCosmosDBHealthy(config: CosmosConfiguration): Boolean { + return if (CosmosClientManager.getCosmosClient(config.uri, config.authKey) == null) + throw Exception("Failed to establish a CosmosDB client.") + else + true } /** diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/queries/ReportDeadLetterQueryService.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/queries/ReportDeadLetterQueryService.kt new file mode 100644 index 00000000..34057298 --- /dev/null +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/queries/ReportDeadLetterQueryService.kt @@ -0,0 +1,38 @@ +package gov.cdc.ocio.processingstatusapi.queries + +import com.expediagroup.graphql.generator.annotations.GraphQLDescription +import com.expediagroup.graphql.server.operations.Query +import gov.cdc.ocio.processingstatusapi.dataloaders.ReportDeadLetterDataLoader +import gov.cdc.ocio.processingstatusapi.loaders.ReportDeadLetterLoader +import gov.cdc.ocio.processingstatusapi.models.ReportDeadLetter +import graphql.schema.DataFetchingEnvironment +import java.util.concurrent.CompletableFuture + +class ReportDeadLetterQueryService : Query { + + @GraphQLDescription("Return all the dead-letter reports associated with the provided uploadId") + @Suppress("unused") + fun getDeadLetterReportsByUploadId(uploadId: String) = ReportDeadLetterLoader().getByUploadId(uploadId) + + @GraphQLDescription("Return all the dead-letter reports associated with the provided datastreamId, datastreamroute and timestamp date range") + @Suppress("unused") + fun getDeadLetterReportsByDataStream(dataStreamId: String, dataStreamRoute:String, startDate:String?, endDate:String?, daysInterval :Int?) + = ReportDeadLetterLoader().getByDataStreamByDateRange(dataStreamId,dataStreamRoute,startDate,endDate,daysInterval) + + @GraphQLDescription("Return count of dead-letter reports associated with the provided datastreamId, (optional) datastreamroute and timestamp date range") + @Suppress("unused") + fun getDeadLetterReportsCountByDataStream(dataStreamId: String, dataStreamRoute:String?, startDate:String?, endDate:String?, daysInterval:Int?) + = ReportDeadLetterLoader().getCountByDataStreamByDateRange(dataStreamId,dataStreamRoute,startDate,endDate,daysInterval) + + @GraphQLDescription("Return list of dead-letter reports based on ReportSearchParameters options") + @Suppress("unused") + fun searchDeadLetterReports(params: ReportDeadLetterSearchParameters, dfe: DataFetchingEnvironment): CompletableFuture> = + dfe.getDataLoader(ReportDeadLetterDataLoader.dataLoaderName) + .loadMany(params.ids) +} + +@GraphQLDescription("Parameters for searching for reports") +data class ReportDeadLetterSearchParameters( + @GraphQLDescription("Array of report IDs to search for and retrieve") + val ids: List +) \ No newline at end of file diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/queries/ReportQueryService.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/queries/ReportQueryService.kt index a91df96f..90fbba51 100644 --- a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/queries/ReportQueryService.kt +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/queries/ReportQueryService.kt @@ -5,15 +5,44 @@ import com.expediagroup.graphql.server.operations.Query import gov.cdc.ocio.processingstatusapi.dataloaders.ReportDataLoader import gov.cdc.ocio.processingstatusapi.models.Report import gov.cdc.ocio.processingstatusapi.loaders.ReportLoader +import gov.cdc.ocio.processingstatusapi.models.SortOrder import graphql.schema.DataFetchingEnvironment import java.util.concurrent.CompletableFuture class ReportQueryService : Query { - @GraphQLDescription("Return all the reports associated with the provided uploadId") + /** + * Query for retrieving reports for the uploadId provided. + * + * @param dataFetchingEnvironment DataFetchingEnvironment + * @param uploadId String + * @param reportsSortedBy String? + * @param sortOrder SortOrder? + * @return List + */ + @GraphQLDescription("Return all the reports associated with the provided upload ID.") @Suppress("unused") - fun getReports(uploadId: String) = ReportLoader().getByUploadId(uploadId) + fun getReports(dataFetchingEnvironment: DataFetchingEnvironment, + @GraphQLDescription("Upload ID to retrieve all the reports for.") + uploadId: String, + @GraphQLDescription("Optional field to specify the field reports should be sorted by. Available fields for sorting are: [`timestamp`].") + reportsSortedBy: String?, + @GraphQLDescription("Optional sort order. When `reportsSortedBy` is provided, the available options are `Ascending` or `Descending`, which defaults to `Ascending` if not provided.") + sortOrder: SortOrder?) = ReportLoader() + .getByUploadId( + dataFetchingEnvironment, + uploadId, + reportsSortedBy, + sortOrder + ) + /** + * Searches for reports with the given search parameters. + * + * @param params ReportSearchParameters + * @param dfe DataFetchingEnvironment + * @return CompletableFuture> + */ @GraphQLDescription("Return list of reports based on ReportSearchParameters options") @Suppress("unused") fun searchReports(params: ReportSearchParameters, dfe: DataFetchingEnvironment): CompletableFuture> = @@ -21,6 +50,12 @@ class ReportQueryService : Query { .loadMany(params.ids) } +/** + * Report search parameters, which is a list of report IDs. + * + * @property ids List + * @constructor + */ @GraphQLDescription("Parameters for searching for reports") data class ReportSearchParameters( @GraphQLDescription("Array of report IDs to search for and retrieve") diff --git a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/queries/UploadQueryService.kt b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/queries/UploadQueryService.kt index fec339c1..ed34e9f5 100644 --- a/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/queries/UploadQueryService.kt +++ b/pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/queries/UploadQueryService.kt @@ -7,6 +7,19 @@ import gov.cdc.ocio.processingstatusapi.loaders.UploadStatusLoader class UploadQueryService : Query { + /** + * Get the upload status for the given search criteria. + * + * @param dataStreamId String + * @param dataStreamRoute String? + * @param dateStart String? + * @param dateEnd String? + * @param pageSize Int + * @param pageNumber Int + * @param sortBy String? + * @param sortOrder String? + * @return UploadsStatus + */ @GraphQLDescription("Get the upload statuses for the given filter, sort, and pagination criteria") @Suppress("unused") fun uploads(dataStreamId: String, @@ -26,7 +39,17 @@ class UploadQueryService : Query { sortBy, sortOrder) - @GraphQLDescription("Return various uploads statistics") + /** + * Provide the upload statistics for the given search criteria. + * + * @param dataStreamId String + * @param dataStreamRoute String + * @param dateStart String? + * @param dateEnd String? + * @param daysInterval Int? + * @return UploadStats + */ + @GraphQLDescription("Return various uploads statistics") @Suppress("unused") fun getUploadStats(@GraphQLDescription("Data stream ID") dataStreamId: String, diff --git a/pstatus-graphql-ktor/src/main/resources/application.conf b/pstatus-graphql-ktor/src/main/resources/application.conf index 65934b11..9a0a3866 100644 --- a/pstatus-graphql-ktor/src/main/resources/application.conf +++ b/pstatus-graphql-ktor/src/main/resources/application.conf @@ -7,11 +7,18 @@ ktor { application { modules = [ gov.cdc.ocio.processingstatusapi.ApplicationKt.module ] } + + version = "0.0.4" +} + +graphql { + path = ${?GRAPHQL_PATH} } jwt { + enabled = ${?SECURITY_ENABLED} secret = "secret" - issuer = "http://0.0.0.0:8080/" + issuer = "CDC B2C/MMS" audience = "http://0.0.0.0:8080/graphql" realm = "Access to 'graphql'" } diff --git a/pstatus-notifications-ktor/.gitignore b/pstatus-notifications-ktor/.gitignore new file mode 100644 index 00000000..c426c32f --- /dev/null +++ b/pstatus-notifications-ktor/.gitignore @@ -0,0 +1,36 @@ +.gradle +build/ +!gradle/wrapper/gradle-wrapper.jar +!**/src/main/**/build/ +!**/src/test/**/build/ + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache +bin/ +!**/src/main/**/bin/ +!**/src/test/**/bin/ + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr +out/ +!**/src/main/**/out/ +!**/src/test/**/out/ + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ + +### VS Code ### +.vscode/ \ No newline at end of file diff --git a/pstatus-notifications-ktor/build.gradle b/pstatus-notifications-ktor/build.gradle new file mode 100644 index 00000000..17ff6237 --- /dev/null +++ b/pstatus-notifications-ktor/build.gradle @@ -0,0 +1,105 @@ + +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +buildscript { + repositories { + mavenCentral() + } + dependencies { + classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" + } +} +plugins { + id 'org.jetbrains.kotlin.jvm' version '1.9.24' + id 'io.ktor.plugin' version '2.3.11' + id("com.google.cloud.tools.jib") version "3.3.0" // Add Jib plugin + id "maven-publish" + id 'java-library' + id "org.jetbrains.kotlin.plugin.serialization" version "1.8.20" +} +apply plugin: "java" +apply plugin: "kotlin" + +java { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 +} + +tasks.withType(KotlinCompile).configureEach { + kotlinOptions { + jvmTarget = "17" + } + +} + +group "gov.cdc.ocio" +version "0.0.1" + +dependencies { + implementation "io.ktor:ktor-server-core:2.3.2" + implementation "io.ktor:ktor-server-netty:2.3.2" + implementation "io.ktor:ktor-server-content-negotiation:2.3.2" + implementation "io.ktor:ktor-serialization-kotlinx-json:2.3.2" + implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.2" + implementation "org.jetbrains.kotlinx:kotlinx-coroutines-jdk8:1.7.2" + implementation "io.github.microutils:kotlin-logging-jvm:3.0.5" + implementation 'com.azure:azure-messaging-servicebus:7.17.1' + implementation 'com.microsoft.azure:azure-servicebus:3.6.7' + implementation "com.google.code.gson:gson:2.10.1" + implementation 'io.github.microutils:kotlin-logging-jvm:3.0.5' + implementation 'org.slf4j:slf4j-api:1.7.36' + implementation 'ch.qos.logback:logback-classic:1.4.12' + implementation 'ch.qos.logback.contrib:logback-json-classic:0.1.5' + implementation group: 'ch.qos.logback.contrib', name: 'logback-jackson', version: '0.1.5' + implementation "io.insert-koin:koin-core:3.5.6" + implementation "io.insert-koin:koin-ktor:3.5.6" + implementation "com.sun.mail:javax.mail:1.6.2" + implementation 'com.expediagroup:graphql-kotlin-ktor-server:7.1.1' + implementation 'com.graphql-java:graphql-java-extended-scalars:22.0' + implementation 'joda-time:joda-time:2.12.7' + implementation 'org.apache.commons:commons-lang3:3.3.1' + implementation "com.expediagroup:graphql-kotlin-server:6.0.0" + implementation "com.expediagroup:graphql-kotlin-schema-generator:6.0.0" + implementation "io.ktor:ktor-server-netty:2.1.0" + implementation "io.ktor:ktor-client-content-negotiation:2.1.0" + testImplementation "org.mockito.kotlin:mockito-kotlin:4.0.0" + testImplementation 'org.testng:testng:7.7.0' + testImplementation "org.mockito:mockito-inline:3.11.2" + testImplementation "io.mockk:mockk:1.13.9" + testImplementation "io.ktor:ktor-server-tests-jvm" + testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlin_version" + +} + +ktor { + docker { + localImageName.set("pstatus-notifications-ktor") + } +} + +jib { + to { + image = 'imagehub.cdc.gov:6989/dex/pstatus/notifications-service' + auth { + username = System.getenv("IMAGEHUB_USERNAME") ?: "" + password = System.getenv("IMAGEHUB_PASSWORD") ?: "" + } + } +} + +test { + useTestNG() + testLogging { + events "passed", "skipped", "failed" + } + //Change this to "true" if we want to execute unit tests + systemProperty("isTestEnvironment", "false") + + // Set the test classpath, if required +} + + +repositories{ + mavenCentral() +} + diff --git a/pstatus-notifications-ktor/gradle.properties b/pstatus-notifications-ktor/gradle.properties new file mode 100644 index 00000000..f107887b --- /dev/null +++ b/pstatus-notifications-ktor/gradle.properties @@ -0,0 +1,4 @@ +ktor_version=2.3.10 +kotlin_version=1.9.23 +logback_version=1.4.14 +kotlin.code.style=official diff --git a/pstatus-notifications-ktor/gradle/wrapper/gradle-wrapper.jar b/pstatus-notifications-ktor/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 00000000..d64cd491 Binary files /dev/null and b/pstatus-notifications-ktor/gradle/wrapper/gradle-wrapper.jar differ diff --git a/pstatus-notifications-ktor/gradle/wrapper/gradle-wrapper.properties b/pstatus-notifications-ktor/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..1af9e093 --- /dev/null +++ b/pstatus-notifications-ktor/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/pstatus-notifications-ktor/gradlew b/pstatus-notifications-ktor/gradlew new file mode 100644 index 00000000..1aa94a42 --- /dev/null +++ b/pstatus-notifications-ktor/gradlew @@ -0,0 +1,249 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/pstatus-notifications-ktor/gradlew.bat b/pstatus-notifications-ktor/gradlew.bat new file mode 100644 index 00000000..93e3f59f --- /dev/null +++ b/pstatus-notifications-ktor/gradlew.bat @@ -0,0 +1,92 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/pstatus-notifications-ktor/src/main/kotlin/Application.kt b/pstatus-notifications-ktor/src/main/kotlin/Application.kt new file mode 100644 index 00000000..14d525b2 --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/Application.kt @@ -0,0 +1,50 @@ +package gov.cdc.ocio.processingstatusnotifications + +import gov.cdc.ocio.processingstatusnotifications.servicebus.AzureServiceBusConfiguration +import io.ktor.serialization.jackson.* +import io.ktor.server.application.* +import io.ktor.server.engine.* +import io.ktor.server.netty.* +import io.ktor.server.plugins.contentnegotiation.* +import io.ktor.server.routing.* +import org.koin.core.KoinApplication +import org.koin.dsl.module +import org.koin.ktor.plugin.Koin + + +/** + * Load the environment configuration values + * Instantiate a singleton CosmosDatabase container instance + * @param environment ApplicationEnvironment + */ +fun KoinApplication.loadKoinModules(environment: ApplicationEnvironment): KoinApplication { + + val asbConfigModule = module { + // Create an azure service bus config that can be dependency injected (for health checks) + single(createdAtStart = true) { AzureServiceBusConfiguration(environment.config, configurationPath = "azure.service_bus") } + } + + return modules(listOf(asbConfigModule)) +} + +fun main(args: Array) { + embeddedServer(Netty, commandLineEnvironment(args)).start(wait = true) +} + +fun Application.module() { + install(Koin) { + loadKoinModules(environment) + } + install(ContentNegotiation) { + jackson() + } + routing { + subscribeEmailNotificationRoute() + unsubscribeEmailNotificationRoute() + subscribeWebhookRoute() + unsubscribeWebhookRoute() + healthCheckRoute() + versionRoute() + } + +} diff --git a/pstatus-notifications-ktor/src/main/kotlin/HealthCheck.kt b/pstatus-notifications-ktor/src/main/kotlin/HealthCheck.kt new file mode 100644 index 00000000..159a596a --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/HealthCheck.kt @@ -0,0 +1,125 @@ +package gov.cdc.ocio.processingstatusnotifications + +import com.azure.core.exception.ResourceNotFoundException +import com.azure.messaging.servicebus.administration.ServiceBusAdministrationClientBuilder +import com.microsoft.azure.servicebus.primitives.ServiceBusException +import gov.cdc.ocio.processingstatusnotifications.servicebus.AzureServiceBusConfiguration +import mu.KotlinLogging +import org.koin.core.component.KoinComponent +import org.koin.core.component.inject +import kotlin.system.measureTimeMillis + + +/** + * Abstract class used for modeling the health issues of an individual service. + * + * @property status String + * @property healthIssues String? + * @property service String + */ +abstract class HealthCheckSystem { + + var status: String = "DOWN" + + var healthIssues: String? = "" + + open val service: String = "" +} + +/** + * Concrete implementation of the Azure Service Bus health check. + * + * @property service String + */ +class HealthCheckServiceBus: HealthCheckSystem() { + override val service: String = "Azure Service Bus" +} + +/** + * Run health checks for the service. + * + * @property status String? + * @property totalChecksDuration String? + * @property dependencyHealthChecks MutableList + */ +class HealthCheck { + + var status: String = "DOWN" + + var totalChecksDuration: String? = null + + var dependencyHealthChecks = mutableListOf() +} + +/** + * Service for querying the health of the report-sink service and its dependencies. + * + * @property logger KLogger + * @property cosmosConfiguration CosmosConfiguration + * @property azureServiceBusConfiguration AzureServiceBusConfiguration + */ +class HealthQueryService: KoinComponent { + + private val logger = KotlinLogging.logger {} + + private val azureServiceBusConfiguration by inject() + + /** + * Returns a HealthCheck object with the overall health of the report-sink service and its dependencies. + * + * @return HealthCheck + */ + fun getHealth(): HealthCheck { + var serviceBusHealthy = false + val serviceBusHealth = HealthCheckServiceBus() + val time = measureTimeMillis { + try { + serviceBusHealthy = isServiceBusHealthy(config = azureServiceBusConfiguration) + serviceBusHealth.status = "UP" + } catch (ex: Exception) { + serviceBusHealth.healthIssues = ex.message + logger.error("Azure Service Bus is not healthy: ${ex.message}") + } + } + + return HealthCheck().apply { + status = if (serviceBusHealthy) "UP" else "DOWN" + totalChecksDuration = formatMillisToHMS(time) + dependencyHealthChecks.add(serviceBusHealth) + } + } + + /** + * Check whether service bus is healthy. + * + * @return Boolean + */ + @Throws(ResourceNotFoundException::class, ServiceBusException::class) + private fun isServiceBusHealthy(config: AzureServiceBusConfiguration): Boolean { + + val adminClient = ServiceBusAdministrationClientBuilder() + .connectionString(config.connectionString) + .buildClient() + + // Get the properties of the topic to check the connection + adminClient.getTopic(config.topicName) + + return true + } + + /** + * Format the time in milliseconds to 00:00:00.000 format. + * + * @param millis Long + * @return String + */ + private fun formatMillisToHMS(millis: Long): String { + val seconds = millis / 1000 + val hours = seconds / 3600 + val minutes = (seconds % 3600) / 60 + val remainingSeconds = seconds % 60 + val remainingMillis = millis % 1000 + + return "%02d:%02d:%02d.%03d".format(hours, minutes, remainingSeconds, remainingMillis / 10) + } +} \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/Routes.kt b/pstatus-notifications-ktor/src/main/kotlin/Routes.kt new file mode 100644 index 00000000..cb8b2c8e --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/Routes.kt @@ -0,0 +1,102 @@ +@file:Suppress("PLUGIN_IS_NOT_ENABLED") + +package gov.cdc.ocio.processingstatusnotifications + +import gov.cdc.ocio.processingstatusnotifications.notifications.UnSubscribeNotifications +import gov.cdc.ocio.processingstatusnotifications.notifications.SubscribeEmailNotifications +import gov.cdc.ocio.processingstatusnotifications.notifications.SubscribeWebhookNotifications +import io.ktor.server.application.* +import io.ktor.server.request.* +import io.ktor.server.response.* +import io.ktor.server.routing.* + + +/** + * Email Subscription data class which is serialized back and forth + */ +data class EmailSubscription(val dataStreamId:String, + val dataStreamRoute:String, + val email: String, + val stageName: String, + val statusType:String) +/** + * Webhook Subscription data class which is serialized back and forth + */ +data class WebhookSubscription(val dataStreamId:String, + val dataStreamRoute:String, + val url: String, + val stageName: String, + val statusType:String) + +/** + * UnSubscription data class which is serialized back and forth when we need to unsubscribe by the subscriptionId + */ +data class UnSubscription(val subscriptionId:String) + +/** + * The resultant class for subscription of email/webhooks + */ +data class SubscriptionResult( + var subscription_id: String? = null, + var timestamp: Long? = null, + var status: Boolean? = false, + var message: String? = "" +) + +/** + * Route to subscribe for email notifications + */ +fun Route.subscribeEmailNotificationRoute() { + post("/subscribe/email") { + val subscription = call.receive() + val emailSubscription =EmailSubscription(subscription.dataStreamId, subscription.dataStreamRoute, subscription.email, subscription.stageName, subscription.statusType) + val result = SubscribeEmailNotifications().run(emailSubscription) + call.respond(result) + + } +} +/** + * Route to Unsubscribe for email notifications + */ +fun Route.unsubscribeEmailNotificationRoute() { + post("/unsubscribe/email") { + val subscription = call.receive() + val result = UnSubscribeNotifications().run(subscription.subscriptionId) + call.respond(result) + } +} +/** + * Route to subscribe for webhook notifications + */ +fun Route.subscribeWebhookRoute() { + post("/subscribe/webhook") { + val subscription = call.receive() + val webhookSubscription = WebhookSubscription(subscription.dataStreamId, subscription.dataStreamRoute, subscription.url, subscription.stageName, subscription.statusType) + val result = SubscribeWebhookNotifications().run(webhookSubscription) + call.respond(result) + + } +} +/** + * Route to unsubscribe for webhook notifications + */ +fun Route.unsubscribeWebhookRoute() { + post("/unsubscribe/webhook") { + val subscription = call.receive() + val result = UnSubscribeNotifications().run(subscription.subscriptionId) + call.respond(result) + } +} + +fun Route.healthCheckRoute() { + get("/health") { + call.respond(HealthQueryService().getHealth()) + } +} + +fun Route.versionRoute() { + val version = environment?.config?.propertyOrNull("ktor.version")?.getString() ?: "unknown" + get("/version") { + call.respondText(version) + } +} \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/cache/InMemoryCache.kt b/pstatus-notifications-ktor/src/main/kotlin/cache/InMemoryCache.kt new file mode 100644 index 00000000..6b15b504 --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/cache/InMemoryCache.kt @@ -0,0 +1,166 @@ +package gov.cdc.ocio.processingstatusnotifications.cache + +import gov.cdc.ocio.processingstatusnotifications.* +import gov.cdc.ocio.processingstatusnotifications.exception.* +import gov.cdc.ocio.processingstatusnotifications.model.cache.* +import mu.KotlinLogging +import java.util.* +import java.util.concurrent.locks.ReentrantReadWriteLock +import kotlin.collections.HashMap + +/** + * This class represents InMemoryCache to maintain state of the data at any given point for + * subscription of rules and subscriber for the rules + */ +object InMemoryCache { + private val logger = KotlinLogging.logger {} + private val readWriteLock = ReentrantReadWriteLock() + + /* + Cache to store "SubscriptionRule HashSet -> SubscriptionId" + subscriptionRuleCache = HashMap() + */ + private val subscriptionRuleCache = HashMap() + + /* + Cache to store "SubscriptionId -> Subscriber Info (Email or Url and type of subscription)" + subscriberCache = HashMap() + */ + private val subscriberCache = HashMap>() + + + /** + * If Success, this method updates Two Caches for New Subscription: + * a. First Cache with subscription Rule and respective subscriptionId, + * if it doesn't exist,or it returns existing subscription id. + * b. Second cache is subscriber cache where the subscription id is mapped to emailId of subscriber + * or websocket url with the type of subscription + * + * @param subscriptionRule String - Hashcode of object with all the required fields + * @param subscriptionType SubscriptionType - Currently supports only 'Email' or 'Websocket' + * @param emailOrUrl String - Valid EmailId or Valid WebSocket Url + * @return String + */ + fun updateCacheForSubscription(subscriptionRule: String, + subscriptionType: SubscriptionType, + emailOrUrl: String): String { + if (subscriptionType == SubscriptionType.EMAIL || subscriptionType == SubscriptionType.WEBSOCKET) { + // If subscription type is EMAIL or WEBSOCKET then proceed else throw BadState Exception + val subscriptionId = updateSubscriptionRuleCache(subscriptionRule) + updateSubscriberCache(subscriptionId, NotificationSubscription(subscriptionId, emailOrUrl, subscriptionType)) + return subscriptionId + } else { + throw BadStateException("Not a valid SubscriptionType") + } + } + + /** + * This method adds a new entry Map[SubscriberRule, SubscriptionId] + * for new rule (if it doesn't exist) in SubscriptionCache + * + * @param subscriptionRule String + * @return String + */ + private fun updateSubscriptionRuleCache(subscriptionRule: String): String { + // Try to read from existing cache to see an existing subscription rule + var existingSubscriptionId: String? = null + readWriteLock.readLock().lock() + try { + existingSubscriptionId = subscriptionRuleCache.get(subscriptionRule) + } finally { + readWriteLock.readLock().unlock() + } + + // if subscription doesn't exist, it will add it else it will return the existing subscription id + return if (existingSubscriptionId != null) { + logger.debug("Subscription Rule exists") + existingSubscriptionId + } else { + // create unique subscription + val subscriptionId = generateUniqueSubscriptionId() + logger.debug("Subscription Id for this new rule has been generated $subscriptionId") + readWriteLock.writeLock().lock() + try { + subscriptionRuleCache.put(subscriptionRule, subscriptionId) + } finally { + readWriteLock.writeLock().unlock() + } + subscriptionId + } + } + + /** + * This method generates new unique susbscriptionId for caches + * @return String + */ + internal fun generateUniqueSubscriptionId(): String { + // TODO: This could be handled to background task to populate + // uniqueSubscriptionIds bucket of size 20 or 50, may be? + var subscriptionId = UUID.randomUUID().toString() + while(subscriberCache.contains(subscriptionId)) { + subscriptionId = UUID.randomUUID().toString() + } + return subscriptionId + } + + /** + * This method adds to the subscriber cache the new entry of subscriptionId to the NotificationSubscriber + * + * @param subscriptionId String + * @param notificationSubscriber NotificationSubscriber + */ + private fun updateSubscriberCache(subscriptionId: String, + notificationSubscriber: NotificationSubscription) { + logger.debug("Subscriber added in subscriber cache") + readWriteLock.writeLock().lock() + try { + subscriberCache.putIfAbsent(subscriptionId, mutableListOf()) + subscriberCache[subscriptionId]?.add(notificationSubscriber) + } finally { + readWriteLock.writeLock().unlock() + } + } + + /** + * This method unsubscribes the subscriber from the subscriber cache + * by removing the Map[subscriptionId, NotificationSubscriber] + * entry from cache but keeps the susbscriptionRule in subscription + * cache for any other existing subscriber needs. + * + * @param subscriptionId String + * @return Boolean + */ + fun unsubscribeSubscriber(subscriptionId: String): Boolean { + if (subscriberCache.containsKey(subscriptionId)) { + val subscribers = subscriberCache[subscriptionId]?.filter { it.subscriptionId == subscriptionId }.orEmpty().toMutableList() + + readWriteLock.writeLock().lock() + try { + subscriberCache.remove(subscriptionId, subscribers) + } finally { + readWriteLock.writeLock().unlock() + } + return true + } else { + logger.info("Subscription $subscriptionId doesn't exist ") + throw BadStateException("Subscription doesn't exist") + } + } + + fun getSubscriptionId(ruleId: String) : String? { + if (subscriptionRuleCache.containsKey(ruleId)) { + return subscriptionRuleCache[ruleId] + } + return null + } + + fun getSubscriptionDetails(subscriptionId: String) : List? { + if (subscriberCache.containsKey(subscriptionId)) { + return subscriberCache[subscriptionId] + } + return emptyList() + } + + + +} \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/cache/InMemoryCacheService.kt b/pstatus-notifications-ktor/src/main/kotlin/cache/InMemoryCacheService.kt new file mode 100644 index 00000000..164ec402 --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/cache/InMemoryCacheService.kt @@ -0,0 +1,77 @@ +package gov.cdc.ocio.processingstatusnotifications.cache + +import gov.cdc.ocio.processingstatusnotifications.* +import gov.cdc.ocio.processingstatusnotifications.exception.* +import gov.cdc.ocio.processingstatusnotifications.model.cache.* + + +/** + * This class is a service that interacts with InMemory Cache in order to subscribe/unsubscribe users + */ +class InMemoryCacheService { + + /** + * This method creates a hash of the rule keys (dataStreamId, stageName, dataStreamRoute, statusType) + * to use as a key for SubscriptionRuleCache and creates a new or existing subscription (if exist) + * and creates a new entry in subscriberCache for the user with the susbscriptionRuleKey + * + * @param dataStreamId String + * @param dataStreamRoute String + * @param stageName String + * @param statusType String + * @param emailOrUrl String + * @param subscriptionType SubscriptionType + * @return String + */ + fun updateNotificationsPreferences( + dataStreamId: String, + dataStreamRoute: String, + stageName: String, + statusType: String, + emailOrUrl: String, + subscriptionType: SubscriptionType + ): String { + try { + val subscriptionRule = SubscriptionRule(dataStreamId, dataStreamRoute, stageName, statusType) + val subscriptionId = + InMemoryCache.updateCacheForSubscription(subscriptionRule.getStringHash(), subscriptionType, emailOrUrl) + return subscriptionId + } catch (e: BadStateException) { + throw e + } + + } + + /** + * This method removes subscriber from subscription rule. + * If the rule doesn't exist then it throws BadStateException + * + * @param subscriptionId String + * @return Boolean + */ + fun unsubscribeNotifications(subscriptionId: String): Boolean { + try { + return InMemoryCache.unsubscribeSubscriber(subscriptionId) + } catch (e: BadStateException) { + throw e + } + } + + /** + * This methods checks for subscription rule and gets the subscriptionId. + * In turn uses the subscription Id to retrieve the NotificationSubscription details + * @param ruleId String + * @return Boolean + */ + fun getSubscription(ruleId: String): List { + try { + val subscriptionId = InMemoryCache.getSubscriptionId(ruleId) + if (subscriptionId != null) { + return InMemoryCache.getSubscriptionDetails(subscriptionId).orEmpty() + } + return emptyList() + } catch (e: BadStateException) { + throw e + } + } +} diff --git a/pstatus-notifications-ktor/src/main/kotlin/dispatcher/EmailDispatcher.kt b/pstatus-notifications-ktor/src/main/kotlin/dispatcher/EmailDispatcher.kt new file mode 100644 index 00000000..9b06bc04 --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/dispatcher/EmailDispatcher.kt @@ -0,0 +1,86 @@ +package gov.cdc.ocio.processingstatusnotifications.dispatcher + + +import gov.cdc.ocio.processingstatusnotifications.model.cache.* + +import mu.KotlinLogging + +import java.io.BufferedReader +import java.io.InputStreamReader +import java.net.Socket +import javax.mail.MessagingException +import javax.mail.Session +import kotlin.math.log + +/** + * EMail dispatcher implements IDispatcher which will implement code to send out emails + * to the subscribers of teh rules (if the rule matches in rule engine evaluation phase) + * @property logger KLogger + */ +class EmailDispatcher(): IDispatcher { + private val logger = KotlinLogging.logger {} + override fun dispatchEvent(subscription: NotificationSubscription): String { + + // Call the function to check SMTP status + return if(checkSMTPStatusWithoutCredentials(subscription)) { + sendEmail(subscription) + logger.info { "Email sent successfully" } + "Email sent successfully" + } else { + logger.info { "Error occurred while sending email" } + "Email server not reachable" + } + + } + + /** + * Method to check teh status of the SMTP server + * @param subscription NotificationSubscription + * @return String + */ + private fun checkSMTPStatusWithoutCredentials(subscription: NotificationSubscription): Boolean { + // This is to get the status from curl statement to see if server is connected + try { + // TODO : Uncomment this later +// val smtpServer = System.getenv("SmtpHostServer") +// val port = System.getenv("SmtpHostPort").toInt() + + val smtpServer = "smtpgw.cdc.gov" + val port = 25 + + val socket = Socket(smtpServer, port) + val reader = BufferedReader(InputStreamReader(socket.getInputStream())) + // Read the server response + val response = reader.readLine() + println("Server response: $response") + // Close the socket + socket.close() + return response !=null + } catch (e: Exception) { + e.printStackTrace() + } + return false + } + + /** + * Method to send email + * @param subscription NotificationSubscription + * @return String + */ + private fun sendEmail(subscription: NotificationSubscription): Unit { + + try{ + // TODO : Change this into properties + val toEmalId = subscription.subscriberAddressOrUrl; + val props = System.getProperties() + props["mail.smtp.host"] = "smtpgw.cdc.gov" + props["mail.smtp.port"] = 25 + val session = Session.getInstance(props, null) + EmailUtil.sendEmail(session, toEmalId, "${System.getenv("ReplyToName")} : Notification for report", "You have signed up for this notification subscription, please check the portal") + } catch(_: MessagingException) { + logger.info("Unable to send email") + } + + } + +} \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/dispatcher/EmailUtil.kt b/pstatus-notifications-ktor/src/main/kotlin/dispatcher/EmailUtil.kt new file mode 100644 index 00000000..6106b475 --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/dispatcher/EmailUtil.kt @@ -0,0 +1,46 @@ +package gov.cdc.ocio.processingstatusnotifications.dispatcher + + +import java.util.* +import javax.mail.Message +import javax.mail.Session +import javax.mail.Transport +import javax.mail.internet.InternetAddress +import javax.mail.internet.MimeMessage + +object EmailUtil { + /** + * Utility method to send simple HTML email + * @param session + * @param toEmail + * @param subject + * @param body + */ + fun sendEmail(session: Session?, toEmail: String?, subject: String?, body: String?) { + try { + val msg = MimeMessage(session) + // TODO: Uncomment this later +// val replyToEmail = System.getenv("ReplyToEmail") +// val replyToName = System.getenv("ReplyToName") + + val replyToEmail = "donotreply@cdc.gov" + val replyToName = "DoNOtReply (DEX Team)" + //set message headers + msg.addHeader("Content-type", "text/HTML; charset=UTF-8") + msg.addHeader("format", "flowed") + msg.addHeader("Content-Transfer-Encoding", "8bit") + + //TODO - Change the from and replyTo address after the new licensed account is created + // Get the email addresses from the property + msg.setFrom(InternetAddress(replyToEmail, replyToName)) + msg.replyTo = InternetAddress.parse(replyToEmail, false) + msg.setSubject(subject, "UTF-8") + msg.setText(body, "UTF-8") + msg.sentDate = Date() + msg.setRecipients(Message.RecipientType.TO, InternetAddress.parse(toEmail, false)) + Transport.send(msg) + } catch (e: Exception) { + e.printStackTrace() + } + } +} \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/dispatcher/IDispatcher.kt b/pstatus-notifications-ktor/src/main/kotlin/dispatcher/IDispatcher.kt new file mode 100644 index 00000000..5a5b04b6 --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/dispatcher/IDispatcher.kt @@ -0,0 +1,8 @@ +package gov.cdc.ocio.processingstatusnotifications.dispatcher + +import gov.cdc.ocio.processingstatusnotifications.model.cache.* + +interface IDispatcher { + + fun dispatchEvent(subscription: NotificationSubscription): String +} \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/exception/BadRequestException.kt b/pstatus-notifications-ktor/src/main/kotlin/exception/BadRequestException.kt new file mode 100644 index 00000000..0a19251e --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/exception/BadRequestException.kt @@ -0,0 +1,9 @@ +package gov.cdc.ocio.processingstatusnotifications.exception + +/** + * Intended use of this exception is for bad requests, such as a record could not be located because an invalid + * identifier was provided. Or, if a required parameter is missing for a request. + * + * @constructor + */ +class BadRequestException(message: String): Exception(message) \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/exception/BadStateException.kt b/pstatus-notifications-ktor/src/main/kotlin/exception/BadStateException.kt new file mode 100644 index 00000000..8819d58e --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/exception/BadStateException.kt @@ -0,0 +1,9 @@ +package gov.cdc.ocio.processingstatusnotifications.exception + +/** + * Intended use of this exception is for internal server issues where we expect to be in a certain state or have + * internal state information that is missing or invalid. + * + * @constructor + */ +class BadStateException(message: String): Exception(message) \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/exception/ContentException.kt b/pstatus-notifications-ktor/src/main/kotlin/exception/ContentException.kt new file mode 100644 index 00000000..370172c7 --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/exception/ContentException.kt @@ -0,0 +1,8 @@ +package gov.cdc.ocio.processingstatusnotifications.exception + +/** + * Intended use of this exception is for content issues. + * + * @constructor + */ +class ContentException(message: String): Exception(message) \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/exception/InvalidSchemaDefException.kt b/pstatus-notifications-ktor/src/main/kotlin/exception/InvalidSchemaDefException.kt new file mode 100644 index 00000000..8a88539d --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/exception/InvalidSchemaDefException.kt @@ -0,0 +1,8 @@ +package gov.cdc.ocio.processingstatusnotifications.exception + +/** + * Intended use of this exception is for invalid schema definitions. + * + * @constructor + */ +class InvalidSchemaDefException(message: String): Exception(message) \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/model/SubscriptionResult.kt b/pstatus-notifications-ktor/src/main/kotlin/model/SubscriptionResult.kt new file mode 100644 index 00000000..ec3ceac6 --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/model/SubscriptionResult.kt @@ -0,0 +1,6 @@ +package gov.cdc.ocio.processingstatusnotifications + +enum class SubscriptionType { + EMAIL, + WEBSOCKET +} \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/model/cache/NotificationSubscription.kt b/pstatus-notifications-ktor/src/main/kotlin/model/cache/NotificationSubscription.kt new file mode 100644 index 00000000..cd5e21c1 --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/model/cache/NotificationSubscription.kt @@ -0,0 +1,8 @@ +package gov.cdc.ocio.processingstatusnotifications.model.cache + +import gov.cdc.ocio.processingstatusnotifications.* + +class NotificationSubscription(val subscriptionId: String, + val subscriberAddressOrUrl: String, + val subscriberType: SubscriptionType +) \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/model/cache/SubscriptionRule.kt b/pstatus-notifications-ktor/src/main/kotlin/model/cache/SubscriptionRule.kt new file mode 100644 index 00000000..be3430b7 --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/model/cache/SubscriptionRule.kt @@ -0,0 +1,33 @@ +package gov.cdc.ocio.processingstatusnotifications.model.cache + +class SubscriptionRule(val dataStreamId: String, + val dataStreamRoute: String, + val stageName: String, + val statusType: String) { + + override fun hashCode(): Int { + var result = dataStreamId.lowercase().hashCode() + result = 31 * result + dataStreamRoute.lowercase().hashCode() + result = 31 * result + stageName.lowercase().hashCode() + result = 31 * result + statusType.lowercase().hashCode() + return result + } + + fun getStringHash(): String { + return Integer.toHexString(hashCode()) + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as SubscriptionRule + + if (dataStreamId != other.dataStreamId) return false + if (dataStreamRoute != other.dataStreamRoute) return false + if (stageName != other.stageName) return false + if (statusType != other.statusType) return false + + return true + } +} \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/model/message/ReportNotificationServiceBusMessage.kt b/pstatus-notifications-ktor/src/main/kotlin/model/message/ReportNotificationServiceBusMessage.kt new file mode 100644 index 00000000..b06e72ab --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/model/message/ReportNotificationServiceBusMessage.kt @@ -0,0 +1,57 @@ +package gov.cdc.ocio.processingstatusnotifications.model.message + +import com.google.gson.Gson +import com.google.gson.annotations.SerializedName +import com.google.gson.reflect.TypeToken +import gov.cdc.ocio.processingstatusnotifications.exception.BadStateException +import java.lang.ClassCastException +import java.util.* + +/** + * Create a report service bus message. + * + * @property uploadId String? + * @property dataStreamId String? + * @property dataStreamRoute String? + * @property stageName String? + * @property contentType String? + * @property content String? + */ +class ReportNotificationServiceBusMessage { + + @SerializedName("upload_id") + val uploadId: String? = null + + @SerializedName("data_stream_id") + val dataStreamId: String? = null + + @SerializedName("data_stream_route") + val dataStreamRoute: String? = null + + @SerializedName("stage_name") + val stageName: String? = null + + @SerializedName("content_type") + val contentType: String? = null + + // content will vary depending on content_type so make it any. For example, if content_type is json then the + // content type will be a Map<*, *>. + val content: Any? = null + + val contentAsString: String? + get() { + if (content == null) return null + + return when (contentType?.lowercase(Locale.getDefault())) { + "json" -> { + val typeObject = object : TypeToken?>() {}.type + try { + Gson().toJson(content as Map<*, *>, typeObject) + } catch (e: ClassCastException) { + throw BadStateException("content_type indicates json, but the content is not in JSON format") + } + } + else -> content.toString() + } + } +} \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/model/message/SchemaDefinition.kt b/pstatus-notifications-ktor/src/main/kotlin/model/message/SchemaDefinition.kt new file mode 100644 index 00000000..80ed85c7 --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/model/message/SchemaDefinition.kt @@ -0,0 +1,83 @@ +package gov.cdc.ocio.processingstatusnotifications.model.message + +import com.google.gson.Gson +import com.google.gson.annotations.SerializedName +import gov.cdc.ocio.processingstatusnotifications.exception.InvalidSchemaDefException + +/** + * Schema definition for all stages. Every stage must inherit this class. + * + * @property schemaName String? + * @property schemaVersion String? + * @constructor + */ +open class SchemaDefinition(@SerializedName("schema_name") var schemaName: String? = null, + @SerializedName("schema_version") var schemaVersion: String? = null, + @Transient private val priority: Int = 0) : + Comparable { + + /** + * Hash operator + * + * @return Int + */ + override fun hashCode(): Int { + var result = schemaName?.hashCode() ?: 0 + result = 31 * result + (schemaVersion?.hashCode() ?: 0) + return result + } + + /** + * Compare operator + * + * @param other SchemaDefinition + * @return Int + */ + override fun compareTo(other: SchemaDefinition): Int { + return compareValues(priority, other.priority) + } + + /** + * Equals operator + * + * @param other Any? + * @return Boolean + */ + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as SchemaDefinition + + if (schemaName != other.schemaName) return false + if (schemaVersion != other.schemaVersion) return false + + return true + } + + /** + * Return the schema definition as a human-readable string. + * + * @return String + */ + override fun toString(): String { + return "SchemaDefinition(schemaName=$schemaName, schemaVersion=$schemaVersion)" + } + + companion object { + + @Throws(InvalidSchemaDefException::class) + fun fromJsonString(jsonContent: String?): SchemaDefinition { + if (jsonContent == null) throw InvalidSchemaDefException("Missing schema definition") + + val schemaDefinition = Gson().fromJson(jsonContent, SchemaDefinition::class.java) + if (schemaDefinition?.schemaName.isNullOrEmpty()) + throw InvalidSchemaDefException("Invalid schema_name provided") + + if (schemaDefinition.schemaVersion.isNullOrEmpty()) + throw InvalidSchemaDefException("Invalid schema_version provided") + + return schemaDefinition + } + } +} diff --git a/pstatus-notifications-ktor/src/main/kotlin/notifications/SubscribeEmailNotifications.kt b/pstatus-notifications-ktor/src/main/kotlin/notifications/SubscribeEmailNotifications.kt new file mode 100644 index 00000000..e7bb0904 --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/notifications/SubscribeEmailNotifications.kt @@ -0,0 +1,90 @@ +package gov.cdc.ocio.processingstatusnotifications.notifications + +import gov.cdc.ocio.processingstatusnotifications.EmailSubscription +import gov.cdc.ocio.processingstatusnotifications.SubscriptionResult +import gov.cdc.ocio.processingstatusnotifications.SubscriptionType +import gov.cdc.ocio.processingstatusnotifications.cache.InMemoryCacheService +import mu.KotlinLogging +import java.time.Instant + +/** + * This method is used by graphL endpoints to subscribe for Webhook notifications + * * based on rules sent in required parameters/arguments + * dataStreamId + * dataStreamRoute + * email + * stageName + * statusType ("warning", "success", "error") + * + */ +class SubscribeEmailNotifications{ + private val logger = KotlinLogging.logger {} + private val cacheService: InMemoryCacheService = InMemoryCacheService() + + /** + * The function which validates and subscribes for email notifications + * @param subscription EmailSubscription + */ + fun run(subscription: EmailSubscription): + SubscriptionResult { + val dataStreamId = subscription.dataStreamId + val dataStreamRoute = subscription.dataStreamRoute + val email = subscription.email + val stageName = subscription.stageName + val statusType = subscription.statusType + + logger.debug("dataStreamId: $dataStreamId") + logger.debug("dataStreamRoute: $dataStreamRoute") + logger.debug("Subscription Email Id: $email") + logger.debug("StageName: $stageName") + logger.debug("StatusType: $statusType") + + val subscriptionResult = subscribeForEmail(dataStreamId, dataStreamRoute, email, stageName, statusType) + if (subscriptionResult.subscription_id == null) { + subscriptionResult.message = "Invalid Request" + subscriptionResult.status = false + } + return subscriptionResult + } + + /** + * This function validates and updates the notification preferences of the cacheService + * @param dataStreamId String + * @param dataStreamRoute String + * @param email String + * @param stageName String + * @param statusType String + */ + private fun subscribeForEmail( + dataStreamId: String, + dataStreamRoute: String, + email: String?, + stageName: String?, + statusType: String? + ): SubscriptionResult { + val result = SubscriptionResult() + if (dataStreamId.isBlank() + || dataStreamRoute.isBlank() + || email.isNullOrBlank() + || stageName.isNullOrBlank() + || statusType.isNullOrBlank()) { + result.status = false + result.message = "Required fields not sent in request" + } else if (!email.contains('@') || email.split(".").size > 2 || !email.matches(Regex("([a-zA-Z0-9_%-]+@[a-zA-Z0-9-]+\\.[a-zA-Z]{2,})\$"))) { + result.status = false + result.message = "Not valid email address" + } else if (!(statusType == "success" || statusType == "warning" || statusType == "error")) { + result.status = false + result.message = "Not valid email address" + } else { + result.subscription_id = cacheService.updateNotificationsPreferences(dataStreamId, dataStreamRoute, stageName, statusType, email, + SubscriptionType.EMAIL + ) + result.timestamp = Instant.now().epochSecond + result.status = true + result.message = "Subscription for Email setup" + } + + return result + } +} \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/notifications/SubscribeWebhookNotifications.kt b/pstatus-notifications-ktor/src/main/kotlin/notifications/SubscribeWebhookNotifications.kt new file mode 100644 index 00000000..002680e8 --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/notifications/SubscribeWebhookNotifications.kt @@ -0,0 +1,100 @@ +package gov.cdc.ocio.processingstatusnotifications.notifications + +import gov.cdc.ocio.processingstatusnotifications.SubscriptionResult +import gov.cdc.ocio.processingstatusnotifications.SubscriptionType +import gov.cdc.ocio.processingstatusnotifications.WebhookSubscription +import gov.cdc.ocio.processingstatusnotifications.cache.InMemoryCacheService +import mu.KotlinLogging +import java.time.Instant + +/** +* This method is used by graphL endpoints to subscribe for Webhook notifications + * based on rules sent in required parameters/arguments + * dataStreamId + * dataStreamRoute + * stageName + * statusType ("warning", "success", "error") + * url (websocket url) + * + * + + * @property logger KLogger + * @property cacheService InMemoryCacheService + * @constructor + */ +class SubscribeWebhookNotifications + { + private val logger = KotlinLogging.logger {} + private val cacheService: InMemoryCacheService = InMemoryCacheService() + + /** + * The function which validates and subscribes for webhook notifications + * @param subscription WebhookSubscription + */ + + fun run(subscription: WebhookSubscription): + SubscriptionResult { + + val dataStreamId = subscription.dataStreamId + val dataStreamRoute = subscription.dataStreamRoute + val url = subscription.url + val stageName = subscription.stageName + val statusType = subscription.statusType + + logger.debug("dataStreamId: $dataStreamId") + logger.debug("dataStreamRoute: $dataStreamRoute") + logger.debug("Subscription Url: $url") + logger.debug("StageName: $stageName") + logger.debug("StatusType: $statusType") + + val subscriptionResult = subscribeForWebhook(dataStreamId, dataStreamRoute, url, stageName, statusType) + if (subscriptionResult.subscription_id != null) { + subscriptionResult.message = "Subscription successful" + return subscriptionResult + } + subscriptionResult.message = "Invalid Request" + subscriptionResult.status = false + + return subscriptionResult + } + + /** + * This function validates and updates the notification preferences of the cacheService + * @param dataStreamId String + * @param dataStreamRoute String + * @param url String + * @param stageName String + * @param statusType String + */ + private fun subscribeForWebhook( + dataStreamId: String, + dataStreamRoute: String, + url: String?, + stageName: String?, + statusType: String? + ): SubscriptionResult { + + val result = SubscriptionResult() + if (dataStreamId.isBlank() + || dataStreamRoute.isBlank() + || url.isNullOrBlank() + || stageName.isNullOrBlank() + || statusType.isNullOrBlank()) { + result.status = false + result.message = "Required fields not sent in request" + } else if (!url.lowercase().startsWith("ws")) { + result.status = false + result.message = "Not valid url address" + } else if (!(statusType.equals("success", true) || statusType.equals("warning", true) || statusType.equals("failure", true))) { + result.status = false + result.message = "Not valid status" + } else { + result.subscription_id = cacheService.updateNotificationsPreferences(dataStreamId, dataStreamRoute, stageName, statusType, url, SubscriptionType.WEBSOCKET) + result.timestamp = Instant.now().epochSecond + result.status = true + result.message = "Subscription for Webhook setup" + } + + return result + } +} \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/notifications/UnsubscribeNotifications.kt b/pstatus-notifications-ktor/src/main/kotlin/notifications/UnsubscribeNotifications.kt new file mode 100644 index 00000000..9cbffa8d --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/notifications/UnsubscribeNotifications.kt @@ -0,0 +1,52 @@ +package gov.cdc.ocio.processingstatusnotifications.notifications + +import gov.cdc.ocio.processingstatusnotifications.SubscriptionResult +import gov.cdc.ocio.processingstatusnotifications.cache.InMemoryCacheService +import java.time.Instant +import mu.KotlinLogging + + +/** + + * @property logger KLogger + * @property cacheService InMemoryCacheService + * @constructor + */ +class UnSubscribeNotifications + { + private val logger = KotlinLogging.logger {} + private val cacheService: InMemoryCacheService = InMemoryCacheService() + + /** + * The function which validates and Unsubscribes for webhook notifications + * @param subscriptionId String + */ + fun run(subscriptionId: String): SubscriptionResult { + logger.debug { "SubscriptionId $subscriptionId" } + + val result = SubscriptionResult() + val unsubscribeSuccessful = unsubscribeNotifications(subscriptionId) + if (subscriptionId.isNotBlank() && unsubscribeSuccessful) { + result.subscription_id = subscriptionId + result.timestamp = Instant.now().epochSecond + result.status = false + result.message = "UnSubscription successful" + + } else { + result.status = false + result.message = "UnSubscription unsuccessful" + + } + return result + } + + /** + * Function which unsubscribes based on subscription id from the cache service + * @param subscriptionId String + */ + private fun unsubscribeNotifications( + subscriptionId: String, + ): Boolean { + return cacheService.unsubscribeNotifications(subscriptionId) + } +} \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/parser/ReportParser.kt b/pstatus-notifications-ktor/src/main/kotlin/parser/ReportParser.kt new file mode 100644 index 00000000..ec5a2a1e --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/parser/ReportParser.kt @@ -0,0 +1,114 @@ +package gov.cdc.ocio.processingstatusnotifications.parser + +import com.fasterxml.jackson.core.JsonFactory +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import gov.cdc.ocio.processingstatusnotifications.exception.ContentException +import kotlin.collections.HashMap + +/** + * Class containing util methods to parse the content of the report + * + */ +class ReportParser { + + private val reportTypeToStatusFieldMapping: MutableMap = mutableMapOf( + "dex-hl7-validation" to "current_status", + "dex-file-copy" to "result", + "dex-metadata-verify" to "issues", + "upload" to "abc" // TODO : find this + ) + + /** + * Method which parses different types of notification reports for report status + * @param content String + * @return Unit + */ + fun parseReportForStatus(content: String, reportType: String?): String { + val reportMetricCollector = HashMap() + val factory = JsonFactory() + val mapper = ObjectMapper(factory) + val rootNode: JsonNode = mapper.readTree(content) + + recurseParseHelper(rootNode, reportMetricCollector, reportType!!) + return reportMetricCollector[reportType]!! + } + + @Throws(ContentException::class) + private fun recurseParseHelper(node: JsonNode, reportMetricMap: HashMap, reportType: String) { + if (node.isNull) { + return + } + + val statusFieldName = reportTypeToStatusFieldMapping[reportType] + val fieldsIterator: Iterator> = node.fields() + while (fieldsIterator.hasNext()) { + val field: Map.Entry = fieldsIterator.next() + if (field.value.isArray) { + // Specifically for Metadata Report type to figure out the status from issues fields + if (field.key.equals(statusFieldName, true) && statusFieldName == "issues") { + if (!field.value.isArray) { + throw ContentException("Invalid content in $reportType") + } else { + val status = if (field.value.size() > 0) "failure" else "success" + reportMetricMap[reportType] = status + return + } + } + + for (element in field.value) { + recurseParseHelper(element, reportMetricMap, reportType) + } + } else if (field.value.isObject) { + recurseParseHelper(field.value, reportMetricMap, reportType) + } else { + if (field.key.equals(statusFieldName, true)) { + processStatusValueInArray(field, reportMetricMap, reportType!!) + } + } + + } + } + + private fun processStatusValueInArray( + field: Map.Entry, + reportMetricMap: HashMap, + reportType: String + ) { + val validStatusValue = getValidStatusValue(field.value.textValue()); + if (!reportMetricMap.containsKey(reportType)) { + reportMetricMap[reportType] = validStatusValue + } else { + val existingStatus = getPrecedenceOfStatus(reportMetricMap.get(reportType)) + val newStatus = getPrecedenceOfStatus(field.value.textValue()) + if (existingStatus < newStatus) { + reportMetricMap[reportType] = validStatusValue + } + } + } + + private fun getPrecedenceOfStatus(status: String?): Int { + return when (status?.lowercase()) { + "success" -> 1 + "warning" -> 2 + "failure" -> 3 + else -> 0 + } + } + + /** + * Method to convert erroneous status types to "SUCCESS", "WARNING" or "FAILURE" + * @param status String + * @return String + */ + private fun getValidStatusValue(status: String): String { + var tempStatus = status.lowercase() + return if (tempStatus == "success" || (!tempStatus.contains("invalid") && tempStatus.contains("valid"))) { + "success" + } else if (tempStatus.contains("invalid") || tempStatus.contains("error") || tempStatus.contains("failure")) { + "failure" + } else { + "warning" + } + } +} \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/rulesEngine/EmailNotificationRule.kt b/pstatus-notifications-ktor/src/main/kotlin/rulesEngine/EmailNotificationRule.kt new file mode 100644 index 00000000..8bce1d1b --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/rulesEngine/EmailNotificationRule.kt @@ -0,0 +1,41 @@ +package gov.cdc.ocio.processingstatusnotifications.rulesEngine + +import gov.cdc.ocio.processingstatusnotifications.SubscriptionType +import gov.cdc.ocio.processingstatusnotifications.cache.InMemoryCacheService +import gov.cdc.ocio.processingstatusnotifications.dispatcher.EmailDispatcher +import gov.cdc.ocio.processingstatusnotifications.model.cache.NotificationSubscription +import mu.KotlinLogging + +/** + * Class to evaluate existing rules in Datastore for email notifications. + * If matching rule exist, we can send an email using EmailDispatcher + */ +class EmailNotificationRule(): Rule { + private val logger = KotlinLogging.logger {} + + /** + * Method to evaluate existing subscription for matching rule for Email notifications + * @param ruleId String + * @param cacheService InMemoryCacheService + * @return String + */ + override fun evaluateAndDispatch(ruleId: String, cacheService: InMemoryCacheService): String { + val subscribers: List = cacheService.getSubscription(ruleId) + for(subscriber in subscribers) { + if (subscriber.subscriberType == SubscriptionType.EMAIL) { + return dispatchEvent(subscriber) + } + } + return "" + } + + /** + * Method to dispatch email to the subscriber using the information saved in Datastore + * @param subscription NotificationSubscription + * @return String + */ + override fun dispatchEvent(subscription: NotificationSubscription): String { + val emailDispatcher = EmailDispatcher() + return emailDispatcher.dispatchEvent(subscription) + } +} \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/rulesEngine/Rule.kt b/pstatus-notifications-ktor/src/main/kotlin/rulesEngine/Rule.kt new file mode 100644 index 00000000..bbc3b215 --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/rulesEngine/Rule.kt @@ -0,0 +1,10 @@ +package gov.cdc.ocio.processingstatusnotifications.rulesEngine + + +import gov.cdc.ocio.processingstatusnotifications.model.cache.NotificationSubscription +import gov.cdc.ocio.processingstatusnotifications.cache.InMemoryCacheService + +interface Rule { + fun evaluateAndDispatch(ruleId: String, cacheService: InMemoryCacheService): String + fun dispatchEvent(subscription: NotificationSubscription): String +} \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/rulesEngine/RuleEngine.kt b/pstatus-notifications-ktor/src/main/kotlin/rulesEngine/RuleEngine.kt new file mode 100644 index 00000000..1c141710 --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/rulesEngine/RuleEngine.kt @@ -0,0 +1,14 @@ +package gov.cdc.ocio.processingstatusnotifications.rulesEngine + +import gov.cdc.ocio.processingstatusnotifications.cache.InMemoryCacheService + +object RuleEngine { + private val rules = listOf(EmailNotificationRule(), WebsocketNotificationRule()) + private val cacheService = InMemoryCacheService() + + fun evaluateAllRules(ruleId: String): List { + val dispatchMsgsForTesting = mutableListOf() + for (rule in rules) dispatchMsgsForTesting += rule.evaluateAndDispatch(ruleId, cacheService) + return dispatchMsgsForTesting + } +} \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/rulesEngine/WebSocketNotificationRule.kt b/pstatus-notifications-ktor/src/main/kotlin/rulesEngine/WebSocketNotificationRule.kt new file mode 100644 index 00000000..0a8bcb4f --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/rulesEngine/WebSocketNotificationRule.kt @@ -0,0 +1,35 @@ +package gov.cdc.ocio.processingstatusnotifications.rulesEngine + +import gov.cdc.ocio.processingstatusnotifications.SubscriptionType +import gov.cdc.ocio.processingstatusnotifications.cache.InMemoryCacheService +import gov.cdc.ocio.processingstatusnotifications.model.cache.NotificationSubscription +import mu.KotlinLogging + +class WebsocketNotificationRule(): Rule { + private val logger = KotlinLogging.logger {} + /** + * Method to evaluate existing subscription for matching rule for Websocket notifications + * @param ruleId String + * @param cacheService InMemoryCacheService + * @return String + */ + override fun evaluateAndDispatch(ruleId: String, cacheService: InMemoryCacheService): String { + val subscribers: List = cacheService.getSubscription(ruleId) + for(subscriber in subscribers) { + if (subscriber.subscriberType == SubscriptionType.WEBSOCKET) { + return dispatchEvent(subscriber) + } + } + return "" + } + + /** + * Method to dispatch websocket event to the subscriber using the information saved in Datastore + * @param subscription NotificationSubscription + * @return String + */ + override fun dispatchEvent(subscription: NotificationSubscription): String { + logger.info("Websocket event dispatched for ${subscription.subscriberAddressOrUrl}") + return "Websocket Event dispatched for ${subscription.subscriberAddressOrUrl}" + } +} \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/servicebus/ReportsNotificationProcessor.kt b/pstatus-notifications-ktor/src/main/kotlin/servicebus/ReportsNotificationProcessor.kt new file mode 100644 index 00000000..cbe0779f --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/servicebus/ReportsNotificationProcessor.kt @@ -0,0 +1,87 @@ +package gov.cdc.ocio.processingstatusnotifications.servicebus + +import com.azure.messaging.servicebus.ServiceBusReceivedMessage +import com.google.gson.GsonBuilder +import com.google.gson.JsonSyntaxException +import com.google.gson.ToNumberPolicy +import gov.cdc.ocio.processingstatusnotifications.exception.BadRequestException +import gov.cdc.ocio.processingstatusnotifications.exception.BadStateException +import gov.cdc.ocio.processingstatusnotifications.exception.ContentException +import gov.cdc.ocio.processingstatusnotifications.exception.InvalidSchemaDefException +import gov.cdc.ocio.processingstatusnotifications.model.cache.SubscriptionRule +import gov.cdc.ocio.processingstatusnotifications.model.message.ReportNotificationServiceBusMessage +import gov.cdc.ocio.processingstatusnotifications.model.message.SchemaDefinition +import gov.cdc.ocio.processingstatusnotifications.parser.ReportParser +import gov.cdc.ocio.processingstatusnotifications.rulesEngine.RuleEngine + +class ReportsNotificationProcessor { + + private val logger = mu.KotlinLogging.logger {} + + // Use the LONG_OR_DOUBLE number policy, which will prevent Longs from being made into Doubles + private val gson = GsonBuilder() + .setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE) + .create() + + /** + * Process a service bus message with the provided message. + * + * @param message String + * @throws BadStateException + */ + @Throws(BadRequestException::class, InvalidSchemaDefException::class) + fun withMessage(message: ServiceBusReceivedMessage): String { + val sbMessage = message.body.toString() + try { + return sendNotificationForReportStatus(gson.fromJson(sbMessage, ReportNotificationServiceBusMessage::class.java)) + } catch (e: JsonSyntaxException) { + logger.error("Failed to parse CreateReportSBMessage: ${e.localizedMessage}") + throw BadRequestException("Unable to interpret the create report message") + } + } + + /** + * Subscribe for notifications from the provided service bus message. + * + * @param reportNotification ReportNotificationMessage + * @throws BadRequestException + */ + @Throws(BadRequestException::class,InvalidSchemaDefException::class) + private fun sendNotificationForReportStatus(reportNotification: ReportNotificationServiceBusMessage): String { + + val dataStreamId = reportNotification.dataStreamId + ?: throw BadRequestException("Missing required field data_stream_id") + + val dataStreamRoute = reportNotification.dataStreamRoute + ?: throw BadRequestException("Missing required field data_stream_route") + + val stageName = reportNotification.stageName + ?: throw BadRequestException("Missing required field stage_name") + + val contentType = reportNotification.contentType + ?: throw BadRequestException("Missing required field content_type") + + val content: String + val status: String + try { + content = reportNotification.contentAsString + ?: throw BadRequestException("Missing required field content") + val schemaDef = SchemaDefinition.fromJsonString(content) + + status = ReportParser().parseReportForStatus(content, schemaDef.schemaName) + logger.debug("Report parsed for status $status") + RuleEngine.evaluateAllRules(SubscriptionRule(dataStreamId, dataStreamRoute, stageName, status).getStringHash()) + return status.lowercase() + } catch (ex: BadStateException) { + // assume a bad request + throw BadRequestException(ex.localizedMessage) + } catch(ex: InvalidSchemaDefException) { + // assume an invalid request + throw InvalidSchemaDefException(ex.localizedMessage) + } catch(ex: ContentException) { + // assume an invalid request + throw ContentException(ex.localizedMessage) + } + } + +} \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/kotlin/servicebus/ServiceBus.kt b/pstatus-notifications-ktor/src/main/kotlin/servicebus/ServiceBus.kt new file mode 100644 index 00000000..4610732f --- /dev/null +++ b/pstatus-notifications-ktor/src/main/kotlin/servicebus/ServiceBus.kt @@ -0,0 +1,187 @@ +package gov.cdc.ocio.processingstatusnotifications.servicebus + +import com.azure.core.amqp.AmqpTransportType +import com.azure.core.amqp.exception.AmqpException +import com.azure.messaging.servicebus.* +import com.azure.messaging.servicebus.models.DeadLetterOptions +import gov.cdc.ocio.processingstatusnotifications.exception.BadRequestException +import io.ktor.server.application.* +import io.ktor.server.application.hooks.* +import io.ktor.server.config.* +import io.ktor.util.logging.* +import org.apache.qpid.proton.engine.TransportException +import java.util.concurrent.TimeUnit + +internal val LOGGER = KtorSimpleLogger("pstatus-notifications") + +/** + * Class which initializes configuration values + * @param config ApplicationConfig + * + */ +class AzureServiceBusConfiguration(config: ApplicationConfig, configurationPath: String? = null) { + private val configPath = if (configurationPath != null) "$configurationPath." else "" + val connectionString = config.tryGetString("${configPath}connection_string") ?: "" + val queueName = config.tryGetString("${configPath}queue_name") ?: "" + val topicName = config.tryGetString("${configPath}topic_name") ?: "" + val subscriptionName = config.tryGetString("${configPath}subscription_name") ?: "" +} + +val AzureServiceBus = createApplicationPlugin( + name = "AzureServiceBus", + configurationPath = "azure.service_bus", + createConfiguration = ::AzureServiceBusConfiguration) { + + val connectionString = pluginConfig.connectionString + val queueName = pluginConfig.queueName + val topicName = pluginConfig.topicName + val subscriptionName = pluginConfig.subscriptionName + + // Initialize Service Bus client for queue + val processorQueueClient by lazy { + ServiceBusClientBuilder() + .connectionString(connectionString) + .transportType(AmqpTransportType.AMQP_WEB_SOCKETS) + .processor() + .queueName(queueName) + .processMessage{ context -> processMessage(context) } + .processError { context -> processError(context) } + .buildProcessorClient() + } + + // Initialize Service Bus client for topics + val processorTopicClient by lazy { + ServiceBusClientBuilder() + .connectionString(connectionString) + .transportType(AmqpTransportType.AMQP_WEB_SOCKETS) + .processor() + .topicName(topicName) + .subscriptionName(subscriptionName) + .processMessage{ context -> processMessage(context) } + .processError { context -> processError(context) } + .buildProcessorClient() + } + + /** + * Function which starts receiving messages from queues and topics + * @throws AmqpException + * @throws TransportException + * @throws Exception generic + */ + @Throws(InterruptedException::class) + fun receiveMessages() { + try { + // Create an instance of the processor through the ServiceBusClientBuilder + println("Starting the Azure service bus processor") + println("connectionString = $connectionString, queueName = $queueName, topicName= $topicName, subscriptionName=$subscriptionName") + processorQueueClient.start() + processorTopicClient.start() + } + + catch (e:AmqpException){ + println("Non-ServiceBusException occurred : ${e.message}") + } + catch (e:TransportException){ + println("Non-ServiceBusException occurred : ${e.message}") + } + + catch (e:Exception){ + println("Non-ServiceBusException occurred : ${e.message}") + } + + } + + on(MonitoringEvent(ApplicationStarted)) { application -> + application.log.info("Server is started") + receiveMessages() + } + on(MonitoringEvent(ApplicationStopped)) { application -> + application.log.info("Server is stopped") + println("Stopping and closing the processor") + processorQueueClient.close() + processorTopicClient.close() + // Release resources and unsubscribe from events + application.environment.monitor.unsubscribe(ApplicationStarted) {} + application.environment.monitor.unsubscribe(ApplicationStopped) {} + } +} + +/** + * Function which processes the message received in the queue or topics + * @param context ServiceBusReceivedMessageContext + * @throws BadRequestException + * @throws IllegalArgumentException + * @throws Exception generic + */ +private fun processMessage(context: ServiceBusReceivedMessageContext) { + val message = context.message + + LOGGER.trace( + "Processing message. Session: {}, Sequence #: {}. Contents: {}", + message.messageId, + message.sequenceNumber, + message.body + ) + try { + ReportsNotificationProcessor().withMessage(message) + } + //This will handle all missing required fields, invalid schema definition and malformed json all under the BadRequest exception and writes to dead-letter queue or topics depending on the context + catch (e: BadRequestException) { + LOGGER.warn("Unable to parse the message: {}", e.localizedMessage) + val deadLetterOptions = DeadLetterOptions() + .setDeadLetterReason("Validation failed") + .setDeadLetterErrorDescription(e.message) + context.deadLetter(deadLetterOptions) + LOGGER.info("Message sent to the dead-letter queue.") + } + catch (e: Exception) { + LOGGER.warn("Failed to process service bus message: {}", e.localizedMessage) + } + +} + +/** + * Function to handle and process the error generated during the processing of messages from queue or topics + * @param context ServiceBusErrorContext + */ +private fun processError(context: ServiceBusErrorContext) { + System.out.printf( + "Error when receiving messages from namespace: '%s'. Entity: '%s'%n", + context.fullyQualifiedNamespace, context.entityPath + ) + if (context.exception !is ServiceBusException) { + System.out.printf("Non-ServiceBusException occurred: %s%n", context.exception) + return + } + val exception = context.exception as ServiceBusException + val reason = exception.reason + if (reason === ServiceBusFailureReason.MESSAGING_ENTITY_DISABLED || reason === ServiceBusFailureReason.MESSAGING_ENTITY_NOT_FOUND || reason === ServiceBusFailureReason.UNAUTHORIZED) { + System.out.printf( + "An unrecoverable error occurred. Stopping processing with reason %s: %s%n", + reason, exception.message + ) + } else if (reason === ServiceBusFailureReason.MESSAGE_LOCK_LOST) { + System.out.printf("Message lock lost for message: %s%n", context.exception) + } else if (reason === ServiceBusFailureReason.SERVICE_BUSY) { + try { + // Choosing an arbitrary amount of time to wait until trying again. + TimeUnit.SECONDS.sleep(1) + } catch (e: InterruptedException) { + System.err.println("Unable to sleep for period of time") + } + } else { + System.out.printf( + "Error source %s, reason %s, message: %s%n", context.errorSource, + reason, context.exception + ) + } +} + +/** + * The main application module which runs always + */ +fun Application.serviceBusModule() { + install(AzureServiceBus) { + // any additional configuration goes here + } +} diff --git a/pstatus-notifications-ktor/src/main/resources/application.conf b/pstatus-notifications-ktor/src/main/resources/application.conf new file mode 100644 index 00000000..670edb2c --- /dev/null +++ b/pstatus-notifications-ktor/src/main/resources/application.conf @@ -0,0 +1,20 @@ +ktor { + deployment { + port = 8080 + host = 0.0.0.0 + } + + application { + modules = [ gov.cdc.ocio.processingstatusnotifications.ApplicationKt.module ] + } + + version = "0.0.1" + } + +azure { + service_bus { + connection_string = ${SERVICE_BUS_CONNECTION_STRING} + topic_name = ${SERVICE_BUS_REPORT_TOPIC_NAME} + subscription_name = ${SERVICE_BUS_NOTIFICATION_TOPIC_SUBSCRIPTION_NAME} + } +} \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/main/resources/logback.xml b/pstatus-notifications-ktor/src/main/resources/logback.xml new file mode 100644 index 00000000..3e11d781 --- /dev/null +++ b/pstatus-notifications-ktor/src/main/resources/logback.xml @@ -0,0 +1,12 @@ + + + + %d{YYYY-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + \ No newline at end of file diff --git a/pstatus-notifications-ktor/src/test/kotlin/cache/InMemoryCacheServiceTest.kt b/pstatus-notifications-ktor/src/test/kotlin/cache/InMemoryCacheServiceTest.kt new file mode 100644 index 00000000..f1e01185 --- /dev/null +++ b/pstatus-notifications-ktor/src/test/kotlin/cache/InMemoryCacheServiceTest.kt @@ -0,0 +1,73 @@ +package cache + + + +import gov.cdc.ocio.processingstatusnotifications.SubscriptionType +import gov.cdc.ocio.processingstatusnotifications.cache.InMemoryCacheService +import gov.cdc.ocio.processingstatusnotifications.exception.BadStateException +import org.testng.Assert +import org.testng.annotations.Test + +class InMemoryCacheServiceTest { + + private var inMemoryCacheService: InMemoryCacheService = InMemoryCacheService() + + @Test(description = "This test asserts true for generating two unique subscriptionIds for same user") + fun testAddingSameNotificationPreferencesSuccess() { + val subscriptionId1 = inMemoryCacheService.updateNotificationsPreferences( + "destination1","dataStreamRoute1", + "stageName1","warning", + "abc@trh.com", SubscriptionType.EMAIL + ) + val subscriptionId2 = inMemoryCacheService.updateNotificationsPreferences( + "destination1","dataStreamRoute1", + "stageName1","warning", + "rty@trh.com", SubscriptionType.EMAIL + ) + Assert.assertEquals(subscriptionId1, subscriptionId2) + } + + @Test(description = "This test asserts true for generating two unique subscriptionIds for different set of rules for same user") + fun testAddingDifferentNotificationPreferencesSuccess() { + val subscriptionId1 = inMemoryCacheService.updateNotificationsPreferences( + "destination1","dataStreamRoute1", + "stageName1","warning", + "abc@trh.com", SubscriptionType.EMAIL + ) + val subscriptionId2 = inMemoryCacheService.updateNotificationsPreferences( + "destination1","dataStreamRoute1", + "stageName1","success", + "abc@trh.com", SubscriptionType.EMAIL + ) + Assert.assertNotEquals(subscriptionId1, subscriptionId2) + } + + @Test(description = "This test asserts true for unsubscribing existing susbcription") + fun testUnsubscribingSubscriptionSuccess() { + val subscriptionId1 = inMemoryCacheService.updateNotificationsPreferences( + "destination1","dataStreamRoute1", + "stageName1","warning", + "abc@trh.com", SubscriptionType.EMAIL + ) + + Assert.assertTrue(inMemoryCacheService.unsubscribeNotifications(subscriptionId1)) + } + + @Test(description = "This test throws exception for unsubscribing susbcriptionId that doesn't exist") + fun testUnsubscribingSubscriptionException() { + val subscriptionId1 = inMemoryCacheService.updateNotificationsPreferences( + "destination1","dataStreamRoute1", + "stageName1","warning", + "abc@trh.com", SubscriptionType.EMAIL + ) + // Remove subscription first + inMemoryCacheService.unsubscribeNotifications(subscriptionId1) + + try { + inMemoryCacheService.unsubscribeNotifications(subscriptionId1) + } catch (e: BadStateException) { + Assert.assertEquals(e.message, "Subscription doesn't exist") + } + } +} + diff --git a/pstatus-notifications-ktor/src/test/kotlin/cache/InMemoryCacheTest.kt b/pstatus-notifications-ktor/src/test/kotlin/cache/InMemoryCacheTest.kt new file mode 100644 index 00000000..6112057c --- /dev/null +++ b/pstatus-notifications-ktor/src/test/kotlin/cache/InMemoryCacheTest.kt @@ -0,0 +1,63 @@ +package cache + + +import gov.cdc.ocio.processingstatusnotifications.SubscriptionType +import gov.cdc.ocio.processingstatusnotifications.cache.InMemoryCache +import gov.cdc.ocio.processingstatusnotifications.exception.BadStateException +import org.testng.Assert +import org.testng.annotations.Test + +class InMemoryCacheTest { + + private var inMemoryCache: InMemoryCache = InMemoryCache + + @Test(description = "This test asserts true for generating two unique subscriptionId") + fun testTwoUniqueSubscriptionIdGenerated() { + val subscriptionId1 = inMemoryCache.generateUniqueSubscriptionId() + val subscriptionId2 = inMemoryCache.generateUniqueSubscriptionId() + Assert.assertNotEquals(subscriptionId1, subscriptionId2) + } + + @Test(description = "This test assert true for generating two unique subscriptionId for two different rules.") + fun testSuccessTwoNewRules() { + val subscriptionRule1 = "subscriptionRuleUnique1" + val subscriptionRule2 = "subscriptionRuleUnique2" + + val subscriptionId1 = inMemoryCache.updateCacheForSubscription(subscriptionRule1, SubscriptionType.EMAIL, "trr@ddf.ccc") + val subscriptionId2 = inMemoryCache.updateCacheForSubscription(subscriptionRule2, SubscriptionType.EMAIL, "trr@ddf.ccc") + Assert.assertNotEquals(subscriptionId1, subscriptionId2) + } + + @Test(description = "This test assert true for generating one unique subscriptionId for single different rules for two users") + fun testSuccessTwoSubscribersSingleRules() { + val subscriptionRule1 = "subscriptionRuleUnique1" + + val subscriptionId1 = inMemoryCache.updateCacheForSubscription(subscriptionRule1, SubscriptionType.EMAIL, "trr@ddf.ccc") + val subscriptionId2 = inMemoryCache.updateCacheForSubscription(subscriptionRule1, SubscriptionType.WEBSOCKET, "tre@ddf.ccc") + Assert.assertEquals(subscriptionId1, subscriptionId2) + } + + @Test(description = "This test assert true for unsubscribing existing subscription which exist") + fun testSuccessUnsubscribeExistingSubscription() { + val subscriptionRule1 = "subscriptionRuleUnique1" + val subscriptionId1 = inMemoryCache.updateCacheForSubscription(subscriptionRule1, SubscriptionType.EMAIL, "trr@ddf.ccc") + + Assert.assertTrue(inMemoryCache.unsubscribeSubscriber(subscriptionId1)) + } + + @Test(description = "This test assert false for unsubscribing subscription which doesn't exist") + fun testFailureUnsubscribeSubscription() { + val subscriptionRule1 = "subscriptionRuleUnique1" + val subscriptionId1 = inMemoryCache.updateCacheForSubscription(subscriptionRule1, SubscriptionType.EMAIL, "trr@ddf.ccc") + var exceptionThrown = false + // Delete once so the subscription for this user doesn't exist + inMemoryCache.unsubscribeSubscriber(subscriptionId1) + try { + inMemoryCache.unsubscribeSubscriber(subscriptionId1) + } catch(ex: BadStateException) { + exceptionThrown = true + } + Assert.assertTrue(exceptionThrown) + + } +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/README.md b/pstatus-report-sink-ktor/README.md index a5e6d231..c7b44e8e 100644 --- a/pstatus-report-sink-ktor/README.md +++ b/pstatus-report-sink-ktor/README.md @@ -1,15 +1,221 @@ # Overview -This project is the processing status report sink. It listens for messages on an Azure Service bus, validates the messages, and if validated persists them to CosmosDB. +Reports are an essential component of the data observability aspect of the CDC Data Exchange (DEX). In DEX, data is ingested to the system typically through a file upload. As the upload progresses through the service line processing occurs. The processing in the service line is made up stages, which can be the upload, routing, data validation, data transformations, etc. Within each of those stages one or more actions may occur. Taking the example of upload, one action within the stage may be to first verify that all the required metadata associated with the uploaded file is provided and reject it if not. Other upload actions may include the file upload itself or the disposition of the upload for further downstream processing. Reports are provided by both services internal to DEX and downstream of DEX as data moves through CDC systems. Those services indicate the processing status of these stages through Reports. + +## Report Sinking +This project is the processing status report sink. It listens for messages on an Azure Service bus queues and topics, validates the messages, and if validated persists them to CosmosDB. If the validation fails due to missing fields or malformed data, then the message is persisted in cosmosdb under a new dead-letter container and the message is also sent to the dead-letter queue under the configured topic subscription(if the message was processed using the topic listener) This is a microservice built using Ktor that can be built as a docker container image. -## Publish to CDC's imagehub +# Publish to CDC's ImageHub With one gradle command you can builds and publish the project's Docker container image to the external container registry, imagehub, which is a nexus repository. -To do this, we use Google [jib](https://cloud.google.com/java/getting-started/jib), which vastly simplies the build process as you don't need a docker daemon running in order to build the Docker container image. +To do this, we use Google [jib](https://cloud.google.com/java/getting-started/jib), which vastly simplifies the build process as you don't need a docker daemon running in order to build the Docker container image. Inside of build.gradle `jib` section are the authentication settings. These use environment variables for the username and password, which are `IMAGEHUB_USERNAME` and `IMAGEHUB_PASSWORD` respectively. ```commandline gradle jib ``` -The location of the deployment will be to the `docker-dev2` repository under the folder `/v2/dex/pstatus`. \ No newline at end of file +The location of the deployment will be to the `docker-dev2` repository under the folder `/v2/dex/pstatus`. + +# Report Delivery Mechanisms +Reports may be provided in one of two ways - either through calls into the Processing Status (PS) API as GraphQL mutations or by way of an Azure Service Bus. There are pros and cons of each summarized below. + +| Azure Service Bus | GraphQL | +|---------------------|--------------------------| +| Fire and forget [1] | Confirmation of delivery | +| Fast | Slower | + +[1] Failed reports are sent to a Report deadletter that can be queried to find out the reason(s) for its rejection. When using ASB there is no direct feedback mechanism to the report provider of the rejection. + +### GraphQL Mutations +GraphQL mutations are writes to a persisted object. In the case of PS API, reports are written to PS API as GraphQL mutations. + +For context, GraphQL does not require any special client and can be communicated to the same as REST endpoints. However, unlike REST there is only one endpoint with path `/graphql` and you POST to it. The main difference is in the request body of the POST. Below is an example of how a Report would be sent to PS API. + +**POST** `{{ps_api_base_url}}/graphql` + +Request body: +```graphql +mutation AddReport($report: Report!) { + addReport(report: $report) { + reportId + result + issues + } +} +``` +In this example, we are asking for the `reportId` of the added report be returned in the response. + +Response of accepted report: +```json +{ + "data": { + "addReport": { + "reportId": "47286e48-2a22-4e26-930e-c7b4115b0cf1", + "result": "SUCCESS", + "issues": null + } + } +} +``` +Response of rejected report: +```json +{ + "data": { + "addReport": { + "reportId": null, + "result": "FAILURE", + "issues": [ + "Missing required field, dex_ingest_datetime" + ] + } + } +} +``` +> NOTE: With GraphQL, every HTTP status code returned is a 200 unless the request is unauthorized or something fails on the server. Clients must inspect the `result` field to determine success. + +There will also be a mutation available to replace an existing report. + +### Azure Service Bus +Reports may be sent to the PS API Azure Service Bus (ASB) queue or topic. Below is an example code snippet in Kotlin. + +```kotlin +val report = MyDEXReport().apply { + // set the report fields +} + +val senderClient = ServiceBusClientBuilder() + .connectionString(sbConnString) + .sender() + .topicName(topicName) + .buildClient() + +// Send the report to the PS API report topic +senderClient.sendMessage(ServiceBusMessage(report)) +``` + +> Sending reports via the PS API ASB report **queue** is being deprecated. The PS API report queue will eventually be removed. Reports should be sent to the PS API report **topic** instead. + +In order to access the ASB from your DEX service there may need be a firewall rule put in place. If your service is running in Kubernetes then no firewall rule should be necessary. + +# Checking on Reports +GraphQL queries are available to look for reports, whether they were accepted by PS API or not. If a report can't be ingested, typically due to failed validations, then it will go to deadletter. The deadletter'd reports can be searched for and the reason(s) for its failure examined. + +The following queries will provide all reports for the given upload ID, whether accepted or not (sent to deadletter): +```graphql +query GetReports($uploadId: String!) { + getReports(uploadId: $uploadId, + reportsSortedBy: "timestamp", + sortOrder: Ascending) { + id + uploadId + dataStreamId + dataStreamRoute + messageId + reportId + stageName + status + timestamp + content + contentType + } + getDeadLetterReportsByUploadId(uploadId: $uploadId) { + id + uploadId + dataStreamId + dataStreamRoute + dispositionType + messageId + reportId + stageName + status + timestamp + validationSchemas + deadLetterReasons + contentType + content + } +} +``` +A response with both a failed report a successful report may look like the following. + +```json +{ + "data": { + "getReports": [ + { + "id": "42b1a3a9-77d6-4436-87fd-a0f07de[SchemaHelper.kt](..%2F..%2F..%2F..%2F..%2FDownloads%2Fapollo-kotlin-main%2Flibraries%2Fapollo-tooling%2Fsrc%2Fmain%2Fkotlin%2Fcom%2Fapollographql%2Fapollo%2Ftooling%2FSchemaHelper.kt)8e861", + "uploadId": "e4361c73-348b-46f2-aad8-3043f8922f1d", + "dataStreamId": "dex-testing", + "dataStreamRoute": "test-event1", + "messageId": null, + "reportId": "42b1a3a9-77d6-4436-87fd-a0f07de8e861", + "stageName": null, + "status": null, + "timestamp": "2024-05-25T17:47:24.360Z", + "contentType": "application/json", + "content": { + "schema_version": "0.0.1", + "metadata": { + "meta_field1": "value1" + }, + "filename": "some_upload1.csv", + "schema_name": "dex-metadata-verify", + "issues": null + } + } + ], + "getDeadLetterReportsByUploadId": [ + { + "id": "392c0eed-4401-494a-8d38-f00a7431ca0f", + "uploadId": "e4361c73-348b-46f2-aad8-3043f8922f1d", + "dataStreamId": "dex-testing", + "dataStreamRoute": null, + "dispositionType": "ADD", + "messageId": null, + "reportId": "392c0eed-4401-494a-8d38-f00a7431ca0f", + "stageName": null, + "status": null, + "timestamp": "2024-07-24T22:08:08.568Z", + "deadLetterReasons": [ + "$.data_stream_route: is missing but it is required", + "JSON is invalid against the content schema base.1.0.0.schema.json.Filename(s) used for validation: base.1.0.0.schema.json" + ], + "validationSchemas": [ + "base.1.0.0.schema.json" + ], + "contentType": "application/json", + "content": { + "report": { + "ingested_file_path": "https://ocioedemessagesatst.blob.core.windows.net/hl7ingress/dex-routing/dex-smoke-test_319101ba2fd7835983f3257713819f7b", + "ingested_file_timestamp": "2024-07-10T15:40:09+00:00", + "ingested_file_size": 10240, + "received_filename": "dex-smoke-test", + "supporting_metadata": { + "meta_ext_source": "test-src", + "meta_ext_filestatus": "test-file-status", + "meta_ext_file_timestamp": "test-timestamp", + "system_provider": "DEX-ROUTING", + "meta_ext_uploadid": "test-upload-id", + "meta_ext_objectkey": "test-obj-key", + "reporting_jurisdiction": "unknown" + }, + "aggregation": "SINGLE", + "number_of_messages": 1, + "number_of_messages_not_propagated": 1, + "error_messages": [ + { + "message_uuid": "378d6660-903f-40b8-b48b-80f9d77aa69c", + "message_index": 1, + "error_message": "No valid message found." + } + ] + }, + "content_schema_name": "hl7v2-debatch", + "content_schema_version": "1.0.0" + } + } + ] + } +} +``` diff --git a/pstatus-report-sink-ktor/build.gradle b/pstatus-report-sink-ktor/build.gradle index 7383d277..d3744a38 100644 --- a/pstatus-report-sink-ktor/build.gradle +++ b/pstatus-report-sink-ktor/build.gradle @@ -36,6 +36,8 @@ configurations { dependencies { implementation("io.ktor:ktor-server-core-jvm") implementation("io.ktor:ktor-server-netty-jvm") + implementation "io.ktor:ktor-server-content-negotiation:$ktor_version" + implementation "io.ktor:ktor-serialization-jackson:$ktor_version" implementation("ch.qos.logback:logback-classic:$logback_version") implementation("com.azure:azure-messaging-servicebus:7.13.3") implementation("com.azure:azure-cosmos:4.55.0") @@ -45,6 +47,56 @@ dependencies { implementation("io.insert-koin:koin-ktor:3.5.6") testImplementation("io.ktor:ktor-server-tests-jvm") testImplementation("org.jetbrains.kotlin:kotlin-test-junit:$kotlin_version") + + implementation "com.microsoft.azure.functions:azure-functions-java-library:3.0.0" + implementation 'com.microsoft.azure:applicationinsights-core:3.4.19' + implementation 'com.azure:azure-cosmos:4.55.0' + implementation "com.fasterxml.jackson.module:jackson-module-kotlin:2.14.2" + implementation 'io.opentelemetry:opentelemetry-api:1.29.0' + implementation 'io.opentelemetry:opentelemetry-sdk:1.29.0' + implementation 'io.opentelemetry:opentelemetry-exporter-logging:1.29.0' + implementation 'io.opentelemetry:opentelemetry-exporter-otlp:1.29.0' + implementation 'io.opentelemetry:opentelemetry-semconv:1.29.0-alpha' + implementation 'com.google.code.gson:gson:2.10.1' + implementation group: 'io.github.microutils', name: 'kotlin-logging-jvm', version: '3.0.5' + implementation group: 'org.slf4j', name: 'slf4j-api', version: '1.7.36' + implementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.3.11' + implementation group: 'ch.qos.logback.contrib', name: 'logback-json-classic', version: '0.1.5' + implementation group: 'ch.qos.logback.contrib', name: 'logback-jackson', version: '0.1.5' + + implementation 'com.azure:azure-messaging-servicebus:7.15.0' + implementation 'com.microsoft.azure:azure-servicebus:3.6.7' + implementation 'com.azure:azure-identity:1.8.0' + + implementation 'org.danilopianini:khttp:1.3.1' + implementation group: 'com.google.code.gson', name: 'gson', version: '2.8.9' + + // JSON validations + implementation("com.networknt:json-schema-validator:1.0.73") + implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.12.+") + + implementation 'org.owasp.encoder:encoder:1.2.3' + implementation group: 'org.apache.commons', name: 'commons-lang3', version: '3.3.1' + + agent "io.opentelemetry.javaagent:opentelemetry-javaagent:1.29.0" + + testImplementation("org.mockito.kotlin:mockito-kotlin:4.0.0") + testImplementation "org.testng:testng:7.4.0" + implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version" + testImplementation "org.mockito:mockito-inline:3.11.2" + testImplementation "io.mockk:mockk:1.13.9" + + +} +test { + useTestNG() + testLogging { + events "passed", "skipped", "failed" + } + //Change this to "true" if we want to execute unit tests + systemProperty("isTestEnvironment", "false") + + // Set the test classpath, if required } jib { diff --git a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/Application.kt b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/Application.kt index 489b3c56..693039a2 100644 --- a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/Application.kt +++ b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/Application.kt @@ -1,36 +1,62 @@ package gov.cdc.ocio.processingstatusapi +import gov.cdc.ocio.processingstatusapi.cosmos.CosmosConfiguration +import gov.cdc.ocio.processingstatusapi.cosmos.CosmosDeadLetterRepository import gov.cdc.ocio.processingstatusapi.cosmos.CosmosRepository +import gov.cdc.ocio.processingstatusapi.plugins.AzureServiceBusConfiguration import gov.cdc.ocio.processingstatusapi.plugins.configureRouting import gov.cdc.ocio.processingstatusapi.plugins.serviceBusModule +import io.ktor.serialization.jackson.* import io.ktor.server.application.* import io.ktor.server.engine.* import io.ktor.server.netty.* +import io.ktor.server.plugins.contentnegotiation.* import org.koin.core.KoinApplication import org.koin.dsl.module import org.koin.ktor.plugin.Koin -import org.koin.mp.KoinPlatform.getKoin + +/** + * Load the environment configuration values + * Instantiate a singleton CosmosDatabase container instance + * @param environment ApplicationEnvironment + */ fun KoinApplication.loadKoinModules(environment: ApplicationEnvironment): KoinApplication { val cosmosModule = module { val uri = environment.config.property("azure.cosmos_db.client.endpoint").getString() val authKey = environment.config.property("azure.cosmos_db.client.key").getString() - single { CosmosRepository(uri, authKey, "Reports", "/uploadId") } + single(createdAtStart = true) { CosmosRepository(uri, authKey, "Reports", "/uploadId") } + single(createdAtStart = true) { CosmosDeadLetterRepository(uri, authKey, "Reports-DeadLetter", "/uploadId") } + + // Create a CosmosDB config that can be dependency injected (for health checks) + single(createdAtStart = true) { CosmosConfiguration(uri, authKey) } + } + val asbConfigModule = module { + // Create an azure service bus config that can be dependency injected (for health checks) + single(createdAtStart = true) { AzureServiceBusConfiguration(environment.config, configurationPath = "azure.service_bus") } } - return modules(listOf(cosmosModule)) + + return modules(listOf(cosmosModule, asbConfigModule)) } +/** + * The main function + * @param args Array + */ fun main(args: Array) { embeddedServer(Netty, commandLineEnvironment(args)).start(wait = true) } +/** + * The main application module which always runs and loads other modules + */ fun Application.module() { configureRouting() serviceBusModule() install(Koin) { loadKoinModules(environment) } - - // Preload the koin module so the CosmosDB client is already initialized on the first call - getKoin().get() + install(ContentNegotiation) { + jackson() + } } diff --git a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/HealthCheck.kt b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/HealthCheck.kt new file mode 100644 index 00000000..376427f2 --- /dev/null +++ b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/HealthCheck.kt @@ -0,0 +1,162 @@ +package gov.cdc.ocio.processingstatusapi + +import com.azure.core.exception.ResourceNotFoundException +import com.azure.messaging.servicebus.administration.ServiceBusAdministrationClientBuilder +import com.microsoft.azure.servicebus.primitives.ServiceBusException +import gov.cdc.ocio.processingstatusapi.cosmos.CosmosClientManager +import gov.cdc.ocio.processingstatusapi.cosmos.CosmosConfiguration +import gov.cdc.ocio.processingstatusapi.plugins.AzureServiceBusConfiguration +import mu.KotlinLogging +import org.koin.core.component.KoinComponent +import org.koin.core.component.inject +import kotlin.system.measureTimeMillis + + +/** + * Abstract class used for modeling the health issues of an individual service. + * + * @property status String + * @property healthIssues String? + * @property service String + */ +abstract class HealthCheckSystem { + + var status: String = "DOWN" + + var healthIssues: String? = "" + + open val service: String = "" +} + +/** + * Concrete implementation of the cosmosdb service health check. + * + * @property service String + */ +class HealthCheckCosmosDb: HealthCheckSystem() { + override val service = "Cosmos DB" +} + +/** + * Concrete implementation of the Azure Service Bus health check. + * + * @property service String + */ +class HealthCheckServiceBus: HealthCheckSystem() { + override val service: String = "Azure Service Bus" +} + +/** + * Run health checks for the service. + * + * @property status String? + * @property totalChecksDuration String? + * @property dependencyHealthChecks MutableList + */ +class HealthCheck { + + var status: String = "DOWN" + + var totalChecksDuration: String? = null + + var dependencyHealthChecks = mutableListOf() +} + +/** + * Service for querying the health of the report-sink service and its dependencies. + * + * @property logger KLogger + * @property cosmosConfiguration CosmosConfiguration + * @property azureServiceBusConfiguration AzureServiceBusConfiguration + */ +class HealthQueryService: KoinComponent { + + private val logger = KotlinLogging.logger {} + + private val cosmosConfiguration by inject() + + private val azureServiceBusConfiguration by inject() + + /** + * Returns a HealthCheck object with the overall health of the report-sink service and its dependencies. + * + * @return HealthCheck + */ + fun getHealth(): HealthCheck { + var cosmosDBHealthy = false + var serviceBusHealthy = false + val cosmosDBHealth = HealthCheckCosmosDb() + val serviceBusHealth = HealthCheckServiceBus() + val time = measureTimeMillis { + try { + cosmosDBHealthy = isCosmosDBHealthy(config = cosmosConfiguration) + cosmosDBHealth.status = "UP" + } catch (ex: Exception) { + cosmosDBHealth.healthIssues = ex.message + logger.error("CosmosDB is not healthy: ${ex.message}") + } + + try { + serviceBusHealthy = isServiceBusHealthy(config = azureServiceBusConfiguration) + serviceBusHealth.status = "UP" + } catch (ex: Exception) { + serviceBusHealth.healthIssues = ex.message + logger.error("Azure Service Bus is not healthy: ${ex.message}") + } + } + + return HealthCheck().apply { + status = if (cosmosDBHealthy && serviceBusHealthy) "UP" else "DOWN" + totalChecksDuration = formatMillisToHMS(time) + dependencyHealthChecks.add(cosmosDBHealth) + dependencyHealthChecks.add(serviceBusHealth) + } + } + + /** + * Check whether CosmosDB is healthy. + * + * @param config CosmosConfiguration + * @return Boolean + */ + private fun isCosmosDBHealthy(config: CosmosConfiguration): Boolean { + return if (CosmosClientManager.getCosmosClient(config.uri, config.authKey) == null) + throw Exception("Failed to establish a CosmosDB client.") + else + true + } + + /** + * Check whether service bus is healthy. + * + * @return Boolean + */ + @Throws(ResourceNotFoundException::class, ServiceBusException::class) + private fun isServiceBusHealthy(config: AzureServiceBusConfiguration): Boolean { + + val adminClient = ServiceBusAdministrationClientBuilder() + .connectionString(config.connectionString) + .buildClient() + + // Get the properties of the topic to check the connection + adminClient.getTopic(config.topicName) + + return true + } + + /** + * Format the time in milliseconds to 00:00:00.000 format. + * + * @param millis Long + * @return String + */ + private fun formatMillisToHMS(millis: Long): String { + val seconds = millis / 1000 + val hours = seconds / 3600 + val minutes = (seconds % 3600) / 60 + val remainingSeconds = seconds % 60 + val remainingMillis = millis % 1000 + + return "%02d:%02d:%02d.%03d".format(hours, minutes, remainingSeconds, remainingMillis / 10) + } +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/ReportManager.kt b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/ReportManager.kt index a829f748..4ba1a496 100644 --- a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/ReportManager.kt +++ b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/ReportManager.kt @@ -3,16 +3,17 @@ package gov.cdc.ocio.processingstatusapi import com.azure.cosmos.models.CosmosItemRequestOptions import com.azure.cosmos.models.CosmosQueryRequestOptions import com.azure.cosmos.models.PartitionKey +import com.google.gson.Gson import com.google.gson.GsonBuilder import com.google.gson.ToNumberPolicy import com.google.gson.reflect.TypeToken +import gov.cdc.ocio.processingstatusapi.cosmos.CosmosDeadLetterRepository import gov.cdc.ocio.processingstatusapi.cosmos.CosmosRepository import gov.cdc.ocio.processingstatusapi.exceptions.BadRequestException import gov.cdc.ocio.processingstatusapi.exceptions.BadStateException -import gov.cdc.ocio.processingstatusapi.exceptions.InvalidSchemaDefException import gov.cdc.ocio.processingstatusapi.models.DispositionType import gov.cdc.ocio.processingstatusapi.models.Report -import gov.cdc.ocio.processingstatusapi.models.reports.SchemaDefinition +import gov.cdc.ocio.processingstatusapi.models.ReportDeadLetter import gov.cdc.ocio.processingstatusapi.models.reports.Source import mu.KotlinLogging import org.koin.core.component.KoinComponent @@ -20,6 +21,7 @@ import org.koin.core.component.inject import java.util.* import io.netty.handler.codec.http.HttpResponseStatus + /** * The report manager interacts directly with CosmosDB to persist and retrieve reports. * @@ -28,6 +30,7 @@ import io.netty.handler.codec.http.HttpResponseStatus class ReportManager: KoinComponent { private val cosmosRepository by inject() + private val cosmosDeadLetterRepository by inject() private val logger = KotlinLogging.logger {} @@ -59,20 +62,33 @@ class ReportManager: KoinComponent { contentType: String, messageId: String?, status: String?, - content: String, + content: Any?, dispositionType: DispositionType, source: Source ): String { // Verify the content contains the minimum schema information - try { + /* try { SchemaDefinition.fromJsonString(content) } catch(e: InvalidSchemaDefException) { throw BadRequestException("Invalid schema definition: ${e.localizedMessage}") } catch(e: Exception) { throw BadRequestException("Malformed message: ${e.localizedMessage}") + }*/ + if (System.getProperty("isTestEnvironment") != "true") { + return createReport( + uploadId, + dataStreamId, + dataStreamRoute, + stageName, + contentType, + messageId, + status, + content, + dispositionType, + source + ) } - - return createReport(uploadId, dataStreamId, dataStreamRoute, stageName, contentType, messageId, status, content, dispositionType, source) + return uploadId // this is just as a fallback } /** @@ -96,7 +112,7 @@ class ReportManager: KoinComponent { contentType: String, messageId: String?, status: String?, - content: String, + content: Any?, dispositionType: DispositionType, source: Source): String { @@ -105,14 +121,14 @@ class ReportManager: KoinComponent { logger.info("Replacing report(s) with stage name = $stageName") // Delete all stages matching the report ID with the same stage name val sqlQuery = "select * from ${reportMgrConfig.reportsContainerName} r where r.uploadId = '$uploadId' and r.stageName = '$stageName'" - val items = cosmosRepository.reportsContainer.queryItems( + val items = cosmosRepository.reportsContainer?.queryItems( sqlQuery, CosmosQueryRequestOptions(), Report::class.java ) if ((items?.count() ?: 0) > 0) { try { items?.forEach { - cosmosRepository.reportsContainer.deleteItem( + cosmosRepository.reportsContainer?.deleteItem( it.id, PartitionKey(it.uploadId), CosmosItemRequestOptions() @@ -154,7 +170,7 @@ class ReportManager: KoinComponent { contentType: String, messageId: String?, status: String?, - content: String, + content: Any?, source: Source): String { val stageReportId = UUID.randomUUID().toString() val stageReport = Report().apply { @@ -170,79 +186,198 @@ class ReportManager: KoinComponent { if (contentType.lowercase() == "json") { val typeObject = object : TypeToken?>() {}.type - val jsonMap: Map = gson.fromJson(content, typeObject) + val jsonMap: Map = gson.fromJson(Gson().toJson(content, MutableMap::class.java).toString(), typeObject) + this.content = jsonMap + } else + this.content = content + } + return createReportItem(uploadId,stageReportId,stageReport) + } + + /** + * Creates a dead-letter report if there is a malformed data or missing required fields + * + * @param uploadId String + * @param dataStreamId String + * @param dataStreamRoute String + * @param stageName String + * @param contentType String + * @param content String + * @return String + * @throws BadStateException + */ + + @Throws(BadStateException::class) + fun createDeadLetterReport(uploadId: String?, + dataStreamId: String?, + dataStreamRoute: String?, + stageName: String?, + dispositionType: DispositionType, + contentType: String?, + content: Any?, + deadLetterReasons: List, + validationSchemaFileNames:List + ): String { + + val deadLetterReportId = UUID.randomUUID().toString() + val deadLetterReport = ReportDeadLetter().apply { + this.id = deadLetterReportId + this.uploadId = uploadId + this.reportId = deadLetterReportId + this.dataStreamId = dataStreamId + this.dataStreamRoute = dataStreamRoute + this.stageName= stageName + this.dispositionType= dispositionType.toString() + this.contentType = contentType + this.deadLetterReasons= deadLetterReasons + this.validationSchemas= validationSchemaFileNames + if (contentType?.lowercase() == "json" && !isNullOrEmpty(content) && !isBase64Encoded(content.toString())) { + val typeObject = object : TypeToken?>() {}.type + val jsonMap: Map = gson.fromJson(Gson().toJson(content, MutableMap::class.java).toString(), typeObject) this.content = jsonMap } else this.content = content } + return createReportItem(uploadId,deadLetterReportId,deadLetterReport) + } + + /** + * Creates a dead-letter report if there is a malformed JSON. This is called if DISABLE_VALIDATION is true + * + * @param deadLetterReason String + * @return String + * @throws BadStateException + */ + + @Throws(BadStateException::class) + fun createDeadLetterReport(deadLetterReason: String): String { + val deadLetterReportId = UUID.randomUUID().toString() + val deadLetterReport = ReportDeadLetter().apply { + this.id = deadLetterReportId + this.deadLetterReasons = listOf(deadLetterReason) + } + return createReportItem(null,deadLetterReportId,deadLetterReport) + } + + /** + * The function which calculates the interval after which the retry should occur + * @param attempt Int + */ + private fun getCalculatedRetryDuration(attempt: Int): Long { + return DEFAULT_RETRY_INTERVAL_MILLIS * (attempt + 1) + } + /** Function to check whether the value is null or empty based on its type + * @param value Any + */ + private fun isNullOrEmpty(value: Any?): Boolean { + return when (value) { + null -> true + is String -> value.isEmpty() + is Collection<*> -> value.isEmpty() + is Map<*, *> -> value.isEmpty() + else -> false // You can adjust this to your needs + } + } + + /** + * The function which checks whether the passed string is Base64 Encoded or not using Regex + * @param value String + */ + private fun isBase64Encoded(value: String): Boolean { + val base64Pattern = "^[A-Za-z0-9+/]+={0,2}$" + return value.matches(base64Pattern.toRegex()) + } + + /** + * The common function which writes to cosmos container based on the report type + * @param uploadId String + * @param reportId String + * @reportType Any + */ + + private fun createReportItem(uploadId: String?, reportId:String, reportType:Any) : String{ + var responseReportId = "" + var reportTypeName = "Report" + var statusCode:Int? = null + var isValidResponse = false + var recommendedDuration:String?= null var attempts = 0 + do { try { - val response = cosmosRepository.reportsContainer.createItem( - stageReport, - PartitionKey(uploadId), - CosmosItemRequestOptions() - ) + //use when here to determing whether report type is StageReport or DeadLetterReport + when (reportType) { + is ReportDeadLetter -> { + val response = cosmosDeadLetterRepository.reportsDeadLetterContainer?.createItem( + reportType, + PartitionKey(uploadId), + CosmosItemRequestOptions()) - logger.info("Creating report, response http status code = ${response?.statusCode}, attempt = ${attempts + 1}, uploadId = $uploadId") - if (response != null) { - when (response.statusCode) { - HttpResponseStatus.OK.code(), HttpResponseStatus.CREATED.code() -> { - logger.info("Created report with reportId = ${response.item?.reportId}, uploadId = $uploadId") - val enableReportForwarding = System.getenv("EnableReportForwarding") -// if (enableReportForwarding.equals("True", ignoreCase = true)) { -// // Send message to reports-notifications-queue -// val message = NotificationReport( -// response.item?.reportId, -// uploadId, dataStreamId, dataStreamRoute, -// stageName, -// contentType, -// content, -// messageId, -// status, -// source -// ) -// reportMgrConfig.serviceBusSender.sendMessage(ServiceBusMessage(message.toString())) -// } - return stageReportId - } + isValidResponse = response!=null + reportTypeName ="dead-letter report" + responseReportId = response?.item?.reportId ?: "0" + statusCode = response?.statusCode + recommendedDuration = response?.responseHeaders?.get("x-ms-retry-after-ms") + } - HttpResponseStatus.TOO_MANY_REQUESTS.code() -> { - // See: https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/performance-tips?tabs=trace-net-core#429 - // https://learn.microsoft.com/en-us/rest/api/cosmos-db/common-cosmosdb-rest-response-headers - // https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/troubleshoot-request-rate-too-large?tabs=resource-specific - val recommendedDuration = response.responseHeaders["x-ms-retry-after-ms"] - logger.warn("Received 429 (too many requests) from cosmosdb, attempt ${attempts + 1}, will retry after $recommendedDuration millis, uploadId = $uploadId") - val waitMillis = recommendedDuration?.toLong() - Thread.sleep(waitMillis ?: DEFAULT_RETRY_INTERVAL_MILLIS) - } + is Report -> { + val response = cosmosRepository.reportsContainer?.createItem( + reportType, + PartitionKey(uploadId), + CosmosItemRequestOptions()) + + isValidResponse = response!=null + responseReportId = response?.item?.reportId ?: "0" + statusCode = response?.statusCode + recommendedDuration = response?.responseHeaders?.get("x-ms-retry-after-ms") - else -> { - // Need to retry regardless - val retryAfterDurationMillis = getCalculatedRetryDuration(attempts) - logger.warn("Received response code ${response.statusCode}, attempt ${attempts + 1}, will retry after $retryAfterDurationMillis millis, uploadId = $uploadId") - Thread.sleep(retryAfterDurationMillis) - } } - } else { - val retryAfterDurationMillis = getCalculatedRetryDuration(attempts) - logger.warn("Received null response from cosmosdb, attempt ${attempts + 1}, will retry after $retryAfterDurationMillis millis, uploadId = $uploadId") - Thread.sleep(retryAfterDurationMillis) + } - } catch (e: Exception) { + logger.info("Creating ${reportTypeName}, response http status code = ${statusCode}, attempt = ${attempts + 1}, uploadId = $uploadId") + if(isValidResponse){ + + when (statusCode) { + HttpResponseStatus.OK.code(), HttpResponseStatus.CREATED.code() -> { + logger.info("Created report with reportId = ${responseReportId}, uploadId = $uploadId") + return reportId + } + + HttpResponseStatus.TOO_MANY_REQUESTS.code() -> { + // See: https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/performance-tips?tabs=trace-net-core#429 + // https://learn.microsoft.com/en-us/rest/api/cosmos-db/common-cosmosdb-rest-response-headers + // https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/troubleshoot-request-rate-too-large?tabs=resource-specific + + logger.warn("Received 429 (too many requests) from cosmosdb, attempt ${attempts + 1}, will retry after $recommendedDuration millis, uploadId = $uploadId") + val waitMillis = recommendedDuration?.toLong() + Thread.sleep(waitMillis ?: DEFAULT_RETRY_INTERVAL_MILLIS) + } + + else -> { + // Need to retry regardless + val retryAfterDurationMillis = getCalculatedRetryDuration(attempts) + logger.warn("Received response code ${statusCode}, attempt ${attempts + 1}, will retry after $retryAfterDurationMillis millis, uploadId = $uploadId") + Thread.sleep(retryAfterDurationMillis) + } + } + } + else { + val retryAfterDurationMillis = getCalculatedRetryDuration(attempts) + logger.warn("Received null response from cosmosdb, attempt ${attempts + 1}, will retry after $retryAfterDurationMillis millis, uploadId = $uploadId") + Thread.sleep(retryAfterDurationMillis) + } + } + catch (e: Exception) { val retryAfterDurationMillis = getCalculatedRetryDuration(attempts) logger.error("CreateReport: Exception: ${e.localizedMessage}, attempt ${attempts + 1}, will retry after $retryAfterDurationMillis millis, uploadId = $uploadId") Thread.sleep(retryAfterDurationMillis) } - } while (attempts++ < MAX_RETRY_ATTEMPTS) - throw BadStateException("Failed to create reportId = ${stageReport.reportId}, uploadId = $uploadId") - } - private fun getCalculatedRetryDuration(attempt: Int): Long { - return DEFAULT_RETRY_INTERVAL_MILLIS * (attempt + 1) + } while (attempts++ < MAX_RETRY_ATTEMPTS) + throw BadStateException("Failed to create dead-letterReport reportId = ${responseReportId}, uploadId = $uploadId") } companion object { diff --git a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/ReportManagerConfig.kt b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/ReportManagerConfig.kt index c899bf2b..56abb9d4 100644 --- a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/ReportManagerConfig.kt +++ b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/ReportManagerConfig.kt @@ -8,5 +8,6 @@ package gov.cdc.ocio.processingstatusapi */ class ReportManagerConfig { val reportsContainerName = "Reports" + val reportsDeadLetterContainerName = "Reports-DeadLetter" private val partitionKey = "/uploadId" } \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosClientManager.kt b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosClientManager.kt index 5e1854c7..4b3e7ba1 100644 --- a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosClientManager.kt +++ b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosClientManager.kt @@ -3,24 +3,50 @@ package gov.cdc.ocio.processingstatusapi.cosmos import com.azure.cosmos.ConsistencyLevel import com.azure.cosmos.CosmosClient import com.azure.cosmos.CosmosClientBuilder +import kotlinx.coroutines.* +import java.time.Duration class CosmosClientManager { companion object { private var client: CosmosClient? = null - fun getCosmosClient(uri: String, authKey: String): CosmosClient { + /** + * Establishes a connection to the CosmosDB and returns a client + * + * @param uri String + * @param authKey String + * @return CosmosClient? + */ + @OptIn(DelicateCoroutinesApi::class) + fun getCosmosClient(uri: String, authKey: String): CosmosClient? { // Initialize a connection to cosmos that will persist across HTTP triggers if (client == null) { - client = CosmosClientBuilder() - .endpoint(uri) - .key(authKey) - .consistencyLevel(ConsistencyLevel.EVENTUAL) - .contentResponseOnWriteEnabled(true) - .clientTelemetryEnabled(false) - .buildClient() + return try { + var d: Deferred? = null + GlobalScope.launch { + d = async { + CosmosClientBuilder() + .endpoint(uri) + .key(authKey) + .consistencyLevel(ConsistencyLevel.EVENTUAL) + .gatewayMode() + .contentResponseOnWriteEnabled(true) + .clientTelemetryEnabled(false) + .buildClient() + } + } + runBlocking { + withTimeout(Duration.ofSeconds(10).toMillis()) { + client = d?.await() + } // wait with timeout + } + client + } catch (ex: TimeoutCancellationException) { + null + } } - return client!! + return client } } } \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosConfiguration.kt b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosConfiguration.kt new file mode 100644 index 00000000..e4e62d30 --- /dev/null +++ b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosConfiguration.kt @@ -0,0 +1,10 @@ +package gov.cdc.ocio.processingstatusapi.cosmos + +/** + * CosmosDB client configuration + * + * @property uri String + * @property authKey String + * @constructor + */ +data class CosmosConfiguration(val uri: String, val authKey: String) \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosContainerManager.kt b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosContainerManager.kt index 0e214e44..20013173 100644 --- a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosContainerManager.kt +++ b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosContainerManager.kt @@ -12,35 +12,50 @@ class CosmosContainerManager { companion object { + /** + * Function which creates the Cosmos db if not exists , and returns the db instance with which we can get a container instance + * @param cosmosClient CosmosClient + * @param databaseName String + */ @Throws(Exception::class) fun createDatabaseIfNotExists(cosmosClient: CosmosClient, databaseName: String): CosmosDatabase? { val logger = KotlinLogging.logger {} logger.info("Create database $databaseName if not exists...") // Create database if not exists - val databaseResponse = cosmosClient.createDatabaseIfNotExists(databaseName) - return cosmosClient.getDatabase(databaseResponse.properties.id) + //Alternate way is : - return cosmosClient.getDatabase(databaseName) + val databaseResponse = cosmosClient.createDatabaseIfNotExists(databaseName) + return cosmosClient.getDatabase(databaseResponse.properties.id) } + /** + * The function which creates the cosmos container instance + * @param uri String + * @param authKey String + * @param containerName String + * @param partitionKey String + */ fun initDatabaseContainer(uri: String, authKey: String, containerName: String, partitionKey: String): CosmosContainer? { val logger = KotlinLogging.logger {} try { logger.info("calling getCosmosClient...") val cosmosClient = CosmosClientManager.getCosmosClient(uri, authKey) - // setup database - logger.info("calling createDatabaseIfNotExists...") - val db = createDatabaseIfNotExists(cosmosClient, "ProcessingStatus")!! + cosmosClient?.run { + // setup database + logger.info("calling createDatabaseIfNotExists...") + val db = createDatabaseIfNotExists(cosmosClient, "ProcessingStatus") - val containerProperties = CosmosContainerProperties(containerName, partitionKey) + val containerProperties = CosmosContainerProperties(containerName, partitionKey) - // Provision throughput - val throughputProperties = ThroughputProperties.createAutoscaledThroughput(1000) + // Provision throughput + val throughputProperties = ThroughputProperties.createAutoscaledThroughput(1000) - // Create container with 1000 RU/s - logger.info("calling createContainerIfNotExists...") - val databaseResponse = db.createContainerIfNotExists(containerProperties, throughputProperties) + // Create container with 1000 RU/s + logger.info("calling createContainerIfNotExists...") + val databaseResponse = db?.createContainerIfNotExists(containerProperties, throughputProperties) - return db.getContainer(databaseResponse.properties.id) + return db?.getContainer(databaseResponse?.properties?.id) + } } catch (ex: CosmosException) { logger.error("exception: ${ex.localizedMessage}") diff --git a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosRepository.kt b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosRepository.kt index 28953ba8..79028294 100644 --- a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosRepository.kt +++ b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/cosmos/CosmosRepository.kt @@ -1,9 +1,27 @@ package gov.cdc.ocio.processingstatusapi.cosmos -import org.koin.core.component.KoinComponent - -class CosmosRepository(uri: String, authKey: String, reportsContainerName: String, partitionKey: String): KoinComponent { - - val reportsContainer = CosmosContainerManager.initDatabaseContainer(uri, authKey, reportsContainerName, partitionKey)!! +/** + * The class which initializes and creates an instance of a cosmos db reports container + * @param uri :String + * @param authKey:String + * @param reportsContainerName:String + * @param partitionKey:String + * + */ +class CosmosRepository(uri: String, authKey: String, reportsContainerName: String, partitionKey: String) { + val reportsContainer = + CosmosContainerManager.initDatabaseContainer(uri, authKey, reportsContainerName, partitionKey) +} +/** + * The class which initializes and creates an instance of a cosmos db reports deadletter container + * @param uri :String + * @param authKey:String + * @param reportsContainerName:String + * @param partitionKey:String + * + */ +class CosmosDeadLetterRepository(uri: String, authKey: String, reportsContainerName: String, partitionKey: String) { + val reportsDeadLetterContainer = + CosmosContainerManager.initDatabaseContainer(uri, authKey, reportsContainerName, partitionKey) } \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/DeadLetterReport.kt b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/DeadLetterReport.kt new file mode 100644 index 00000000..5016dcef --- /dev/null +++ b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/DeadLetterReport.kt @@ -0,0 +1,28 @@ +package gov.cdc.ocio.processingstatusapi.models + + +import com.google.gson.annotations.SerializedName + + +/** + * Dead-LetterReport when there is missing fields or malformed data. + * + * @property uploadId String? + * @property reportId String? + * @property dataStreamId String? + * @property dataStreamRoute String? + * @property dispositionType DispositionType? + * @property timestamp Date + * @property contentType String? + * @property content String? + * @property deadLetterReasons List + */ + class ReportDeadLetter : Report() { + + @SerializedName("disposition_type") + var dispositionType: String? = null + + var deadLetterReasons: List? = null + + var validationSchemas: List? = null +} diff --git a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/Report.kt b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/Report.kt index 697e438e..bd61e51e 100644 --- a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/Report.kt +++ b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/Report.kt @@ -1,6 +1,6 @@ package gov.cdc.ocio.processingstatusapi.models -import com.google.gson.* + import com.google.gson.annotations.SerializedName import java.util.* @@ -18,7 +18,7 @@ import java.util.* * @property content String? * @property timestamp Date */ -data class Report( +open class Report( var id : String? = null, @@ -49,19 +49,4 @@ data class Report( var content: Any? = null, val timestamp: Date = Date() -) { - val contentAsString: String? - get() { - if (content == null) return null - - return when (contentType?.lowercase(Locale.getDefault())) { - "json" -> { - if (content is LinkedHashMap<*, *>) - Gson().toJson(content, MutableMap::class.java).toString() - else - content.toString() - } - else -> content.toString() - } - } -} +) diff --git a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/reports/CreateReportSBMessage.kt b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/reports/CreateReportSBMessage.kt index 1b127713..dc32c817 100644 --- a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/reports/CreateReportSBMessage.kt +++ b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/reports/CreateReportSBMessage.kt @@ -1,12 +1,8 @@ package gov.cdc.ocio.processingstatusapi.models.reports -import com.google.gson.Gson import com.google.gson.annotations.SerializedName -import com.google.gson.reflect.TypeToken -import gov.cdc.ocio.processingstatusapi.exceptions.BadStateException import gov.cdc.ocio.processingstatusapi.models.ServiceBusMessage -import java.lang.ClassCastException -import java.util.* + /** * Create a report service bus message. @@ -43,22 +39,7 @@ class CreateReportSBMessage: ServiceBusMessage() { // content will vary depending on content_type so make it any. For example, if content_type is json then the // content type will be a Map<*, *>. - val content: Any? = null + var content: Any? = null - val contentAsString: String? - get() { - if (content == null) return null - return when (contentType?.lowercase(Locale.getDefault())) { - "json" -> { - val typeObject = object : TypeToken?>() {}.type - try { - Gson().toJson(content as Map<*, *>, typeObject) - } catch (e: ClassCastException) { - throw BadStateException("content_type indicates json, but the content is not in JSON format") - } - } - else -> content.toString() - } - } } \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/reports/SchemaDefinition.kt b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/reports/SchemaDefinition.kt index 11d41bce..450e640a 100644 --- a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/reports/SchemaDefinition.kt +++ b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/reports/SchemaDefinition.kt @@ -67,10 +67,9 @@ open class SchemaDefinition(@SerializedName("schema_name") var schemaName: Strin companion object { @Throws(InvalidSchemaDefException::class) - fun fromJsonString(jsonContent: String?): SchemaDefinition { + fun fromJsonString(jsonContent: Any?): SchemaDefinition { if (jsonContent == null) throw InvalidSchemaDefException("Missing schema definition") - - val schemaDefinition = Gson().fromJson(jsonContent, SchemaDefinition::class.java) + val schemaDefinition= Gson().fromJson(Gson().toJson(jsonContent, MutableMap::class.java).toString(), SchemaDefinition::class.java) if (schemaDefinition?.schemaName.isNullOrEmpty()) throw InvalidSchemaDefException("Invalid schema_name provided") diff --git a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/Routing.kt b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/Routing.kt index d93c52d7..dbb7919c 100644 --- a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/Routing.kt +++ b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/Routing.kt @@ -1,13 +1,23 @@ package gov.cdc.ocio.processingstatusapi.plugins +import gov.cdc.ocio.processingstatusapi.HealthQueryService import io.ktor.server.application.* import io.ktor.server.response.* import io.ktor.server.routing.* +/** + * Configures the REST routing endpoints for the application. + * + * @receiver Application + */ fun Application.configureRouting() { + val version = environment.config.propertyOrNull("ktor.version")?.getString() ?: "unknown" routing { - get("/") { - call.respondText("Hello World!") + get("/health") { + call.respond(HealthQueryService().getHealth()) + } + get("/version") { + call.respondText(version) } } } diff --git a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/ServiceBus.kt b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/ServiceBus.kt index fe716d63..48a87990 100644 --- a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/ServiceBus.kt +++ b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/ServiceBus.kt @@ -1,18 +1,30 @@ package gov.cdc.ocio.processingstatusapi.plugins +import com.azure.core.amqp.AmqpTransportType +import com.azure.core.amqp.exception.AmqpException import com.azure.messaging.servicebus.* +import com.azure.messaging.servicebus.models.DeadLetterOptions import gov.cdc.ocio.processingstatusapi.exceptions.BadRequestException import io.ktor.server.application.* import io.ktor.server.application.hooks.* import io.ktor.server.config.* import io.ktor.util.logging.* +import org.apache.qpid.proton.engine.TransportException import java.util.concurrent.TimeUnit internal val LOGGER = KtorSimpleLogger("pstatus-report-sink") -class AzureServiceBusConfiguration(config: ApplicationConfig) { - var connectionString: String = config.tryGetString("connection_string") ?: "" - var queueName: String = config.tryGetString("queue_name") ?: "" +/** + * Class which initializes configuration values + * @param config ApplicationConfig + * + */ +class AzureServiceBusConfiguration(config: ApplicationConfig, configurationPath: String? = null) { + private val configPath = if (configurationPath != null) "$configurationPath." else "" + val connectionString = config.tryGetString("${configPath}connection_string") ?: "" + val queueName = config.tryGetString("${configPath}queue_name") ?: "" + val topicName = config.tryGetString("${configPath}topic_name") ?: "" + val subscriptionName = config.tryGetString("${configPath}subscription_name") ?: "" } val AzureServiceBus = createApplicationPlugin( @@ -22,10 +34,14 @@ val AzureServiceBus = createApplicationPlugin( val connectionString = pluginConfig.connectionString val queueName = pluginConfig.queueName + val topicName = pluginConfig.topicName + val subscriptionName = pluginConfig.subscriptionName - val processorClient by lazy { + // Initialize Service Bus client for queue + val processorQueueClient by lazy { ServiceBusClientBuilder() .connectionString(connectionString) + .transportType(AmqpTransportType.AMQP_WEB_SOCKETS) .processor() .queueName(queueName) .processMessage{ context -> processMessage(context) } @@ -33,13 +49,46 @@ val AzureServiceBus = createApplicationPlugin( .buildProcessorClient() } - // handles received messages + // Initialize Service Bus client for topics + val processorTopicClient by lazy { + ServiceBusClientBuilder() + .connectionString(connectionString) + .transportType(AmqpTransportType.AMQP_WEB_SOCKETS) + .processor() + .topicName(topicName) + .subscriptionName(subscriptionName) + .processMessage{ context -> processMessage(context) } + .processError { context -> processError(context) } + .buildProcessorClient() + } + + /** + * Function which starts receiving messages from queues and topics + * @throws AmqpException + * @throws TransportException + * @throws Exception generic + */ @Throws(InterruptedException::class) fun receiveMessages() { - // Create an instance of the processor through the ServiceBusClientBuilder - println("Starting the Azure service bus processor") - println("connectionString = $connectionString, queueName = $queueName") - processorClient.start() + try { + // Create an instance of the processor through the ServiceBusClientBuilder + println("Starting the Azure service bus processor") + println("connectionString = $connectionString, queueName = $queueName, topicName= $topicName, subscriptionName=$subscriptionName") + processorQueueClient.start() + processorTopicClient.start() + } + + catch (e:AmqpException){ + println("Non-ServiceBusException occurred : ${e.message}") + } + catch (e:TransportException){ + println("Non-ServiceBusException occurred : ${e.message}") + } + + catch (e:Exception){ + println("Non-ServiceBusException occurred : ${e.message}") + } + } on(MonitoringEvent(ApplicationStarted)) { application -> @@ -49,15 +98,24 @@ val AzureServiceBus = createApplicationPlugin( on(MonitoringEvent(ApplicationStopped)) { application -> application.log.info("Server is stopped") println("Stopping and closing the processor") - processorClient.close() + processorQueueClient.close() + processorTopicClient.close() // Release resources and unsubscribe from events application.environment.monitor.unsubscribe(ApplicationStarted) {} application.environment.monitor.unsubscribe(ApplicationStopped) {} } } +/** + * Function which processes the message received in the queue or topics + * @param context ServiceBusReceivedMessageContext + * @throws BadRequestException + * @throws IllegalArgumentException + * @throws Exception generic + */ private fun processMessage(context: ServiceBusReceivedMessageContext) { val message = context.message + LOGGER.trace( "Processing message. Session: {}, Sequence #: {}. Contents: {}", message.messageId, @@ -65,14 +123,22 @@ private fun processMessage(context: ServiceBusReceivedMessageContext) { message.body ) try { - ServiceBusProcessor().withMessage(message.body.toString()) - } catch (e: BadRequestException) { + ServiceBusProcessor().withMessage(message) + } + //This will handle all missing required fields, invalid schema definition and malformed json all under the BadRequest exception and writes to dead-letter queue or topics depending on the context + catch (e: BadRequestException) { LOGGER.warn("Unable to parse the message: {}", e.localizedMessage) - } catch (e: Exception) { + } + catch (e: Exception) { LOGGER.warn("Failed to process service bus message: {}", e.localizedMessage) } + } +/** + * Function to handle and process the error generated during the processing of messages from queue or topics + * @param context ServiceBusErrorContext + */ private fun processError(context: ServiceBusErrorContext) { System.out.printf( "Error when receiving messages from namespace: '%s'. Entity: '%s'%n", @@ -106,6 +172,9 @@ private fun processError(context: ServiceBusErrorContext) { } } +/** + * The main application module which runs always + */ fun Application.serviceBusModule() { install(AzureServiceBus) { // any additional configuration goes here diff --git a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/ServiceBusProcessor.kt b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/ServiceBusProcessor.kt index a4c294d4..2a492072 100644 --- a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/ServiceBusProcessor.kt +++ b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/ServiceBusProcessor.kt @@ -1,5 +1,7 @@ package gov.cdc.ocio.processingstatusapi.plugins + +import com.azure.messaging.servicebus.ServiceBusReceivedMessage import com.google.gson.GsonBuilder import com.google.gson.JsonSyntaxException import com.google.gson.ToNumberPolicy @@ -9,7 +11,17 @@ import gov.cdc.ocio.processingstatusapi.exceptions.BadStateException import gov.cdc.ocio.processingstatusapi.models.reports.CreateReportSBMessage import gov.cdc.ocio.processingstatusapi.models.reports.Source import mu.KotlinLogging +import com.fasterxml.jackson.databind.ObjectMapper import java.util.* +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.networknt.schema.JsonSchema +import com.networknt.schema.JsonSchemaFactory +import com.networknt.schema.SpecVersion +import com.networknt.schema.ValidationMessage +import java.awt.datatransfer.MimeTypeParseException +import java.io.File +import javax.activation.MimeType /** * The service bus is another interface for receiving reports. @@ -33,8 +45,10 @@ class ServiceBusProcessor { * @throws BadRequestException */ @Throws(BadRequestException::class) - fun withMessage(message: String) { - var sbMessage = message + fun withMessage(message: ServiceBusReceivedMessage) { + val sbMessageId = message.messageId + var sbMessage =String(message.body.toBytes()) + val sbMessageStatus = message.state.name try { logger.info { "Before Message received = $sbMessage" } if (sbMessage.contains("destination_id")) { @@ -44,9 +58,23 @@ class ServiceBusProcessor { sbMessage = sbMessage.replace("event_type", "data_stream_route") } logger.info { "After Message received = $sbMessage" } - createReport(gson.fromJson(sbMessage, CreateReportSBMessage::class.java)) + val disableValidation = System.getenv("DISABLE_VALIDATION")?.toBoolean() ?: false + + if(disableValidation){ + val isValid = isJsonValid(sbMessage) + if(!isValid){ + sendToDeadLetter("Validation failed.The message is not in JSON format.") + } + } + else { + validateJsonSchema(message) + } + createReport(sbMessageId, sbMessageStatus, gson.fromJson(sbMessage, CreateReportSBMessage::class.java)) + } catch (e: BadRequestException) { + logger.error("Validation failed: ${e.message}") + throw e } catch (e: JsonSyntaxException) { - logger.error("Failed to parse CreateReportSBMessage: ${e.localizedMessage}") + logger.error("Failed to parse service bus message: ${e.localizedMessage}") throw BadStateException("Unable to interpret the create report message") } } @@ -58,45 +86,231 @@ class ServiceBusProcessor { * @throws BadRequestException */ @Throws(BadRequestException::class) - private fun createReport(createReportMessage: CreateReportSBMessage) { - - val uploadId = createReportMessage.uploadId - ?: throw BadRequestException("Missing required field upload_id") + private fun createReport(messageId: String, messageStatus: String, createReportMessage: CreateReportSBMessage) { + try { + val uploadId = createReportMessage.uploadId + var stageName = createReportMessage.stageName + if (stageName.isNullOrEmpty()) { + stageName = "" + } + logger.info("Creating report for uploadId = ${uploadId} with stageName = $stageName") - val dataStreamId = createReportMessage.dataStreamId - ?: throw BadRequestException("Missing required field data_stream_id") + ReportManager().createReportWithUploadId( + uploadId!!, + createReportMessage.dataStreamId!!, + createReportMessage.dataStreamRoute!!, + stageName, + createReportMessage.contentType!!, + messageId, //createReportMessage.messageId is null + messageStatus, //createReportMessage.status is null + createReportMessage.content!!, // it was Content I changed to ContentAsString + createReportMessage.dispositionType, + Source.SERVICEBUS + ) - val dataStreamRoute = createReportMessage.dataStreamRoute - ?: throw BadRequestException("Missing required field data_stream_route") + } catch (e: BadRequestException) { + throw e + } catch (e: Exception) { + logger.error("Failed to process service bus message:${e.message}") - val stageName = createReportMessage.stageName - ?: throw BadRequestException("Missing required field stage_name") + } - val contentType = createReportMessage.contentType - ?: throw BadRequestException("Missing required field content_type") + } + /** + * Function to validate report attributes for missing required fields, for schema validation and malformed content message using networknt/json-schema-validator + * @param message ServiceBusReceivedMessage + * @throws BadRequestException + */ + private fun validateJsonSchema(message: ServiceBusReceivedMessage){ + val invalidData = mutableListOf() + val schemaFileNames = mutableListOf() + var reason: String + //TODO : this needs to be replaced with more robust source or a URL of some sorts + val schemaDirectoryPath = "/schema" + // Convert the message body to a JSON string + val messageBody = String(message.body.toBytes()) + val objectMapper: ObjectMapper = jacksonObjectMapper() + var reportSchemaVersion = "0.0.1" // for backward compatibility - this schema will load if report_schema_version is not found - val content: String + // Check for the presence of `report_schema_version` try { - content = createReportMessage.contentAsString - ?: throw BadRequestException("Missing required field content") - } catch (ex: BadStateException) { - // assume a bad request - throw BadRequestException(ex.localizedMessage) + + val createReportMessage: CreateReportSBMessage = gson.fromJson(messageBody, CreateReportSBMessage::class.java) + //Convert to JSON + val jsonNode: JsonNode =objectMapper.readTree(messageBody) + // Check for the presence of `report_schema_version` + val reportSchemaVersionNode = jsonNode.get("report_schema_version") + if (reportSchemaVersionNode == null || reportSchemaVersionNode.asText().isEmpty()) { + LOGGER.info("Report schema version node missing. Backward compatibility mode enabled ") + + } else { + reportSchemaVersion = reportSchemaVersionNode.asText() + } + val fileName ="base.$reportSchemaVersion.schema.json" + val schemaFilePath = javaClass.getResource( "$schemaDirectoryPath/$fileName") + ?: throw IllegalArgumentException("File not found: $fileName") + + // Attempt to load the schema + val schemaFile = File(schemaFilePath.toURI()) + if (!schemaFile.exists()) { + reason ="Report rejected: Schema file not found for base schema version $reportSchemaVersion." + processError(fileName,reason,invalidData,schemaFileNames,createReportMessage) + } + //Validate report schema version schema + validateSchema(fileName,jsonNode,schemaFile,objectMapper,invalidData,schemaFileNames,createReportMessage) + // Check if the content_type is JSON + val contentTypeNode = jsonNode.get("content_type") + if (contentTypeNode == null) { + reason="Report rejected: `content_type` is not JSON or is missing." + processError(fileName,reason,invalidData,schemaFileNames,createReportMessage) + } + else{ + if(!isJsonMimeType(contentTypeNode.asText())) + { + //don't need to go further down if the mimetype is other than json. i.e. xml or text etc.... + return + } + } + // Open the content as JSON + val contentNode = jsonNode.get("content") + if (contentNode == null) { + reason="Report rejected: `content` is not JSON or is missing." + processError(fileName,reason,invalidData, schemaFileNames,createReportMessage) + } + // Check for `content_schema_name` and `content_schema_version` + val contentSchemaNameNode = contentNode.get("content_schema_name") + val contentSchemaVersionNode = contentNode.get("content_schema_version") + if (contentSchemaNameNode == null || contentSchemaNameNode.asText().isEmpty() || + contentSchemaVersionNode == null || contentSchemaVersionNode.asText().isEmpty() + ) { + reason= "Report rejected: `content_schema_name` or `content_schema_version` is missing or empty." + processError(fileName,reason,invalidData, schemaFileNames,createReportMessage) + } + //ContentSchema validation + val contentSchemaName = contentSchemaNameNode.asText() + val contentSchemaVersion = contentSchemaVersionNode.asText() + + //TODO Will this be from the same source??? + val contentSchemaFileName ="$contentSchemaName.$contentSchemaVersion.schema.json" + val contentSchemaFilePath =javaClass.getResource( "$schemaDirectoryPath/$contentSchemaFileName") + ?: throw IllegalArgumentException("File not found: $contentSchemaFileName") + + // Attempt to load the schema + val contentSchemaFile = File(contentSchemaFilePath.toURI()) + if (!contentSchemaFile .exists()) { + reason ="Report rejected: Content schema file not found for content schema name $contentSchemaName and schema version $contentSchemaVersion." + processError(contentSchemaFileName,reason,invalidData,schemaFileNames,createReportMessage) + } + //Validate content schema + validateSchema(contentSchemaFileName,contentNode,contentSchemaFile,objectMapper,invalidData, schemaFileNames, createReportMessage) + + } catch (e: Exception) { + LOGGER.error("Report rejected: Malformed JSON or error processing the report - ${e.message}") + throw e } + } - logger.info("Creating report for uploadId = $uploadId with stageName = $stageName") - ReportManager().createReportWithUploadId( - uploadId, - dataStreamId, - dataStreamRoute, - stageName, - contentType, - createReportMessage.messageId, - createReportMessage.status, - content, - createReportMessage.dispositionType, - Source.SERVICEBUS - ) + /** + * Function to send the invalid data reasons to the deadLetter queue + * @param invalidData MutableList + * @param createReportMessage CreateReportSBMessage + */ + private fun sendToDeadLetter(invalidData:MutableList, validationSchemaFileNames:MutableList, createReportMessage: CreateReportSBMessage){ + if (invalidData.isNotEmpty()) { + //This should not run for unit tests + if (System.getProperty("isTestEnvironment") != "true") { + // Write the content of the dead-letter reports to CosmosDb + ReportManager().createDeadLetterReport( + createReportMessage.uploadId, + createReportMessage.dataStreamId, + createReportMessage.dataStreamRoute, + createReportMessage.stageName, + createReportMessage.dispositionType, + createReportMessage.contentType, + createReportMessage.content, + invalidData, + validationSchemaFileNames + ) + } + throw BadRequestException(invalidData.joinToString(separator = ",")) + } + } + + /** + * Function to send the invalid data reasons to the deadLetter queue + * @param reason String + */ + private fun sendToDeadLetter(reason:String){ + //This should not run for unit tests + if (System.getProperty("isTestEnvironment") != "true") { + // Write the content of the dead-letter reports to CosmosDb + ReportManager().createDeadLetterReport(reason) + throw BadRequestException(reason) + } + } + + /** + * Function to process the error by logging it and adding to the invalidData list and sending it to deadletter + * @param reason String + * @param invalidData MutableList + * @param createReportMessage CreateReportSBMessage + */ + private fun processError(schemaFileName:String, reason:String, invalidData:MutableList, validationSchemaFileNames:MutableList, + createReportMessage: CreateReportSBMessage) { + + validationSchemaFileNames.add(schemaFileName) + val updatedReason = reason + "Filename(s) used for validation: " + validationSchemaFileNames.joinToString(separator = ",") + LOGGER.error(updatedReason) + invalidData.add(updatedReason) + sendToDeadLetter(invalidData, validationSchemaFileNames, createReportMessage) + } + + /** + * Function to validate the schema based on the schema file and the json contents passed into it + * @param schemaFile String + * @param objectMapper ObjectMapper + + */ + private fun validateSchema(schemaFileName: String, jsonNode:JsonNode, schemaFile:File, objectMapper: ObjectMapper, invalidData:MutableList, + validationSchemaFileNames: MutableList, createReportMessage: CreateReportSBMessage) { + val schemaNode: JsonNode = objectMapper.readTree(schemaFile) + val schemaFactory = JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V7) + val schema: JsonSchema = schemaFactory.getSchema(schemaNode) + val schemaValidationMessages: Set = schema.validate(jsonNode) + + if (schemaValidationMessages.isEmpty()) { + LOGGER.info("JSON is valid against the content schema $schema.") + } else { + val reason ="JSON is invalid against the content schema $schemaFileName." + schemaValidationMessages.forEach { invalidData.add(it.message) } + processError(schemaFileName,reason, invalidData,validationSchemaFileNames,createReportMessage) + } + } + + /** + * Function to check whether the content type is json or application/json using MimeType + * @param contentType String + */ + private fun isJsonMimeType(contentType: String): Boolean { + return try { + val mimeType = MimeType(contentType) + mimeType.primaryType == "json" || (mimeType.primaryType == "application" && mimeType.subType == "json") + } catch (e: MimeTypeParseException) { + false + } + } + + /** + * Function to check whether the message is in JSON format or not + */ + private fun isJsonValid(jsonString: String): Boolean { + return try { + val mapper = jacksonObjectMapper() + mapper.readTree(jsonString) + true + } catch (e: Exception) { + false + } } } \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/sample_reports/Report-invalid.json b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/sample_reports/Report-invalid.json new file mode 100644 index 00000000..7ff6974c --- /dev/null +++ b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/sample_reports/Report-invalid.json @@ -0,0 +1,53 @@ +{ + "upload_id": "319101ba2fd7835983f3257713819f7b", + "user_id": null, + "data_stream_id": "celr", + "data_stream_route": "hl7_out_recdeb", + "jurisdiction": "SMOKE", + "sender_id": "APHL", + "data_producer_id": "smoke-test-data-producer", + "dex_ingest_timestamp": "2024-07-10T15:40:01Z", + "message_metadata": null, + "stage_info": { + "service": "HL7v2 Pipeline", + "stage": "RECEIVER", + "version": "0.0.49-SNAPSHOT", + "status": "SUCCESS", + "issues": null, + "start_processing_time": "2024-07-10T15:40:10.162+00:00", + "end_processing_time": "2024-07-10T15:40:10.228+00:00" + }, + "tags": null, + "data": null, + "content": { + "report": { + "ingested_file_path": "https://ocioedemessagesatst.blob.core.windows.net/hl7ingress/dex-routing/dex-smoke-test_319101ba2fd7835983f3257713819f7b", + "ingested_file_timestamp": "2024-07-10T15:40:09+00:00", + "ingested_file_size": 10240, + "received_filename": "dex-smoke-test", + "supporting_metadata": { + "meta_ext_source": "test-src", + "meta_ext_filestatus": "test-file-status", + "meta_ext_file_timestamp": "test-timestamp", + "system_provider": "DEX-ROUTING", + "meta_ext_uploadid": "test-upload-id", + "meta_ext_objectkey": "test-obj-key", + "reporting_jurisdiction": "unknown" + }, + "aggregation": "SINGLE", + "number_of_messages": 1, + "number_of_messages_not_propagated": 1, + "error_messages": [ + { + "message_uuid": "378d6660-903f-40b8-b48b-80f9d77aa69c", + "message_index": 1, + "error_message": "No valid message found." + } + ] + }, + "content_schema_name": "DEX HL7v2 RECEIVER", + "content_schema_version": "2.0.0" + }, + "report_schema_version": "1.0.0", + "content_type": "application/json" +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/sample_reports/Report-valid.json b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/sample_reports/Report-valid.json new file mode 100644 index 00000000..54975357 --- /dev/null +++ b/pstatus-report-sink-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/plugins/sample_reports/Report-valid.json @@ -0,0 +1,85 @@ +{ + "upload_id": "fe85fdb7-2f07-4ad1-b73a-646cc2e25bbc", + "user_id": "test-event1", + "data_stream_id": "celr", + "data_stream_route": "hl7_out_recdeb", + "jurisdiction": "INTEGRATION", + "sender_id": "APHL", + "data_producer_id": "smoke-test-data-producer", + "dex_ingest_timestamp": "2024-07-10T15:40:01Z", + "status": "SUCCESS", + "stageName": "dex-upload", + "messageMetadata": { + "type": "object", + "properties": { + "message_uuid": { + "type": "UUID", + "description": "Unique identifier for the message associated with this report. Null if not applicable." + }, + "message_hash": { + "type": "string", + "description": "MD5 hash of the message content." + }, + "aggregation": { + "type": "string", + "enum": ["SINGLE", "BATCH"], + "description": "Enumeration: [SINGLE, BATCH]." + }, + "message_index": { + "type": "integer", + "description": "Index of the message; e.g. row if csv." + } + + } + }, + "stage_info": { + "service": "HL7v2 Pipeline", + "stage": "RECEIVER", + "version": "0.0.49-SNAPSHOT", + "status": "SUCCESS", + "issues": null, + "start_processing_time": "2024-07-10T15:40:10.162+00:00", + "end_processing_time": "2024-07-10T15:40:10.228+00:00" + }, + "tags": { + "$ref": "test-ref", + "description": "Optional tag(s) associated with this report." + }, + "data": { + "$ref": "keyValueMap", + "description": "Optional data associated with this report." + }, + + "content": { + "report": { + "ingested_file_path": "https://ocioedemessagesatst.blob.core.windows.net/hl7ingress/dex-routing/dex-smoke-test_319101ba2fd7835983f3257713819f7b", + "ingested_file_timestamp": "2024-07-10T15:40:09+00:00", + "ingested_file_size": 10240, + "received_filename": "dex-smoke-test", + "supporting_metadata": { + "meta_ext_source": "test-src", + "meta_ext_filestatus": "test-file-status", + "meta_ext_file_timestamp": "test-timestamp", + "system_provider": "DEX-ROUTING", + "meta_ext_uploadid": "test-upload-id", + "meta_ext_objectkey": "test-obj-key", + "reporting_jurisdiction": "unknown" + }, + "aggregation": "SINGLE", + "number_of_messages": 1, + "number_of_messages_not_propagated": 1, + "error_messages": [ + { + "message_uuid": "378d6660-903f-40b8-b48b-80f9d77aa69c", + "message_index": 1, + "error_message": "No valid message found." + } + ] + }, + "content_schema_name": "hl7v2-debatch", + "content_schema_version": "1.0.0" + }, + "report_schema_version": "1.0.0", + "content_type": "application/json" + +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/main/resources/application.conf b/pstatus-report-sink-ktor/src/main/resources/application.conf index 1a143ca4..7a16ce97 100644 --- a/pstatus-report-sink-ktor/src/main/resources/application.conf +++ b/pstatus-report-sink-ktor/src/main/resources/application.conf @@ -7,12 +7,16 @@ ktor { application { modules = [ gov.cdc.ocio.processingstatusapi.ApplicationKt.module ] } + + version = "0.0.5" } azure { service_bus { connection_string = ${SERVICE_BUS_CONNECTION_STRING} queue_name = ${SERVICE_BUS_REPORT_QUEUE_NAME} + topic_name = ${SERVICE_BUS_REPORT_TOPIC_NAME} + subscription_name = ${SERVICE_BUS_REPORT_TOPIC_SUBSCRIPTION_NAME} } cosmos_db { client { diff --git a/pstatus-report-sink-ktor/src/main/resources/schema/base.0.0.1.schema.json b/pstatus-report-sink-ktor/src/main/resources/schema/base.0.0.1.schema.json new file mode 100644 index 00000000..7f0e510f --- /dev/null +++ b/pstatus-report-sink-ktor/src/main/resources/schema/base.0.0.1.schema.json @@ -0,0 +1,53 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://github.com/cdcent/data-exchange-messages/reports/base", + "title": "Base Report", + "type": "object", + "required": ["upload_id", "data_stream_id", "data_stream_route", "content_type"], + "if": { + "properties": { + "content_type": { + "const": "json" + } + } + }, + "then": { + "properties": { + "content": { + "$ref": "#/$defs/jsonContent" + } + } + }, + "properties": { + "upload_id": { + "$ref": "#/$defs/uuid", + "description": "Unique upload identifier associated with this report." + }, + "data_stream_id": { + "type": "string" + }, + "data_stream_route": { + "type": "string" + }, + "content_type": { + "type": "string" + } + }, + "$defs": { + "uuid": { + "type": "string", + "format": "uuid" + }, + "jsonContent": { + "type": "object", + "properties": { + "schema_name": { + "type": "string" + }, + "schema_version": { + "type": "string" + } + } + } + } +} diff --git a/pstatus-report-sink-ktor/src/main/resources/schema/base.1.0.0.schema.json b/pstatus-report-sink-ktor/src/main/resources/schema/base.1.0.0.schema.json new file mode 100644 index 00000000..3ce6a91e --- /dev/null +++ b/pstatus-report-sink-ktor/src/main/resources/schema/base.1.0.0.schema.json @@ -0,0 +1,160 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://github.com/cdcent/data-exchange-messages/reports/base", + "title": "Base Report", + "type": "object", + "required": ["report_schema_version", "upload_id", "data_stream_id", "data_stream_route", "dex_ingest_timestamp", "sender_id", "status", "stage_info", "content_type"], + "if": { + "properties": { + "content_type": { + "const": "json" + } + } + }, + "then": { + "properties": { + "content": { + "$ref": "#/$defs/jsonContent" + } + } + }, + "properties": { + "report_schema_version": { + "type": "string" + }, + "upload_id": { + "$ref": "#/$defs/uuid", + "description": "Unique upload identifier associated with this report." + }, + "user_id": { + "type": "string", + "description": "User or system id that uploaded the file, not the provider of this report." + }, + "data_stream_id": { + "type": "string" + }, + "data_stream_route": { + "type": "string" + }, + "jurisdiction": { + "type": "string" + }, + "sender_id": { + "type": "string" + }, + "data_producer_id": { + "type": "string" + }, + "dex_ingest_timestamp": { + "$ref": "#/$defs/timestamp" + }, + "message_metadata": { + "$ref": "#/$defs/messageMetadata", + "description": "Metadata associated with the message this report belong to." + }, + "stage_info": { + "$ref": "#/$defs/stageInfo", + "description": "Describes the stage that is providing this report." + }, + "tags": { + "$ref": "#/$defs/keyValueMap", + "description": "Optional tag(s) associated with this report." + }, + "data": { + "$ref": "#/$defs/keyValueMap", + "description": "Optional data associated with this report." + }, + "content_type": { + "type": "string" + } + }, + "$defs": { + "uuid": { + "type": "string", + "format": "uuid" + }, + "timestamp": { + "type": "string", + "format": "date-time" + }, + "messageMetadata": { + "type": "object", + "properties": { + "message_uuid": { + "type": "string", + "description": "Unique identifier for the message associated with this report. Null if not applicable." + }, + "message_hash": { + "type": "string", + "description": "MD5 hash of the message content." + }, + "aggregation": { + "type": "string", + "enum": ["SINGLE", "BATCH"], + "description": "Enumeration: [SINGLE, BATCH]." + }, + "message_index": { + "type": "integer", + "description": "Index of the message; e.g. row if csv." + } + } + }, + "stageInfo": { + "service": { + "type": "string", + "description": "Name of the service associated with this report." + }, + "stage": { + "type": "string", + "description": "Action the stage was conducting when providing this report." + }, + "version": { + "type": "string", + "description": "Version of the stage providing this report" + }, + "status": { + "type": "string", + "enum": ["SUCCESS", "FAILURE"] + }, + "issues": { + "type": "array", + "items": { + "type": "object", + "properties": { + "level": { + "type": "string", + "enum": [ + "WARNING", + "ERROR" + ] + }, + "message": { + "type": "string" + } + } + } + }, + "start_processing_time": { + "$ref": "#/$defs/timestamp" + }, + "end_processing_time": { + "$ref": "#/$defs/timestamp" + } + }, + "keyValueMap": { + "type" : "object", + "existingJavaType" : "java.util.Map" + }, + "jsonContent": { + "type": "object", + "properties": { + "content_schema_name": { + "type": "string" + }, + "content_schema_version": { + "type": "string" + } + } + } + } +} diff --git a/pstatus-report-sink-ktor/src/main/resources/schema/blob-file-copy.1.0.0.schema.json b/pstatus-report-sink-ktor/src/main/resources/schema/blob-file-copy.1.0.0.schema.json new file mode 100644 index 00000000..729f85d2 --- /dev/null +++ b/pstatus-report-sink-ktor/src/main/resources/schema/blob-file-copy.1.0.0.schema.json @@ -0,0 +1,28 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://github.com/cdcent/data-exchange-messages/reports/blob-file-copy", + "title": "Blob File Copy Report", + "type": "object", + "required": ["content_schema_name", "content_schema_version", "file_source_blob_url", "file_destination_blob_url"], + "properties": { + "content_schema_name": { + "type": "string" + }, + "content_schema_version": { + "type": "string" + }, + "file_source_blob_url": { + "type": "string", + "description": "URL of the source blob file to be copied." + }, + "file_destination_blob_url": { + "type": "string", + "description": "URL of destination blob file." + }, + "timestamp": { + "type": "string", + "format": "date-time", + "description": "Timestamp of when the file copy was executed." + } + } +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/main/resources/schema/buzz-file-capture.1.0.0.schema.json b/pstatus-report-sink-ktor/src/main/resources/schema/buzz-file-capture.1.0.0.schema.json new file mode 100644 index 00000000..69a66042 --- /dev/null +++ b/pstatus-report-sink-ktor/src/main/resources/schema/buzz-file-capture.1.0.0.schema.json @@ -0,0 +1,86 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://github.com/cdcent/data-exchange-messages/reports/buzz-file-capture", + "title": "Buzz File Capture", + "type": "object", + "required": ["content_schema_name", "content_schema_version", "source", "name", "id", "src_url", "date_created", "date_modified", "user_email", "parent_name", "parent_id", "mime"], + "properties": { + "content_schema_name": { + "type": "string" + }, + "content_schema_version": { + "type": "string" + }, + "source": { + "type": "string", + "description": "Source of the content." + }, + "timestamp": { + "type": "string", + "format": "date-time", + "description": "Timestamp of the content." + }, + "name": { + "type": "string", + "description": "Name of the file." + }, + "id": { + "type": "string", + "description": "Unique identifier for the file." + }, + "src_url": { + "type": "string", + "format": "uri", + "description": "Source URL of the file." + }, + "date_created": { + "type": "string", + "format": "date-time", + "description": "Creation date of the file." + }, + "date_modified": { + "type": "string", + "format": "date-time", + "description": "Last modification date of the file." + }, + "user_email": { + "type": "string", + "format": "email", + "description": "Email of the user associated with the file." + }, + "parent_name": { + "type": "string", + "description": "Name of the parent directory or REDCap project." + }, + "parent_id": { + "type": "string", + "description": "Unique identifier of the parent directory or REDCap project." + }, + "file_size": { + "type": ["integer", "null"], + "description": "Size of the file in bytes." + }, + "mime": { + "type": "string", + "description": "MIME type of the file." + }, + "fingerprints": { + "type": ["array", "null"], + "items": { + "type": "object", + "required": ["algo", "hash"], + "properties": { + "algo": { + "type": "string", + "description": "Algorithm used for the hash." + }, + "hash": { + "type": "string", + "description": "Hash value of the file." + } + } + }, + "description": "Optional list of fingerprints associated with the file." + } + } +} diff --git a/pstatus-report-sink-ktor/src/main/resources/schema/hl7v2-debatch.1.0.0.schema.json b/pstatus-report-sink-ktor/src/main/resources/schema/hl7v2-debatch.1.0.0.schema.json new file mode 100644 index 00000000..13c9958a --- /dev/null +++ b/pstatus-report-sink-ktor/src/main/resources/schema/hl7v2-debatch.1.0.0.schema.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://github.com/cdcent/data-exchange-messages/reports/hl7-receiver-report", + "title": "HL7v2 Receiver Schema", + "type": "object", + "properties": { + "content_schema_name": { + "type": "string", + "const": "hl7v2-debatch" + }, + "content_schema_version": { + "type": "string", + "const": "1.0.0" + }, + + "report": { + "type": "object", + "properties": { + "ingested_file_path": { + "type": "string" + }, + "ingested_file_timestamp": { + "type": "string" + }, + "ingested_file_size": { + "type": "integer" + }, + "received_filename": { + "type": "string" + }, + "supporting_metadata": { + "$ref": "#/$defs/keyValueMap", + "description": "Extra metadata provided by the sender" + }, + "aggregation": { + "type": "string", + "enum": ["SINGLE", "BATCH"] + }, + "number_of_messages": { + "type": "integer" + }, + "number_of_messages_not_propagated": { + "type": "integer" + }, + "error_messages": { + "type": "array", + "items": { + "$ref": "#/$defs/ingest_error" + } + } + } + } +}, + "$defs": { + "keyValueMap": { + "type" : "object", + "existingJavaType" : "java.util.Map" + }, + "ingest_error": { + "type": "object", + "properties": { + "message_uuid": { + "type": "string" + }, + "message_index": { + "type": "integer" + }, + "error_message": { + "type": "string" + } + } + } + } +} + diff --git a/pstatus-report-sink-ktor/src/main/resources/schema/hl7v2-json-lake-transformer.1.0.0.schema.json b/pstatus-report-sink-ktor/src/main/resources/schema/hl7v2-json-lake-transformer.1.0.0.schema.json new file mode 100644 index 00000000..afd438f6 --- /dev/null +++ b/pstatus-report-sink-ktor/src/main/resources/schema/hl7v2-json-lake-transformer.1.0.0.schema.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://github.com/cdcent/data-exchange-messages/reports/base", + "title": "HL7v2 JSON Lake Transformer Report", + "type": "object", + "required": ["content_schema_name", "content_schema_version"], + "properties": { + "content_schema_name": { + "type": "string", + "const": "hl7v2-json-lake-transformer" + }, + "content_schema_version": { + "type": "string", + "const": "1.0.0" + }, + "configs": { + "type": "array", + "items": { + "type": "string" + } + } + } +} diff --git a/pstatus-report-sink-ktor/src/main/resources/schema/hl7v2-lake-segments-transformer.1.0.0.schema.json b/pstatus-report-sink-ktor/src/main/resources/schema/hl7v2-lake-segments-transformer.1.0.0.schema.json new file mode 100644 index 00000000..bd76c7a8 --- /dev/null +++ b/pstatus-report-sink-ktor/src/main/resources/schema/hl7v2-lake-segments-transformer.1.0.0.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "title": "HL7 v2 Message Schema", + "type": "object", + "properties": { + "content_schema_name": { + "const": "hl7v2-lake-segments-transformer" + }, + "content_schema_version": { + "type": "string", + "const": "1.0.0" + }, + + "configs": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": ["content_schema_name", "content_schema_version"] +} diff --git a/pstatus-report-sink-ktor/src/main/resources/schema/hl7v2-redact.1.0.0.schema.json b/pstatus-report-sink-ktor/src/main/resources/schema/hl7v2-redact.1.0.0.schema.json new file mode 100644 index 00000000..d0cbf644 --- /dev/null +++ b/pstatus-report-sink-ktor/src/main/resources/schema/hl7v2-redact.1.0.0.schema.json @@ -0,0 +1,49 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://github.com/cdcent/data-exchange-messages/reports/hl7-redaction-report", + "title": "HL7v2 Redactor Schema", + "type": "object", + "properties": { + "content_schema_name": { + "type": "string", + "const": "hl7v2-redact" + }, + "content_schema_version": { + "type": "string", + "const": "1.0.0" + }, + "report": { + "type": "object", + "properties": { + "entries": { + "type": "array", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "rule": { + "type": "string" + }, + "lineNumber": { + "type": "integer" + }, + "fieldIndex": { + "type": "integer" + } + }, + "required": ["path", "rule", "lineNumber"] + } + } + } + }, + "configs": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": ["content_schema_name", "content_schema_version", "report", "configs"] +} diff --git a/pstatus-report-sink-ktor/src/main/resources/schema/hl7v2-structure-validation.1.0.0.schema.json b/pstatus-report-sink-ktor/src/main/resources/schema/hl7v2-structure-validation.1.0.0.schema.json new file mode 100644 index 00000000..e770c00d --- /dev/null +++ b/pstatus-report-sink-ktor/src/main/resources/schema/hl7v2-structure-validation.1.0.0.schema.json @@ -0,0 +1,69 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://github.com/cdcent/data-exchange-messages/reports/hl7-validation-report", + "title": "HL7v2 Validation Report Schema", + "type": "object", + "properties": { + "content_schema_name": { + "type" : "string", + "const": "hl7v2-structure-validation" + }, + "content_schema_version": {"type": "string", "const": "1.0.0"}, + "report": { + "type": "object", + "properties": { + "entries": { + "structure": {"$ref": "#/$defs/reportItem"}, + "content" : {"$ref": "#/$defs/reportItem"}, + "value-set": {"$ref": "#/$defs/reportItem"} + }, + "error-count": { + "type": "object", + "properties": { + "structure": {"type": "integer"}, + "value-set": {"type": "integer"}, + "content" : {"type": "integer"} + }, + "required": ["structure", "value-set", "content"] + }, + "warning-count": { + "type": "object", + "properties": { + "structure": {"type": "integer"}, + "value-set": {"type": "integer"}, + "content" : {"type": "integer"} + }, + "required": ["structure", "value-set", "content"] + }, + "status": {"type": "string"} + }, + "required": ["entries", "error-count", "warning-count", "status"] + }, + "configs": { + "type": "array", + "items": {"type": "string"} + } + }, + "$defs": { + "reportItem": { + "type": "object", + "properties": { + "structure": { + "type": "array", + "items": { + "type": "object", + "properties": { + "line" : { "type": "integer" }, + "column" : { "type": "integer" }, + "path" : { "type": "string" }, + "description" : { "type": "string" }, + "category" : { "type": "string" }, + "classification": { "type": "string" }, + "stackTrace" : { "type": ["null", "array"] } + } + } + } + } + } + } +} diff --git a/pstatus-report-sink-ktor/src/main/resources/schema/metadata-verify.1.0.0.schema.json b/pstatus-report-sink-ktor/src/main/resources/schema/metadata-verify.1.0.0.schema.json new file mode 100644 index 00000000..fcd303dd --- /dev/null +++ b/pstatus-report-sink-ktor/src/main/resources/schema/metadata-verify.1.0.0.schema.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://github.com/cdcent/data-exchange-messages/reports/base", + "title": "Metadata Verify Report", + "type": "object", + "required": ["content_schema_name", "content_schema_version", "filename", "metadata"], + "properties": { + "content_schema_name": { + "type": "string" + }, + "content_schema_version": { + "type": "string" + }, + "filename": { + "type": "string" + }, + "metadata": { + "$ref": "#/$defs/metadataMap" + }, + "timestamp": { + "type": "string", + "format": "date-time", + "description": "Timestamp of when the metadata verification step was executed." + } + }, + "$defs": { + "metadataMap": { + "type" : "object", + "existingJavaType" : "java.util.Map" + } + } +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/main/resources/schema/upload-status.1.0.0.schema.json b/pstatus-report-sink-ktor/src/main/resources/schema/upload-status.1.0.0.schema.json new file mode 100644 index 00000000..d55b2863 --- /dev/null +++ b/pstatus-report-sink-ktor/src/main/resources/schema/upload-status.1.0.0.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://github.com/cdcent/data-exchange-messages/reports/base", + "title": "Upload Status Report", + "type": "object", + "required": ["schema_name", "schema_version", "tguid", "offset", "size", "filename"], + "properties": { + "schema_name": { + "type": "string" + }, + "schema_version": { + "type": "string" + }, + "tguid": { + "$ref": "#/$defs/uuid" + }, + "offset": { + "type": "integer" + }, + "size": { + "type": "integer" + }, + "filename": { + "type": "string" + } + }, + "$defs": { + "uuid": { + "type": "string", + "format": "uuid" + } + } +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_contentSchemaVersion_validation.json b/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_contentSchemaVersion_validation.json new file mode 100644 index 00000000..e6629c11 --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_contentSchemaVersion_validation.json @@ -0,0 +1,84 @@ +{ + "upload_id": "fe85fdb7-2f07-4ad1-b73a-646cc2e25bbc", + "user_id": "test-event1", + "data_stream_id": "celr", + "data_stream_route": "hl7_out_recdeb", + "jurisdiction": "SMOKE", + "sender_id": "APHL", + "data_producer_id": "smoke-test-data-producer", + "dex_ingest_timestamp": "2024-07-10T15:40:01Z", + "status": "SUCCESS", + "messageMetadata": { + "type": "object", + "properties": { + "message_uuid": { + "type": "UUID", + "description": "Unique identifier for the message associated with this report. Null if not applicable." + }, + "message_hash": { + "type": "string", + "description": "MD5 hash of the message content." + }, + "aggregation": { + "type": "string", + "enum": ["SINGLE", "BATCH"], + "description": "Enumeration: [SINGLE, BATCH]." + }, + "message_index": { + "type": "integer", + "description": "Index of the message; e.g. row if csv." + } + + } + }, + "stage_info": { + "service": "HL7v2 Pipeline", + "stage": "RECEIVER", + "version": "0.0.49-SNAPSHOT", + "status": "SUCCESS", + "issues": null, + "start_processing_time": "2024-07-10T15:40:10.162+00:00", + "end_processing_time": "2024-07-10T15:40:10.228+00:00" + }, + "tags": { + "$ref": "test-ref", + "description": "Optional tag(s) associated with this report." + }, + "data": { + "$ref": "keyValueMap", + "description": "Optional data associated with this report." + }, + + "content": { + "report": { + "ingested_file_path": "https://ocioedemessagesatst.blob.core.windows.net/hl7ingress/dex-routing/dex-smoke-test_319101ba2fd7835983f3257713819f7b", + "ingested_file_timestamp": "2024-07-10T15:40:09+00:00", + "ingested_file_size": 10240, + "received_filename": "dex-smoke-test", + "supporting_metadata": { + "meta_ext_source": "test-src", + "meta_ext_filestatus": "test-file-status", + "meta_ext_file_timestamp": "test-timestamp", + "system_provider": "DEX-ROUTING", + "meta_ext_uploadid": "test-upload-id", + "meta_ext_objectkey": "test-obj-key", + "reporting_jurisdiction": "unknown" + }, + "aggregation": "SINGLE", + "number_of_messages": 1, + "number_of_messages_not_propagated": 1, + "error_messages": [ + { + "message_uuid": "378d6660-903f-40b8-b48b-80f9d77aa69c", + "message_index": 1, + "error_message": "No valid message found." + } + ] + }, + "content_schema_name": "hl7v2-debatch", + "content_schema_version": "2.0.0" + }, + "report_schema_version": "1.0.0", + "content_type": "application/json" + +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_contentType_validation.json b/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_contentType_validation.json new file mode 100644 index 00000000..3f4eb248 --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_contentType_validation.json @@ -0,0 +1,53 @@ +{ + "upload_id": "fe85fdb7-2f07-4ad1-b73a-646cc2e25bbc", + "user_id": null, + "data_stream_id": "celr", + "data_stream_route": "hl7_out_recdeb", + "jurisdiction": "SMOKE", + "sender_id": "APHL", + "data_producer_id": "smoke-test-data-producer", + "dex_ingest_timestamp": "2024-07-10T15:40:01Z", + "message_metadata": null, + "stage_info": { + "service": "HL7v2 Pipeline", + "stage": "RECEIVER", + "version": "0.0.49-SNAPSHOT", + "status": "SUCCESS", + "issues": null, + "start_processing_time": "2024-07-10T15:40:10.162+00:00", + "end_processing_time": "2024-07-10T15:40:10.228+00:00" + }, + "tags": null, + "data": null, + "content": { + "report": { + "ingested_file_path": "https://ocioedemessagesatst.blob.core.windows.net/hl7ingress/dex-routing/dex-smoke-test_319101ba2fd7835983f3257713819f7b", + "ingested_file_timestamp": "2024-07-10T15:40:09+00:00", + "ingested_file_size": 10240, + "received_filename": "dex-smoke-test", + "supporting_metadata": { + "meta_ext_source": "test-src", + "meta_ext_filestatus": "test-file-status", + "meta_ext_file_timestamp": "test-timestamp", + "system_provider": "DEX-ROUTING", + "meta_ext_uploadid": "test-upload-id", + "meta_ext_objectkey": "test-obj-key", + "reporting_jurisdiction": "unknown" + }, + "aggregation": "SINGLE", + "number_of_messages": 1, + "number_of_messages_not_propagated": 1, + "error_messages": [ + { + "message_uuid": "378d6660-903f-40b8-b48b-80f9d77aa69c", + "message_index": 1, + "error_message": "No valid message found." + } + ] + }, + "content_schema_name": "DEX HL7v2 RECEIVER", + "content_schema_version": "2.0.0" + }, + "report_schema_version": "1.0.0" + +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_content_validation.json b/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_content_validation.json new file mode 100644 index 00000000..54546c40 --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_content_validation.json @@ -0,0 +1,53 @@ +{ + "upload_id": "fe85fdb7-2f07-4ad1-b73a-646cc2e25bbc", + "user_id": "test-event1", + "data_stream_id": "celr", + "data_stream_route": "hl7_out_recdeb", + "jurisdiction": "SMOKE", + "sender_id": "APHL", + "status": "SUCCESS", + "data_producer_id": "smoke-test-data-producer", + "dex_ingest_timestamp": "2024-07-10T15:40:01Z", + "messageMetadata": { + "type": "object", + "properties": { + "message_uuid": { + "type": "UUID", + "description": "Unique identifier for the message associated with this report. Null if not applicable." + }, + "message_hash": { + "type": "string", + "description": "MD5 hash of the message content." + }, + "aggregation": { + "type": "string", + "enum": ["SINGLE", "BATCH"], + "description": "Enumeration: [SINGLE, BATCH]." + }, + "message_index": { + "type": "integer", + "description": "Index of the message; e.g. row if csv." + } + + } + }, + "stage_info": { + "service": "HL7v2 Pipeline", + "stage": "RECEIVER", + "version": "0.0.49-SNAPSHOT", + "status": "SUCCESS", + "issues": null, + "start_processing_time": "2024-07-10T15:40:10.162+00:00", + "end_processing_time": "2024-07-10T15:40:10.228+00:00" + }, + "tags": { + "$ref": "test-ref", + "description": "Optional tag(s) associated with this report." + }, + "data": { + "$ref": "keyValueMap", + "description": "Optional data associated with this report." + }, + "report_schema_version": "1.0.0", + "content_type": "application/json" +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_dataStreamId_validation.json b/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_dataStreamId_validation.json new file mode 100644 index 00000000..aa3641f1 --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_dataStreamId_validation.json @@ -0,0 +1,83 @@ +{ + "upload_id": "fe85fdb7-2f07-4ad1-b73a-646cc2e25bbc", + "user_id": "test-event1", + "data_stream_route": "hl7_out_recdeb", + "jurisdiction": "SMOKE", + "sender_id": "APHL", + "data_producer_id": "smoke-test-data-producer", + "dex_ingest_timestamp": "2024-07-10T15:40:01Z", + "status": "SUCCESS", + "messageMetadata": { + "type": "object", + "properties": { + "message_uuid": { + "type": "UUID", + "description": "Unique identifier for the message associated with this report. Null if not applicable." + }, + "message_hash": { + "type": "string", + "description": "MD5 hash of the message content." + }, + "aggregation": { + "type": "string", + "enum": ["SINGLE", "BATCH"], + "description": "Enumeration: [SINGLE, BATCH]." + }, + "message_index": { + "type": "integer", + "description": "Index of the message; e.g. row if csv." + } + + } + }, + "stage_info": { + "service": "HL7v2 Pipeline", + "stage": "RECEIVER", + "version": "0.0.49-SNAPSHOT", + "status": "SUCCESS", + "issues": null, + "start_processing_time": "2024-07-10T15:40:10.162+00:00", + "end_processing_time": "2024-07-10T15:40:10.228+00:00" + }, + "tags": { + "$ref": "test-ref", + "description": "Optional tag(s) associated with this report." + }, + "data": { + "$ref": "keyValueMap", + "description": "Optional data associated with this report." + }, + + "content": { + "report": { + "ingested_file_path": "https://ocioedemessagesatst.blob.core.windows.net/hl7ingress/dex-routing/dex-smoke-test_319101ba2fd7835983f3257713819f7b", + "ingested_file_timestamp": "2024-07-10T15:40:09+00:00", + "ingested_file_size": 10240, + "received_filename": "dex-smoke-test", + "supporting_metadata": { + "meta_ext_source": "test-src", + "meta_ext_filestatus": "test-file-status", + "meta_ext_file_timestamp": "test-timestamp", + "system_provider": "DEX-ROUTING", + "meta_ext_uploadid": "test-upload-id", + "meta_ext_objectkey": "test-obj-key", + "reporting_jurisdiction": "unknown" + }, + "aggregation": "SINGLE", + "number_of_messages": 1, + "number_of_messages_not_propagated": 1, + "error_messages": [ + { + "message_uuid": "378d6660-903f-40b8-b48b-80f9d77aa69c", + "message_index": 1, + "error_message": "No valid message found." + } + ] + }, + "content_schema_name": "hl7v2-debatch", + "content_schema_version": "1.0.0" + }, + "report_schema_version": "1.0.0", + "content_type": "application/json" + +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_dataStreamRoute_validation.json b/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_dataStreamRoute_validation.json new file mode 100644 index 00000000..54762b32 --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_dataStreamRoute_validation.json @@ -0,0 +1,83 @@ +{ + "upload_id": "fe85fdb7-2f07-4ad1-b73a-646cc2e25bbc", + "user_id": "test-event1", + "data_stream_id": "celr", + "jurisdiction": "SMOKE", + "sender_id": "APHL", + "data_producer_id": "smoke-test-data-producer", + "dex_ingest_timestamp": "2024-07-10T15:40:01Z", + "status": "SUCCESS", + "messageMetadata": { + "type": "object", + "properties": { + "message_uuid": { + "type": "UUID", + "description": "Unique identifier for the message associated with this report. Null if not applicable." + }, + "message_hash": { + "type": "string", + "description": "MD5 hash of the message content." + }, + "aggregation": { + "type": "string", + "enum": ["SINGLE", "BATCH"], + "description": "Enumeration: [SINGLE, BATCH]." + }, + "message_index": { + "type": "integer", + "description": "Index of the message; e.g. row if csv." + } + + } + }, + "stage_info": { + "service": "HL7v2 Pipeline", + "stage": "RECEIVER", + "version": "0.0.49-SNAPSHOT", + "status": "SUCCESS", + "issues": null, + "start_processing_time": "2024-07-10T15:40:10.162+00:00", + "end_processing_time": "2024-07-10T15:40:10.228+00:00" + }, + "tags": { + "$ref": "test-ref", + "description": "Optional tag(s) associated with this report." + }, + "data": { + "$ref": "keyValueMap", + "description": "Optional data associated with this report." + }, + + "content": { + "report": { + "ingested_file_path": "https://ocioedemessagesatst.blob.core.windows.net/hl7ingress/dex-routing/dex-smoke-test_319101ba2fd7835983f3257713819f7b", + "ingested_file_timestamp": "2024-07-10T15:40:09+00:00", + "ingested_file_size": 10240, + "received_filename": "dex-smoke-test", + "supporting_metadata": { + "meta_ext_source": "test-src", + "meta_ext_filestatus": "test-file-status", + "meta_ext_file_timestamp": "test-timestamp", + "system_provider": "DEX-ROUTING", + "meta_ext_uploadid": "test-upload-id", + "meta_ext_objectkey": "test-obj-key", + "reporting_jurisdiction": "unknown" + }, + "aggregation": "SINGLE", + "number_of_messages": 1, + "number_of_messages_not_propagated": 1, + "error_messages": [ + { + "message_uuid": "378d6660-903f-40b8-b48b-80f9d77aa69c", + "message_index": 1, + "error_message": "No valid message found." + } + ] + }, + "content_schema_name": "hl7v2-debatch", + "content_schema_version": "1.0.0" + }, + "report_schema_version": "1.0.0", + "content_type": "application/json" + +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_schemaName_validation.json b/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_schemaName_validation.json new file mode 100644 index 00000000..70ea4b61 --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_schemaName_validation.json @@ -0,0 +1,83 @@ +{ + "upload_id": "fe85fdb7-2f07-4ad1-b73a-646cc2e25bbc", + "user_id": "test-event1", + "data_stream_id": "celr", + "data_stream_route": "hl7_out_recdeb", + "jurisdiction": "SMOKE", + "sender_id": "APHL", + "data_producer_id": "smoke-test-data-producer", + "dex_ingest_timestamp": "2024-07-10T15:40:01Z", + "status": "SUCCESS", + "messageMetadata": { + "type": "object", + "properties": { + "message_uuid": { + "type": "UUID", + "description": "Unique identifier for the message associated with this report. Null if not applicable." + }, + "message_hash": { + "type": "string", + "description": "MD5 hash of the message content." + }, + "aggregation": { + "type": "string", + "enum": ["SINGLE", "BATCH"], + "description": "Enumeration: [SINGLE, BATCH]." + }, + "message_index": { + "type": "integer", + "description": "Index of the message; e.g. row if csv." + } + + } + }, + "stage_info": { + "service": "HL7v2 Pipeline", + "stage": "RECEIVER", + "version": "0.0.49-SNAPSHOT", + "status": "SUCCESS", + "issues": null, + "start_processing_time": "2024-07-10T15:40:10.162+00:00", + "end_processing_time": "2024-07-10T15:40:10.228+00:00" + }, + "tags": { + "$ref": "test-ref", + "description": "Optional tag(s) associated with this report." + }, + "data": { + "$ref": "keyValueMap", + "description": "Optional data associated with this report." + }, + + "content": { + "report": { + "ingested_file_path": "https://ocioedemessagesatst.blob.core.windows.net/hl7ingress/dex-routing/dex-smoke-test_319101ba2fd7835983f3257713819f7b", + "ingested_file_timestamp": "2024-07-10T15:40:09+00:00", + "ingested_file_size": 10240, + "received_filename": "dex-smoke-test", + "supporting_metadata": { + "meta_ext_source": "test-src", + "meta_ext_filestatus": "test-file-status", + "meta_ext_file_timestamp": "test-timestamp", + "system_provider": "DEX-ROUTING", + "meta_ext_uploadid": "test-upload-id", + "meta_ext_objectkey": "test-obj-key", + "reporting_jurisdiction": "unknown" + }, + "aggregation": "SINGLE", + "number_of_messages": 1, + "number_of_messages_not_propagated": 1, + "error_messages": [ + { + "message_uuid": "378d6660-903f-40b8-b48b-80f9d77aa69c", + "message_index": 1, + "error_message": "No valid message found." + } + ] + }, + "content_schema_version": "1.0.0" + }, + "report_schema_version": "1.0.0", + "content_type": "application/json" + +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_schemaVersion_validation.json b/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_schemaVersion_validation.json new file mode 100644 index 00000000..76c09444 --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_schemaVersion_validation.json @@ -0,0 +1,83 @@ +{ + "upload_id": "fe85fdb7-2f07-4ad1-b73a-646cc2e25bbc", + "user_id": "test-event1", + "data_stream_id": "celr", + "data_stream_route": "hl7_out_recdeb", + "jurisdiction": "SMOKE", + "sender_id": "APHL", + "data_producer_id": "smoke-test-data-producer", + "dex_ingest_timestamp": "2024-07-10T15:40:01Z", + "status": "SUCCESS", + "messageMetadata": { + "type": "object", + "properties": { + "message_uuid": { + "type": "UUID", + "description": "Unique identifier for the message associated with this report. Null if not applicable." + }, + "message_hash": { + "type": "string", + "description": "MD5 hash of the message content." + }, + "aggregation": { + "type": "string", + "enum": ["SINGLE", "BATCH"], + "description": "Enumeration: [SINGLE, BATCH]." + }, + "message_index": { + "type": "integer", + "description": "Index of the message; e.g. row if csv." + } + + } + }, + "stage_info": { + "service": "HL7v2 Pipeline", + "stage": "RECEIVER", + "version": "0.0.49-SNAPSHOT", + "status": "SUCCESS", + "issues": null, + "start_processing_time": "2024-07-10T15:40:10.162+00:00", + "end_processing_time": "2024-07-10T15:40:10.228+00:00" + }, + "tags": { + "$ref": "test-ref", + "description": "Optional tag(s) associated with this report." + }, + "data": { + "$ref": "keyValueMap", + "description": "Optional data associated with this report." + }, + + "content": { + "report": { + "ingested_file_path": "https://ocioedemessagesatst.blob.core.windows.net/hl7ingress/dex-routing/dex-smoke-test_319101ba2fd7835983f3257713819f7b", + "ingested_file_timestamp": "2024-07-10T15:40:09+00:00", + "ingested_file_size": 10240, + "received_filename": "dex-smoke-test", + "supporting_metadata": { + "meta_ext_source": "test-src", + "meta_ext_filestatus": "test-file-status", + "meta_ext_file_timestamp": "test-timestamp", + "system_provider": "DEX-ROUTING", + "meta_ext_uploadid": "test-upload-id", + "meta_ext_objectkey": "test-obj-key", + "reporting_jurisdiction": "unknown" + }, + "aggregation": "SINGLE", + "number_of_messages": 1, + "number_of_messages_not_propagated": 1, + "error_messages": [ + { + "message_uuid": "378d6660-903f-40b8-b48b-80f9d77aa69c", + "message_index": 1, + "error_message": "No valid message found." + } + ] + }, + "content_schema_name": "hl7v2-debatch" + }, + "report_schema_version": "1.0.0", + "content_type": "application/json" + +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_stageInfo_validation.json b/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_stageInfo_validation.json new file mode 100644 index 00000000..9007ea06 --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_stageInfo_validation.json @@ -0,0 +1,76 @@ +{ + "upload_id": "fe85fdb7-2f07-4ad1-b73a-646cc2e25bbc", + "user_id": "test-event1", + "data_stream_id": "celr", + "data_stream_route": "hl7_out_recdeb", + "jurisdiction": "SMOKE", + "sender_id": "APHL", + "data_producer_id": "smoke-test-data-producer", + "dex_ingest_timestamp": "2024-07-10T15:40:01Z", + "status": "SUCCESS", + "messageMetadata": { + "type": "object", + "properties": { + "message_uuid": { + "type": "UUID", + "description": "Unique identifier for the message associated with this report. Null if not applicable." + }, + "message_hash": { + "type": "string", + "description": "MD5 hash of the message content." + }, + "aggregation": { + "type": "string", + "enum": ["SINGLE", "BATCH"], + "description": "Enumeration: [SINGLE, BATCH]." + }, + "message_index": { + "type": "integer", + "description": "Index of the message; e.g. row if csv." + } + + } + }, + + "tags": { + "$ref": "test-ref", + "description": "Optional tag(s) associated with this report." + }, + "data": { + "$ref": "keyValueMap", + "description": "Optional data associated with this report." + }, + + "content": { + "report": { + "ingested_file_path": "https://ocioedemessagesatst.blob.core.windows.net/hl7ingress/dex-routing/dex-smoke-test_319101ba2fd7835983f3257713819f7b", + "ingested_file_timestamp": "2024-07-10T15:40:09+00:00", + "ingested_file_size": 10240, + "received_filename": "dex-smoke-test", + "supporting_metadata": { + "meta_ext_source": "test-src", + "meta_ext_filestatus": "test-file-status", + "meta_ext_file_timestamp": "test-timestamp", + "system_provider": "DEX-ROUTING", + "meta_ext_uploadid": "test-upload-id", + "meta_ext_objectkey": "test-obj-key", + "reporting_jurisdiction": "unknown" + }, + "aggregation": "SINGLE", + "number_of_messages": 1, + "number_of_messages_not_propagated": 1, + "error_messages": [ + { + "message_uuid": "378d6660-903f-40b8-b48b-80f9d77aa69c", + "message_index": 1, + "error_message": "No valid message found." + } + ] + }, + "content_schema_name": "hl7v2-debatch", + "content_schema_version": "1.0.0" + }, + "report_schema_version": "1.0.0", + "content_type": "application/json" + +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_uploadId_validation.json b/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_uploadId_validation.json new file mode 100644 index 00000000..7262e51c --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_missing_uploadId_validation.json @@ -0,0 +1,53 @@ +{ + + "user_id": null, + "data_stream_id": "celr", + "data_stream_route": "hl7_out_recdeb", + "jurisdiction": "SMOKE", + "sender_id": "APHL", + "data_producer_id": "smoke-test-data-producer", + "dex_ingest_timestamp": "2024-07-10T15:40:01Z", + "message_metadata": null, + "stage_info": { + "service": "HL7v2 Pipeline", + "stage": "RECEIVER", + "version": "0.0.49-SNAPSHOT", + "status": "SUCCESS", + "issues": null, + "start_processing_time": "2024-07-10T15:40:10.162+00:00", + "end_processing_time": "2024-07-10T15:40:10.228+00:00" + }, + "tags": null, + "data": null, + "content": { + "report": { + "ingested_file_path": "https://ocioedemessagesatst.blob.core.windows.net/hl7ingress/dex-routing/dex-smoke-test_319101ba2fd7835983f3257713819f7b", + "ingested_file_timestamp": "2024-07-10T15:40:09+00:00", + "ingested_file_size": 10240, + "received_filename": "dex-smoke-test", + "supporting_metadata": { + "meta_ext_source": "test-src", + "meta_ext_filestatus": "test-file-status", + "meta_ext_file_timestamp": "test-timestamp", + "system_provider": "DEX-ROUTING", + "meta_ext_uploadid": "test-upload-id", + "meta_ext_objectkey": "test-obj-key", + "reporting_jurisdiction": "unknown" + }, + "aggregation": "SINGLE", + "number_of_messages": 1, + "number_of_messages_not_propagated": 1, + "error_messages": [ + { + "message_uuid": "378d6660-903f-40b8-b48b-80f9d77aa69c", + "message_index": 1, + "error_message": "No valid message found." + } + ] + }, + "content_schema_name": "DEX HL7v2 RECEIVER", + "content_schema_version": "2.0.0" + }, + "report_schema_version": "1.0.0", + "content_type": "application/json" +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_validation_pass.json b/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_validation_pass.json new file mode 100644 index 00000000..68d6441b --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/report_schema_validation_pass.json @@ -0,0 +1,84 @@ +{ + "upload_id": "fe85fdb7-2f07-4ad1-b73a-646cc2e25bbc", + "user_id": "test-event1", + "data_stream_id": "celr", + "data_stream_route": "hl7_out_recdeb", + "jurisdiction": "SMOKE", + "sender_id": "APHL", + "data_producer_id": "smoke-test-data-producer", + "dex_ingest_timestamp": "2024-07-10T15:40:01Z", + "status": "SUCCESS", + "messageMetadata": { + "type": "object", + "properties": { + "message_uuid": { + "type": "UUID", + "description": "Unique identifier for the message associated with this report. Null if not applicable." + }, + "message_hash": { + "type": "string", + "description": "MD5 hash of the message content." + }, + "aggregation": { + "type": "string", + "enum": ["SINGLE", "BATCH"], + "description": "Enumeration: [SINGLE, BATCH]." + }, + "message_index": { + "type": "integer", + "description": "Index of the message; e.g. row if csv." + } + + } + }, + "stage_info": { + "service": "HL7v2 Pipeline", + "stage": "RECEIVER", + "version": "0.0.49-SNAPSHOT", + "status": "SUCCESS", + "issues": null, + "start_processing_time": "2024-07-10T15:40:10.162+00:00", + "end_processing_time": "2024-07-10T15:40:10.228+00:00" + }, + "tags": { + "$ref": "test-ref", + "description": "Optional tag(s) associated with this report." + }, + "data": { + "$ref": "keyValueMap", + "description": "Optional data associated with this report." + }, + + "content": { + "report": { + "ingested_file_path": "https://ocioedemessagesatst.blob.core.windows.net/hl7ingress/dex-routing/dex-smoke-test_319101ba2fd7835983f3257713819f7b", + "ingested_file_timestamp": "2024-07-10T15:40:09+00:00", + "ingested_file_size": 10240, + "received_filename": "dex-smoke-test", + "supporting_metadata": { + "meta_ext_source": "test-src", + "meta_ext_filestatus": "test-file-status", + "meta_ext_file_timestamp": "test-timestamp", + "system_provider": "DEX-ROUTING", + "meta_ext_uploadid": "test-upload-id", + "meta_ext_objectkey": "test-obj-key", + "reporting_jurisdiction": "unknown" + }, + "aggregation": "SINGLE", + "number_of_messages": 1, + "number_of_messages_not_propagated": 1, + "error_messages": [ + { + "message_uuid": "378d6660-903f-40b8-b48b-80f9d77aa69c", + "message_index": 1, + "error_message": "No valid message found." + } + ] + }, + "content_schema_name": "hl7v2-debatch", + "content_schema_version": "1.0.0" + }, + "report_schema_version": "1.0.0", + "content_type": "application/json" + +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_content_missing_schema_name.json b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_content_missing_schema_name.json new file mode 100644 index 00000000..9bd417c0 --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_content_missing_schema_name.json @@ -0,0 +1,10 @@ +{ + "upload_id": "4f99cfde-7c83-44e5-bd49-206ca4df7e16", + "data_stream_id": "dex-testing", + "data_stream_route": "test-event1", + "stage_name": "dex-hl7-validation", + "content_type": "json", + "content": { + "schema_version": "0.0.1" + } +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_content_missing_schema_version.json b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_content_missing_schema_version.json new file mode 100644 index 00000000..f39dd2c0 --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_content_missing_schema_version.json @@ -0,0 +1,10 @@ +{ + "upload_id": "4f99cfde-7c83-44e5-bd49-206ca4df7e16", + "data_stream_id": "dex-testing", + "data_stream_route": "test-event1", + "stage_name": "dex-hl7-validation", + "content_type": "json", + "content": { + "schema_name": "dex-hl7-validation" + } +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_escape_quoted_json.json b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_escape_quoted_json.json new file mode 100644 index 00000000..75b1c8a4 --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_escape_quoted_json.json @@ -0,0 +1,8 @@ +{ + "upload_id": "12345678", + "data_stream_id": "dex-testing", + "data_stream_route": "test-event1", + "stage_name": "dex-upload", + "content_type": "json", + "content": "{\"schema_name\":\"upload\",\"schema_version\":\"1.0\",\"tguid\":\"12345678\",\"offset\":27472691,\"size\":27472691,\"filename\":\"some_upload1.csv\",\"meta_destination_id\":\"ndlp\",\"meta_ext_event\":\"routineImmunization\",\"end_time_epoch_millis\":1700009141546,\"start_time_epoch_millis\":1700009137234,\"metadata\":{\"filename\":\"10MB-test-file\",\"filetype\":\"text\/plain\",\"meta_destination_id\":\"ndlp\",\"meta_ext_event\":\"routineImmunization\",\"meta_ext_source\":\"IZGW\",\"meta_ext_sourceversion\":\"V2022-12-31\",\"meta_ext_entity\":\"DD2\",\"meta_username\":\"ygj6@cdc.gov\",\"meta_ext_objectkey\":\"2b18d70c-8559-11ee-b9d1-0242ac120002\",\"meta_ext_filename\":\"10MB-test-file\",\"meta_ext_submissionperiod\":\"1\"}}" +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_good_message.json b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_good_message.json new file mode 100644 index 00000000..cb416e79 --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_good_message.json @@ -0,0 +1,144 @@ +{ + "upload_id": "4f99cfde-7c83-44e5-bd49-206ca4df7e16", + "data_stream_id": "dex-testing", + "data_stream_route": "test-event1", + "stage_name": "dex-hl7-validation", + "content_type": "json", + "content": { + "id": "dd1c8c4b-a18d-4449-aaf3-5f0ff505bf38", + "message_uuid": "dd1c8c4b-a18d-4449-aaf3-5f0ff505bf38", + "message_info": { + "event_code": "10150", + "route": "vaccine_preventable_diseases", + "reporting_jurisdiction": "13", + "type": "CASE", + "local_record_id": "SAI" + }, + "metadata": { + "provenance": { + "event_id": "9b36fb63-e01e-0066-11ab-2ee31d06a24b", + "event_timestamp": "2023-12-14T16:36:47.910526Z", + "file_uuid": "03cb333c-46c0-4a96-8bab-bf494df87528", + "file_path": "https://tfedemessagestoragedev.blob.core.windows.net/hl7ingress/postman-dtx0-c679a16e-fbfc-4388-9147-18d8a605abc0.txt", + "file_timestamp": "2023-12-14T11:36:47-05:00", + "file_size": 14813, + "single_or_batch": "SINGLE", + "message_hash": "74ada45a7c41c98c574b6c4f5c8e1d26", + "ext_system_provider": "POSTMAN", + "ext_original_file_name": "postman-dtx0-c679a16e-fbfc-4388-9147-18d8a605abc0.txt", + "message_index": 1, + "ext_original_file_timestamp": "2023-03-09T12:05:04.074", + "source_metadata": { + "orginal_file_name": "test-d4731f4d-b119-4d78-a0b1-d322658d1d3d.txt" + } + }, + "processes": [ + { + "status": "SUCCESS", + "process_name": "RECEIVER", + "process_version": "1.0.0", + "eventhub_queued_time": "2023-12-14T16:36:48.847", + "eventhub_offset": 1125281432952, + "eventhub_sequence_number": 89766, + "configs": [], + "start_processing_time": "2023-12-14T11:36:48.892-05:00", + "end_processing_time": "2023-12-14T11:36:48.982-05:00" + }, + { + "status": "SUCCESS", + "report": { + "entries": [], + "status": "SUCCESS" + }, + "process_name": "REDACTOR", + "process_version": "1.0.0", + "eventhub_queued_time": "2023-12-14T16:36:49.253", + "eventhub_offset": 1189705961896, + "eventhub_sequence_number": 86965, + "configs": [ + "case_config.txt" + ], + "start_processing_time": "2023-12-14T11:36:49.34-05:00", + "end_processing_time": "2023-12-14T11:36:49.436-05:00" + }, + { + "status": "SUCCESS", + "report": { + "entries": { + "structure": [ + { + "line": 4, + "column": 64, + "path": "OBX[1]-5[1]", + "description": "The primitive Field OBX-5 (Observation Value) contains at least one unescaped delimiter", + "category": "Unescaped Separator", + "classification": "Warning" + }, + { + "line": 6, + "column": 68, + "path": "OBX[3]-5[1]", + "description": "The primitive Field OBX-5 (Observation Value) contains at least one unescaped delimiter", + "category": "Unescaped Separator", + "classification": "Warning" + }, + { + "line": 2, + "column": 236, + "path": "PID[1]-11[1].9", + "description": "The primitive Component PID-11.9 (County/Parish Code) contains at least one unescaped delimiter", + "category": "Unescaped Separator", + "classification": "Warning" + }, + { + "line": 2, + "column": 195, + "path": "PID[1]-11[1].4", + "description": "The primitive Component PID-11.4 (State or Province) contains at least one unescaped delimiter", + "category": "Unescaped Separator", + "classification": "Warning" + }, + { + "line": 1, + "column": 226, + "path": "MSH[1]-10[1]", + "description": "The length of Field MSH-10 (Message Control ID) must be within the range [1, 20]. Value \\u003d \\u0027RIBD_N_Meningitidis_Case_Invalid_03\\u0027", + "category": "Length", + "classification": "Warning" + } + ], + "content": [], + "value-set": [] + }, + "error-count": { + "structure": 0, + "value-set": 0, + "content": 0 + }, + "warning-count": { + "structure": 5, + "value-set": 0, + "content": 0 + }, + "status": "VALID_MESSAGE" + }, + "process_name": "STRUCTURE-VALIDATOR", + "process_version": "1.0.0", + "eventhub_queued_time": "2023-12-14T16:36:50.097", + "eventhub_offset": 1120986506816, + "eventhub_sequence_number": 96087, + "configs": [ + "NOTF_ORU_V3.0" + ], + "start_processing_time": "2023-12-14T11:36:50.133-05:00", + "end_processing_time": "2023-12-14T11:36:50.205-05:00" + } + ] + }, + "summary": { + "current_status": "VALID_MESSAGE" + }, + "schema_version": "0.0.1", + "schema_name": "dex-hl7-validation" + } +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_good_message_V1.json b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_good_message_V1.json new file mode 100644 index 00000000..74b6cd46 --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_good_message_V1.json @@ -0,0 +1,144 @@ +{ + "upload_id": "4f99cfde-7c83-44e5-bd49-206ca4df7e16", + "destination_id": "dex-testing", + "event_type": "test-event1", + "stage_name": "dex-hl7-validation", + "content_type": "json", + "content": { + "id": "dd1c8c4b-a18d-4449-aaf3-5f0ff505bf38", + "message_uuid": "dd1c8c4b-a18d-4449-aaf3-5f0ff505bf38", + "message_info": { + "event_code": "10150", + "route": "vaccine_preventable_diseases", + "reporting_jurisdiction": "13", + "type": "CASE", + "local_record_id": "SAI" + }, + "metadata": { + "provenance": { + "event_id": "9b36fb63-e01e-0066-11ab-2ee31d06a24b", + "event_timestamp": "2023-12-14T16:36:47.910526Z", + "file_uuid": "03cb333c-46c0-4a96-8bab-bf494df87528", + "file_path": "https://tfedemessagestoragedev.blob.core.windows.net/hl7ingress/postman-dtx0-c679a16e-fbfc-4388-9147-18d8a605abc0.txt", + "file_timestamp": "2023-12-14T11:36:47-05:00", + "file_size": 14813, + "single_or_batch": "SINGLE", + "message_hash": "74ada45a7c41c98c574b6c4f5c8e1d26", + "ext_system_provider": "POSTMAN", + "ext_original_file_name": "postman-dtx0-c679a16e-fbfc-4388-9147-18d8a605abc0.txt", + "message_index": 1, + "ext_original_file_timestamp": "2023-03-09T12:05:04.074", + "source_metadata": { + "orginal_file_name": "test-d4731f4d-b119-4d78-a0b1-d322658d1d3d.txt" + } + }, + "processes": [ + { + "status": "SUCCESS", + "process_name": "RECEIVER", + "process_version": "1.0.0", + "eventhub_queued_time": "2023-12-14T16:36:48.847", + "eventhub_offset": 1125281432952, + "eventhub_sequence_number": 89766, + "configs": [], + "start_processing_time": "2023-12-14T11:36:48.892-05:00", + "end_processing_time": "2023-12-14T11:36:48.982-05:00" + }, + { + "status": "SUCCESS", + "report": { + "entries": [], + "status": "SUCCESS" + }, + "process_name": "REDACTOR", + "process_version": "1.0.0", + "eventhub_queued_time": "2023-12-14T16:36:49.253", + "eventhub_offset": 1189705961896, + "eventhub_sequence_number": 86965, + "configs": [ + "case_config.txt" + ], + "start_processing_time": "2023-12-14T11:36:49.34-05:00", + "end_processing_time": "2023-12-14T11:36:49.436-05:00" + }, + { + "status": "SUCCESS", + "report": { + "entries": { + "structure": [ + { + "line": 4, + "column": 64, + "path": "OBX[1]-5[1]", + "description": "The primitive Field OBX-5 (Observation Value) contains at least one unescaped delimiter", + "category": "Unescaped Separator", + "classification": "Warning" + }, + { + "line": 6, + "column": 68, + "path": "OBX[3]-5[1]", + "description": "The primitive Field OBX-5 (Observation Value) contains at least one unescaped delimiter", + "category": "Unescaped Separator", + "classification": "Warning" + }, + { + "line": 2, + "column": 236, + "path": "PID[1]-11[1].9", + "description": "The primitive Component PID-11.9 (County/Parish Code) contains at least one unescaped delimiter", + "category": "Unescaped Separator", + "classification": "Warning" + }, + { + "line": 2, + "column": 195, + "path": "PID[1]-11[1].4", + "description": "The primitive Component PID-11.4 (State or Province) contains at least one unescaped delimiter", + "category": "Unescaped Separator", + "classification": "Warning" + }, + { + "line": 1, + "column": 226, + "path": "MSH[1]-10[1]", + "description": "The length of Field MSH-10 (Message Control ID) must be within the range [1, 20]. Value \\u003d \\u0027RIBD_N_Meningitidis_Case_Invalid_03\\u0027", + "category": "Length", + "classification": "Warning" + } + ], + "content": [], + "value-set": [] + }, + "error-count": { + "structure": 0, + "value-set": 0, + "content": 0 + }, + "warning-count": { + "structure": 5, + "value-set": 0, + "content": 0 + }, + "status": "VALID_MESSAGE" + }, + "process_name": "STRUCTURE-VALIDATOR", + "process_version": "1.0.0", + "eventhub_queued_time": "2023-12-14T16:36:50.097", + "eventhub_offset": 1120986506816, + "eventhub_sequence_number": 96087, + "configs": [ + "NOTF_ORU_V3.0" + ], + "start_processing_time": "2023-12-14T11:36:50.133-05:00", + "end_processing_time": "2023-12-14T11:36:50.205-05:00" + } + ] + }, + "summary": { + "current_status": "VALID_MESSAGE" + }, + "schema_version": "0.0.1", + "schema_name": "dex-hl7-validation" + } +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_missing_content.json b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_missing_content.json new file mode 100644 index 00000000..34bfc52b --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_missing_content.json @@ -0,0 +1,7 @@ +{ + "upload_id": "12345678", + "stage_name": "dex-stage1", + "data_stream_id": "dex-testing", + "data_stream_route": "test-event1", + "content_type": "json" +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_missing_content_type.json b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_missing_content_type.json new file mode 100644 index 00000000..d863dc53 --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_missing_content_type.json @@ -0,0 +1,7 @@ +{ + "upload_id": "12345678", + "stage_name": "dex-stage1", + "data_stream_id": "dex-testing", + "data_stream_route": "test-event1" + +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_missing_data_stream_id.json b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_missing_data_stream_id.json new file mode 100644 index 00000000..5033bc4d --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_missing_data_stream_id.json @@ -0,0 +1,7 @@ +{ + "upload_id": "12345678", + "data_stream_route": "test-event1", + "stage_name": "dex-stage1", + "content_type": "json", + "content": "" +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_missing_data_stream_route.json b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_missing_data_stream_route.json new file mode 100644 index 00000000..bbe6926a --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_missing_data_stream_route.json @@ -0,0 +1,7 @@ +{ + "upload_id": "12345678", + "stage_name": "dex-stage1", + "data_stream_id": "dex-testing", + "content_type": "json", + "content": "" +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_missing_stage_name.json b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_missing_stage_name.json new file mode 100644 index 00000000..4dda4740 --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_missing_stage_name.json @@ -0,0 +1,7 @@ +{ + "upload_id": "12345678", + "data_stream_id": "dex-testing", + "data_stream_route": "test-event1", + "content_type": "json", + "content": "" +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_missing_upload_id.json b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_missing_upload_id.json new file mode 100644 index 00000000..df0e2405 --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/data/service_bus_missing_upload_id.json @@ -0,0 +1,143 @@ +{ + "data_stream_route": "test-event1", + "stage_name": "dex-stage1", + "data_stream_id": "dex-testing", + "content_type": "json", + "content": { + "id": "dd1c8c4b-a18d-4449-aaf3-5f0ff505bf38", + "message_uuid": "dd1c8c4b-a18d-4449-aaf3-5f0ff505bf38", + "message_info": { + "event_code": "10150", + "route": "vaccine_preventable_diseases", + "reporting_jurisdiction": "13", + "type": "CASE", + "local_record_id": "SAI" + }, + "metadata": { + "provenance": { + "event_id": "9b36fb63-e01e-0066-11ab-2ee31d06a24b", + "event_timestamp": "2023-12-14T16:36:47.910526Z", + "file_uuid": "03cb333c-46c0-4a96-8bab-bf494df87528", + "file_path": "https://tfedemessagestoragedev.blob.core.windows.net/hl7ingress/postman-dtx0-c679a16e-fbfc-4388-9147-18d8a605abc0.txt", + "file_timestamp": "2023-12-14T11:36:47-05:00", + "file_size": 14813, + "single_or_batch": "SINGLE", + "message_hash": "74ada45a7c41c98c574b6c4f5c8e1d26", + "ext_system_provider": "POSTMAN", + "ext_original_file_name": "postman-dtx0-c679a16e-fbfc-4388-9147-18d8a605abc0.txt", + "message_index": 1, + "ext_original_file_timestamp": "2023-03-09T12:05:04.074", + "source_metadata": { + "orginal_file_name": "test-d4731f4d-b119-4d78-a0b1-d322658d1d3d.txt" + } + }, + "processes": [ + { + "status": "SUCCESS", + "process_name": "RECEIVER", + "process_version": "1.0.0", + "eventhub_queued_time": "2023-12-14T16:36:48.847", + "eventhub_offset": 1125281432952, + "eventhub_sequence_number": 89766, + "configs": [], + "start_processing_time": "2023-12-14T11:36:48.892-05:00", + "end_processing_time": "2023-12-14T11:36:48.982-05:00" + }, + { + "status": "SUCCESS", + "report": { + "entries": [], + "status": "SUCCESS" + }, + "process_name": "REDACTOR", + "process_version": "1.0.0", + "eventhub_queued_time": "2023-12-14T16:36:49.253", + "eventhub_offset": 1189705961896, + "eventhub_sequence_number": 86965, + "configs": [ + "case_config.txt" + ], + "start_processing_time": "2023-12-14T11:36:49.34-05:00", + "end_processing_time": "2023-12-14T11:36:49.436-05:00" + }, + { + "status": "SUCCESS", + "report": { + "entries": { + "structure": [ + { + "line": 4, + "column": 64, + "path": "OBX[1]-5[1]", + "description": "The primitive Field OBX-5 (Observation Value) contains at least one unescaped delimiter", + "category": "Unescaped Separator", + "classification": "Warning" + }, + { + "line": 6, + "column": 68, + "path": "OBX[3]-5[1]", + "description": "The primitive Field OBX-5 (Observation Value) contains at least one unescaped delimiter", + "category": "Unescaped Separator", + "classification": "Warning" + }, + { + "line": 2, + "column": 236, + "path": "PID[1]-11[1].9", + "description": "The primitive Component PID-11.9 (County/Parish Code) contains at least one unescaped delimiter", + "category": "Unescaped Separator", + "classification": "Warning" + }, + { + "line": 2, + "column": 195, + "path": "PID[1]-11[1].4", + "description": "The primitive Component PID-11.4 (State or Province) contains at least one unescaped delimiter", + "category": "Unescaped Separator", + "classification": "Warning" + }, + { + "line": 1, + "column": 226, + "path": "MSH[1]-10[1]", + "description": "The length of Field MSH-10 (Message Control ID) must be within the range [1, 20]. Value \\u003d \\u0027RIBD_N_Meningitidis_Case_Invalid_03\\u0027", + "category": "Length", + "classification": "Warning" + } + ], + "content": [], + "value-set": [] + }, + "error-count": { + "structure": 0, + "value-set": 0, + "content": 0 + }, + "warning-count": { + "structure": 5, + "value-set": 0, + "content": 0 + }, + "status": "VALID_MESSAGE" + }, + "process_name": "STRUCTURE-VALIDATOR", + "process_version": "1.0.0", + "eventhub_queued_time": "2023-12-14T16:36:50.097", + "eventhub_offset": 1120986506816, + "eventhub_sequence_number": 96087, + "configs": [ + "NOTF_ORU_V3.0" + ], + "start_processing_time": "2023-12-14T11:36:50.133-05:00", + "end_processing_time": "2023-12-14T11:36:50.205-05:00" + } + ] + }, + "summary": { + "current_status": "VALID_MESSAGE" + }, + "schema_version": "0.0.1", + "schema_name": "dex-hl7-validation" + } +} \ No newline at end of file diff --git a/pstatus-report-sink-ktor/src/test/kotlin/test/ReportSchemaValidationTests.kt b/pstatus-report-sink-ktor/src/test/kotlin/test/ReportSchemaValidationTests.kt new file mode 100644 index 00000000..d3ddd4aa --- /dev/null +++ b/pstatus-report-sink-ktor/src/test/kotlin/test/ReportSchemaValidationTests.kt @@ -0,0 +1,169 @@ + +package test + +import com.azure.messaging.servicebus.ServiceBusReceivedMessage +import com.microsoft.azure.functions.ExecutionContext +import gov.cdc.ocio.processingstatusapi.cosmos.CosmosContainerManager +import gov.cdc.ocio.processingstatusapi.exceptions.BadRequestException +import gov.cdc.ocio.processingstatusapi.plugins.ServiceBusProcessor +import io.mockk.* +import org.mockito.Mockito +import org.testng.Assert +import org.testng.annotations.BeforeMethod +import org.testng.annotations.Test +import java.io.File + +class ReportSchemaValidationTests { + + private lateinit var context: ExecutionContext + + @BeforeMethod + fun setUp() { + context = Mockito.mock(ExecutionContext::class.java) + mockkObject(CosmosContainerManager) + every { CosmosContainerManager.initDatabaseContainer(any(), any(),any(), any()) } returns null + + } + + @Test + fun testReportSchemaValidationMissingUploadId() { + val testMessage =File("./src/test/kotlin/data/report_schema_missing_uploadId_validation.json").readBytes() + val serviceBusReceivedMessage = createServiceBusReceivedMessageFromBinary(testMessage) + var exceptionThrown = false + try { + // Mock file + ServiceBusProcessor().withMessage(serviceBusReceivedMessage) + } catch(ex: BadRequestException) { + exceptionThrown = ex.localizedMessage.contains("$.upload_id: is missing but it is required") + } + Assert.assertTrue(exceptionThrown) + } + @Test + fun testReportSchemaValidationMissingDataStreamId() { + val testMessage =File("./src/test/kotlin/data/report_schema_missing_dataStreamId_validation.json").readBytes() + val serviceBusReceivedMessage = createServiceBusReceivedMessageFromBinary(testMessage) + var exceptionThrown = false + try { + ServiceBusProcessor().withMessage(serviceBusReceivedMessage) + } catch(ex: BadRequestException) { + exceptionThrown = ex.localizedMessage.contains("$.data_stream_id: is missing but it is required") + } + Assert.assertTrue(exceptionThrown) + } + @Test + fun testReportSchemaValidationMissingRoute() { + val testMessage =File("./src/test/kotlin/data/report_schema_missing_dataStreamRoute_validation.json").readBytes() + val serviceBusReceivedMessage = createServiceBusReceivedMessageFromBinary(testMessage) + var exceptionThrown = false + try { + ServiceBusProcessor().withMessage(serviceBusReceivedMessage) + } catch(ex: BadRequestException) { + exceptionThrown = ex.localizedMessage.contains("$.data_stream_route: is missing but it is required") + } + Assert.assertTrue(exceptionThrown) + } + @Test + fun testReportSchemaValidationMissingStageInfo() { + val testMessage =File("./src/test/kotlin/data/report_schema_missing_stageInfo_validation.json").readBytes() + val serviceBusReceivedMessage = createServiceBusReceivedMessageFromBinary(testMessage) + + var exceptionThrown = false + try { + ServiceBusProcessor().withMessage(serviceBusReceivedMessage) + } catch(ex: BadRequestException) { + exceptionThrown = ex.localizedMessage.contains("$.stage_info: is missing but it is required") + } + Assert.assertTrue(exceptionThrown) + } + + @Test + fun testReportSchemaValidationMissingContentType() { + val testMessage =File("./src/test/kotlin/data/report_schema_missing_contentType_validation.json").readBytes() + val serviceBusReceivedMessage = createServiceBusReceivedMessageFromBinary(testMessage) + var exceptionThrown = false + try { + ServiceBusProcessor().withMessage(serviceBusReceivedMessage) + } catch(ex: BadRequestException) { + exceptionThrown = ex.localizedMessage.contains("$.content_type: is missing but it is required") + } + Assert.assertTrue(exceptionThrown) + } + + @Test + fun testReportSchemaValidationMissingContent() { + val testMessage =File("./src/test/kotlin/data/report_schema_missing_content_validation.json").readBytes() + val serviceBusReceivedMessage = createServiceBusReceivedMessageFromBinary(testMessage) + var exceptionThrown = false + try { + ServiceBusProcessor().withMessage(serviceBusReceivedMessage) + } catch(ex: BadRequestException) { + exceptionThrown = ex.localizedMessage.contains("Report rejected: `content` is not JSON or is missing.") + } + Assert.assertTrue(exceptionThrown) + } + + @Test + fun testReportSchemaValidationMissingSchemaName() { + val testMessage =File("./src/test/kotlin/data/report_schema_missing_schemaName_validation.json").readBytes() + val serviceBusReceivedMessage = createServiceBusReceivedMessageFromBinary(testMessage) + var exceptionThrown = false + try { + ServiceBusProcessor().withMessage(serviceBusReceivedMessage) + } catch(ex: BadRequestException) { + exceptionThrown = ex.localizedMessage.contains("Report rejected: `content_schema_name` or `content_schema_version` is missing or empty.") + } + Assert.assertTrue(exceptionThrown) + } + + @Test + fun testReportSchemaValidationMissingSchemaVersion() { + val testMessage =File("./src/test/kotlin/data/report_schema_missing_schemaVersion_validation.json").readBytes() + val serviceBusReceivedMessage = createServiceBusReceivedMessageFromBinary(testMessage) + var exceptionThrown = false + try { + ServiceBusProcessor().withMessage(serviceBusReceivedMessage) + } catch(ex: BadRequestException) { + exceptionThrown = ex.localizedMessage.contains("Report rejected: `content_schema_name` or `content_schema_version` is missing or empty.") + } + Assert.assertTrue(exceptionThrown) + } + @Test + fun testReportContentSchemaValidationFileNotFound() { + val testMessage =File("./src/test/kotlin/data/report_schema_contentSchemaVersion_validation.json").readBytes() + val serviceBusReceivedMessage = createServiceBusReceivedMessageFromBinary(testMessage) + var exceptionThrown = false + try { + ServiceBusProcessor().withMessage(serviceBusReceivedMessage) + } + catch (ex :IllegalArgumentException){ + exceptionThrown = ex.localizedMessage.contains("File not found: hl7v2-debatch.2.0.0.schema.json") + } + catch(ex: BadRequestException) { + exceptionThrown = ex.localizedMessage.contains("Report rejected: Content schema file not found for content schema name hl7v2-debatch and schema version 2.0.0.") + } + Assert.assertTrue(exceptionThrown) + } + + @Test + fun testReportSchemaValidationPass() { + val testMessage =File("./src/test/kotlin/data/report_schema_validation_pass.json").readBytes() + val serviceBusReceivedMessage = createServiceBusReceivedMessageFromBinary(testMessage) + ServiceBusProcessor().withMessage(serviceBusReceivedMessage) + } + + private fun createServiceBusReceivedMessageFromBinary(messageBody: ByteArray): ServiceBusReceivedMessage { + val message = mockk(relaxed = true) + val messageId ="MessageId123" + val status ="Active" + every{ message.messageId } returns messageId + every{ message.state.name } returns status + every { message.body.toBytes() } returns messageBody + return message + } + + +} + + + +