diff --git a/processing-status-api-function-app/src/main/java/gov/cdc/ocio/processingstatusapi/FunctionJavaWrappers.java b/processing-status-api-function-app/src/main/java/gov/cdc/ocio/processingstatusapi/FunctionJavaWrappers.java index ac4497f0..5c73f1ab 100644 --- a/processing-status-api-function-app/src/main/java/gov/cdc/ocio/processingstatusapi/FunctionJavaWrappers.java +++ b/processing-status-api-function-app/src/main/java/gov/cdc/ocio/processingstatusapi/FunctionJavaWrappers.java @@ -220,6 +220,28 @@ public HttpResponseMessage getReportCountsWithQueryParams( return new GetReportCountsFunction(request).withQueryParams(); } + @FunctionName("GetSubmissionCounts") + public HttpResponseMessage getSubmissionCounts( + @HttpTrigger( + name = "req", + methods = {HttpMethod.GET}, + route = "report/counts/submissions/summary", + authLevel = AuthorizationLevel.ANONYMOUS + ) HttpRequestMessage> request) { + return new GetReportCountsFunction(request).getSubmissionCounts(); + } + + @FunctionName("GetHL7InvalidStructureValidationCounts") + public HttpResponseMessage getHL7InvalidStructureValidationCounts( + @HttpTrigger( + name = "req", + methods = {HttpMethod.GET}, + route = "report/counts/hl7/invalidStructureValidation", + authLevel = AuthorizationLevel.ANONYMOUS + ) HttpRequestMessage> request) { + return new GetReportCountsFunction(request).getHL7InvalidStructureValidationCounts(); + } + @FunctionName("GetStatusByUploadId") public HttpResponseMessage getStatusByUploadId( @HttpTrigger( diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/status/GetReportCountsFunction.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/status/GetReportCountsFunction.kt index 1b85da74..4ee49a18 100644 --- a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/status/GetReportCountsFunction.kt +++ b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/status/GetReportCountsFunction.kt @@ -19,7 +19,9 @@ import gov.cdc.ocio.processingstatusapi.utils.PageUtils import mu.KotlinLogging import org.joda.time.DateTime import org.joda.time.DateTimeZone +import org.json.JSONObject import java.util.* +import java.util.AbstractMap.SimpleEntry /** @@ -152,8 +154,8 @@ class GetReportCountsFunction( val hl7ValidationCountsQuery = ( "select " + "r.uploadId, r.stageName, " - + "count(contains(upper(r.content.summary.current_status), 'VALID') ? 1 : undefined) as valid, " - + "count(not contains(upper(r.content.summary.current_status), 'VALID') ? 1 : undefined) as invalid " + + "count(contains(upper(r.content.summary.current_status), 'VALID_') ? 1 : undefined) as valid, " + + "count(not contains(upper(r.content.summary.current_status), 'VALID_') ? 1 : undefined) as invalid " + "from $reportsContainerName r " + "where r.content.schema_name = '$hl7ValidationSchemaName' and r.uploadId in ($quotedUploadIds) " + "group by r.uploadId, r.stageName" @@ -417,4 +419,234 @@ class GetReportCountsFunction( .body(gson.toJson(aggregateReportCounts)) .build() } + + /** + * Gets the total number of HL7 reports found with an invalid structure validation for the filter criteria + * provided. + * + * @return HttpResponseMessage + */ + fun getHL7InvalidStructureValidationCounts(): HttpResponseMessage { + + val dataStreamId = request.queryParameters["data_stream_id"] + val dataStreamRoute = request.queryParameters["data_stream_route"] + + val dateStart = request.queryParameters["date_start"] + val dateEnd = request.queryParameters["date_end"] + + val daysInterval = request.queryParameters["days_interval"] + + // Verify the request is complete and properly formatted + checkRequiredCountsQueryParams( + dataStreamId, + dataStreamRoute, + dateStart, + dateEnd, + daysInterval, + true + )?.let { return it } + + val timeRangeWhereClause: String + try { + timeRangeWhereClause = buildSqlClauseForDateRange(daysInterval, dateStart, dateEnd) + } catch (e: Exception) { + logger.error(e.localizedMessage) + return request + .createResponseBuilder(HttpStatus.BAD_REQUEST) + .body(e.localizedMessage) + .build() + } + + val reportsSqlQuery = ( + "select " + + "value count(not contains(upper(r.content.summary.current_status), 'VALID_') ? 1 : undefined) " + + "from $reportsContainerName r " + + "where r.content.schema_name = '${HL7Validation.schemaDefinition.schemaName}' and " + + "r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and $timeRangeWhereClause" + ) + + val startTime = System.currentTimeMillis() + val countResult = reportsContainer.queryItems( + reportsSqlQuery, CosmosQueryRequestOptions(), + Long::class.java + ) + val totalItems = countResult.firstOrNull() ?: 0 + val endTime = System.currentTimeMillis() + val countsJson = JSONObject() + .put("counts", totalItems) + .put("query_time_millis", endTime - startTime) + + return request + .createResponseBuilder(HttpStatus.OK) + .header("Content-Type", "application/json") + .body(countsJson.toString()) + .build() + } + + /** + * Get summary submission counts + * + * @return HttpResponseMessage + */ + fun getSubmissionCounts(): HttpResponseMessage { + + val dataStreamId = request.queryParameters["data_stream_id"] + val dataStreamRoute = request.queryParameters["data_stream_route"] + + val dateStart = request.queryParameters["date_start"] + val dateEnd = request.queryParameters["date_end"] + + val daysInterval = request.queryParameters["days_interval"] + + // Verify the request is complete and properly formatted + checkRequiredCountsQueryParams( + dataStreamId, + dataStreamRoute, + dateStart, + dateEnd, + daysInterval, + true + )?.let { return it } + + val timeRangeWhereClause: String + try { + timeRangeWhereClause = buildSqlClauseForDateRange(daysInterval, dateStart, dateEnd) + } catch (e: Exception) { + logger.error(e.localizedMessage) + return request + .createResponseBuilder(HttpStatus.BAD_REQUEST) + .body(e.localizedMessage) + .build() + } + + // Get number completed uploading + val numCompletedUploadingSqlQuery = ( + "select " + + "value count(1) " + + "from Reports r " + + "where r.content.schema_name = 'upload' and r.content['offset'] = r.content.size and " + + "r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and $timeRangeWhereClause" + ) + + val completedUploadingCountResult = reportsContainer.queryItems( + numCompletedUploadingSqlQuery, CosmosQueryRequestOptions(), + Long::class.java + ) + val totalCompletedUploading = completedUploadingCountResult.firstOrNull() ?: 0 + + val numUploadingSqlQuery = ( + "select " + + "value count(1) " + + "from Reports r " + + "where r.content.schema_name = 'upload' and r.content['offset'] != r.content.size and " + + "r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and $timeRangeWhereClause" + ) + + val uploadingCountResult = reportsContainer.queryItems( + numUploadingSqlQuery, CosmosQueryRequestOptions(), + Long::class.java + ) + val totalUploading = uploadingCountResult.firstOrNull() ?: 0 + + val numFailedSqlQuery = ( + "select " + + "value count(1) " + + "from Reports r " + + "where r.content.schema_name = 'dex-metadata-verify' and r.content.issues != null and " + + "r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and $timeRangeWhereClause" + ) + + val failedCountResult = reportsContainer.queryItems( + numFailedSqlQuery, CosmosQueryRequestOptions(), + Long::class.java + ) + val totalFailed = failedCountResult.firstOrNull() ?: 0 + + val counts = ProcessingCounts().apply { + totalCounts = totalCompletedUploading + totalUploading + totalFailed + statusCounts.apply { + uploaded.counts = totalCompletedUploading + failed.counts = totalFailed + failed.reasons = mapOf("metadata" to totalFailed) + uploading.counts = totalUploading + } + } + + return request + .createResponseBuilder(HttpStatus.OK) + .header("Content-Type", "application/json") + .body(gson.toJson(counts)) + .build() + } + + @Throws(NumberFormatException::class, BadRequestException::class) + private fun buildSqlClauseForDateRange(daysInterval: String?, + dateStart: String?, + dateEnd: String?): String { + + val timeRangeSqlPortion = StringBuilder() + if (!daysInterval.isNullOrBlank()) { + val dateStartEpochSecs = DateTime + .now(DateTimeZone.UTC) + .minusDays(Integer.parseInt(daysInterval)) + .withTimeAtStartOfDay() + .toDate() + .time / 1000 + timeRangeSqlPortion.append("r._ts >= $dateStartEpochSecs") + } else { + dateStart?.run { + val dateStartEpochSecs = DateUtils.getEpochFromDateString(dateStart, "date_start") + timeRangeSqlPortion.append("r._ts >= $dateStartEpochSecs") + } + dateEnd?.run { + val dateEndEpochSecs = DateUtils.getEpochFromDateString(dateEnd, "date_end") + timeRangeSqlPortion.append(" and r._ts < $dateEndEpochSecs") + } + } + return timeRangeSqlPortion.toString() + } + + /** + * Checks that all the required query parameters are present in order to process the request. If not, + * an appropriate HTTP response message is generated with the details. + * + * @return HttpResponseMessage? + */ + private fun checkRequiredCountsQueryParams(dataStreamId: String?, + dataStreamRoute: String?, + dateStart: String?, + dateEnd: String?, + daysInterval: String?, + dateRangeRequired: Boolean): HttpResponseMessage? { + + if (dataStreamId == null) { + return request + .createResponseBuilder(HttpStatus.BAD_REQUEST) + .body("destination_id or data_stream_id is required") + .build() + } + + if (dataStreamRoute == null) { + return request + .createResponseBuilder(HttpStatus.BAD_REQUEST) + .body("event_type or data_stream_route is required") + .build() + } + + if (!daysInterval.isNullOrBlank() && (!dateStart.isNullOrBlank() || !dateEnd.isNullOrBlank())) { + return request + .createResponseBuilder(HttpStatus.BAD_REQUEST) + .body("date_interval and date_start/date_end can't be used simultaneously") + .build() + } + + if (dateRangeRequired && daysInterval.isNullOrBlank() && dateStart.isNullOrBlank()) { + return request + .createResponseBuilder(HttpStatus.BAD_REQUEST) + .body("days_interval or date_start must be provided") + .build() + } + + return null + } } \ No newline at end of file diff --git a/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/reports/ProcessingCounts.kt b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/reports/ProcessingCounts.kt new file mode 100644 index 00000000..0d5952b6 --- /dev/null +++ b/processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/reports/ProcessingCounts.kt @@ -0,0 +1,22 @@ +package gov.cdc.ocio.processingstatusapi.model.reports + +import com.google.gson.annotations.SerializedName + +data class ProcessingCounts( + @SerializedName("total_counts") + var totalCounts: Long = 0, + + @SerializedName("status_counts") + var statusCounts: StatusCounts = StatusCounts() +) + +data class StatusCounts( + var uploading: CountsDetails = CountsDetails(), + var failed: CountsDetails = CountsDetails(), + var uploaded: CountsDetails = CountsDetails() +) + +data class CountsDetails( + var counts: Long = 0, + var reasons: Map? = null +)