DASB-310 - HL7 report counts

processing-status-api-function-app/src/main/java/gov/cdc/ocio/processingstatusapi/FunctionJavaWrappers.java

@@ -220,6 +220,28 @@ public HttpResponseMessage getReportCountsWithQueryParams(
         return new GetReportCountsFunction(request).withQueryParams();
     }
 
+    @FunctionName("GetSubmissionCounts")
+    public HttpResponseMessage getSubmissionCounts(
+            @HttpTrigger(
+                    name = "req",
+                    methods = {HttpMethod.GET},
+                    route = "report/counts/submissions/summary",
+                    authLevel = AuthorizationLevel.ANONYMOUS
+            ) HttpRequestMessage<Optional<String>> request) {
+        return new GetReportCountsFunction(request).getSubmissionCounts();
+    }
+
+    @FunctionName("GetHL7InvalidStructureValidationCounts")
+    public HttpResponseMessage getHL7InvalidStructureValidationCounts(
+            @HttpTrigger(
+                    name = "req",
+                    methods = {HttpMethod.GET},
+                    route = "report/counts/hl7/invalidStructureValidation",
+                    authLevel = AuthorizationLevel.ANONYMOUS
+            ) HttpRequestMessage<Optional<String>> request) {
+        return new GetReportCountsFunction(request).getHL7InvalidStructureValidationCounts();
+    }
+
     @FunctionName("GetStatusByUploadId")
     public HttpResponseMessage getStatusByUploadId(
             @HttpTrigger(

processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/functions/status/GetReportCountsFunction.kt

@@ -19,7 +19,9 @@ import gov.cdc.ocio.processingstatusapi.utils.PageUtils
 import mu.KotlinLogging
 import org.joda.time.DateTime
 import org.joda.time.DateTimeZone
+import org.json.JSONObject
 import java.util.*
+import java.util.AbstractMap.SimpleEntry
 
 
 /**
@@ -152,8 +154,8 @@ class GetReportCountsFunction(
         val hl7ValidationCountsQuery = (
             "select "
             + "r.uploadId, r.stageName, "
-            + "count(contains(upper(r.content.summary.current_status), 'VALID') ? 1 : undefined) as valid, "
-            + "count(not contains(upper(r.content.summary.current_status), 'VALID') ? 1 : undefined) as invalid "
+            + "count(contains(upper(r.content.summary.current_status), 'VALID_') ? 1 : undefined) as valid, "
+            + "count(not contains(upper(r.content.summary.current_status), 'VALID_') ? 1 : undefined) as invalid "
             + "from $reportsContainerName r "
             + "where r.content.schema_name = '$hl7ValidationSchemaName' and r.uploadId in ($quotedUploadIds) "
             + "group by r.uploadId, r.stageName"
@@ -417,4 +419,234 @@ class GetReportCountsFunction(
             .body(gson.toJson(aggregateReportCounts))
             .build()
     }
+
+    /**
+     * Gets the total number of HL7 reports found with an invalid structure validation for the filter criteria
+     * provided.
+     *
+     * @return HttpResponseMessage
+     */
+    fun getHL7InvalidStructureValidationCounts(): HttpResponseMessage {
+
+        val dataStreamId = request.queryParameters["data_stream_id"]
+        val dataStreamRoute = request.queryParameters["data_stream_route"]
+
+        val dateStart = request.queryParameters["date_start"]
+        val dateEnd = request.queryParameters["date_end"]
+
+        val daysInterval = request.queryParameters["days_interval"]
+
+        // Verify the request is complete and properly formatted
+        checkRequiredCountsQueryParams(
+            dataStreamId,
+            dataStreamRoute,
+            dateStart,
+            dateEnd,
+            daysInterval,
+            true
+        )?.let { return it }
+
+        val timeRangeWhereClause: String
+        try {
+            timeRangeWhereClause = buildSqlClauseForDateRange(daysInterval, dateStart, dateEnd)
+        } catch (e: Exception) {
+            logger.error(e.localizedMessage)
+            return request
+                .createResponseBuilder(HttpStatus.BAD_REQUEST)
+                .body(e.localizedMessage)
+                .build()
+        }
+
+        val reportsSqlQuery = (
+            "select "
+            + "value count(not contains(upper(r.content.summary.current_status), 'VALID_') ? 1 : undefined) "
+            + "from $reportsContainerName r "
+            + "where r.content.schema_name = '${HL7Validation.schemaDefinition.schemaName}' and "
+            + "r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and $timeRangeWhereClause"
+        )
+
+        val startTime = System.currentTimeMillis()
+        val countResult = reportsContainer.queryItems(
+            reportsSqlQuery, CosmosQueryRequestOptions(),
+            Long::class.java
+        )
+        val totalItems = countResult.firstOrNull() ?: 0
+        val endTime = System.currentTimeMillis()
+        val countsJson = JSONObject()
+            .put("counts", totalItems)
+            .put("query_time_millis", endTime - startTime)
+
+        return request
+            .createResponseBuilder(HttpStatus.OK)
+            .header("Content-Type", "application/json")
+            .body(countsJson.toString())
+            .build()
+    }
+
+    /**
+     * Get summary submission counts
+     *
+     * @return HttpResponseMessage
+     */
+    fun getSubmissionCounts(): HttpResponseMessage {
+
+        val dataStreamId = request.queryParameters["data_stream_id"]
+        val dataStreamRoute = request.queryParameters["data_stream_route"]
+
+        val dateStart = request.queryParameters["date_start"]
+        val dateEnd = request.queryParameters["date_end"]
+
+        val daysInterval = request.queryParameters["days_interval"]
+
+        // Verify the request is complete and properly formatted
+        checkRequiredCountsQueryParams(
+            dataStreamId,
+            dataStreamRoute,
+            dateStart,
+            dateEnd,
+            daysInterval,
+            true
+        )?.let { return it }
+
+        val timeRangeWhereClause: String
+        try {
+            timeRangeWhereClause = buildSqlClauseForDateRange(daysInterval, dateStart, dateEnd)
+        } catch (e: Exception) {
+            logger.error(e.localizedMessage)
+            return request
+                .createResponseBuilder(HttpStatus.BAD_REQUEST)
+                .body(e.localizedMessage)
+                .build()
+        }
+
+        // Get number completed uploading
+        val numCompletedUploadingSqlQuery = (
+            "select "
+                + "value count(1) "
+                + "from Reports r "
+                + "where r.content.schema_name = 'upload' and r.content['offset'] = r.content.size and "
+                + "r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and $timeRangeWhereClause"
+        )
+
+        val completedUploadingCountResult = reportsContainer.queryItems(
+            numCompletedUploadingSqlQuery, CosmosQueryRequestOptions(),
+            Long::class.java
+        )
+        val totalCompletedUploading = completedUploadingCountResult.firstOrNull() ?: 0
+
+        val numUploadingSqlQuery = (
+            "select "
+                + "value count(1) "
+                + "from Reports r "
+                + "where r.content.schema_name = 'upload' and r.content['offset'] != r.content.size and "
+                + "r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and $timeRangeWhereClause"
+        )
+
+        val uploadingCountResult = reportsContainer.queryItems(
+            numUploadingSqlQuery, CosmosQueryRequestOptions(),
+            Long::class.java
+        )
+        val totalUploading = uploadingCountResult.firstOrNull() ?: 0
+
+        val numFailedSqlQuery = (
+            "select "
+                + "value count(1) "
+                + "from Reports r "
+                + "where r.content.schema_name = 'dex-metadata-verify' and r.content.issues != null and "
+                + "r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and $timeRangeWhereClause"
+        )
+
+        val failedCountResult = reportsContainer.queryItems(
+            numFailedSqlQuery, CosmosQueryRequestOptions(),
+            Long::class.java
+        )
+        val totalFailed = failedCountResult.firstOrNull() ?: 0
+
+        val counts = ProcessingCounts().apply {
+            totalCounts = totalCompletedUploading + totalUploading + totalFailed
+            statusCounts.apply {
+                uploaded.counts = totalCompletedUploading
+                failed.counts = totalFailed
+                failed.reasons = mapOf("metadata" to totalFailed)
+                uploading.counts = totalUploading
+            }
+        }
+
+        return request
+            .createResponseBuilder(HttpStatus.OK)
+            .header("Content-Type", "application/json")
+            .body(gson.toJson(counts))
+            .build()
+    }
+
+    @Throws(NumberFormatException::class, BadRequestException::class)
+    private fun buildSqlClauseForDateRange(daysInterval: String?,
+                                           dateStart: String?,
+                                           dateEnd: String?): String {
+
+        val timeRangeSqlPortion = StringBuilder()
+        if (!daysInterval.isNullOrBlank()) {
+            val dateStartEpochSecs = DateTime
+                .now(DateTimeZone.UTC)
+                .minusDays(Integer.parseInt(daysInterval))
+                .withTimeAtStartOfDay()
+                .toDate()
+                .time / 1000
+            timeRangeSqlPortion.append("r._ts >= $dateStartEpochSecs")
+        } else {
+            dateStart?.run {
+                val dateStartEpochSecs = DateUtils.getEpochFromDateString(dateStart, "date_start")
+                timeRangeSqlPortion.append("r._ts >= $dateStartEpochSecs")
+            }
+            dateEnd?.run {
+                val dateEndEpochSecs = DateUtils.getEpochFromDateString(dateEnd, "date_end")
+                timeRangeSqlPortion.append(" and r._ts < $dateEndEpochSecs")
+            }
+        }
+        return timeRangeSqlPortion.toString()
+    }
+
+    /**
+     * Checks that all the required query parameters are present in order to process the request.  If not,
+     * an appropriate HTTP response message is generated with the details.
+     *
+     * @return HttpResponseMessage?
+     */
+    private fun checkRequiredCountsQueryParams(dataStreamId: String?,
+                                               dataStreamRoute: String?,
+                                               dateStart: String?,
+                                               dateEnd: String?,
+                                               daysInterval: String?,
+                                               dateRangeRequired: Boolean): HttpResponseMessage? {
+
+        if (dataStreamId == null) {
+            return request
+                .createResponseBuilder(HttpStatus.BAD_REQUEST)
+                .body("destination_id or data_stream_id is required")
+                .build()
+        }
+
+        if (dataStreamRoute == null) {
+            return request
+                .createResponseBuilder(HttpStatus.BAD_REQUEST)
+                .body("event_type or data_stream_route is required")
+                .build()
+        }
+
+        if (!daysInterval.isNullOrBlank() && (!dateStart.isNullOrBlank() || !dateEnd.isNullOrBlank())) {
+            return request
+                .createResponseBuilder(HttpStatus.BAD_REQUEST)
+                .body("date_interval and date_start/date_end can't be used simultaneously")
+                .build()
+        }
+
+        if (dateRangeRequired && daysInterval.isNullOrBlank() && dateStart.isNullOrBlank()) {
+            return request
+                .createResponseBuilder(HttpStatus.BAD_REQUEST)
+                .body("days_interval or date_start must be provided")
+                .build()
+        }
+
+        return null
+    }
 }

processing-status-api-function-app/src/main/kotlin/gov/cdc/ocio/processingstatusapi/model/reports/ProcessingCounts.kt

@@ -0,0 +1,22 @@
+package gov.cdc.ocio.processingstatusapi.model.reports
+
+import com.google.gson.annotations.SerializedName
+
+data class ProcessingCounts(
+    @SerializedName("total_counts")
+    var totalCounts: Long = 0,
+
+    @SerializedName("status_counts")
+    var statusCounts: StatusCounts = StatusCounts()
+)
+
+data class StatusCounts(
+    var uploading: CountsDetails = CountsDetails(),
+    var failed: CountsDetails = CountsDetails(),
+    var uploaded: CountsDetails = CountsDetails()
+)
+
+data class CountsDetails(
+    var counts: Long = 0,
+    var reasons: Map<String, Long>? = null
+)