Merged develop into main

pstatus-graphql-ktor/build.gradle

@@ -72,6 +72,17 @@ dependencies {
     implementation "org.jetbrains.kotlinx:kotlinx-serialization-json:1.6.0"
     testImplementation "io.ktor:ktor-server-tests-jvm:$ktor_version"
     testImplementation "org.jetbrains.kotlin:kotlin-test-junit:$kotlin_version"
+
+//    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.2'
+//    testImplementation 'org.mockito:mockito-core:4.6.1'
+//    testImplementation 'org.mockito:mockito-junit-jupiter:4.6.1'
+
+    testImplementation "org.junit.jupiter:junit-jupiter-api:5.8.1"
+    testImplementation "org.junit.jupiter:junit-jupiter-engine:5.8.1"
+    testImplementation "org.mockito:mockito-core:4.5.1"
+    testImplementation "org.mockito:mockito-inline:4.5.1"
+    testImplementation "io.mockk:mockk:1.13.9"
+    testImplementation 'io.insert-koin:koin-test:3.4.3'
 }
 
 ktor {
@@ -89,7 +100,7 @@ jib {
     }
     to {
         image = 'imagehub.cdc.gov:6989/dex/pstatus/graphql-service'
-        tags = ['v1.5.0']
+        tags = [ System.getenv("IMAGE_TAG") ?: "latest" ]
         auth {
             username = System.getenv("IMAGEHUB_USERNAME") ?: ""
             password = System.getenv("IMAGEHUB_PASSWORD") ?: ""

pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/CosmosLoader.kt

@@ -9,7 +9,7 @@ open class CosmosLoader: KoinComponent {
 
     private val cosmosRepository by inject<CosmosRepository>()
 
-    protected val reportsContainer = cosmosRepository.reportsContainer
+    protected var reportsContainer = cosmosRepository.reportsContainer
 
     protected val logger = KotlinLogging.logger {}
 }

pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/ReportLoader.kt

@@ -176,6 +176,7 @@ class ReportLoader: CosmosLoader() {
             dataStreamRoute = firstReport?.dataStreamRoute,
             jurisdiction = firstReport?.jurisdiction,
             senderId = firstReport?.senderId,
+            dataProducerId= firstReport?.dataProducerId,
             reports = reports
         )
     }

pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/UploadStatsLoader.kt

@@ -33,42 +33,42 @@ class UploadStatsLoader: CosmosLoader() {
                 "select value count(1) "
                         + "from r "
                         + "where r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and "
-                        + "r.content.schema_name = 'upload' and "
+                        + "r.stageInfo.service = 'UPLOAD API' and r.stageInfo.action = 'upload-status' and "
                         + timeRangeWhereClause
                 )
 
         val badMetadataCountQuery = (
                 "select value count(1) "
                         + "from r "
                         + "where r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and "
-                        + "r.content.schema_name = 'dex-metadata-verify' and "
-                        + "ARRAY_LENGTH(r.content.issues) > 0 and $timeRangeWhereClause"
+                        + "r.stageInfo.service = 'UPLOAD API' and r.stageInfo.action = 'metadata-verify' and "
+                        + "ARRAY_LENGTH(r.stageInfo.issues) > 0 and $timeRangeWhereClause"
                 )
 
         val inProgressUploadsCountQuery = (
                 "select value count(1) "
                         + "from r "
-                        + "where r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = 'dataStreamRoute' and "
-                        + "r.content.schema_name = 'upload' and "
+                        + "where r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and "
+                        + "r.stageInfo.service = 'UPLOAD API' and r.stageInfo.action = 'upload-status' and "
                         + "r.content['offset'] < r.content['size'] and $timeRangeWhereClause"
                 )
 
         val completedUploadsCountQuery = (
                 "select value count(1) "
                         + "from r "
                         + "where r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and "
-                        + "r.content.schema_name = 'upload' and "
+                        + "r.stageInfo.service = 'UPLOAD API' and r.stageInfo.action = 'upload-status' and "
                         + "r.content['offset'] = r.content['size'] and $timeRangeWhereClause"
                 )
 
         val duplicateFilenameCountQuery = (
                 "select * from "
-                        + "(select r.content.metadata.filename, count(1) as totalCount "
+                        + "(select r.content.metadata.received_filename, count(1) as totalCount "
                         + "from r "
-                        + "where r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = 'dataStreamRoute' and "
-                        + "r.content.schema_name = 'dex-metadata-verify' and "
+                        + "where r.dataStreamId = '$dataStreamId' and r.dataStreamRoute = '$dataStreamRoute' and "
+                        + "r.stageInfo.service = 'UPLOAD API' and r.stageInfo.action = 'metadata-verify' and "
                         + "$timeRangeWhereClause "
-                        + "group by r.content.metadata.filename"
+                        + "group by r.content.metadata.received_filename"
                         + ") r where r.totalCount > 1"
                 )
 

pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/loaders/UploadStatusLoader.kt

@@ -10,6 +10,7 @@ import gov.cdc.ocio.processingstatusapi.models.dao.ReportDao
 import gov.cdc.ocio.processingstatusapi.models.query.UploadCounts
 import gov.cdc.ocio.processingstatusapi.utils.DateUtils
 import gov.cdc.ocio.processingstatusapi.utils.PageUtils
+import gov.cdc.ocio.processingstatusapi.utils.SortUtils
 
 class UploadStatusLoader: CosmosLoader() {
 
@@ -38,8 +39,7 @@ class UploadStatusLoader: CosmosLoader() {
                         pageNumber: Int,
                         sortBy: String?,
                         sortOrder: String?,
-                        fileName:String?,
-                        status: String?
+                        fileName:String?
     ): UploadsStatus {
 
         logger.info("dataStreamId = $dataStreamId")
@@ -63,10 +63,6 @@ class UploadStatusLoader: CosmosLoader() {
             sqlQuery.append(" and r.content.filename = '$fileName'")
         }
 
-        status?.run {
-            sqlQuery.append(" and r.stageInfo.status = '$status'")
-        }
-
         dateStart?.run {
             val dateStartEpochSecs = DateUtils.getEpochFromDateString(dateStart, "date_start")
             sqlQuery.append(" and r.dexIngestDateTime >= $dateStartEpochSecs")
@@ -76,39 +72,48 @@ class UploadStatusLoader: CosmosLoader() {
             sqlQuery.append(" and r.dexIngestDateTime < $dateEndEpochSecs")
         }
 
-        sqlQuery.append(" group by r.uploadId")
+        sqlQuery.append(" group by r.uploadId, r.jurisdiction, r.senderId")
 
         // Check the sort field as well to add them to the group by clause
         var sortByQueryStr = StringBuilder()
         sortBy.run {
-            val sortField = when (sortBy) {
-                "date" -> "dexIngestDateTime" // "group  by _ts" is already added by default above
-                "fileName" -> "content.filename"
-                "dataStreamId" -> "dataStreamId"
-                "dataStreamRoute" -> "dataStreamRoute"
-                "service" -> "stageInfo.service"
-                "action" -> "stageInfo.action"
-                "jurisdiction" -> "jurisdiction"
-                "status" -> "stageInfo.status"
-                else -> {
-                    return@run
-                }
-            }
-            //Add the sort by fields to grouping
-            sqlQuery.append(" , r.$sortField, r.jurisdiction, r.senderId")
-
-            var sortOrderVal = DEFAULT_SORT_ORDER
-            sortOrder?.run {
-                sortOrderVal = when (sortOrder) {
-                    "ascending" -> "asc"
-                    "descending" -> "desc"
-                    else -> {
-                        return@run
+            if (sortBy != null) {
+                if (sortBy.isNotEmpty()) {
+                    val sortField = when (sortBy) {
+                        "date" -> "dexIngestDateTime" // "group  by _ts" is already added by default above
+                        "fileName" -> "content.filename"
+                        "dataStreamId" -> "dataStreamId"
+                        "dataStreamRoute" -> "dataStreamRoute"
+                        "service" -> "stageInfo.service"
+                        "action" -> "stageInfo.action"
+                        "jurisdiction" -> "jurisdiction"
+                        "status" -> "stageInfo.status"
+                        else -> {
+                            throw BadRequestException("Invalid sort field: $sortBy")
+                        }
                     }
+                    //Add the sort by fields to grouping
+                    sqlQuery.append(" , r.$sortField")
+
+                    var sortOrderVal = DEFAULT_SORT_ORDER
+
+                    sortOrder?.run {
+                        if (sortOrder.isNotEmpty()) {
+                            sortOrderVal = when (sortOrder.lowercase()) {
+                                "asc", "ascending" -> "asc"
+                                "desc", "descending" -> "desc"
+                                else -> {
+                                    throw BadRequestException("Invalid sort order: $sortOrder")
+                                }
+                            }
+                        }
+
+                    }
+                    //Sort By/ Order By the given sort field
+                    sortByQueryStr.append(" order by r.$sortField $sortOrderVal")
                 }
             }
-            //Sort By/ Order By the given sort field
-            sortByQueryStr.append(" order by r.$sortField $sortOrderVal")
+
 
         }
 
@@ -186,8 +191,14 @@ class UploadStatusLoader: CosmosLoader() {
         uploadsStatus.summary.pageSize = pageSize
         uploadsStatus.summary.numberOfPages = numberOfPages
         uploadsStatus.summary.totalItems = totalItems
-        uploadsStatus.summary.senderIds = senderIds
-        uploadsStatus.summary.jurisdictions = jurisdictions
+        uploadsStatus.summary.senderIds = senderIds.toMutableList()
+        uploadsStatus.summary.jurisdictions = jurisdictions.toMutableList()
+
+        if(sortOrder.isNullOrEmpty()){
+            SortUtils.sortByField(uploadsStatus, sortBy.toString(), DEFAULT_SORT_ORDER)
+        }else{
+            SortUtils.sortByField(uploadsStatus, sortBy.toString(), sortOrder.toString())
+        }
 
         return uploadsStatus
     }
@@ -197,6 +208,6 @@ class UploadStatusLoader: CosmosLoader() {
         private const val MAX_PAGE_SIZE = 10000
         const val DEFAULT_PAGE_SIZE = 100
 
-        private const val DEFAULT_SORT_ORDER = "asc"
+        private const val DEFAULT_SORT_ORDER = "desc"
     }
 }

pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/Report.kt

@@ -22,6 +22,7 @@ import java.time.OffsetDateTime
  * @property contentType String?
  * @property jurisdiction String?
  * @property senderId String?
+ * @property dataProducerId String?
  * @property content Map<*, *>?
  * @property timestamp OffsetDateTime?
  * @constructor
@@ -68,6 +69,9 @@ data class Report(
     @GraphQLDescription("SenderId this report belongs to; set to null if not applicable")
     var senderId: String? = null,
 
+    @GraphQLDescription("DataProducerId stated in the report; set to null if not applicable")
+    var dataProducerId: String? = null,
+
     @GraphQLDescription("Content of the report.  If the report is JSON then the content will be shown as JSON.  Otherwise, the content is a base64 encoded string.")
     var content : Map<*, *>? = null,
 

pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/ReportDeadLetter.kt

@@ -71,6 +71,9 @@ data class ReportDeadLetter(
     @GraphQLDescription("SenderId this report belongs to; set to null if not applicable")
     var senderId: String? = null,
 
+    @GraphQLDescription("DataProducerId stated in the report; set to null if not applicable")
+    var dataProducerId: String? = null,
+
     @GraphQLDescription("Content of the report.  If the report is JSON then the content will be shown as JSON.  Otherwise, the content is a base64 encoded string.")
     var content : Map<*, *>? = null,
 
@@ -84,5 +87,5 @@ data class ReportDeadLetter(
     var deadLetterReasons: List<String>? = null,
 
     @GraphQLDescription("Schemas used to validate the report")
-    var validationSchemas: List<String>? = null
+    var validationSchemas: List<String>? = null,
 )

pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/dao/ReportDao.kt

@@ -23,6 +23,7 @@ import java.util.*
  * @property content Any?
  * @property jurisdiction String?
  * @property senderId String?
+ * @property dataProducerId String?
  * @property contentAsString String?
  * @constructor
  */
@@ -54,6 +55,8 @@ open class ReportDao(
 
     var senderId:String? = null,
 
+    var dataProducerId:String? = null,
+
     var timestamp: Date? = null,
 
     var content: Any? = null
@@ -92,6 +95,7 @@ open class ReportDao(
         this.data = this@ReportDao.data
         this.jurisdiction = this@ReportDao.jurisdiction
         this.senderId = this@ReportDao.senderId
+        this.dataProducerId= this@ReportDao.dataProducerId
         this.timestamp = this@ReportDao.timestamp?.toInstant()?.atOffset(ZoneOffset.UTC)
         this.contentType = this@ReportDao.contentType
         this.content = this@ReportDao.content as? Map<*, *>

pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/dao/ReportDeadLetterDao.kt

@@ -32,12 +32,15 @@ data class ReportDeadLetterDao(
         this.data = this@ReportDeadLetterDao.data
         this.jurisdiction = this@ReportDeadLetterDao.jurisdiction
         this.senderId = this@ReportDeadLetterDao.senderId
+        this.dataProducerId = this@ReportDeadLetterDao.dataProducerId
         this.timestamp = this@ReportDeadLetterDao.timestamp?.toInstant()?.atOffset(ZoneOffset.UTC)
         this.contentType = this@ReportDeadLetterDao.contentType
         this.content = this@ReportDeadLetterDao.content as? Map<*, *>
         this.dispositionType = this@ReportDeadLetterDao.dispositionType
         this.deadLetterReasons = this@ReportDeadLetterDao.deadLetterReasons
         this.validationSchemas = this@ReportDeadLetterDao.validationSchemas
+
+
     }
 
 }

pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/query/PageSummary.kt

@@ -27,8 +27,8 @@ data class PageSummary(
     var totalItems: Int = 0,
 
     @GraphQLDescription("List of all the senderIds in the entire dataset matching the search criteria, not just this page.")
-    var senderIds: List<String> = listOf(),
+    var senderIds: MutableList<String> = mutableListOf(),
 
     @GraphQLDescription("List of all the jurisdictions in the entire dataset matching the search criteria, not just this page.")
-    var jurisdictions: List<String> = listOf()
+    var jurisdictions: MutableList<String> = mutableListOf()
 )

pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/query/UploadStatus.kt

@@ -63,6 +63,9 @@ class UploadStatus {
     @GraphQLDescription("The senderId of the unit/organization from which the file was sent")
     var senderId: String? = null
 
+    @GraphQLDescription("The dataProducerId from which the file was sent")
+    var dataProducerId: String? = null
+
     @GraphQLDescription("The jurisdiction from which the file was sent")
     var jurisdiction: String? = null
 

pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/models/submission/SubmissionDetails.kt

@@ -17,6 +17,7 @@ import java.time.OffsetDateTime
  * @property dataStreamRoute String?
  * @property jurisdiction String?
  * @property senderId String?
+ * @property dataProducerId String?
  * @property reports List<Report>
  */
 @GraphQLDescription("Contains upload details")
@@ -52,6 +53,9 @@ data class SubmissionDetails(
     @GraphQLDescription("SenderId mentioned in the report")
     var senderId: String? = null,
 
+    @GraphQLDescription("DataProducerId mentioned in the report")
+    var dataProducerId: String? = null,
+
     @GraphQLDescription("Array of the raw reports provided for this upload ID.")
     var reports: List<Report>? = null
 )

pstatus-graphql-ktor/src/main/kotlin/gov/cdc/ocio/processingstatusapi/queries/UploadQueryService.kt

@@ -58,17 +58,12 @@ class UploadQueryService : Query {
                             + "`desc`: Descending order\n"
                             + "If a value is provided that is not supported then a bad request response is returned."
                 )
-                sortOrder: String?,
+                sortOrder: String? = null,
 
                 @GraphQLDescription(
                     "*File Name* Search by the provided File Name:\n"
                 )
-                fileName: String? = null,
-
-                @GraphQLDescription(
-                    "*Status* Search by the status of the upload:\n"
-                )
-                status: String? = null) =
+                fileName: String? = null) =
         UploadStatusLoader().getUploadStatus(dataStreamId,
                                         dataStreamRoute,
                                         dateStart,
@@ -77,8 +72,7 @@ class UploadQueryService : Query {
                                         pageNumber,
                                         sortBy,
                                         sortOrder,
-                                        fileName,
-                                        status)
+                                        fileName)
 
     @GraphQLDescription("Return various uploads statistics")
     @Suppress("unused")