feature: add filter to query csr (+ query csr tests) (#1337)

* fix: add filter to query csr (+ query csr tests)

* fix: detekt

* fix: PR comments

search-service/config/detekt/baseline.xml

@@ -4,7 +4,7 @@
   <CurrentIssues>
     <ID>ClassNaming:V0_29_JsonLd_migrationTests.kt$V0_29_JsonLd_migrationTests</ID>
     <ID>ClassNaming:V0_29__JsonLd_migration.kt$V0_29__JsonLd_migration : BaseJavaMigration</ID>
-    <ID>ComplexCondition:EntitiesQueryUtils.kt$geoQuery == null &amp;&amp; q.isNullOrEmpty() &amp;&amp; typeSelection.isNullOrEmpty() &amp;&amp; attrs.isEmpty()</ID>
+    <ID>ComplexCondition:EntitiesQueryUtils.kt$geoQuery == null &amp;&amp; q.isNullOrEmpty() &amp;&amp; typeSelection.isNullOrEmpty() &amp;&amp; attrs.isEmpty() &amp;&amp; !local</ID>
     <ID>Filename:V0_29__JsonLd_migration.kt$db.migration.V0_29__JsonLd_migration.kt</ID>
     <ID>LongMethod:AttributeInstanceService.kt$AttributeInstanceService$@Transactional suspend fun create(attributeInstance: AttributeInstance): Either&lt;APIException, Unit&gt;</ID>
     <ID>LongMethod:EnabledAuthorizationServiceTests.kt$EnabledAuthorizationServiceTests$@Test fun `it should return serialized access control entities with other rigths if user is owner`()</ID>
@@ -19,16 +19,14 @@
     <ID>LongParameterList:AttributeInstance.kt$AttributeInstance.Companion$( attributeUuid: UUID, instanceId: URI = generateRandomInstanceId(), timeAndProperty: Pair&lt;ZonedDateTime, TemporalProperty&gt;, value: Triple&lt;String?, Double?, WKTCoordinates?&gt;, payload: ExpandedAttributeInstance, sub: String? )</ID>
     <ID>LongParameterList:AttributeInstance.kt$AttributeInstance.Companion$( attributeUuid: UUID, instanceId: URI = generateRandomInstanceId(), timeProperty: TemporalProperty? = TemporalProperty.OBSERVED_AT, modifiedAt: ZonedDateTime? = null, attributeMetadata: AttributeMetadata, payload: ExpandedAttributeInstance, time: ZonedDateTime, sub: String? = null )</ID>
     <ID>LongParameterList:BusinessObjectsFactory.kt$( attributeUuid: UUID, timeProperty: AttributeInstance.TemporalProperty = AttributeInstance.TemporalProperty.OBSERVED_AT, measuredValue: Double? = Random.nextDouble(), value: String? = null, time: ZonedDateTime = ngsiLdDateTime(), sub: Sub? = null )</ID>
-    <ID>LongParameterList:EntitiesQuery.kt$EntitiesQuery$( open val q: String?, open val scopeQ: String?, open val paginationQuery: PaginationQuery, open val attrs: Set&lt;ExpandedTerm&gt;, open val datasetId: Set&lt;String&gt;, open val geoQuery: GeoQuery?, open val linkedEntityQuery: LinkedEntityQuery?, open val contexts: List&lt;String&gt; )</ID>
+    <ID>LongParameterList:EntitiesQuery.kt$EntitiesQuery$( open val q: String?, open val scopeQ: String?, open val paginationQuery: PaginationQuery, open val attrs: Set&lt;ExpandedTerm&gt;, open val datasetId: Set&lt;String&gt;, open val geoQuery: GeoQuery?, open val linkedEntityQuery: LinkedEntityQuery?, open val local: Boolean = false, open val contexts: List&lt;String&gt; )</ID>
     <ID>LongParameterList:EntityAttributeService.kt$EntityAttributeService$( attribute: Attribute, attributeName: ExpandedTerm, attributeMetadata: AttributeMetadata, mergedAt: ZonedDateTime, observedAt: ZonedDateTime?, attributePayload: ExpandedAttributeInstance, sub: Sub? )</ID>
     <ID>LongParameterList:EntityAttributeService.kt$EntityAttributeService$( attribute: Attribute, ngsiLdAttribute: NgsiLdAttribute, attributeMetadata: AttributeMetadata, createdAt: ZonedDateTime, attributePayload: ExpandedAttributeInstance, sub: Sub? )</ID>
     <ID>LongParameterList:EntityAttributeService.kt$EntityAttributeService$( entityId: URI, attributeName: ExpandedTerm, attributeMetadata: AttributeMetadata, createdAt: ZonedDateTime, attributePayload: ExpandedAttributeInstance, sub: Sub? )</ID>
     <ID>LongParameterList:EntityAttributeService.kt$EntityAttributeService$( entityUri: URI, ngsiLdAttributes: List&lt;NgsiLdAttribute&gt;, expandedAttributes: ExpandedAttributes, createdAt: ZonedDateTime, observedAt: ZonedDateTime?, sub: Sub? )</ID>
     <ID>LongParameterList:EntityAttributeService.kt$EntityAttributeService$( entityUri: URI, ngsiLdAttributes: List&lt;NgsiLdAttribute&gt;, expandedAttributes: ExpandedAttributes, disallowOverwrite: Boolean, createdAt: ZonedDateTime, sub: Sub? )</ID>
     <ID>LongParameterList:TemporalEntityHandler.kt$TemporalEntityHandler$( @RequestHeader httpHeaders: HttpHeaders, @PathVariable entityId: URI, @PathVariable attrId: String, @PathVariable instanceId: URI, @RequestBody requestBody: Mono&lt;String&gt;, @AllowedParameters(notImplemented = [QP.LOCAL, QP.VIA]) @RequestParam queryParams: MultiValueMap&lt;String, String&gt; )</ID>
     <ID>LongParameterList:V0_29__JsonLd_migration.kt$V0_29__JsonLd_migration$( entityId: URI, attributeName: ExpandedTerm, datasetId: URI?, attributePayload: ExpandedAttributeInstance, ngsiLdAttributeInstance: NgsiLdAttributeInstance, defaultCreatedAt: ZonedDateTime )</ID>
-    <ID>MaxLineLength:DistributedEntityProvisionServiceTests.kt$DistributedEntityProvisionServiceTests$fun</ID>
-    <ID>MaximumLineLength:DistributedEntityProvisionServiceTests.kt$DistributedEntityProvisionServiceTests$</ID>
     <ID>NestedBlockDepth:V0_29__JsonLd_migration.kt$V0_29__JsonLd_migration$override fun migrate(context: Context)</ID>
     <ID>SwallowedException:TemporalQueryUtils.kt$e: IllegalArgumentException</ID>
   </CurrentIssues>

search-service/src/main/kotlin/com/egm/stellio/search/csr/model/CSRFilters.kt

@@ -1,6 +1,14 @@
 package com.egm.stellio.search.csr.model
 
+import arrow.core.Either
+import arrow.core.raise.either
+import com.egm.stellio.shared.model.APIException
 import com.egm.stellio.shared.model.EntityTypeSelection
+import com.egm.stellio.shared.queryparameter.QueryParameter
+import com.egm.stellio.shared.util.expandTypeSelection
+import com.egm.stellio.shared.util.toListOfUri
+import com.egm.stellio.shared.util.validateIdPattern
+import org.springframework.util.MultiValueMap
 import java.net.URI
 
 open class CSRFilters( // we should use a combination of EntitiesQuery TemporalQuery (when we implement all operations)
@@ -16,9 +24,9 @@ open class CSRFilters( // we should use a combination of EntitiesQuery TemporalQ
         operations: List<Operation>?
     ) :
         this(
-            ids = ids,
-            typeSelection = typeSelection,
-            idPattern = idPattern,
+            ids,
+            typeSelection,
+            idPattern,
             csf = operations?.joinToString("|") { "${ContextSourceRegistration::operations.name}==${it.key}" }
         )
 
@@ -28,9 +36,22 @@ open class CSRFilters( // we should use a combination of EntitiesQuery TemporalQ
         idPattern: String? = null,
         operations: List<Operation>? = null
     ) : this(
-        ids = ids,
-        typeSelection = types.joinToString("|"),
-        idPattern = idPattern,
+        ids,
+        types.joinToString("|"),
+        idPattern,
         operations = operations
     )
+
+    companion object {
+        fun fromQueryParameters(
+            queryParams: MultiValueMap<String, String>,
+            contexts: List<String>
+        ): Either<APIException, CSRFilters> = either {
+            val ids = queryParams.getFirst(QueryParameter.ID.key)?.split(",").orEmpty().toListOfUri().toSet()
+            val typeSelection = expandTypeSelection(queryParams.getFirst(QueryParameter.TYPE.key), contexts)
+            val idPattern = validateIdPattern(queryParams.getFirst(QueryParameter.ID_PATTERN.key)).bind()
+
+            CSRFilters(ids, typeSelection, idPattern)
+        }
+    }
 }

search-service/src/main/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationService.kt

@@ -215,15 +215,22 @@ class ContextSourceRegistrationService(
             .allToMappedList { rowToContextSourceRegistration(it) }
     }
 
-    suspend fun getContextSourceRegistrationsCount(sub: Option<Sub>): Either<APIException, Int> {
+    suspend fun getContextSourceRegistrationsCount(
+        filters: CSRFilters = CSRFilters(),
+    ): Either<APIException, Int> {
+        val filterQuery = buildWhereStatement(filters)
+
         val selectStatement =
             """
-            SELECT count(*)
-            FROM context_source_registration
-            WHERE sub = :sub
+            SELECT count(distinct csr.id)
+            FROM context_source_registration as csr
+            LEFT JOIN jsonb_to_recordset(information)
+                as information(entities jsonb, propertyNames text[], relationshipNames text[]) on true
+            LEFT JOIN jsonb_to_recordset(entities)
+                as entity_info(id text, idPattern text, type text[]) on true
+            WHERE $filterQuery
             """.trimIndent()
         return databaseClient.sql(selectStatement)
-            .bind("sub", sub.toStringValue())
             .oneToResult { toInt(it["count"]) }
     }
 

search-service/src/main/kotlin/com/egm/stellio/search/csr/web/ContextSourceRegistrationHandler.kt

@@ -6,6 +6,7 @@ import arrow.core.flatMap
 import arrow.core.left
 import arrow.core.raise.either
 import arrow.core.right
+import com.egm.stellio.search.csr.model.CSRFilters
 import com.egm.stellio.search.csr.model.ContextSourceRegistration.Companion.deserialize
 import com.egm.stellio.search.csr.model.ContextSourceRegistration.Companion.unauthorizedMessage
 import com.egm.stellio.search.csr.model.serialize
@@ -83,12 +84,15 @@ class ContextSourceRegistrationHandler(
      * Implements 6.8.3.2 - Query ContextSourceRegistrations
      */
     @GetMapping(produces = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE])
-    suspend fun get(
+    suspend fun query(
         @RequestHeader httpHeaders: HttpHeaders,
         @AllowedParameters(
-            implemented = [QP.OPTIONS, QP.COUNT, QP.OFFSET, QP.LIMIT],
+            implemented = [
+                QP.OPTIONS, QP.COUNT, QP.OFFSET, QP.LIMIT,
+                QP.ID, QP.TYPE, QP.ID_PATTERN
+            ],
             notImplemented = [
-                QP.ID, QP.TYPE, QP.ID_PATTERN, QP.ATTRS, QP.Q, QP.CSF,
+                QP.ATTRS, QP.Q, QP.CSF,
                 QP.GEOMETRY, QP.GEOREL, QP.COORDINATES, QP.GEOPROPERTY,
                 QP.TIMEPROPERTY, QP.TIMEREL, QP.TIMEAT, QP.ENDTIMEAT,
                 QP.GEOMETRY_PROPERTY, QP.LANG, QP.SCOPEQ,
@@ -98,7 +102,7 @@ class ContextSourceRegistrationHandler(
     ): ResponseEntity<*> = either {
         val contexts = getContextFromLinkHeaderOrDefault(httpHeaders, applicationProperties.contexts.core).bind()
         val mediaType = getApplicableMediaType(httpHeaders).bind()
-        val sub = getSubFromSecurityContext()
+        val csrFilters = CSRFilters.fromQueryParameters(queryParams, contexts).bind()
 
         val includeSysAttrs = queryParams.getOrDefault(QueryParameter.OPTIONS.key, emptyList())
             .contains(OptionsValue.SYS_ATTRS.value)
@@ -108,11 +112,12 @@ class ContextSourceRegistrationHandler(
             applicationProperties.pagination.limitMax
         ).bind()
         val contextSourceRegistrations = contextSourceRegistrationService.getContextSourceRegistrations(
-            limit = paginationQuery.limit,
-            offset = paginationQuery.offset,
+            csrFilters,
+            paginationQuery.limit,
+            paginationQuery.offset,
         ).serialize(contexts, mediaType, includeSysAttrs)
         val contextSourceRegistrationsCount = contextSourceRegistrationService.getContextSourceRegistrationsCount(
-            sub
+            csrFilters
         ).bind()
 
         buildQueryResponse(
@@ -133,7 +138,7 @@ class ContextSourceRegistrationHandler(
      * Implements 6.9.3.1 - Retrieve ContextSourceRegistration
      */
     @GetMapping("/{contextSourceRegistrationId}", produces = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE])
-    suspend fun getByURI(
+    suspend fun retrieve(
         @RequestHeader httpHeaders: HttpHeaders,
         @PathVariable contextSourceRegistrationId: URI,
         @AllowedParameters(implemented = [QP.OPTIONS])

search-service/src/main/kotlin/com/egm/stellio/search/entity/model/EntitiesQuery.kt

@@ -16,6 +16,7 @@ sealed class EntitiesQuery(
     open val datasetId: Set<String>,
     open val geoQuery: GeoQuery?,
     open val linkedEntityQuery: LinkedEntityQuery?,
+    open val local: Boolean = false,
     open val contexts: List<String>
 )
 
@@ -30,8 +31,9 @@ data class EntitiesQueryFromGet(
     override val datasetId: Set<String> = emptySet(),
     override val geoQuery: GeoQuery? = null,
     override val linkedEntityQuery: LinkedEntityQuery? = null,
-    override val contexts: List<String>
-) : EntitiesQuery(q, scopeQ, paginationQuery, attrs, datasetId, geoQuery, linkedEntityQuery, contexts)
+    override val contexts: List<String>,
+    override val local: Boolean = false,
+) : EntitiesQuery(q, scopeQ, paginationQuery, attrs, datasetId, geoQuery, linkedEntityQuery, local, contexts)
 
 data class EntitiesQueryFromPost(
     val entitySelectors: List<EntitySelector>? = null,
@@ -42,5 +44,6 @@ data class EntitiesQueryFromPost(
     override val datasetId: Set<String> = emptySet(),
     override val geoQuery: GeoQuery? = null,
     override val linkedEntityQuery: LinkedEntityQuery? = null,
+    override val local: Boolean = false,
     override val contexts: List<String>
-) : EntitiesQuery(q, scopeQ, paginationQuery, attrs, datasetId, geoQuery, linkedEntityQuery, contexts)
+) : EntitiesQuery(q, scopeQ, paginationQuery, attrs, datasetId, geoQuery, linkedEntityQuery, local, contexts)

search-service/src/main/kotlin/com/egm/stellio/search/entity/util/EntitiesQueryUtils.kt

@@ -52,6 +52,7 @@ fun composeEntitiesQueryFromGet(
         queryParams.getFirst(QueryParameter.JOIN_LEVEL.key),
         queryParams.getFirst(QueryParameter.CONTAINED_BY.key)
     ).bind()
+    val local = queryParams.getFirst(QueryParameter.LOCAL.key)?.toBoolean() ?: false
 
     EntitiesQueryFromGet(
         ids = ids,
@@ -64,6 +65,7 @@ fun composeEntitiesQueryFromGet(
         datasetId = datasetId,
         geoQuery = geoQuery,
         linkedEntityQuery = linkedEntityQuery,
+        local = local,
         contexts = contexts
     )
 }
@@ -73,10 +75,11 @@ fun EntitiesQueryFromGet.validateMinimalQueryEntitiesParameters(): Either<APIExc
         geoQuery == null &&
         q.isNullOrEmpty() &&
         typeSelection.isNullOrEmpty() &&
-        attrs.isEmpty()
+        attrs.isEmpty() &&
+        !local
     )
         return@either BadRequestDataException(
-            "One of 'type', 'attrs', 'q', 'geoQ' must be provided in the query"
+            "One of 'type', 'attrs', 'q', 'geoQ' must be provided in the query unless local is true"
         ).left().bind<EntitiesQueryFromGet>()
 
     this@validateMinimalQueryEntitiesParameters

search-service/src/main/kotlin/com/egm/stellio/search/entity/web/EntityHandler.kt

@@ -231,7 +231,7 @@ class EntityHandler(
             }
 
         val (warnings, entities, count) =
-            if (queryParams.getFirst(QP.LOCAL.key)?.toBoolean() != true) {
+            if (entitiesQuery.local != true) {
                 val (queryWarnings, remoteEntitiesWithCSR, remoteCounts) =
                     distributedEntityConsumptionService.distributeQueryEntitiesOperation(
                         entitiesQuery,

search-service/src/test/kotlin/com/egm/stellio/search/csr/service/ContextSourceRegistrationServiceTests.kt

@@ -324,6 +324,23 @@ class ContextSourceRegistrationServiceTests : WithTimescaleContainer, WithKafkaC
         assertTrue(notMatchingCsr.isEmpty())
     }
 
+    @Test
+    fun `count should apply the filter`() = runTest {
+        val contextSourceRegistration =
+            loadAndDeserializeContextSourceRegistration("csr/contextSourceRegistration_minimal_entities.json")
+        contextSourceRegistrationService.create(contextSourceRegistration, mockUserSub).shouldSucceed()
+
+        val count = contextSourceRegistrationService.getContextSourceRegistrationsCount(
+            CSRFilters(idPattern = ".*")
+        )
+        assertEquals(1, count.getOrNull())
+
+        val countEmpty = contextSourceRegistrationService.getContextSourceRegistrationsCount(
+            CSRFilters(idPattern = "INVALID")
+        )
+        assertEquals(0, countEmpty.getOrNull())
+    }
+
     @Test
     fun `delete an existing CSR should succeed`() = runTest {
         val contextSourceRegistration =

search-service/src/test/kotlin/com/egm/stellio/search/csr/web/ContextSourceRegistrationHandlerTests.kt

@@ -11,6 +11,7 @@ import com.egm.stellio.shared.model.ResourceNotFoundException
 import com.egm.stellio.shared.util.AQUAC_HEADER_LINK
 import com.egm.stellio.shared.util.JSON_LD_MEDIA_TYPE
 import com.egm.stellio.shared.util.MOCK_USER_SUB
+import com.egm.stellio.shared.util.RESULTS_COUNT_HEADER
 import com.egm.stellio.shared.util.toUri
 import com.ninjasquad.springmockk.MockkBean
 import io.mockk.coEvery
@@ -108,6 +109,46 @@ class ContextSourceRegistrationHandlerTests {
         coVerify { contextSourceRegistrationService.getById(contextSourceRegistration.id) }
     }
 
+    @Test
+    fun `query CSR should return 200 whether a CSR exists or not`() = runTest {
+        val contextSourceRegistration = ContextSourceRegistration(id = id, endpoint = endpoint)
+
+        coEvery {
+            contextSourceRegistrationService.getContextSourceRegistrations(any(), any(), any())
+        } returns listOf(contextSourceRegistration)
+
+        coEvery { contextSourceRegistrationService.getContextSourceRegistrationsCount(any()) } returns 1.right()
+
+        webClient.get()
+            .uri("$csrUri?id=$id")
+            .exchange()
+            .expectStatus().isOk
+            .expectBody()
+
+        coVerify { contextSourceRegistrationService.getContextSourceRegistrations(any(), any()) }
+    }
+
+    @Test
+    fun `query CSR should return the count if it was asked`() = runTest {
+        val contextSourceRegistration = ContextSourceRegistration(id = id, endpoint = endpoint)
+
+        coEvery { contextSourceRegistrationService.isCreatorOf(any(), any()) } returns true.right()
+        coEvery {
+            contextSourceRegistrationService.getContextSourceRegistrations(any(), any(), any())
+        } returns listOf(contextSourceRegistration)
+
+        coEvery { contextSourceRegistrationService.getContextSourceRegistrationsCount(any()) } returns 1.right()
+
+        webClient.get()
+            .uri("$csrUri?id=$id&count=true")
+            .exchange()
+            .expectStatus().isOk
+            .expectHeader().exists(RESULTS_COUNT_HEADER)
+            .expectBody()
+
+        coVerify { contextSourceRegistrationService.getContextSourceRegistrations(any(), any()) }
+    }
+
     @Test
     fun `delete CSR should return the errors from the service`() = runTest {
         coEvery { contextSourceRegistrationService.isCreatorOf(any(), any()) } returns true.right()

search-service/src/test/kotlin/com/egm/stellio/search/entity/web/EntityHandlerTests.kt

@@ -1390,7 +1390,7 @@ class EntityHandlerTests {
                 """
                 {
                     "type": "https://uri.etsi.org/ngsi-ld/errors/BadRequestData",
-                    "title": "One of 'type', 'attrs', 'q', 'geoQ' must be provided in the query",
+                    "title": "One of 'type', 'attrs', 'q', 'geoQ' must be provided in the query unless local is true",
                     "detail": "$DEFAULT_DETAIL"
                 }
                 """.trimIndent()

search-service/src/test/kotlin/com/egm/stellio/search/temporal/util/TemporalQueryUtilsTests.kt

@@ -51,7 +51,10 @@ class TemporalQueryUtilsTests {
             true
         ).shouldFail {
             assertInstanceOf(BadRequestDataException::class.java, it)
-            assertEquals("One of 'type', 'attrs', 'q', 'geoQ' must be provided in the query", it.message)
+            assertEquals(
+                "One of 'type', 'attrs', 'q', 'geoQ' must be provided in the query unless local is true",
+                it.message
+            )
         }
     }
 

search-service/src/test/kotlin/com/egm/stellio/search/temporal/web/TemporalEntityHandlerTests.kt

@@ -1475,7 +1475,7 @@ class TemporalEntityHandlerTests : TemporalEntityHandlerTestCommon() {
                 """
                 {
                     "type": "https://uri.etsi.org/ngsi-ld/errors/BadRequestData",
-                    "title": "One of 'type', 'attrs', 'q', 'geoQ' must be provided in the query",
+                    "title": "One of 'type', 'attrs', 'q', 'geoQ' must be provided in the query unless local is true",
                     "detail": "$DEFAULT_DETAIL"
                 }
                 """.trimIndent()