From 05abe1cfbf3f61863c9d41c8dda94f613a6ed2af Mon Sep 17 00:00:00 2001 From: Fabian Engelniederhammer Date: Wed, 17 Apr 2024 17:30:01 +0200 Subject: [PATCH] wip --- .../genspectrum/lapis/controller/CsvWriter.kt | 24 ++-- .../lapis/controller/LapisController.kt | 95 ++++++------- .../genspectrum/lapis/model/SiloQueryModel.kt | 13 +- .../org/genspectrum/lapis/silo/SiloClient.kt | 81 ++++++----- .../org/genspectrum/lapis/silo/SiloQuery.kt | 42 +++--- .../auth/ProtectedDataAuthorizationTest.kt | 5 +- .../LapisControllerCommonFieldsTest.kt | 21 +-- .../controller/LapisControllerCsvTest.kt | 5 +- .../lapis/controller/LapisControllerTest.kt | 43 ++++-- .../genspectrum/lapis/controller/MockData.kt | 7 +- .../lapis/model/SiloQueryModelTest.kt | 17 +-- .../genspectrum/lapis/silo/SiloClientTest.kt | 130 +++++------------- 12 files changed, 217 insertions(+), 266 deletions(-) diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CsvWriter.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CsvWriter.kt index 95eb41055..ee7fff650 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CsvWriter.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CsvWriter.kt @@ -4,7 +4,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVPrinter import org.springframework.stereotype.Component -import java.io.StringWriter +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody +import java.util.stream.Stream interface CsvRecord { @JsonIgnore @@ -17,29 +18,28 @@ interface CsvRecord { @Component class CsvWriter { fun write( - headers: Array?, - data: List, + includeHeaders: Boolean, + data: Stream, delimiter: Delimiter, - ): String { - val stringWriter = StringWriter() + ): StreamingResponseBody = StreamingResponseBody { stream-> + var shouldWriteHeaders = includeHeaders + CSVPrinter( - stringWriter, + stream.writer(), CSVFormat.DEFAULT.builder() .setRecordSeparator("\n") .setDelimiter(delimiter.value) .setNullString("") - .also { - when { - headers != null -> it.setHeader(*headers) - } - } .build(), ).use { for (datum in data) { + if (shouldWriteHeaders) { + it.printRecord(datum.getHeader()) + shouldWriteHeaders = false + } it.printRecord(datum.getValuesList()) } } - return stringWriter.toString().trimEnd('\n') } } diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/LapisController.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/LapisController.kt index 0292b8cd7..9cbde6eb9 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/LapisController.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/LapisController.kt @@ -69,6 +69,8 @@ import org.springframework.web.bind.annotation.RequestHeader import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody +import java.util.stream.Stream @RestController @RequestMapping("/sample") @@ -126,7 +128,7 @@ class LapisController( requestContext.filter = request - return LapisResponse(siloQueryModel.getAggregated(request)) + return LapisResponse(siloQueryModel.getAggregated(request).toList()) } @GetMapping(AGGREGATED_ROUTE, produces = [TEXT_CSV]) @@ -167,7 +169,7 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val request = SequenceFiltersRequestWithFields( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -221,7 +223,7 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val request = SequenceFiltersRequestWithFields( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -249,7 +251,7 @@ class LapisController( ): LapisResponse> { requestContext.filter = request - return LapisResponse(siloQueryModel.getAggregated(request)) + return LapisResponse(siloQueryModel.getAggregated(request).toList()) } @PostMapping(AGGREGATED_ROUTE, produces = [TEXT_CSV]) @@ -263,7 +265,7 @@ class LapisController( @RequestBody request: SequenceFiltersRequestWithFields, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { return getResponseAsCsv(request, httpHeaders.accept, COMMA, siloQueryModel::getAggregated) } @@ -278,7 +280,7 @@ class LapisController( @RequestBody request: SequenceFiltersRequestWithFields, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { return getResponseAsCsv(request, httpHeaders.accept, TAB, siloQueryModel::getAggregated) } @@ -330,7 +332,7 @@ class LapisController( requestContext.filter = mutationProportionsRequest val result = siloQueryModel.computeNucleotideMutationProportions(mutationProportionsRequest) - return LapisResponse(result) + return LapisResponse(result.toList()) } @GetMapping(NUCLEOTIDE_MUTATIONS_ROUTE, produces = [TEXT_CSV]) @@ -366,7 +368,7 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val request = MutationProportionsRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -421,7 +423,7 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val request = MutationProportionsRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -451,7 +453,7 @@ class LapisController( requestContext.filter = mutationProportionsRequest val result = siloQueryModel.computeNucleotideMutationProportions(mutationProportionsRequest) - return LapisResponse(result) + return LapisResponse(result.toList()) } @PostMapping(NUCLEOTIDE_MUTATIONS_ROUTE, produces = [TEXT_CSV]) @@ -465,7 +467,7 @@ class LapisController( @RequestBody mutationProportionsRequest: MutationProportionsRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { return getResponseAsCsv( mutationProportionsRequest, httpHeaders.accept, @@ -485,7 +487,7 @@ class LapisController( @RequestBody mutationProportionsRequest: MutationProportionsRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { return getResponseAsCsv( mutationProportionsRequest, httpHeaders.accept, @@ -538,7 +540,7 @@ class LapisController( requestContext.filter = mutationProportionsRequest val result = siloQueryModel.computeAminoAcidMutationProportions(mutationProportionsRequest) - return LapisResponse(result) + return LapisResponse(result.toList()) } @GetMapping(AMINO_ACID_MUTATIONS_ROUTE, produces = [TEXT_CSV]) @@ -574,7 +576,7 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val mutationProportionsRequest = MutationProportionsRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -629,7 +631,7 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val mutationProportionsRequest = MutationProportionsRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -664,7 +666,7 @@ class LapisController( requestContext.filter = mutationProportionsRequest val result = siloQueryModel.computeAminoAcidMutationProportions(mutationProportionsRequest) - return LapisResponse(result) + return LapisResponse(result.toList()) } @PostMapping(AMINO_ACID_MUTATIONS_ROUTE, produces = [TEXT_CSV]) @@ -678,7 +680,7 @@ class LapisController( @RequestBody mutationProportionsRequest: MutationProportionsRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { requestContext.filter = mutationProportionsRequest return getResponseAsCsv( @@ -700,7 +702,7 @@ class LapisController( @RequestBody mutationProportionsRequest: MutationProportionsRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { requestContext.filter = mutationProportionsRequest return getResponseAsCsv( @@ -758,7 +760,7 @@ class LapisController( ) requestContext.filter = request - return LapisResponse(siloQueryModel.getDetails(request)) + return LapisResponse(siloQueryModel.getDetails(request).toList()) } @GetMapping(DETAILS_ROUTE, produces = [TEXT_CSV]) @@ -795,7 +797,7 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val request = SequenceFiltersRequestWithFields( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -846,7 +848,7 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val request = SequenceFiltersRequestWithFields( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -874,7 +876,7 @@ class LapisController( ): LapisResponse> { requestContext.filter = request - return LapisResponse(siloQueryModel.getDetails(request)) + return LapisResponse(siloQueryModel.getDetails(request).toList()) } @PostMapping(DETAILS_ROUTE, produces = [TEXT_CSV]) @@ -888,7 +890,7 @@ class LapisController( @RequestBody request: SequenceFiltersRequestWithFields, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { return getResponseAsCsv(request, httpHeaders.accept, COMMA, siloQueryModel::getDetails) } @@ -903,7 +905,7 @@ class LapisController( @RequestBody request: SequenceFiltersRequestWithFields, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { return getResponseAsCsv(request, httpHeaders.accept, TAB, siloQueryModel::getDetails) } @@ -952,7 +954,7 @@ class LapisController( requestContext.filter = request val result = siloQueryModel.getNucleotideInsertions(request) - return LapisResponse(result) + return LapisResponse(result.toList()) } @GetMapping(NUCLEOTIDE_INSERTIONS_ROUTE, produces = [TEXT_CSV]) @@ -990,7 +992,7 @@ class LapisController( @RequestParam dataFormat: String? = null, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val request = SequenceFiltersRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -1042,7 +1044,7 @@ class LapisController( @RequestParam dataFormat: String? = null, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val request = SequenceFiltersRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -1072,7 +1074,7 @@ class LapisController( requestContext.filter = request val result = siloQueryModel.getNucleotideInsertions(request) - return LapisResponse(result) + return LapisResponse(result.toList()) } @PostMapping(NUCLEOTIDE_INSERTIONS_ROUTE, produces = [TEXT_CSV]) @@ -1086,7 +1088,7 @@ class LapisController( @RequestBody request: SequenceFiltersRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { requestContext.filter = request return getResponseAsCsv(request, httpHeaders.accept, COMMA, siloQueryModel::getNucleotideInsertions) @@ -1103,7 +1105,7 @@ class LapisController( @RequestBody request: SequenceFiltersRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { requestContext.filter = request return getResponseAsCsv(request, httpHeaders.accept, TAB, siloQueryModel::getNucleotideInsertions) @@ -1154,7 +1156,7 @@ class LapisController( requestContext.filter = request val result = siloQueryModel.getAminoAcidInsertions(request) - return LapisResponse(result) + return LapisResponse(result.toList()) } @GetMapping(AMINO_ACID_INSERTIONS_ROUTE, produces = [TEXT_CSV]) @@ -1192,7 +1194,7 @@ class LapisController( @RequestParam dataFormat: String? = null, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val request = SequenceFiltersRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -1244,7 +1246,7 @@ class LapisController( @RequestParam dataFormat: String? = null, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val request = SequenceFiltersRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -1274,7 +1276,7 @@ class LapisController( requestContext.filter = request val result = siloQueryModel.getAminoAcidInsertions(request) - return LapisResponse(result) + return LapisResponse(result.toList()) } @PostMapping(AMINO_ACID_INSERTIONS_ROUTE, produces = [TEXT_CSV]) @@ -1288,7 +1290,7 @@ class LapisController( @RequestBody request: SequenceFiltersRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { requestContext.filter = request return getResponseAsCsv(request, httpHeaders.accept, COMMA, siloQueryModel::getAminoAcidInsertions) @@ -1305,7 +1307,7 @@ class LapisController( @RequestBody request: SequenceFiltersRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { requestContext.filter = request return getResponseAsCsv(request, httpHeaders.accept, TAB, siloQueryModel::getAminoAcidInsertions) @@ -1377,23 +1379,16 @@ class LapisController( request: Request, acceptHeader: List, delimiter: Delimiter, - getResponse: (request: Request) -> List, - ): String { + getResponse: (request: Request) -> Stream, + ): StreamingResponseBody { requestContext.filter = request - val data = getResponse(request) - - if (data.isEmpty()) { - return "" - } val headersParameter = getHeadersParameter(delimiter, acceptHeader) - val dontIncludeHeaders = headersParameter == "false" - - val headers = when (dontIncludeHeaders) { - true -> null - false -> data[0].getHeader() - } - return csvWriter.write(headers, data, delimiter) + return csvWriter.write( + includeHeaders = headersParameter != "false", + data = getResponse(request), + delimiter = delimiter, + ) } private fun getHeadersParameter( diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/model/SiloQueryModel.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/model/SiloQueryModel.kt index 5991dbcca..668b1a7f1 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/model/SiloQueryModel.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/model/SiloQueryModel.kt @@ -15,6 +15,7 @@ import org.genspectrum.lapis.silo.SiloAction import org.genspectrum.lapis.silo.SiloClient import org.genspectrum.lapis.silo.SiloQuery import org.springframework.stereotype.Component +import java.util.stream.Stream @Component class SiloQueryModel( @@ -37,7 +38,7 @@ class SiloQueryModel( fun computeNucleotideMutationProportions( sequenceFilters: MutationProportionsRequest, - ): List { + ): Stream { val data = siloClient.sendQuery( SiloQuery( SiloAction.mutations( @@ -73,7 +74,7 @@ class SiloQueryModel( fun computeAminoAcidMutationProportions( sequenceFilters: MutationProportionsRequest, - ): List { + ): Stream { val data = siloClient.sendQuery( SiloQuery( SiloAction.aminoAcidMutations( @@ -98,7 +99,7 @@ class SiloQueryModel( } } - fun getDetails(sequenceFilters: SequenceFiltersRequestWithFields): List = + fun getDetails(sequenceFilters: SequenceFiltersRequestWithFields): Stream = siloClient.sendQuery( SiloQuery( SiloAction.details( @@ -111,7 +112,7 @@ class SiloQueryModel( ), ) - fun getNucleotideInsertions(sequenceFilters: SequenceFiltersRequest): List { + fun getNucleotideInsertions(sequenceFilters: SequenceFiltersRequest): Stream { val data = siloClient.sendQuery( SiloQuery( SiloAction.nucleotideInsertions( @@ -137,7 +138,7 @@ class SiloQueryModel( } } - fun getAminoAcidInsertions(sequenceFilters: SequenceFiltersRequest): List { + fun getAminoAcidInsertions(sequenceFilters: SequenceFiltersRequest): Stream { val data = siloClient.sendQuery( SiloQuery( SiloAction.aminoAcidInsertions( @@ -176,7 +177,7 @@ class SiloQueryModel( ), siloFilterExpressionMapper.map(sequenceFilters), ), - ).joinToString("\n") { ">${it.sequenceKey}\n${it.sequence}" } + ).toList().joinToString("\n") { ">${it.sequenceKey}\n${it.sequence}" } } fun getInfo(): InfoData = siloClient.callInfo() diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/silo/SiloClient.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/silo/SiloClient.kt index c012a09d3..e3fea4f4b 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/silo/SiloClient.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/silo/SiloClient.kt @@ -19,6 +19,7 @@ import java.net.http.HttpClient import java.net.http.HttpRequest import java.net.http.HttpResponse import java.net.http.HttpResponse.BodyHandlers +import java.util.stream.Stream private val log = KotlinLogging.logger {} @@ -30,7 +31,7 @@ class SiloClient( private val dataVersion: DataVersion, private val requestContext: RequestContext, ) { - fun sendQuery(query: SiloQuery): ResponseType { + fun sendQuery(query: SiloQuery): Stream { val result = cachedSiloClient.sendQuery(query) dataVersion.dataVersion = result.dataVersion @@ -71,32 +72,43 @@ class CachedSiloClient( log.info { "Calling SILO: $queryJson" } - val response = send(URI("$siloUrl/query")) { + val response = send( + uri = URI("$siloUrl/query"), + bodyHandler = BodyHandlers.ofLines(), + tryToReadSiloErrorFromBody = { tryToReadSiloErrorFromString(it.findFirst().orElse("")) }, + ) { it.header(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) .POST(HttpRequest.BodyPublishers.ofString(queryJson)) } - try { - return WithDataVersion( - queryResult = objectMapper.readValue(response.body(), query.action.typeReference).queryResult, - dataVersion = getDataVersion(response), - ) - } catch (exception: Exception) { - val message = "Could not parse response from silo: " + exception::class.toString() + " " + exception.message - throw RuntimeException(message, exception) - } + return WithDataVersion( + queryResult = response.body() + .filter { it.isNotBlank() } + .map { + try { + objectMapper.readValue(it, query.action.typeReference) + } catch (exception: Exception) { + val message = "Could not parse response from silo: " + + exception::class.toString() + " " + exception.message + throw RuntimeException(message, exception) + } + }, + dataVersion = getDataVersion(response), + ) } fun callInfo(): InfoData { - val response = send(URI("$siloUrl/info")) { it.GET() } + val response = send(URI("$siloUrl/info"), BodyHandlers.ofString(), ::tryToReadSiloErrorFromString) { it.GET() } return InfoData(getDataVersion(response)) } - private fun send( + private fun send( uri: URI, + bodyHandler: HttpResponse.BodyHandler, + tryToReadSiloErrorFromBody: (ResponseBodyType) -> SiloErrorResponse, buildRequest: (HttpRequest.Builder) -> Unit, - ): HttpResponse { + ): HttpResponse { val request = HttpRequest.newBuilder(uri) .apply(buildRequest) .apply { @@ -107,7 +119,7 @@ class CachedSiloClient( .build() val response = try { - httpClient.send(request, BodyHandlers.ofString()) + httpClient.send(request, bodyHandler) } catch (exception: Exception) { val message = "Could not connect to silo: " + exception::class.toString() + " " + exception.message throw RuntimeException(message, exception) @@ -116,34 +128,18 @@ class CachedSiloClient( if (!uri.toString().endsWith("info")) { log.info { "Response from SILO: ${response.statusCode()}" } } - log.debug { - val body = response.body() - val truncationPostfix = when { - body.length > SILO_RESPONSE_MAX_LOG_LENGTH -> "(...truncated)" - else -> "" - } - "Data from SILO: ${body.take(SILO_RESPONSE_MAX_LOG_LENGTH)}$truncationPostfix" - } if (response.statusCode() != 200) { - val siloErrorResponse = tryToReadSiloError(response) + val siloErrorResponse = tryToReadSiloErrorFromBody(response.body()) if (response.statusCode() == 503) { - val message = siloErrorResponse?.message ?: "Unknown reason." + val message = siloErrorResponse.message throw SiloUnavailableException( "SILO is currently unavailable: $message", response.headers().firstValue("retry-after").orElse(null), ) } - if (siloErrorResponse == null) { - throw SiloException( - HttpStatus.INTERNAL_SERVER_ERROR.value(), - "Internal Server Error", - "Unexpected error from SILO: ${response.body()}", - ) - } - throw SiloException( response.statusCode(), siloErrorResponse.error, @@ -154,15 +150,20 @@ class CachedSiloClient( return response } - private fun tryToReadSiloError(response: HttpResponse) = + private fun tryToReadSiloErrorFromString(responseBody: String) = try { - objectMapper.readValue(response.body()) + objectMapper.readValue(responseBody) } catch (e: Exception) { log.error { "Failed to deserialize error response from SILO: $e" } - null + + throw SiloException( + HttpStatus.INTERNAL_SERVER_ERROR.value(), + "Internal Server Error", + "Unexpected error from SILO: $responseBody", + ) } - private fun getDataVersion(response: HttpResponse): String { + private fun getDataVersion(response: HttpResponse<*>): String { return response.headers().firstValue("data-version").orElse("") } } @@ -171,13 +172,9 @@ class SiloException(val statusCode: Int, val title: String, override val message class SiloUnavailableException(override val message: String, val retryAfter: String?) : Exception(message) -data class SiloQueryResponse( - val queryResult: ResponseType, -) - data class WithDataVersion( val dataVersion: String, - val queryResult: ResponseType, + val queryResult: Stream, ) data class SiloErrorResponse(val error: String, val message: String) diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/silo/SiloQuery.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/silo/SiloQuery.kt index 2854ddbf3..e71a282ae 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/silo/SiloQuery.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/silo/SiloQuery.kt @@ -14,17 +14,17 @@ import java.time.LocalDate data class SiloQuery(val action: SiloAction, val filterExpression: SiloFilterExpression) -class AggregationDataTypeReference : TypeReference>>() +class AggregationDataTypeReference : TypeReference() -class MutationDataTypeReference : TypeReference>>() +class MutationDataTypeReference : TypeReference() -class AminoAcidMutationDataTypeReference : TypeReference>>() +class AminoAcidMutationDataTypeReference : TypeReference() -class DetailsDataTypeReference : TypeReference>>() +class DetailsDataTypeReference : TypeReference() -class InsertionDataTypeReference : TypeReference>>() +class InsertionDataTypeReference : TypeReference() -class SequenceDataTypeReference : TypeReference>>() +class SequenceDataTypeReference : TypeReference() interface CommonActionFields { val orderByFields: List @@ -36,7 +36,7 @@ interface CommonActionFields { const val ORDER_BY_RANDOM_FIELD_NAME = "random" sealed class SiloAction( - @JsonIgnore val typeReference: TypeReference>, + @JsonIgnore val typeReference: TypeReference, @JsonIgnore val cacheable: Boolean, ) : CommonActionFields { companion object { @@ -45,7 +45,7 @@ sealed class SiloAction( orderByFields: List = emptyList(), limit: Int? = null, offset: Int? = null, - ): SiloAction> = + ): SiloAction = AggregatedAction( groupByFields = groupByFields, orderByFields = getNonRandomizedOrderByFields(orderByFields), @@ -59,7 +59,7 @@ sealed class SiloAction( orderByFields: List = emptyList(), limit: Int? = null, offset: Int? = null, - ): SiloAction> = + ): SiloAction = MutationsAction( minProportion = minProportion, orderByFields = getNonRandomizedOrderByFields(orderByFields), @@ -73,7 +73,7 @@ sealed class SiloAction( orderByFields: List = emptyList(), limit: Int? = null, offset: Int? = null, - ): SiloAction> = + ): SiloAction = AminoAcidMutationsAction( minProportion = minProportion, orderByFields = getNonRandomizedOrderByFields(orderByFields), @@ -87,7 +87,7 @@ sealed class SiloAction( orderByFields: List = emptyList(), limit: Int? = null, offset: Int? = null, - ): SiloAction> = + ): SiloAction = DetailsAction( fields = fields, orderByFields = getNonRandomizedOrderByFields(orderByFields), @@ -100,7 +100,7 @@ sealed class SiloAction( orderByFields: List = emptyList(), limit: Int? = null, offset: Int? = null, - ): SiloAction> = + ): SiloAction = NucleotideInsertionsAction( orderByFields = getNonRandomizedOrderByFields(orderByFields), limit = limit, @@ -112,7 +112,7 @@ sealed class SiloAction( orderByFields: List = emptyList(), limit: Int? = null, offset: Int? = null, - ): SiloAction> = + ): SiloAction = AminoAcidInsertionsAction( orderByFields = getNonRandomizedOrderByFields(orderByFields), limit = limit, @@ -126,7 +126,7 @@ sealed class SiloAction( orderByFields: List = emptyList(), limit: Int? = null, offset: Int? = null, - ): SiloAction> = + ): SiloAction = SequenceAction( type = type, sequenceName = sequenceName, @@ -151,7 +151,7 @@ sealed class SiloAction( override val limit: Int? = null, override val offset: Int? = null, val type: String = "Aggregated", - ) : SiloAction>(AggregationDataTypeReference(), cacheable = true) + ) : SiloAction(AggregationDataTypeReference(), cacheable = true) @JsonInclude(JsonInclude.Include.NON_EMPTY) private data class MutationsAction( @@ -161,7 +161,7 @@ sealed class SiloAction( override val limit: Int? = null, override val offset: Int? = null, val type: String = "Mutations", - ) : SiloAction>(MutationDataTypeReference(), cacheable = true) + ) : SiloAction(MutationDataTypeReference(), cacheable = true) @JsonInclude(JsonInclude.Include.NON_EMPTY) private data class AminoAcidMutationsAction( @@ -171,7 +171,7 @@ sealed class SiloAction( override val limit: Int? = null, override val offset: Int? = null, val type: String = "AminoAcidMutations", - ) : SiloAction>(AminoAcidMutationDataTypeReference(), cacheable = true) + ) : SiloAction(AminoAcidMutationDataTypeReference(), cacheable = true) @JsonInclude(JsonInclude.Include.NON_EMPTY) private data class DetailsAction( @@ -181,7 +181,7 @@ sealed class SiloAction( override val limit: Int? = null, override val offset: Int? = null, val type: String = "Details", - ) : SiloAction>(DetailsDataTypeReference(), cacheable = false) + ) : SiloAction(DetailsDataTypeReference(), cacheable = false) @JsonInclude(JsonInclude.Include.NON_EMPTY) private data class NucleotideInsertionsAction( @@ -190,7 +190,7 @@ sealed class SiloAction( override val limit: Int? = null, override val offset: Int? = null, val type: String = "Insertions", - ) : SiloAction>(InsertionDataTypeReference(), cacheable = true) + ) : SiloAction(InsertionDataTypeReference(), cacheable = true) @JsonInclude(JsonInclude.Include.NON_EMPTY) private data class AminoAcidInsertionsAction( @@ -199,7 +199,7 @@ sealed class SiloAction( override val limit: Int? = null, override val offset: Int? = null, val type: String = "AminoAcidInsertions", - ) : SiloAction>(InsertionDataTypeReference(), cacheable = true) + ) : SiloAction(InsertionDataTypeReference(), cacheable = true) @JsonInclude(JsonInclude.Include.NON_EMPTY) private data class SequenceAction( @@ -209,7 +209,7 @@ sealed class SiloAction( override val offset: Int? = null, val type: SequenceType, val sequenceName: String, - ) : SiloAction>(SequenceDataTypeReference(), cacheable = false) + ) : SiloAction(SequenceDataTypeReference(), cacheable = false) } sealed class SiloFilterExpression(val type: String) diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/auth/ProtectedDataAuthorizationTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/auth/ProtectedDataAuthorizationTest.kt index 43770e156..bb533c5da 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/auth/ProtectedDataAuthorizationTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/auth/ProtectedDataAuthorizationTest.kt @@ -26,6 +26,7 @@ import org.springframework.test.web.servlet.MockMvc import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get import org.springframework.test.web.servlet.result.MockMvcResultMatchers.content import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status +import java.util.stream.Stream private const val NOT_AUTHORIZED_TO_ACCESS_ENDPOINT_ERROR = """ { @@ -60,7 +61,7 @@ class ProtectedDataAuthorizationTest( @BeforeEach fun setUp() { - every { siloQueryModelMock.getAggregated(any()) } returns emptyList() + every { siloQueryModelMock.getAggregated(any()) } returns Stream.empty() every { lapisInfo.dataVersion @@ -231,7 +232,7 @@ class ProtectedDataAuthorizationTest( @Test fun `GIVEN aggregated accessKey in details request where fields only contains primaryKey THEN access is granted`() { - every { siloQueryModelMock.getDetails(any()) } returns emptyList() + every { siloQueryModelMock.getDetails(any()) } returns Stream.empty() mockMvc.perform( postSample(DETAILS_ROUTE) diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCommonFieldsTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCommonFieldsTest.kt index 75f8c6da5..95f4a8151 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCommonFieldsTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCommonFieldsTest.kt @@ -29,6 +29,7 @@ import org.springframework.http.MediaType.APPLICATION_JSON import org.springframework.test.web.servlet.MockMvc import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status +import java.util.stream.Stream @SpringBootTest @AutoConfigureMockMvc @@ -62,7 +63,7 @@ class LapisControllerCommonFieldsTest( listOf(OrderByField("country", Order.ASCENDING)), ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) mockMvc.perform(getSample("$AGGREGATED_ROUTE?orderBy=country")) .andExpect(status().isOk) @@ -88,7 +89,7 @@ class LapisControllerCommonFieldsTest( ), ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) val uppercaseField = FIELD_WITH_ONLY_LOWERCASE_LETTERS.uppercase() val lowercaseField = FIELD_WITH_UPPERCASE_LETTER.lowercase() @@ -116,7 +117,7 @@ class LapisControllerCommonFieldsTest( ), ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) val request = postSample(AGGREGATED_ROUTE) .content( @@ -155,7 +156,7 @@ class LapisControllerCommonFieldsTest( ), ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) val request = postSample(AGGREGATED_ROUTE) .content( @@ -207,7 +208,7 @@ class LapisControllerCommonFieldsTest( 100, ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) mockMvc.perform(getSample("$AGGREGATED_ROUTE?limit=100")) .andExpect(status().isOk) @@ -230,7 +231,7 @@ class LapisControllerCommonFieldsTest( 100, ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) val request = postSample(AGGREGATED_ROUTE) .content("""{"limit": 100}""") @@ -268,7 +269,7 @@ class LapisControllerCommonFieldsTest( 5, ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) mockMvc.perform(getSample("$AGGREGATED_ROUTE?offset=5")) .andExpect(status().isOk) @@ -292,7 +293,7 @@ class LapisControllerCommonFieldsTest( 5, ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) val request = postSample(AGGREGATED_ROUTE) .content("""{"offset": 5}""") @@ -327,7 +328,7 @@ class LapisControllerCommonFieldsTest( emptyList(), ), ) - } returns listOf(AggregationData(5, emptyMap())) + } returns Stream.of(AggregationData(5, emptyMap())) mockMvc.perform(getSample("$AGGREGATED_ROUTE?nucleotideInsertions=ins_123:ABC,ins_other_segment:124:DEF")) .andExpect(status().isOk) @@ -347,7 +348,7 @@ class LapisControllerCommonFieldsTest( emptyList(), ), ) - } returns listOf(AggregationData(5, emptyMap())) + } returns Stream.of(AggregationData(5, emptyMap())) mockMvc.perform(getSample("$AGGREGATED_ROUTE?aminoAcidInsertions=ins_gene1:123:ABC,ins_gene2:124:DEF")) .andExpect(status().isOk) diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCsvTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCsvTest.kt index a99426e6c..3f847bebc 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCsvTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCsvTest.kt @@ -26,6 +26,7 @@ import org.springframework.test.web.servlet.result.MockMvcResultMatchers.content import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status +import java.util.stream.Stream @SpringBootTest @AutoConfigureMockMvc @@ -140,7 +141,7 @@ class LapisControllerCsvTest( @Test fun `GIVEN aggregated endpoint returns result with null values THEN CSV contains empty strings instead`() { - every { siloQueryModelMock.getAggregated(any()) } returns listOf( + every { siloQueryModelMock.getAggregated(any()) } returns Stream.of( AggregationData( 1, mapOf("firstKey" to TextNode("someValue"), "keyWithNullValue" to NullNode.instance), @@ -160,7 +161,7 @@ class LapisControllerCsvTest( @Test fun `GIVEN details endpoint returns result with null values THEN CSV contains empty strings instead`() { - every { siloQueryModelMock.getDetails(any()) } returns listOf( + every { siloQueryModelMock.getDetails(any()) } returns Stream.of( DetailsData( mapOf( "firstKey" to TextNode("some first value"), diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerTest.kt index 6bfa834e6..e37b6f64a 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerTest.kt @@ -36,6 +36,7 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status +import java.util.stream.Stream @SpringBootTest @AutoConfigureMockMvc @@ -59,7 +60,7 @@ class LapisControllerTest( fun `GET aggregated`() { every { siloQueryModelMock.getAggregated(sequenceFiltersRequestWithFields(mapOf("country" to "Switzerland"))) - } returns listOf( + } returns Stream.of( AggregationData( 0, mapOf("country" to TextNode("Switzerland"), "age" to IntNode(42)), @@ -79,7 +80,7 @@ class LapisControllerTest( fun `POST aggregated`() { every { siloQueryModelMock.getAggregated(sequenceFiltersRequestWithFields(mapOf("country" to "Switzerland"))) - } returns listOf( + } returns Stream.of( AggregationData( 0, emptyMap(), @@ -105,7 +106,7 @@ class LapisControllerTest( listOf("country", "date"), ), ) - } returns listOf( + } returns Stream.of( AggregationData( 0, mapOf("country" to TextNode("Switzerland"), "date" to TextNode("a date")), @@ -131,7 +132,7 @@ class LapisControllerTest( mapOf("country" to listOf("Switzerland", "Germany")), ), ) - } returns listOf( + } returns Stream.of( AggregationData( 0, mapOf("country" to TextNode("Switzerland")), @@ -157,7 +158,7 @@ class LapisControllerTest( emptyList(), ), ) - } returns listOf(AggregationData(5, emptyMap())) + } returns Stream.of(AggregationData(5, emptyMap())) mockMvc.perform(getSample("$AGGREGATED_ROUTE?nucleotideMutations=123A,124B")) .andExpect(status().isOk) @@ -173,7 +174,7 @@ class LapisControllerTest( listOf("country", "date"), ), ) - } returns listOf( + } returns Stream.of( AggregationData( 0, mapOf("country" to TextNode("Switzerland"), "date" to TextNode("a date")), @@ -306,7 +307,7 @@ class LapisControllerTest( minProportion, ), ) - } returns listOf(someNucleotideMutationProportion()) + } returns Stream.of(someNucleotideMutationProportion()) } if (endpoint == "/aminoAcidMutations") { every { @@ -316,7 +317,7 @@ class LapisControllerTest( minProportion, ), ) - } returns listOf(someAminoAcidMutationProportion()) + } returns Stream.of(someAminoAcidMutationProportion()) } } @@ -379,13 +380,13 @@ class LapisControllerTest( siloQueryModelMock.getNucleotideInsertions( sequenceFiltersRequest(mapOf("country" to "Switzerland")), ) - } returns listOf(someNucleotideInsertion()) + } returns Stream.of(someNucleotideInsertion()) } AMINO_ACID_INSERTIONS_ROUTE -> { every { siloQueryModelMock.getAminoAcidInsertions(sequenceFiltersRequest(mapOf("country" to "Switzerland"))) - } returns listOf(someAminoAcidInsertion()) + } returns Stream.of(someAminoAcidInsertion()) } else -> throw IllegalArgumentException("Unknown endpoint: $endpoint") @@ -426,7 +427,7 @@ class LapisControllerTest( fun `GET details`() { every { siloQueryModelMock.getDetails(sequenceFiltersRequestWithFields(mapOf("country" to "Switzerland"))) - } returns listOf(DetailsData(mapOf("country" to TextNode("Switzerland"), "age" to IntNode(42)))) + } returns Stream.of(DetailsData(mapOf("country" to TextNode("Switzerland"), "age" to IntNode(42)))) mockMvc.perform(getSample("$DETAILS_ROUTE?country=Switzerland")) .andExpect(status().isOk) @@ -445,7 +446,14 @@ class LapisControllerTest( listOf("country", "date"), ), ) - } returns listOf(DetailsData(mapOf("country" to TextNode("Switzerland"), "date" to TextNode("a date")))) + } returns Stream.of( + DetailsData( + mapOf( + "country" to TextNode("Switzerland"), + "date" to TextNode("a date"), + ), + ), + ) mockMvc.perform(getSample("$DETAILS_ROUTE?country=Switzerland&fields=country&fields=date")) .andExpect(status().isOk) @@ -457,7 +465,7 @@ class LapisControllerTest( fun `POST details`() { every { siloQueryModelMock.getDetails(sequenceFiltersRequestWithFields(mapOf("country" to "Switzerland"))) - } returns listOf(DetailsData(mapOf("country" to TextNode("Switzerland"), "age" to IntNode(42)))) + } returns Stream.of(DetailsData(mapOf("country" to TextNode("Switzerland"), "age" to IntNode(42)))) val request = postSample(DETAILS_ROUTE) .content("""{"country": "Switzerland"}""") @@ -480,7 +488,14 @@ class LapisControllerTest( listOf("country", "date"), ), ) - } returns listOf(DetailsData(mapOf("country" to TextNode("Switzerland"), "date" to TextNode("a date")))) + } returns Stream.of( + DetailsData( + mapOf( + "country" to TextNode("Switzerland"), + "date" to TextNode("a date"), + ), + ), + ) val request = postSample(DETAILS_ROUTE) .content("""{"country": "Switzerland", "fields": ["country", "date"]}""") diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/MockData.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/MockData.kt index 594a3010d..218e5c3d4 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/MockData.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/MockData.kt @@ -18,6 +18,7 @@ import org.genspectrum.lapis.response.NucleotideMutationResponse import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.`is` import org.springframework.http.MediaType.APPLICATION_JSON_VALUE +import java.util.stream.Stream data class MockDataCollection( val mockToReturnEmptyData: (SiloQueryModel) -> Unit, @@ -35,14 +36,14 @@ data class MockDataCollection( companion object { inline fun create( - crossinline siloQueryModelMockCall: (SiloQueryModel) -> (Arg) -> List, + crossinline siloQueryModelMockCall: (SiloQueryModel) -> (Arg) -> Stream, modelData: List, expectedJson: String, expectedCsv: String, expectedTsv: String, ) = MockDataCollection( - { modelMock -> every { siloQueryModelMockCall(modelMock)(any()) } returns emptyList() }, - { modelMock -> every { siloQueryModelMockCall(modelMock)(any()) } returns modelData }, + { modelMock -> every { siloQueryModelMockCall(modelMock)(any()) } returns Stream.empty() }, + { modelMock -> every { siloQueryModelMockCall(modelMock)(any()) } returns modelData.stream() }, expectedJson, expectedCsv, expectedTsv, diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/model/SiloQueryModelTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/model/SiloQueryModelTest.kt index 44692f450..ac54109f8 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/model/SiloQueryModelTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/model/SiloQueryModelTest.kt @@ -26,6 +26,7 @@ import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.equalTo import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import java.util.stream.Stream private val someMutationData = MutationData( mutation = "A1234B", @@ -65,7 +66,7 @@ class SiloQueryModelTest { @Test fun `aggregate calls the SILO client with an aggregated action`() { - every { siloClientMock.sendQuery(any>>()) } returns emptyList() + every { siloClientMock.sendQuery(any>>()) } returns Stream.empty() every { siloFilterExpressionMapperMock.map(any()) } returns True every { referenceGenomeSchemaMock.isSingleSegmented() } returns true @@ -89,7 +90,7 @@ class SiloQueryModelTest { @Test fun `computeNucleotideMutationProportions calls the SILO client with a mutations action`() { - every { siloClientMock.sendQuery(any>>()) } returns emptyList() + every { siloClientMock.sendQuery(any>>()) } returns Stream.empty() every { siloFilterExpressionMapperMock.map(any()) } returns True every { referenceGenomeSchemaMock.isSingleSegmented() } returns true @@ -106,7 +107,7 @@ class SiloQueryModelTest { @Test fun `computeNucleotideMutationProportions ignores the segmentName if singleSegmentedSequenceFeature is enabled`() { - every { siloClientMock.sendQuery(any>>()) } returns listOf(someMutationData) + every { siloClientMock.sendQuery(any>()) } returns Stream.of(someMutationData) every { siloFilterExpressionMapperMock.map(any()) } returns True every { referenceGenomeSchemaMock.isSingleSegmented() } returns true @@ -128,7 +129,7 @@ class SiloQueryModelTest { @Test fun `computeNucleotideMutationProportions includes segmentName if singleSegmentedSequenceFeature is not enabled`() { - every { siloClientMock.sendQuery(any>>()) } returns listOf(someMutationData) + every { siloClientMock.sendQuery(any>()) } returns Stream.of(someMutationData) every { siloFilterExpressionMapperMock.map(any()) } returns True every { referenceGenomeSchemaMock.isSingleSegmented() } returns false @@ -150,7 +151,7 @@ class SiloQueryModelTest { @Test fun `computeAminoAcidMutationsProportions returns the sequenceName with the position`() { - every { siloClientMock.sendQuery(any>>()) } returns listOf(someMutationData) + every { siloClientMock.sendQuery(any>()) } returns Stream.of(someMutationData) every { siloFilterExpressionMapperMock.map(any()) } returns True val result = underTest.computeAminoAcidMutationProportions( @@ -171,7 +172,7 @@ class SiloQueryModelTest { @Test fun `getNucleotideInsertions ignores the field sequenceName if the nucleotide sequence has one segment`() { - every { siloClientMock.sendQuery(any>>()) } returns listOf(someInsertionData) + every { siloClientMock.sendQuery(any>()) } returns Stream.of(someInsertionData) every { siloFilterExpressionMapperMock.map(any()) } returns True every { referenceGenomeSchemaMock.isSingleSegmented() } returns true @@ -198,7 +199,7 @@ class SiloQueryModelTest { @Test fun `getAminoAcidInsertions returns the sequenceName with the position`() { - every { siloClientMock.sendQuery(any>>()) } returns listOf(someInsertionData) + every { siloClientMock.sendQuery(any>()) } returns Stream.of(someInsertionData) every { siloFilterExpressionMapperMock.map(any()) } returns True val result = underTest.getAminoAcidInsertions( @@ -224,7 +225,7 @@ class SiloQueryModelTest { @Test fun `getGenomicSequence calls the SILO client with a sequence action`() { - every { siloClientMock.sendQuery(any>>()) } returns emptyList() + every { siloClientMock.sendQuery(any>>()) } returns Stream.empty() every { siloFilterExpressionMapperMock.map(any()) } returns True underTest.getGenomicSequence( diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/silo/SiloClientTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/silo/SiloClientTest.kt index a99400d89..6e05c2045 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/silo/SiloClientTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/silo/SiloClientTest.kt @@ -76,23 +76,15 @@ class SiloClientTest( response() .withContentType(MediaType.APPLICATION_JSON_UTF_8) .withBody( - """{ - "queryResult": [ - { - "count": 6, - "division": "Aargau" - }, - { - "count": 8, - "division": "Basel-Land" - } - ] - }""", + """ + {"count": 6,"division": "Aargau"} + {"count": 8,"division": "Basel-Land"} + """, ), ) val query = SiloQuery(SiloAction.aggregated(), StringEquals("theColumn", "theValue")) - val result = underTest.sendQuery(query) + val result = underTest.sendQuery(query).toList() assertThat( result, @@ -108,39 +100,21 @@ class SiloClientTest( @ParameterizedTest @MethodSource("getMutationActions") fun `given server returns mutations response then response can be deserialized`( - action: SiloAction>, + action: SiloAction, ) { expectQueryRequestAndRespondWith( response() .withContentType(MediaType.APPLICATION_JSON_UTF_8) .withBody( - """{ - "queryResult": [ - { - "count": 51, - "mutation": "C3037T", - "mutationFrom": "C", - "mutationTo": "T", - "position": 3037, - "proportion": 1, - "sequenceName": "main" - }, - { - "count": 52, - "mutation": "C14408T", - "mutationFrom": "C", - "mutationTo": "T", - "position": 14408, - "proportion": 1, - "sequenceName": "main" - } - ] - }""", + """ +{"count": 51,"mutation": "C3037T","mutationFrom": "C","mutationTo": "T","position": 3037,"proportion": 1,"sequenceName": "main"} +{"count": 52,"mutation": "C14408T","mutationFrom": "C","mutationTo": "T","position": 14408,"proportion": 1,"sequenceName": "main"} + """, ), ) val query = SiloQuery(action, StringEquals("theColumn", "theValue")) - val result = underTest.sendQuery(query) + val result = underTest.sendQuery(query).toList() assertThat(result, hasSize(2)) assertThat( @@ -174,18 +148,10 @@ class SiloClientTest( response() .withContentType(MediaType.APPLICATION_JSON_UTF_8) .withBody( - """{ - "queryResult": [ - { - "primaryKey": "key1", - "someSequenceName": "ABCD" - }, - { - "primaryKey": "key2", - "someSequenceName": "DEFG" - } - ] - }""", + """ + {"primaryKey": "key1","someSequenceName": "ABCD"} + {"primaryKey": "key2","someSequenceName": "DEFG"} + """, ), ) @@ -193,7 +159,7 @@ class SiloClientTest( SiloAction.genomicSequence(SequenceType.ALIGNED, "someSequenceName"), StringEquals("theColumn", "theValue"), ) - val result = underTest.sendQuery(query) + val result = underTest.sendQuery(query).toList() assertThat(result, hasSize(2)) assertThat( @@ -211,29 +177,15 @@ class SiloClientTest( response() .withContentType(MediaType.APPLICATION_JSON_UTF_8) .withBody( - """{ - "queryResult": [ - { - "age": 50, - "country": "Switzerland", - "date": "2021-02-23", - "pango_lineage": "B.1.1.7", - "qc_value": 0.95 - }, - { - "age": 54, - "country": "Switzerland", - "date": "2021-03-19", - "pango_lineage": "B.1.1.7", - "qc_value": 0.94 - } - ] - }""", + """ +{ "age": 50, "country": "Switzerland", "date": "2021-02-23", "pango_lineage": "B.1.1.7", "qc_value": 0.95 } +{ "age": 54, "country": "Switzerland", "date": "2021-03-19", "pango_lineage": "B.1.1.7", "qc_value": 0.94 } + """, ), ) val query = SiloQuery(SiloAction.details(), StringEquals("theColumn", "theValue")) - val result = underTest.sendQuery(query) + val result = underTest.sendQuery(query).toList() assertThat(result, hasSize(2)) assertThat( @@ -270,29 +222,15 @@ class SiloClientTest( response() .withContentType(MediaType.APPLICATION_JSON_UTF_8) .withBody( - """{ - "queryResult": [ - { - "count": 1, - "insertedSymbols": "SGE", - "position": 143, - "insertion": "ins_S:247:SGE", - "sequenceName": "S" - }, - { - "count": 2, - "insertedSymbols": "EPE", - "position": 214, - "insertion": "ins_S:214:EPE", - "sequenceName": "S" - } - ] - }""", + """ +{ "count": 1, "insertedSymbols": "SGE", "position": 143, "insertion": "ins_S:247:SGE", "sequenceName": "S" } +{ "count": 2, "insertedSymbols": "EPE", "position": 214, "insertion": "ins_S:214:EPE", "sequenceName": "S" } + """, ), ) val query = SiloQuery(action, True) - val result = underTest.sendQuery(query) + val result = underTest.sendQuery(query).toList() assertThat(result, hasSize(2)) assertThat( @@ -343,7 +281,7 @@ class SiloClientTest( .withBody("""{"error": "Test Error", "message": "test message with details"}"""), ) - val exception = assertThrows { underTest.sendQuery(someQuery) } + val exception = assertThrows { underTest.sendQuery(someQuery).toList() } assertThat(exception.statusCode, equalTo(432)) assertThat(exception.message, equalTo("Error from SILO: test message with details")) } @@ -357,8 +295,8 @@ class SiloClientTest( .withBody("""{"unexpectedField": "some message"}"""), ) - val exception = assertThrows { underTest.sendQuery(someQuery) } - assertThat(exception.message, containsString("value failed for JSON property")) + val exception = assertThrows { underTest.sendQuery(someQuery).toList() } + assertThat(exception.message, containsString("Could not parse response from silo")) } @Test @@ -373,7 +311,7 @@ class SiloClientTest( .withBody("""{"error": "Test Error", "message": "$errorMessage"}"""), ) - val exception = assertThrows { underTest.sendQuery(someQuery) } + val exception = assertThrows { underTest.sendQuery(someQuery).toList() } assertThat(exception.message, `is`("SILO is currently unavailable: $errorMessage")) assertThat(exception.retryAfter, `is`(retryAfterValue)) @@ -389,7 +327,7 @@ class SiloClientTest( .withBody("""{"error": "Test Error", "message": "$errorMessage"}"""), ) - val exception = assertThrows { underTest.sendQuery(someQuery) } + val exception = assertThrows { underTest.sendQuery(someQuery).toList() } assertThat(exception.message, `is`("SILO is currently unavailable: $errorMessage")) assertThat(exception.retryAfter, `is`(nullValue())) @@ -405,7 +343,7 @@ class SiloClientTest( expectQueryRequestAndRespondWith( response() .withStatusCode(200) - .withBody("""{"queryResult": []}"""), + .withBody(""), Times.exactly(1), ) expectQueryRequestAndRespondWith( @@ -469,7 +407,7 @@ class SiloClientTest( expectQueryRequestAndRespondWith( response() .withStatusCode(200) - .withBody("""{"queryResult": []}"""), + .withBody(""), Times.once(), ) expectQueryRequestAndRespondWith( @@ -486,7 +424,7 @@ class SiloClientTest( val query = SiloQuery(SiloAction.mutations(orderByFields = listOf(orderByRandom)), True) assertThat(query.action.cacheable, `is`(true)) - val result = underTest.sendQuery(query) + val result = underTest.sendQuery(query).toList() assertThat(result, hasSize(0)) val exception = assertThrows { underTest.sendQuery(query) }