From b6b7f3da908b7c4d1c8b96f49cdf35bfd6cc9746 Mon Sep 17 00:00:00 2001 From: Fabian Engelniederhammer Date: Thu, 18 Apr 2024 09:11:13 +0200 Subject: [PATCH] wip --- .../genspectrum/lapis/controller/CsvWriter.kt | 24 ++--- .../lapis/controller/LapisController.kt | 95 +++++++++---------- .../genspectrum/lapis/model/SiloQueryModel.kt | 13 +-- .../org/genspectrum/lapis/silo/SiloClient.kt | 8 +- .../auth/ProtectedDataAuthorizationTest.kt | 5 +- .../LapisControllerCommonFieldsTest.kt | 21 ++-- .../controller/LapisControllerCsvTest.kt | 5 +- .../lapis/controller/LapisControllerTest.kt | 43 ++++++--- .../genspectrum/lapis/controller/MockData.kt | 7 +- .../lapis/model/SiloQueryModelTest.kt | 17 ++-- .../genspectrum/lapis/silo/SiloClientTest.kt | 30 +++--- 11 files changed, 142 insertions(+), 126 deletions(-) diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CsvWriter.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CsvWriter.kt index 95eb41055..ee7fff650 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CsvWriter.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CsvWriter.kt @@ -4,7 +4,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVPrinter import org.springframework.stereotype.Component -import java.io.StringWriter +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody +import java.util.stream.Stream interface CsvRecord { @JsonIgnore @@ -17,29 +18,28 @@ interface CsvRecord { @Component class CsvWriter { fun write( - headers: Array?, - data: List, + includeHeaders: Boolean, + data: Stream, delimiter: Delimiter, - ): String { - val stringWriter = StringWriter() + ): StreamingResponseBody = StreamingResponseBody { stream-> + var shouldWriteHeaders = includeHeaders + CSVPrinter( - stringWriter, + stream.writer(), CSVFormat.DEFAULT.builder() .setRecordSeparator("\n") .setDelimiter(delimiter.value) .setNullString("") - .also { - when { - headers != null -> it.setHeader(*headers) - } - } .build(), ).use { for (datum in data) { + if (shouldWriteHeaders) { + it.printRecord(datum.getHeader()) + shouldWriteHeaders = false + } it.printRecord(datum.getValuesList()) } } - return stringWriter.toString().trimEnd('\n') } } diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/LapisController.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/LapisController.kt index 0292b8cd7..9cbde6eb9 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/LapisController.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/LapisController.kt @@ -69,6 +69,8 @@ import org.springframework.web.bind.annotation.RequestHeader import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody +import java.util.stream.Stream @RestController @RequestMapping("/sample") @@ -126,7 +128,7 @@ class LapisController( requestContext.filter = request - return LapisResponse(siloQueryModel.getAggregated(request)) + return LapisResponse(siloQueryModel.getAggregated(request).toList()) } @GetMapping(AGGREGATED_ROUTE, produces = [TEXT_CSV]) @@ -167,7 +169,7 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val request = SequenceFiltersRequestWithFields( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -221,7 +223,7 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val request = SequenceFiltersRequestWithFields( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -249,7 +251,7 @@ class LapisController( ): LapisResponse> { requestContext.filter = request - return LapisResponse(siloQueryModel.getAggregated(request)) + return LapisResponse(siloQueryModel.getAggregated(request).toList()) } @PostMapping(AGGREGATED_ROUTE, produces = [TEXT_CSV]) @@ -263,7 +265,7 @@ class LapisController( @RequestBody request: SequenceFiltersRequestWithFields, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { return getResponseAsCsv(request, httpHeaders.accept, COMMA, siloQueryModel::getAggregated) } @@ -278,7 +280,7 @@ class LapisController( @RequestBody request: SequenceFiltersRequestWithFields, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { return getResponseAsCsv(request, httpHeaders.accept, TAB, siloQueryModel::getAggregated) } @@ -330,7 +332,7 @@ class LapisController( requestContext.filter = mutationProportionsRequest val result = siloQueryModel.computeNucleotideMutationProportions(mutationProportionsRequest) - return LapisResponse(result) + return LapisResponse(result.toList()) } @GetMapping(NUCLEOTIDE_MUTATIONS_ROUTE, produces = [TEXT_CSV]) @@ -366,7 +368,7 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val request = MutationProportionsRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -421,7 +423,7 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val request = MutationProportionsRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -451,7 +453,7 @@ class LapisController( requestContext.filter = mutationProportionsRequest val result = siloQueryModel.computeNucleotideMutationProportions(mutationProportionsRequest) - return LapisResponse(result) + return LapisResponse(result.toList()) } @PostMapping(NUCLEOTIDE_MUTATIONS_ROUTE, produces = [TEXT_CSV]) @@ -465,7 +467,7 @@ class LapisController( @RequestBody mutationProportionsRequest: MutationProportionsRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { return getResponseAsCsv( mutationProportionsRequest, httpHeaders.accept, @@ -485,7 +487,7 @@ class LapisController( @RequestBody mutationProportionsRequest: MutationProportionsRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { return getResponseAsCsv( mutationProportionsRequest, httpHeaders.accept, @@ -538,7 +540,7 @@ class LapisController( requestContext.filter = mutationProportionsRequest val result = siloQueryModel.computeAminoAcidMutationProportions(mutationProportionsRequest) - return LapisResponse(result) + return LapisResponse(result.toList()) } @GetMapping(AMINO_ACID_MUTATIONS_ROUTE, produces = [TEXT_CSV]) @@ -574,7 +576,7 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val mutationProportionsRequest = MutationProportionsRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -629,7 +631,7 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val mutationProportionsRequest = MutationProportionsRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -664,7 +666,7 @@ class LapisController( requestContext.filter = mutationProportionsRequest val result = siloQueryModel.computeAminoAcidMutationProportions(mutationProportionsRequest) - return LapisResponse(result) + return LapisResponse(result.toList()) } @PostMapping(AMINO_ACID_MUTATIONS_ROUTE, produces = [TEXT_CSV]) @@ -678,7 +680,7 @@ class LapisController( @RequestBody mutationProportionsRequest: MutationProportionsRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { requestContext.filter = mutationProportionsRequest return getResponseAsCsv( @@ -700,7 +702,7 @@ class LapisController( @RequestBody mutationProportionsRequest: MutationProportionsRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { requestContext.filter = mutationProportionsRequest return getResponseAsCsv( @@ -758,7 +760,7 @@ class LapisController( ) requestContext.filter = request - return LapisResponse(siloQueryModel.getDetails(request)) + return LapisResponse(siloQueryModel.getDetails(request).toList()) } @GetMapping(DETAILS_ROUTE, produces = [TEXT_CSV]) @@ -795,7 +797,7 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val request = SequenceFiltersRequestWithFields( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -846,7 +848,7 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val request = SequenceFiltersRequestWithFields( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -874,7 +876,7 @@ class LapisController( ): LapisResponse> { requestContext.filter = request - return LapisResponse(siloQueryModel.getDetails(request)) + return LapisResponse(siloQueryModel.getDetails(request).toList()) } @PostMapping(DETAILS_ROUTE, produces = [TEXT_CSV]) @@ -888,7 +890,7 @@ class LapisController( @RequestBody request: SequenceFiltersRequestWithFields, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { return getResponseAsCsv(request, httpHeaders.accept, COMMA, siloQueryModel::getDetails) } @@ -903,7 +905,7 @@ class LapisController( @RequestBody request: SequenceFiltersRequestWithFields, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { return getResponseAsCsv(request, httpHeaders.accept, TAB, siloQueryModel::getDetails) } @@ -952,7 +954,7 @@ class LapisController( requestContext.filter = request val result = siloQueryModel.getNucleotideInsertions(request) - return LapisResponse(result) + return LapisResponse(result.toList()) } @GetMapping(NUCLEOTIDE_INSERTIONS_ROUTE, produces = [TEXT_CSV]) @@ -990,7 +992,7 @@ class LapisController( @RequestParam dataFormat: String? = null, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val request = SequenceFiltersRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -1042,7 +1044,7 @@ class LapisController( @RequestParam dataFormat: String? = null, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val request = SequenceFiltersRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -1072,7 +1074,7 @@ class LapisController( requestContext.filter = request val result = siloQueryModel.getNucleotideInsertions(request) - return LapisResponse(result) + return LapisResponse(result.toList()) } @PostMapping(NUCLEOTIDE_INSERTIONS_ROUTE, produces = [TEXT_CSV]) @@ -1086,7 +1088,7 @@ class LapisController( @RequestBody request: SequenceFiltersRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { requestContext.filter = request return getResponseAsCsv(request, httpHeaders.accept, COMMA, siloQueryModel::getNucleotideInsertions) @@ -1103,7 +1105,7 @@ class LapisController( @RequestBody request: SequenceFiltersRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { requestContext.filter = request return getResponseAsCsv(request, httpHeaders.accept, TAB, siloQueryModel::getNucleotideInsertions) @@ -1154,7 +1156,7 @@ class LapisController( requestContext.filter = request val result = siloQueryModel.getAminoAcidInsertions(request) - return LapisResponse(result) + return LapisResponse(result.toList()) } @GetMapping(AMINO_ACID_INSERTIONS_ROUTE, produces = [TEXT_CSV]) @@ -1192,7 +1194,7 @@ class LapisController( @RequestParam dataFormat: String? = null, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val request = SequenceFiltersRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -1244,7 +1246,7 @@ class LapisController( @RequestParam dataFormat: String? = null, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { val request = SequenceFiltersRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -1274,7 +1276,7 @@ class LapisController( requestContext.filter = request val result = siloQueryModel.getAminoAcidInsertions(request) - return LapisResponse(result) + return LapisResponse(result.toList()) } @PostMapping(AMINO_ACID_INSERTIONS_ROUTE, produces = [TEXT_CSV]) @@ -1288,7 +1290,7 @@ class LapisController( @RequestBody request: SequenceFiltersRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { requestContext.filter = request return getResponseAsCsv(request, httpHeaders.accept, COMMA, siloQueryModel::getAminoAcidInsertions) @@ -1305,7 +1307,7 @@ class LapisController( @RequestBody request: SequenceFiltersRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { + ): StreamingResponseBody { requestContext.filter = request return getResponseAsCsv(request, httpHeaders.accept, TAB, siloQueryModel::getAminoAcidInsertions) @@ -1377,23 +1379,16 @@ class LapisController( request: Request, acceptHeader: List, delimiter: Delimiter, - getResponse: (request: Request) -> List, - ): String { + getResponse: (request: Request) -> Stream, + ): StreamingResponseBody { requestContext.filter = request - val data = getResponse(request) - - if (data.isEmpty()) { - return "" - } val headersParameter = getHeadersParameter(delimiter, acceptHeader) - val dontIncludeHeaders = headersParameter == "false" - - val headers = when (dontIncludeHeaders) { - true -> null - false -> data[0].getHeader() - } - return csvWriter.write(headers, data, delimiter) + return csvWriter.write( + includeHeaders = headersParameter != "false", + data = getResponse(request), + delimiter = delimiter, + ) } private fun getHeadersParameter( diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/model/SiloQueryModel.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/model/SiloQueryModel.kt index 5991dbcca..668b1a7f1 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/model/SiloQueryModel.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/model/SiloQueryModel.kt @@ -15,6 +15,7 @@ import org.genspectrum.lapis.silo.SiloAction import org.genspectrum.lapis.silo.SiloClient import org.genspectrum.lapis.silo.SiloQuery import org.springframework.stereotype.Component +import java.util.stream.Stream @Component class SiloQueryModel( @@ -37,7 +38,7 @@ class SiloQueryModel( fun computeNucleotideMutationProportions( sequenceFilters: MutationProportionsRequest, - ): List { + ): Stream { val data = siloClient.sendQuery( SiloQuery( SiloAction.mutations( @@ -73,7 +74,7 @@ class SiloQueryModel( fun computeAminoAcidMutationProportions( sequenceFilters: MutationProportionsRequest, - ): List { + ): Stream { val data = siloClient.sendQuery( SiloQuery( SiloAction.aminoAcidMutations( @@ -98,7 +99,7 @@ class SiloQueryModel( } } - fun getDetails(sequenceFilters: SequenceFiltersRequestWithFields): List = + fun getDetails(sequenceFilters: SequenceFiltersRequestWithFields): Stream = siloClient.sendQuery( SiloQuery( SiloAction.details( @@ -111,7 +112,7 @@ class SiloQueryModel( ), ) - fun getNucleotideInsertions(sequenceFilters: SequenceFiltersRequest): List { + fun getNucleotideInsertions(sequenceFilters: SequenceFiltersRequest): Stream { val data = siloClient.sendQuery( SiloQuery( SiloAction.nucleotideInsertions( @@ -137,7 +138,7 @@ class SiloQueryModel( } } - fun getAminoAcidInsertions(sequenceFilters: SequenceFiltersRequest): List { + fun getAminoAcidInsertions(sequenceFilters: SequenceFiltersRequest): Stream { val data = siloClient.sendQuery( SiloQuery( SiloAction.aminoAcidInsertions( @@ -176,7 +177,7 @@ class SiloQueryModel( ), siloFilterExpressionMapper.map(sequenceFilters), ), - ).joinToString("\n") { ">${it.sequenceKey}\n${it.sequence}" } + ).toList().joinToString("\n") { ">${it.sequenceKey}\n${it.sequence}" } } fun getInfo(): InfoData = siloClient.callInfo() diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/silo/SiloClient.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/silo/SiloClient.kt index 79681fbbd..64c612d0a 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/silo/SiloClient.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/silo/SiloClient.kt @@ -19,6 +19,7 @@ import java.net.http.HttpClient import java.net.http.HttpRequest import java.net.http.HttpResponse import java.net.http.HttpResponse.BodyHandlers +import java.util.stream.Stream private val log = KotlinLogging.logger {} @@ -28,7 +29,7 @@ class SiloClient( private val dataVersion: DataVersion, private val requestContext: RequestContext, ) { - fun sendQuery(query: SiloQuery): List { + fun sendQuery(query: SiloQuery): Stream { val result = cachedSiloClient.sendQuery(query) dataVersion.dataVersion = result.dataVersion @@ -89,8 +90,7 @@ class CachedSiloClient( exception::class.toString() + " " + exception.message throw RuntimeException(message, exception) } - } - .toList(), + }, dataVersion = getDataVersion(response), ) } @@ -172,7 +172,7 @@ class SiloUnavailableException(override val message: String, val retryAfter: Str data class WithDataVersion( val dataVersion: String, - val queryResult: List, + val queryResult: Stream, ) data class SiloErrorResponse(val error: String, val message: String) diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/auth/ProtectedDataAuthorizationTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/auth/ProtectedDataAuthorizationTest.kt index 43770e156..bb533c5da 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/auth/ProtectedDataAuthorizationTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/auth/ProtectedDataAuthorizationTest.kt @@ -26,6 +26,7 @@ import org.springframework.test.web.servlet.MockMvc import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get import org.springframework.test.web.servlet.result.MockMvcResultMatchers.content import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status +import java.util.stream.Stream private const val NOT_AUTHORIZED_TO_ACCESS_ENDPOINT_ERROR = """ { @@ -60,7 +61,7 @@ class ProtectedDataAuthorizationTest( @BeforeEach fun setUp() { - every { siloQueryModelMock.getAggregated(any()) } returns emptyList() + every { siloQueryModelMock.getAggregated(any()) } returns Stream.empty() every { lapisInfo.dataVersion @@ -231,7 +232,7 @@ class ProtectedDataAuthorizationTest( @Test fun `GIVEN aggregated accessKey in details request where fields only contains primaryKey THEN access is granted`() { - every { siloQueryModelMock.getDetails(any()) } returns emptyList() + every { siloQueryModelMock.getDetails(any()) } returns Stream.empty() mockMvc.perform( postSample(DETAILS_ROUTE) diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCommonFieldsTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCommonFieldsTest.kt index 75f8c6da5..95f4a8151 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCommonFieldsTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCommonFieldsTest.kt @@ -29,6 +29,7 @@ import org.springframework.http.MediaType.APPLICATION_JSON import org.springframework.test.web.servlet.MockMvc import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status +import java.util.stream.Stream @SpringBootTest @AutoConfigureMockMvc @@ -62,7 +63,7 @@ class LapisControllerCommonFieldsTest( listOf(OrderByField("country", Order.ASCENDING)), ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) mockMvc.perform(getSample("$AGGREGATED_ROUTE?orderBy=country")) .andExpect(status().isOk) @@ -88,7 +89,7 @@ class LapisControllerCommonFieldsTest( ), ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) val uppercaseField = FIELD_WITH_ONLY_LOWERCASE_LETTERS.uppercase() val lowercaseField = FIELD_WITH_UPPERCASE_LETTER.lowercase() @@ -116,7 +117,7 @@ class LapisControllerCommonFieldsTest( ), ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) val request = postSample(AGGREGATED_ROUTE) .content( @@ -155,7 +156,7 @@ class LapisControllerCommonFieldsTest( ), ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) val request = postSample(AGGREGATED_ROUTE) .content( @@ -207,7 +208,7 @@ class LapisControllerCommonFieldsTest( 100, ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) mockMvc.perform(getSample("$AGGREGATED_ROUTE?limit=100")) .andExpect(status().isOk) @@ -230,7 +231,7 @@ class LapisControllerCommonFieldsTest( 100, ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) val request = postSample(AGGREGATED_ROUTE) .content("""{"limit": 100}""") @@ -268,7 +269,7 @@ class LapisControllerCommonFieldsTest( 5, ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) mockMvc.perform(getSample("$AGGREGATED_ROUTE?offset=5")) .andExpect(status().isOk) @@ -292,7 +293,7 @@ class LapisControllerCommonFieldsTest( 5, ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) val request = postSample(AGGREGATED_ROUTE) .content("""{"offset": 5}""") @@ -327,7 +328,7 @@ class LapisControllerCommonFieldsTest( emptyList(), ), ) - } returns listOf(AggregationData(5, emptyMap())) + } returns Stream.of(AggregationData(5, emptyMap())) mockMvc.perform(getSample("$AGGREGATED_ROUTE?nucleotideInsertions=ins_123:ABC,ins_other_segment:124:DEF")) .andExpect(status().isOk) @@ -347,7 +348,7 @@ class LapisControllerCommonFieldsTest( emptyList(), ), ) - } returns listOf(AggregationData(5, emptyMap())) + } returns Stream.of(AggregationData(5, emptyMap())) mockMvc.perform(getSample("$AGGREGATED_ROUTE?aminoAcidInsertions=ins_gene1:123:ABC,ins_gene2:124:DEF")) .andExpect(status().isOk) diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCsvTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCsvTest.kt index a99426e6c..3f847bebc 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCsvTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCsvTest.kt @@ -26,6 +26,7 @@ import org.springframework.test.web.servlet.result.MockMvcResultMatchers.content import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status +import java.util.stream.Stream @SpringBootTest @AutoConfigureMockMvc @@ -140,7 +141,7 @@ class LapisControllerCsvTest( @Test fun `GIVEN aggregated endpoint returns result with null values THEN CSV contains empty strings instead`() { - every { siloQueryModelMock.getAggregated(any()) } returns listOf( + every { siloQueryModelMock.getAggregated(any()) } returns Stream.of( AggregationData( 1, mapOf("firstKey" to TextNode("someValue"), "keyWithNullValue" to NullNode.instance), @@ -160,7 +161,7 @@ class LapisControllerCsvTest( @Test fun `GIVEN details endpoint returns result with null values THEN CSV contains empty strings instead`() { - every { siloQueryModelMock.getDetails(any()) } returns listOf( + every { siloQueryModelMock.getDetails(any()) } returns Stream.of( DetailsData( mapOf( "firstKey" to TextNode("some first value"), diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerTest.kt index 6bfa834e6..e37b6f64a 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerTest.kt @@ -36,6 +36,7 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status +import java.util.stream.Stream @SpringBootTest @AutoConfigureMockMvc @@ -59,7 +60,7 @@ class LapisControllerTest( fun `GET aggregated`() { every { siloQueryModelMock.getAggregated(sequenceFiltersRequestWithFields(mapOf("country" to "Switzerland"))) - } returns listOf( + } returns Stream.of( AggregationData( 0, mapOf("country" to TextNode("Switzerland"), "age" to IntNode(42)), @@ -79,7 +80,7 @@ class LapisControllerTest( fun `POST aggregated`() { every { siloQueryModelMock.getAggregated(sequenceFiltersRequestWithFields(mapOf("country" to "Switzerland"))) - } returns listOf( + } returns Stream.of( AggregationData( 0, emptyMap(), @@ -105,7 +106,7 @@ class LapisControllerTest( listOf("country", "date"), ), ) - } returns listOf( + } returns Stream.of( AggregationData( 0, mapOf("country" to TextNode("Switzerland"), "date" to TextNode("a date")), @@ -131,7 +132,7 @@ class LapisControllerTest( mapOf("country" to listOf("Switzerland", "Germany")), ), ) - } returns listOf( + } returns Stream.of( AggregationData( 0, mapOf("country" to TextNode("Switzerland")), @@ -157,7 +158,7 @@ class LapisControllerTest( emptyList(), ), ) - } returns listOf(AggregationData(5, emptyMap())) + } returns Stream.of(AggregationData(5, emptyMap())) mockMvc.perform(getSample("$AGGREGATED_ROUTE?nucleotideMutations=123A,124B")) .andExpect(status().isOk) @@ -173,7 +174,7 @@ class LapisControllerTest( listOf("country", "date"), ), ) - } returns listOf( + } returns Stream.of( AggregationData( 0, mapOf("country" to TextNode("Switzerland"), "date" to TextNode("a date")), @@ -306,7 +307,7 @@ class LapisControllerTest( minProportion, ), ) - } returns listOf(someNucleotideMutationProportion()) + } returns Stream.of(someNucleotideMutationProportion()) } if (endpoint == "/aminoAcidMutations") { every { @@ -316,7 +317,7 @@ class LapisControllerTest( minProportion, ), ) - } returns listOf(someAminoAcidMutationProportion()) + } returns Stream.of(someAminoAcidMutationProportion()) } } @@ -379,13 +380,13 @@ class LapisControllerTest( siloQueryModelMock.getNucleotideInsertions( sequenceFiltersRequest(mapOf("country" to "Switzerland")), ) - } returns listOf(someNucleotideInsertion()) + } returns Stream.of(someNucleotideInsertion()) } AMINO_ACID_INSERTIONS_ROUTE -> { every { siloQueryModelMock.getAminoAcidInsertions(sequenceFiltersRequest(mapOf("country" to "Switzerland"))) - } returns listOf(someAminoAcidInsertion()) + } returns Stream.of(someAminoAcidInsertion()) } else -> throw IllegalArgumentException("Unknown endpoint: $endpoint") @@ -426,7 +427,7 @@ class LapisControllerTest( fun `GET details`() { every { siloQueryModelMock.getDetails(sequenceFiltersRequestWithFields(mapOf("country" to "Switzerland"))) - } returns listOf(DetailsData(mapOf("country" to TextNode("Switzerland"), "age" to IntNode(42)))) + } returns Stream.of(DetailsData(mapOf("country" to TextNode("Switzerland"), "age" to IntNode(42)))) mockMvc.perform(getSample("$DETAILS_ROUTE?country=Switzerland")) .andExpect(status().isOk) @@ -445,7 +446,14 @@ class LapisControllerTest( listOf("country", "date"), ), ) - } returns listOf(DetailsData(mapOf("country" to TextNode("Switzerland"), "date" to TextNode("a date")))) + } returns Stream.of( + DetailsData( + mapOf( + "country" to TextNode("Switzerland"), + "date" to TextNode("a date"), + ), + ), + ) mockMvc.perform(getSample("$DETAILS_ROUTE?country=Switzerland&fields=country&fields=date")) .andExpect(status().isOk) @@ -457,7 +465,7 @@ class LapisControllerTest( fun `POST details`() { every { siloQueryModelMock.getDetails(sequenceFiltersRequestWithFields(mapOf("country" to "Switzerland"))) - } returns listOf(DetailsData(mapOf("country" to TextNode("Switzerland"), "age" to IntNode(42)))) + } returns Stream.of(DetailsData(mapOf("country" to TextNode("Switzerland"), "age" to IntNode(42)))) val request = postSample(DETAILS_ROUTE) .content("""{"country": "Switzerland"}""") @@ -480,7 +488,14 @@ class LapisControllerTest( listOf("country", "date"), ), ) - } returns listOf(DetailsData(mapOf("country" to TextNode("Switzerland"), "date" to TextNode("a date")))) + } returns Stream.of( + DetailsData( + mapOf( + "country" to TextNode("Switzerland"), + "date" to TextNode("a date"), + ), + ), + ) val request = postSample(DETAILS_ROUTE) .content("""{"country": "Switzerland", "fields": ["country", "date"]}""") diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/MockData.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/MockData.kt index 594a3010d..218e5c3d4 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/MockData.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/MockData.kt @@ -18,6 +18,7 @@ import org.genspectrum.lapis.response.NucleotideMutationResponse import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.`is` import org.springframework.http.MediaType.APPLICATION_JSON_VALUE +import java.util.stream.Stream data class MockDataCollection( val mockToReturnEmptyData: (SiloQueryModel) -> Unit, @@ -35,14 +36,14 @@ data class MockDataCollection( companion object { inline fun create( - crossinline siloQueryModelMockCall: (SiloQueryModel) -> (Arg) -> List, + crossinline siloQueryModelMockCall: (SiloQueryModel) -> (Arg) -> Stream, modelData: List, expectedJson: String, expectedCsv: String, expectedTsv: String, ) = MockDataCollection( - { modelMock -> every { siloQueryModelMockCall(modelMock)(any()) } returns emptyList() }, - { modelMock -> every { siloQueryModelMockCall(modelMock)(any()) } returns modelData }, + { modelMock -> every { siloQueryModelMockCall(modelMock)(any()) } returns Stream.empty() }, + { modelMock -> every { siloQueryModelMockCall(modelMock)(any()) } returns modelData.stream() }, expectedJson, expectedCsv, expectedTsv, diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/model/SiloQueryModelTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/model/SiloQueryModelTest.kt index 44692f450..ac54109f8 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/model/SiloQueryModelTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/model/SiloQueryModelTest.kt @@ -26,6 +26,7 @@ import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.equalTo import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import java.util.stream.Stream private val someMutationData = MutationData( mutation = "A1234B", @@ -65,7 +66,7 @@ class SiloQueryModelTest { @Test fun `aggregate calls the SILO client with an aggregated action`() { - every { siloClientMock.sendQuery(any>>()) } returns emptyList() + every { siloClientMock.sendQuery(any>>()) } returns Stream.empty() every { siloFilterExpressionMapperMock.map(any()) } returns True every { referenceGenomeSchemaMock.isSingleSegmented() } returns true @@ -89,7 +90,7 @@ class SiloQueryModelTest { @Test fun `computeNucleotideMutationProportions calls the SILO client with a mutations action`() { - every { siloClientMock.sendQuery(any>>()) } returns emptyList() + every { siloClientMock.sendQuery(any>>()) } returns Stream.empty() every { siloFilterExpressionMapperMock.map(any()) } returns True every { referenceGenomeSchemaMock.isSingleSegmented() } returns true @@ -106,7 +107,7 @@ class SiloQueryModelTest { @Test fun `computeNucleotideMutationProportions ignores the segmentName if singleSegmentedSequenceFeature is enabled`() { - every { siloClientMock.sendQuery(any>>()) } returns listOf(someMutationData) + every { siloClientMock.sendQuery(any>()) } returns Stream.of(someMutationData) every { siloFilterExpressionMapperMock.map(any()) } returns True every { referenceGenomeSchemaMock.isSingleSegmented() } returns true @@ -128,7 +129,7 @@ class SiloQueryModelTest { @Test fun `computeNucleotideMutationProportions includes segmentName if singleSegmentedSequenceFeature is not enabled`() { - every { siloClientMock.sendQuery(any>>()) } returns listOf(someMutationData) + every { siloClientMock.sendQuery(any>()) } returns Stream.of(someMutationData) every { siloFilterExpressionMapperMock.map(any()) } returns True every { referenceGenomeSchemaMock.isSingleSegmented() } returns false @@ -150,7 +151,7 @@ class SiloQueryModelTest { @Test fun `computeAminoAcidMutationsProportions returns the sequenceName with the position`() { - every { siloClientMock.sendQuery(any>>()) } returns listOf(someMutationData) + every { siloClientMock.sendQuery(any>()) } returns Stream.of(someMutationData) every { siloFilterExpressionMapperMock.map(any()) } returns True val result = underTest.computeAminoAcidMutationProportions( @@ -171,7 +172,7 @@ class SiloQueryModelTest { @Test fun `getNucleotideInsertions ignores the field sequenceName if the nucleotide sequence has one segment`() { - every { siloClientMock.sendQuery(any>>()) } returns listOf(someInsertionData) + every { siloClientMock.sendQuery(any>()) } returns Stream.of(someInsertionData) every { siloFilterExpressionMapperMock.map(any()) } returns True every { referenceGenomeSchemaMock.isSingleSegmented() } returns true @@ -198,7 +199,7 @@ class SiloQueryModelTest { @Test fun `getAminoAcidInsertions returns the sequenceName with the position`() { - every { siloClientMock.sendQuery(any>>()) } returns listOf(someInsertionData) + every { siloClientMock.sendQuery(any>()) } returns Stream.of(someInsertionData) every { siloFilterExpressionMapperMock.map(any()) } returns True val result = underTest.getAminoAcidInsertions( @@ -224,7 +225,7 @@ class SiloQueryModelTest { @Test fun `getGenomicSequence calls the SILO client with a sequence action`() { - every { siloClientMock.sendQuery(any>>()) } returns emptyList() + every { siloClientMock.sendQuery(any>>()) } returns Stream.empty() every { siloFilterExpressionMapperMock.map(any()) } returns True underTest.getGenomicSequence( diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/silo/SiloClientTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/silo/SiloClientTest.kt index 95cb71259..5d185314c 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/silo/SiloClientTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/silo/SiloClientTest.kt @@ -84,7 +84,7 @@ class SiloClientTest( ) val query = SiloQuery(SiloAction.aggregated(), StringEquals("theColumn", "theValue")) - val result = underTest.sendQuery(query) + val result = underTest.sendQuery(query).toList() assertThat( result, @@ -114,7 +114,7 @@ class SiloClientTest( ) val query = SiloQuery(action, StringEquals("theColumn", "theValue")) - val result = underTest.sendQuery(query) + val result = underTest.sendQuery(query).toList() assertThat(result, hasSize(2)) assertThat( @@ -159,7 +159,7 @@ class SiloClientTest( SiloAction.genomicSequence(SequenceType.ALIGNED, "someSequenceName"), StringEquals("theColumn", "theValue"), ) - val result = underTest.sendQuery(query) + val result = underTest.sendQuery(query).toList() assertThat(result, hasSize(2)) assertThat( @@ -185,7 +185,7 @@ class SiloClientTest( ) val query = SiloQuery(SiloAction.details(), StringEquals("theColumn", "theValue")) - val result = underTest.sendQuery(query) + val result = underTest.sendQuery(query).toList() assertThat(result, hasSize(2)) assertThat( @@ -230,7 +230,7 @@ class SiloClientTest( ) val query = SiloQuery(action, True) - val result = underTest.sendQuery(query) + val result = underTest.sendQuery(query).toList() assertThat(result, hasSize(2)) assertThat( @@ -353,7 +353,7 @@ class SiloClientTest( Times.exactly(1), ) - underTest.sendQuery(query) + underTest.sendQuery(query).toList() val exception = assertThrows { underTest.sendQuery(query) } assertThat(exception.message, containsString(errorMessage)) @@ -371,8 +371,8 @@ class SiloClientTest( Times.once(), ) - val result1 = underTest.sendQuery(query) - val result2 = underTest.sendQuery(query) + val result1 = underTest.sendQuery(query).toList() + val result2 = underTest.sendQuery(query).toList() assertThat(result1, `is`(result2)) } @@ -393,11 +393,11 @@ class SiloClientTest( val query = queriesThatShouldBeCached[0] assertThat(dataVersion.dataVersion, `is`(nullValue())) - underTest.sendQuery(query) + underTest.sendQuery(query).toList() assertThat(dataVersion.dataVersion, `is`(dataVersionValue)) dataVersion.dataVersion = null - underTest.sendQuery(query) + underTest.sendQuery(query).toList() assertThat(dataVersion.dataVersion, `is`(dataVersionValue)) } @@ -424,10 +424,10 @@ class SiloClientTest( val query = SiloQuery(SiloAction.mutations(orderByFields = listOf(orderByRandom)), True) assertThat(query.action.cacheable, `is`(true)) - val result = underTest.sendQuery(query) + val result = underTest.sendQuery(query).toList() assertThat(result, hasSize(0)) - val exception = assertThrows { underTest.sendQuery(query) } + val exception = assertThrows { underTest.sendQuery(query).toList() } assertThat(exception.message, containsString(errorMessage)) } @@ -508,8 +508,8 @@ class SiloClientAndCacheInvalidatorTest( Times.once(), ) - siloClient.sendQuery(someQuery) - siloClient.sendQuery(someQuery) + siloClient.sendQuery(someQuery).toList() + siloClient.sendQuery(someQuery).toList() assertThat(dataVersion.dataVersion, `is`(firstDataVersion)) } @@ -523,7 +523,7 @@ class SiloClientAndCacheInvalidatorTest( Times.once(), ) - val exception = assertThrows { siloClient.sendQuery(someQuery) } + val exception = assertThrows { siloClient.sendQuery(someQuery).toList() } assertThat(exception.message, containsString(errorMessage)) } }