From c31e53cdba726729bf7828470846a52f2df3ad2b Mon Sep 17 00:00:00 2001 From: Fabian Engelniederhammer Date: Thu, 18 Apr 2024 16:41:52 +0200 Subject: [PATCH] feat(lapis2): stream data from SILO Refs: #744 --- .../lapis/controller/CompressionFilter.kt | 9 +- .../controller/ControllerDescriptions.kt | 4 +- .../genspectrum/lapis/controller/CsvWriter.kt | 26 +- .../controller/DataFormatParameterFilter.kt | 31 +- .../lapis/controller/DownloadAsFileFilter.kt | 8 +- .../genspectrum/lapis/controller/Headers.kt | 13 +- .../lapis/controller/InfoController.kt | 4 +- .../lapis/controller/LapisController.kt | 358 ++++++++++-------- .../MultiSegmentedSequenceController.kt | 30 +- .../SingleSegmentedSequenceController.kt | 32 +- .../controller/YamlHttpMessageConverter.kt | 2 +- .../genspectrum/lapis/model/SiloQueryModel.kt | 36 +- .../org/genspectrum/lapis/openApi/Schemas.kt | 17 +- .../genspectrum/lapis/request/LapisInfo.kt | 2 +- .../lapis/response/LapisResponse.kt | 8 +- .../lapis/response/SiloResponse.kt | 30 +- .../org/genspectrum/lapis/silo/SiloClient.kt | 27 +- .../auth/ProtectedDataAuthorizationTest.kt | 5 +- .../lapis/controller/InfoControllerTest.kt | 6 +- .../LapisControllerCommonFieldsTest.kt | 21 +- .../LapisControllerCompressionTest.kt | 14 +- .../controller/LapisControllerCsvTest.kt | 34 +- .../LapisControllerDownloadAsFileTest.kt | 6 + .../lapis/controller/LapisControllerTest.kt | 29 +- .../genspectrum/lapis/controller/MockData.kt | 57 ++- .../MultiSegmentedSequenceControllerTest.kt | 35 +- .../SingleSegmentedSequenceControllerTest.kt | 39 +- .../lapis/model/SiloQueryModelTest.kt | 27 +- .../genspectrum/lapis/silo/SiloClientTest.kt | 44 +-- siloLapisTests/test/aggregated.spec.ts | 4 +- siloLapisTests/test/common.spec.ts | 18 +- siloLapisTests/test/common.ts | 2 +- siloLapisTests/test/details.spec.ts | 6 +- .../test/unalignedNucleotideSequence.spec.ts | 4 +- 34 files changed, 564 insertions(+), 424 deletions(-) diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CompressionFilter.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CompressionFilter.kt index 2a90aaae5..8b79bbb35 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CompressionFilter.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CompressionFilter.kt @@ -25,6 +25,7 @@ import org.springframework.http.converter.json.MappingJackson2HttpMessageConvert import org.springframework.stereotype.Component import org.springframework.web.context.annotation.RequestScope import org.springframework.web.filter.OncePerRequestFilter +import java.io.IOException import java.io.OutputStream import java.nio.charset.Charset import java.util.Enumeration @@ -154,8 +155,12 @@ class CompressionFilter(val objectMapper: ObjectMapper, val requestCompression: maybeCompressingResponse, ) - maybeCompressingResponse.outputStream.flush() - maybeCompressingResponse.outputStream.close() + try { + maybeCompressingResponse.outputStream.flush() + maybeCompressingResponse.outputStream.close() + } catch (e: IOException) { + log.debug { "Failed to flush and close the compressing output stream: ${e.message}" } + } } private fun getValidatedCompressionProperty(reReadableRequest: CachedBodyHttpServletRequest): Compression? { diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/ControllerDescriptions.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/ControllerDescriptions.kt index c8831f895..d6a9a1485 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/ControllerDescriptions.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/ControllerDescriptions.kt @@ -1,6 +1,6 @@ package org.genspectrum.lapis.controller -import org.genspectrum.lapis.controller.LapisMediaType.TEXT_CSV_WITHOUT_HEADERS +import org.genspectrum.lapis.controller.LapisMediaType.TEXT_CSV_WITHOUT_HEADERS_VALUE const val DETAILS_ENDPOINT_DESCRIPTION = """Returns the specified metadata fields of sequences matching the filter.""" const val AGGREGATED_ENDPOINT_DESCRIPTION = @@ -53,7 +53,7 @@ const val OFFSET_DESCRIPTION = This is useful for pagination in combination with \"limit\".""" const val FORMAT_DESCRIPTION = """The data format of the response. Alternatively, the data format can be specified by setting the \"Accept\"-header. -You can include the parameter to return the CSV/TSV without headers: "$TEXT_CSV_WITHOUT_HEADERS". +You can include the parameter to return the CSV/TSV without headers: "$TEXT_CSV_WITHOUT_HEADERS_VALUE". When both are specified, the request parameter takes precedence over the header.""" private const val MAYBE_DESCRIPTION = """ diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CsvWriter.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CsvWriter.kt index 95eb41055..267022a4d 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CsvWriter.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/CsvWriter.kt @@ -4,42 +4,42 @@ import com.fasterxml.jackson.annotation.JsonIgnore import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVPrinter import org.springframework.stereotype.Component -import java.io.StringWriter +import java.util.stream.Stream interface CsvRecord { @JsonIgnore fun getValuesList(): List @JsonIgnore - fun getHeader(): Array + fun getHeader(): Iterable } @Component class CsvWriter { fun write( - headers: Array?, - data: List, + appendable: Appendable, + includeHeaders: Boolean, + data: Stream, delimiter: Delimiter, - ): String { - val stringWriter = StringWriter() + ) { + var shouldWriteHeaders = includeHeaders + CSVPrinter( - stringWriter, + appendable, CSVFormat.DEFAULT.builder() .setRecordSeparator("\n") .setDelimiter(delimiter.value) .setNullString("") - .also { - when { - headers != null -> it.setHeader(*headers) - } - } .build(), ).use { for (datum in data) { + if (shouldWriteHeaders) { + it.printRecord(datum.getHeader()) + shouldWriteHeaders = false + } it.printRecord(datum.getValuesList()) } } - return stringWriter.toString().trimEnd('\n') } } diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/DataFormatParameterFilter.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/DataFormatParameterFilter.kt index c37109333..991736b97 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/DataFormatParameterFilter.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/DataFormatParameterFilter.kt @@ -7,10 +7,8 @@ import jakarta.servlet.http.HttpServletResponse import mu.KotlinLogging import org.genspectrum.lapis.util.CachedBodyHttpServletRequest import org.genspectrum.lapis.util.HeaderModifyingRequestWrapper -import org.genspectrum.lapis.util.ResponseWithContentType import org.springframework.core.annotation.Order import org.springframework.http.HttpHeaders.ACCEPT -import org.springframework.http.InvalidMediaTypeException import org.springframework.http.MediaType import org.springframework.stereotype.Component import org.springframework.web.filter.OncePerRequestFilter @@ -44,37 +42,16 @@ class DataFormatParameterFilter(val objectMapper: ObjectMapper) : OncePerRequest filterChain.doFilter( requestWithModifiedAcceptHeader, - when (isCsvOrTsvWithoutHeaders(requestWithModifiedAcceptHeader)) { - true -> ResponseWithContentType(response, MediaType.TEXT_PLAIN_VALUE) - false -> response - }, + response, ) } private fun findAcceptHeaderOverwriteValue(reReadableRequest: CachedBodyHttpServletRequest) = when (reReadableRequest.getStringField(FORMAT_PROPERTY)?.uppercase()) { - DataFormat.CSV -> LapisMediaType.TEXT_CSV - DataFormat.CSV_WITHOUT_HEADERS -> LapisMediaType.TEXT_CSV_WITHOUT_HEADERS - DataFormat.TSV -> LapisMediaType.TEXT_TSV + DataFormat.CSV -> LapisMediaType.TEXT_CSV_VALUE + DataFormat.CSV_WITHOUT_HEADERS -> LapisMediaType.TEXT_CSV_WITHOUT_HEADERS_VALUE + DataFormat.TSV -> LapisMediaType.TEXT_TSV_VALUE DataFormat.JSON -> MediaType.APPLICATION_JSON_VALUE else -> null } - - private fun isCsvOrTsvWithoutHeaders(requestWithModifiedAcceptHeader: HeaderModifyingRequestWrapper): Boolean { - val acceptHeader = requestWithModifiedAcceptHeader.getHeader(ACCEPT) ?: return false - - val acceptMediaType = try { - MediaType.parseMediaType(acceptHeader) - } catch (e: InvalidMediaTypeException) { - log.info { "failed to parse accept header: " + e.message } - return false - } - - if (acceptMediaType.parameters[HEADERS_ACCEPT_HEADER_PARAMETER] == "false") { - log.info { "Setting response content type to plain text due to 'headers=false'" } - return true - } - - return false - } } diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/DownloadAsFileFilter.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/DownloadAsFileFilter.kt index 70ef7e0c1..3cbfd3941 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/DownloadAsFileFilter.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/DownloadAsFileFilter.kt @@ -4,8 +4,8 @@ import com.fasterxml.jackson.databind.ObjectMapper import jakarta.servlet.FilterChain import jakarta.servlet.http.HttpServletRequest import jakarta.servlet.http.HttpServletResponse -import org.genspectrum.lapis.controller.LapisMediaType.TEXT_CSV -import org.genspectrum.lapis.controller.LapisMediaType.TEXT_TSV +import org.genspectrum.lapis.controller.LapisMediaType.TEXT_CSV_VALUE +import org.genspectrum.lapis.controller.LapisMediaType.TEXT_TSV_VALUE import org.genspectrum.lapis.util.CachedBodyHttpServletRequest import org.springframework.core.annotation.Order import org.springframework.http.HttpHeaders.ACCEPT @@ -45,8 +45,8 @@ class DownloadAsFileFilter( } val fileEnding = when (request.getHeader(ACCEPT)) { - TEXT_CSV -> "csv" - TEXT_TSV -> "tsv" + TEXT_CSV_VALUE -> "csv" + TEXT_TSV_VALUE -> "tsv" else -> when (matchingRoute?.servesFasta) { true -> "fasta" else -> "json" diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/Headers.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/Headers.kt index de8ce02d5..ce9ee4c6b 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/Headers.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/Headers.kt @@ -1,14 +1,17 @@ package org.genspectrum.lapis.controller +import org.springframework.http.MediaType + object LapisHeaders { const val REQUEST_ID = "X-Request-ID" const val LAPIS_DATA_VERSION = "Lapis-Data-Version" } object LapisMediaType { - const val TEXT_X_FASTA = "text/x-fasta" - const val TEXT_CSV = "text/csv" - const val TEXT_CSV_WITHOUT_HEADERS = "text/csv;$HEADERS_ACCEPT_HEADER_PARAMETER=false" - const val TEXT_TSV = "text/tab-separated-values" - const val APPLICATION_YAML = "application/yaml" + const val TEXT_X_FASTA_VALUE = "text/x-fasta" + val TEXT_X_FASTA = MediaType.parseMediaType(TEXT_X_FASTA_VALUE) + const val TEXT_CSV_VALUE = "text/csv" + const val TEXT_CSV_WITHOUT_HEADERS_VALUE = "text/csv;$HEADERS_ACCEPT_HEADER_PARAMETER=false" + const val TEXT_TSV_VALUE = "text/tab-separated-values" + const val APPLICATION_YAML_VALUE = "application/yaml" } diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/InfoController.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/InfoController.kt index b5b2436ad..926fc7bf7 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/InfoController.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/InfoController.kt @@ -3,7 +3,7 @@ package org.genspectrum.lapis.controller import io.swagger.v3.oas.annotations.Operation import org.genspectrum.lapis.config.DatabaseConfig import org.genspectrum.lapis.config.ReferenceGenome -import org.genspectrum.lapis.controller.LapisMediaType.APPLICATION_YAML +import org.genspectrum.lapis.controller.LapisMediaType.APPLICATION_YAML_VALUE import org.genspectrum.lapis.model.SiloQueryModel import org.genspectrum.lapis.request.LapisInfo import org.springframework.http.MediaType @@ -29,7 +29,7 @@ class InfoController( return LapisInfo(siloInfo.dataVersion) } - @GetMapping(DATABASE_CONFIG_ROUTE, produces = [MediaType.APPLICATION_JSON_VALUE, APPLICATION_YAML]) + @GetMapping(DATABASE_CONFIG_ROUTE, produces = [MediaType.APPLICATION_JSON_VALUE, APPLICATION_YAML_VALUE]) @Operation(description = DATABASE_CONFIG_ENDPOINT_DESCRIPTION) fun getDatabaseConfigAsJson(): DatabaseConfig = databaseConfig diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/LapisController.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/LapisController.kt index 0292b8cd7..5e6bf7943 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/LapisController.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/LapisController.kt @@ -3,13 +3,14 @@ package org.genspectrum.lapis.controller import io.swagger.v3.oas.annotations.Operation import io.swagger.v3.oas.annotations.Parameter import io.swagger.v3.oas.annotations.media.Schema -import io.swagger.v3.oas.annotations.responses.ApiResponse import jakarta.servlet.http.HttpServletRequest +import jakarta.servlet.http.HttpServletResponse import org.genspectrum.lapis.controller.Delimiter.COMMA import org.genspectrum.lapis.controller.Delimiter.TAB -import org.genspectrum.lapis.controller.LapisMediaType.TEXT_CSV -import org.genspectrum.lapis.controller.LapisMediaType.TEXT_TSV -import org.genspectrum.lapis.controller.LapisMediaType.TEXT_X_FASTA +import org.genspectrum.lapis.controller.LapisHeaders.LAPIS_DATA_VERSION +import org.genspectrum.lapis.controller.LapisMediaType.TEXT_CSV_VALUE +import org.genspectrum.lapis.controller.LapisMediaType.TEXT_TSV_VALUE +import org.genspectrum.lapis.controller.LapisMediaType.TEXT_X_FASTA_VALUE import org.genspectrum.lapis.logging.RequestContext import org.genspectrum.lapis.model.SiloQueryModel import org.genspectrum.lapis.openApi.AGGREGATED_REQUEST_SCHEMA @@ -40,6 +41,7 @@ import org.genspectrum.lapis.openApi.NucleotideMutations import org.genspectrum.lapis.openApi.Offset import org.genspectrum.lapis.openApi.PrimitiveFieldFilters import org.genspectrum.lapis.openApi.REQUEST_SCHEMA_WITH_MIN_PROPORTION +import org.genspectrum.lapis.openApi.StringResponseOperation import org.genspectrum.lapis.request.AminoAcidInsertion import org.genspectrum.lapis.request.AminoAcidMutation import org.genspectrum.lapis.request.CommonSequenceFilters @@ -58,6 +60,8 @@ import org.genspectrum.lapis.response.AminoAcidMutationResponse import org.genspectrum.lapis.response.DetailsData import org.genspectrum.lapis.response.NucleotideInsertionResponse import org.genspectrum.lapis.response.NucleotideMutationResponse +import org.genspectrum.lapis.response.writeFastaTo +import org.genspectrum.lapis.silo.DataVersion import org.genspectrum.lapis.silo.SequenceType import org.springframework.http.HttpHeaders import org.springframework.http.MediaType @@ -69,6 +73,8 @@ import org.springframework.web.bind.annotation.RequestHeader import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController +import java.nio.charset.Charset +import java.util.stream.Stream @RestController @RequestMapping("/sample") @@ -77,6 +83,7 @@ class LapisController( private val requestContext: RequestContext, private val csvWriter: CsvWriter, private val fieldConverter: FieldConverter, + private val dataVersion: DataVersion, ) { @GetMapping(AGGREGATED_ROUTE, produces = [MediaType.APPLICATION_JSON_VALUE]) @LapisAggregatedResponse @@ -126,14 +133,13 @@ class LapisController( requestContext.filter = request - return LapisResponse(siloQueryModel.getAggregated(request)) + return LapisResponse(siloQueryModel.getAggregated(request).toList()) } - @GetMapping(AGGREGATED_ROUTE, produces = [TEXT_CSV]) - @Operation( + @GetMapping(AGGREGATED_ROUTE, produces = [TEXT_CSV_VALUE]) + @StringResponseOperation( description = AGGREGATED_ENDPOINT_DESCRIPTION, operationId = "getAggregatedAsCsv", - responses = [ApiResponse(responseCode = "200")], ) fun getAggregatedAsCsv( @PrimitiveFieldFilters @@ -167,7 +173,8 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + response: HttpServletResponse, + ) { val request = SequenceFiltersRequestWithFields( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -180,14 +187,13 @@ class LapisController( offset, ) - return getResponseAsCsv(request, httpHeaders.accept, COMMA, siloQueryModel::getAggregated) + writeCsvToResponse(response, request, httpHeaders.accept, COMMA, siloQueryModel::getAggregated) } - @GetMapping(AGGREGATED_ROUTE, produces = [TEXT_TSV]) - @Operation( + @GetMapping(AGGREGATED_ROUTE, produces = [TEXT_TSV_VALUE]) + @StringResponseOperation( description = AGGREGATED_ENDPOINT_DESCRIPTION, operationId = "getAggregatedAsTsv", - responses = [ApiResponse(responseCode = "200")], ) fun getAggregatedAsTsv( @PrimitiveFieldFilters @@ -221,7 +227,8 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + response: HttpServletResponse, + ) { val request = SequenceFiltersRequestWithFields( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -234,7 +241,7 @@ class LapisController( offset, ) - return getResponseAsCsv(request, httpHeaders.accept, TAB, siloQueryModel::getAggregated) + writeCsvToResponse(response, request, httpHeaders.accept, TAB, siloQueryModel::getAggregated) } @PostMapping(AGGREGATED_ROUTE, produces = [MediaType.APPLICATION_JSON_VALUE]) @@ -249,37 +256,37 @@ class LapisController( ): LapisResponse> { requestContext.filter = request - return LapisResponse(siloQueryModel.getAggregated(request)) + return LapisResponse(siloQueryModel.getAggregated(request).toList()) } - @PostMapping(AGGREGATED_ROUTE, produces = [TEXT_CSV]) - @Operation( + @PostMapping(AGGREGATED_ROUTE, produces = [TEXT_CSV_VALUE]) + @StringResponseOperation( description = AGGREGATED_ENDPOINT_DESCRIPTION, operationId = "postAggregatedAsCsv", - responses = [ApiResponse(responseCode = "200")], ) fun postAggregatedAsCsv( @Parameter(schema = Schema(ref = "#/components/schemas/$AGGREGATED_REQUEST_SCHEMA")) @RequestBody request: SequenceFiltersRequestWithFields, @RequestHeader httpHeaders: HttpHeaders, - ): String { - return getResponseAsCsv(request, httpHeaders.accept, COMMA, siloQueryModel::getAggregated) + response: HttpServletResponse, + ) { + writeCsvToResponse(response, request, httpHeaders.accept, COMMA, siloQueryModel::getAggregated) } - @PostMapping(AGGREGATED_ROUTE, produces = [TEXT_TSV]) - @Operation( + @PostMapping(AGGREGATED_ROUTE, produces = [TEXT_TSV_VALUE]) + @StringResponseOperation( description = AGGREGATED_ENDPOINT_DESCRIPTION, operationId = "postAggregatedAsTsv", - responses = [ApiResponse(responseCode = "200")], ) fun postAggregatedAsTsv( @Parameter(schema = Schema(ref = "#/components/schemas/$AGGREGATED_REQUEST_SCHEMA")) @RequestBody request: SequenceFiltersRequestWithFields, @RequestHeader httpHeaders: HttpHeaders, - ): String { - return getResponseAsCsv(request, httpHeaders.accept, TAB, siloQueryModel::getAggregated) + response: HttpServletResponse, + ) { + writeCsvToResponse(response, request, httpHeaders.accept, TAB, siloQueryModel::getAggregated) } @GetMapping(NUCLEOTIDE_MUTATIONS_ROUTE, produces = [MediaType.APPLICATION_JSON_VALUE]) @@ -330,14 +337,13 @@ class LapisController( requestContext.filter = mutationProportionsRequest val result = siloQueryModel.computeNucleotideMutationProportions(mutationProportionsRequest) - return LapisResponse(result) + return LapisResponse(result.toList()) } - @GetMapping(NUCLEOTIDE_MUTATIONS_ROUTE, produces = [TEXT_CSV]) - @Operation( + @GetMapping(NUCLEOTIDE_MUTATIONS_ROUTE, produces = [TEXT_CSV_VALUE]) + @StringResponseOperation( description = NUCLEOTIDE_MUTATION_ENDPOINT_DESCRIPTION, operationId = "getNucleotideMutationsAsCsv", - responses = [ApiResponse(responseCode = "200")], ) fun getNucleotideMutationsAsCsv( @PrimitiveFieldFilters @@ -366,7 +372,8 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + response: HttpServletResponse, + ) { val request = MutationProportionsRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -380,7 +387,8 @@ class LapisController( ) requestContext.filter = request - return getResponseAsCsv( + writeCsvToResponse( + response, request, httpHeaders.accept, COMMA, @@ -388,11 +396,10 @@ class LapisController( ) } - @GetMapping(NUCLEOTIDE_MUTATIONS_ROUTE, produces = [TEXT_TSV]) - @Operation( + @GetMapping(NUCLEOTIDE_MUTATIONS_ROUTE, produces = [TEXT_TSV_VALUE]) + @StringResponseOperation( description = NUCLEOTIDE_MUTATION_ENDPOINT_DESCRIPTION, operationId = "getNucleotideMutationsAsTsv", - responses = [ApiResponse(responseCode = "200")], ) fun getNucleotideMutationsAsTsv( @PrimitiveFieldFilters @@ -421,7 +428,8 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + response: HttpServletResponse, + ) { val request = MutationProportionsRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -435,7 +443,13 @@ class LapisController( ) requestContext.filter = request - return getResponseAsCsv(request, httpHeaders.accept, TAB, siloQueryModel::computeNucleotideMutationProportions) + writeCsvToResponse( + response, + request, + httpHeaders.accept, + TAB, + siloQueryModel::computeNucleotideMutationProportions, + ) } @PostMapping(NUCLEOTIDE_MUTATIONS_ROUTE, produces = [MediaType.APPLICATION_JSON_VALUE]) @@ -451,22 +465,23 @@ class LapisController( requestContext.filter = mutationProportionsRequest val result = siloQueryModel.computeNucleotideMutationProportions(mutationProportionsRequest) - return LapisResponse(result) + return LapisResponse(result.toList()) } - @PostMapping(NUCLEOTIDE_MUTATIONS_ROUTE, produces = [TEXT_CSV]) - @Operation( + @PostMapping(NUCLEOTIDE_MUTATIONS_ROUTE, produces = [TEXT_CSV_VALUE]) + @StringResponseOperation( description = NUCLEOTIDE_MUTATION_ENDPOINT_DESCRIPTION, operationId = "postNucleotideMutationsAsCsv", - responses = [ApiResponse(responseCode = "200")], ) fun postNucleotideMutationsAsCsv( @Parameter(schema = Schema(ref = "#/components/schemas/$REQUEST_SCHEMA_WITH_MIN_PROPORTION")) @RequestBody mutationProportionsRequest: MutationProportionsRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { - return getResponseAsCsv( + response: HttpServletResponse, + ) { + writeCsvToResponse( + response, mutationProportionsRequest, httpHeaders.accept, COMMA, @@ -474,19 +489,20 @@ class LapisController( ) } - @PostMapping(NUCLEOTIDE_MUTATIONS_ROUTE, produces = [TEXT_TSV]) - @Operation( + @PostMapping(NUCLEOTIDE_MUTATIONS_ROUTE, produces = [TEXT_TSV_VALUE]) + @StringResponseOperation( description = NUCLEOTIDE_MUTATION_ENDPOINT_DESCRIPTION, operationId = "postNucleotideMutationsAsTsv", - responses = [ApiResponse(responseCode = "200")], ) fun postNucleotideMutationsAsTsv( @Parameter(schema = Schema(ref = "#/components/schemas/$REQUEST_SCHEMA_WITH_MIN_PROPORTION")) @RequestBody mutationProportionsRequest: MutationProportionsRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { - return getResponseAsCsv( + response: HttpServletResponse, + ) { + writeCsvToResponse( + response, mutationProportionsRequest, httpHeaders.accept, TAB, @@ -538,14 +554,13 @@ class LapisController( requestContext.filter = mutationProportionsRequest val result = siloQueryModel.computeAminoAcidMutationProportions(mutationProportionsRequest) - return LapisResponse(result) + return LapisResponse(result.toList()) } - @GetMapping(AMINO_ACID_MUTATIONS_ROUTE, produces = [TEXT_CSV]) - @Operation( + @GetMapping(AMINO_ACID_MUTATIONS_ROUTE, produces = [TEXT_CSV_VALUE]) + @StringResponseOperation( description = AMINO_ACID_MUTATIONS_ENDPOINT_DESCRIPTION, operationId = "getAminoAcidMutationsAsCsv", - responses = [ApiResponse(responseCode = "200")], ) fun getAminoAcidMutationsAsCsv( @PrimitiveFieldFilters @@ -574,7 +589,8 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + response: HttpServletResponse, + ) { val mutationProportionsRequest = MutationProportionsRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -588,7 +604,8 @@ class LapisController( ) requestContext.filter = mutationProportionsRequest - return getResponseAsCsv( + writeCsvToResponse( + response, mutationProportionsRequest, httpHeaders.accept, COMMA, @@ -596,11 +613,10 @@ class LapisController( ) } - @GetMapping(AMINO_ACID_MUTATIONS_ROUTE, produces = [TEXT_TSV]) - @Operation( + @GetMapping(AMINO_ACID_MUTATIONS_ROUTE, produces = [TEXT_TSV_VALUE]) + @StringResponseOperation( description = AMINO_ACID_MUTATIONS_ENDPOINT_DESCRIPTION, operationId = "getAminoAcidMutationsAsTsv", - responses = [ApiResponse(responseCode = "200")], ) fun getAminoAcidMutationsAsTsv( @PrimitiveFieldFilters @@ -629,7 +645,8 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + response: HttpServletResponse, + ) { val mutationProportionsRequest = MutationProportionsRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -643,7 +660,8 @@ class LapisController( ) requestContext.filter = mutationProportionsRequest - return getResponseAsCsv( + writeCsvToResponse( + response, mutationProportionsRequest, httpHeaders.accept, TAB, @@ -664,24 +682,25 @@ class LapisController( requestContext.filter = mutationProportionsRequest val result = siloQueryModel.computeAminoAcidMutationProportions(mutationProportionsRequest) - return LapisResponse(result) + return LapisResponse(result.toList()) } - @PostMapping(AMINO_ACID_MUTATIONS_ROUTE, produces = [TEXT_CSV]) - @Operation( + @PostMapping(AMINO_ACID_MUTATIONS_ROUTE, produces = [TEXT_CSV_VALUE]) + @StringResponseOperation( description = AMINO_ACID_MUTATIONS_ENDPOINT_DESCRIPTION, operationId = "postAminoAcidMutationsAsCsv", - responses = [ApiResponse(responseCode = "200")], ) fun postAminoAcidMutationsAsCsv( @Parameter(schema = Schema(ref = "#/components/schemas/$REQUEST_SCHEMA_WITH_MIN_PROPORTION")) @RequestBody mutationProportionsRequest: MutationProportionsRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { + response: HttpServletResponse, + ) { requestContext.filter = mutationProportionsRequest - return getResponseAsCsv( + return writeCsvToResponse( + response, mutationProportionsRequest, httpHeaders.accept, COMMA, @@ -689,21 +708,22 @@ class LapisController( ) } - @PostMapping(AMINO_ACID_MUTATIONS_ROUTE, produces = [TEXT_TSV]) - @Operation( + @PostMapping(AMINO_ACID_MUTATIONS_ROUTE, produces = [TEXT_TSV_VALUE]) + @StringResponseOperation( description = AMINO_ACID_MUTATIONS_ENDPOINT_DESCRIPTION, operationId = "postAminoAcidMutationsAsCsv", - responses = [ApiResponse(responseCode = "200")], ) fun postAminoAcidMutationsAsTsv( @Parameter(schema = Schema(ref = "#/components/schemas/$REQUEST_SCHEMA_WITH_MIN_PROPORTION")) @RequestBody mutationProportionsRequest: MutationProportionsRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { + response: HttpServletResponse, + ) { requestContext.filter = mutationProportionsRequest - return getResponseAsCsv( + return writeCsvToResponse( + response, mutationProportionsRequest, httpHeaders.accept, TAB, @@ -758,13 +778,12 @@ class LapisController( ) requestContext.filter = request - return LapisResponse(siloQueryModel.getDetails(request)) + return LapisResponse(siloQueryModel.getDetails(request).toList()) } - @GetMapping(DETAILS_ROUTE, produces = [TEXT_CSV]) - @Operation( + @GetMapping(DETAILS_ROUTE, produces = [TEXT_CSV_VALUE]) + @StringResponseOperation( operationId = "getDetailsAsCsv", - responses = [ApiResponse(responseCode = "200")], ) fun getDetailsAsCsv( @PrimitiveFieldFilters @@ -795,7 +814,8 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + response: HttpServletResponse, + ) { val request = SequenceFiltersRequestWithFields( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -808,14 +828,13 @@ class LapisController( offset, ) requestContext.filter = request - return getResponseAsCsv(request, httpHeaders.accept, COMMA, siloQueryModel::getDetails) + return writeCsvToResponse(response, request, httpHeaders.accept, COMMA, siloQueryModel::getDetails) } - @GetMapping(DETAILS_ROUTE, produces = [TEXT_TSV]) - @Operation( + @GetMapping(DETAILS_ROUTE, produces = [TEXT_TSV_VALUE]) + @StringResponseOperation( description = DETAILS_ENDPOINT_DESCRIPTION, operationId = "getDetailsAsTsv", - responses = [ApiResponse(responseCode = "200")], ) fun getDetailsAsTsv( @PrimitiveFieldFilters @@ -846,7 +865,8 @@ class LapisController( @RequestParam aminoAcidInsertions: List?, @RequestHeader httpHeaders: HttpHeaders, - ): String { + response: HttpServletResponse, + ) { val request = SequenceFiltersRequestWithFields( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -859,7 +879,7 @@ class LapisController( offset, ) - return getResponseAsCsv(request, httpHeaders.accept, TAB, siloQueryModel::getDetails) + return writeCsvToResponse(response, request, httpHeaders.accept, TAB, siloQueryModel::getDetails) } @PostMapping(DETAILS_ROUTE, produces = [MediaType.APPLICATION_JSON_VALUE]) @@ -874,37 +894,37 @@ class LapisController( ): LapisResponse> { requestContext.filter = request - return LapisResponse(siloQueryModel.getDetails(request)) + return LapisResponse(siloQueryModel.getDetails(request).toList()) } - @PostMapping(DETAILS_ROUTE, produces = [TEXT_CSV]) - @Operation( + @PostMapping(DETAILS_ROUTE, produces = [TEXT_CSV_VALUE]) + @StringResponseOperation( description = DETAILS_ENDPOINT_DESCRIPTION, operationId = "postDetailsAsCsv", - responses = [ApiResponse(responseCode = "200")], ) fun postDetailsAsCsv( @Parameter(schema = Schema(ref = "#/components/schemas/$DETAILS_REQUEST_SCHEMA")) @RequestBody request: SequenceFiltersRequestWithFields, @RequestHeader httpHeaders: HttpHeaders, - ): String { - return getResponseAsCsv(request, httpHeaders.accept, COMMA, siloQueryModel::getDetails) + response: HttpServletResponse, + ) { + writeCsvToResponse(response, request, httpHeaders.accept, COMMA, siloQueryModel::getDetails) } - @PostMapping(DETAILS_ROUTE, produces = [TEXT_TSV]) - @Operation( + @PostMapping(DETAILS_ROUTE, produces = [TEXT_TSV_VALUE]) + @StringResponseOperation( description = DETAILS_ENDPOINT_DESCRIPTION, operationId = "postDetailsAsTsv", - responses = [ApiResponse(responseCode = "200")], ) fun postDetailsAsTsv( @Parameter(schema = Schema(ref = "#/components/schemas/$DETAILS_REQUEST_SCHEMA")) @RequestBody request: SequenceFiltersRequestWithFields, @RequestHeader httpHeaders: HttpHeaders, - ): String { - return getResponseAsCsv(request, httpHeaders.accept, TAB, siloQueryModel::getDetails) + response: HttpServletResponse, + ) { + writeCsvToResponse(response, request, httpHeaders.accept, TAB, siloQueryModel::getDetails) } @GetMapping(NUCLEOTIDE_INSERTIONS_ROUTE, produces = [MediaType.APPLICATION_JSON_VALUE]) @@ -952,14 +972,13 @@ class LapisController( requestContext.filter = request val result = siloQueryModel.getNucleotideInsertions(request) - return LapisResponse(result) + return LapisResponse(result.toList()) } - @GetMapping(NUCLEOTIDE_INSERTIONS_ROUTE, produces = [TEXT_CSV]) - @Operation( + @GetMapping(NUCLEOTIDE_INSERTIONS_ROUTE, produces = [TEXT_CSV_VALUE]) + @StringResponseOperation( description = NUCLEOTIDE_INSERTIONS_ENDPOINT_DESCRIPTION, operationId = "getNucleotideInsertionsAsCsv", - responses = [ApiResponse(responseCode = "200")], ) fun getNucleotideInsertionsAsCsv( @PrimitiveFieldFilters @@ -990,7 +1009,8 @@ class LapisController( @RequestParam dataFormat: String? = null, @RequestHeader httpHeaders: HttpHeaders, - ): String { + response: HttpServletResponse, + ) { val request = SequenceFiltersRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -1004,14 +1024,13 @@ class LapisController( requestContext.filter = request - return getResponseAsCsv(request, httpHeaders.accept, COMMA, siloQueryModel::getNucleotideInsertions) + writeCsvToResponse(response, request, httpHeaders.accept, COMMA, siloQueryModel::getNucleotideInsertions) } - @GetMapping(NUCLEOTIDE_INSERTIONS_ROUTE, produces = [TEXT_TSV]) - @Operation( + @GetMapping(NUCLEOTIDE_INSERTIONS_ROUTE, produces = [TEXT_TSV_VALUE]) + @StringResponseOperation( description = NUCLEOTIDE_INSERTIONS_ENDPOINT_DESCRIPTION, operationId = "getNucleotideInsertionsAsTsv", - responses = [ApiResponse(responseCode = "200")], ) fun getNucleotideInsertionsAsTsv( @PrimitiveFieldFilters @@ -1042,7 +1061,8 @@ class LapisController( @RequestParam dataFormat: String? = null, @RequestHeader httpHeaders: HttpHeaders, - ): String { + response: HttpServletResponse, + ) { val request = SequenceFiltersRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -1056,7 +1076,7 @@ class LapisController( requestContext.filter = request - return getResponseAsCsv(request, httpHeaders.accept, TAB, siloQueryModel::getNucleotideInsertions) + writeCsvToResponse(response, request, httpHeaders.accept, TAB, siloQueryModel::getNucleotideInsertions) } @PostMapping(NUCLEOTIDE_INSERTIONS_ROUTE, produces = [MediaType.APPLICATION_JSON_VALUE]) @@ -1072,41 +1092,41 @@ class LapisController( requestContext.filter = request val result = siloQueryModel.getNucleotideInsertions(request) - return LapisResponse(result) + return LapisResponse(result.toList()) } - @PostMapping(NUCLEOTIDE_INSERTIONS_ROUTE, produces = [TEXT_CSV]) - @Operation( + @PostMapping(NUCLEOTIDE_INSERTIONS_ROUTE, produces = [TEXT_CSV_VALUE]) + @StringResponseOperation( description = NUCLEOTIDE_INSERTIONS_ENDPOINT_DESCRIPTION, operationId = "postNucleotideInsertionsAsCsv", - responses = [ApiResponse(responseCode = "200")], ) fun postNucleotideInsertionsAsCsv( @Parameter(schema = Schema(ref = "#/components/schemas/$INSERTIONS_REQUEST_SCHEMA")) @RequestBody request: SequenceFiltersRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { + response: HttpServletResponse, + ) { requestContext.filter = request - return getResponseAsCsv(request, httpHeaders.accept, COMMA, siloQueryModel::getNucleotideInsertions) + writeCsvToResponse(response, request, httpHeaders.accept, COMMA, siloQueryModel::getNucleotideInsertions) } - @PostMapping(NUCLEOTIDE_INSERTIONS_ROUTE, produces = [TEXT_TSV]) - @Operation( + @PostMapping(NUCLEOTIDE_INSERTIONS_ROUTE, produces = [TEXT_TSV_VALUE]) + @StringResponseOperation( description = NUCLEOTIDE_INSERTIONS_ENDPOINT_DESCRIPTION, operationId = "postNucleotideInsertionsAsTsv", - responses = [ApiResponse(responseCode = "200")], ) fun postNucleotideInsertionsAsTsv( @Parameter(schema = Schema(ref = "#/components/schemas/$INSERTIONS_REQUEST_SCHEMA")) @RequestBody request: SequenceFiltersRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { + response: HttpServletResponse, + ) { requestContext.filter = request - return getResponseAsCsv(request, httpHeaders.accept, TAB, siloQueryModel::getNucleotideInsertions) + writeCsvToResponse(response, request, httpHeaders.accept, TAB, siloQueryModel::getNucleotideInsertions) } @GetMapping(AMINO_ACID_INSERTIONS_ROUTE, produces = [MediaType.APPLICATION_JSON_VALUE]) @@ -1154,14 +1174,13 @@ class LapisController( requestContext.filter = request val result = siloQueryModel.getAminoAcidInsertions(request) - return LapisResponse(result) + return LapisResponse(result.toList()) } - @GetMapping(AMINO_ACID_INSERTIONS_ROUTE, produces = [TEXT_CSV]) - @Operation( + @GetMapping(AMINO_ACID_INSERTIONS_ROUTE, produces = [TEXT_CSV_VALUE]) + @StringResponseOperation( description = AMINO_ACID_INSERTIONS_ENDPOINT_DESCRIPTION, operationId = "getAminoAcidInsertionsAsCsv", - responses = [ApiResponse(responseCode = "200")], ) fun getAminoAcidInsertionsAsCsv( @PrimitiveFieldFilters @@ -1192,7 +1211,8 @@ class LapisController( @RequestParam dataFormat: String? = null, @RequestHeader httpHeaders: HttpHeaders, - ): String { + response: HttpServletResponse, + ) { val request = SequenceFiltersRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -1206,14 +1226,13 @@ class LapisController( requestContext.filter = request - return getResponseAsCsv(request, httpHeaders.accept, COMMA, siloQueryModel::getAminoAcidInsertions) + writeCsvToResponse(response, request, httpHeaders.accept, COMMA, siloQueryModel::getAminoAcidInsertions) } - @GetMapping(AMINO_ACID_INSERTIONS_ROUTE, produces = [TEXT_TSV]) - @Operation( + @GetMapping(AMINO_ACID_INSERTIONS_ROUTE, produces = [TEXT_TSV_VALUE]) + @StringResponseOperation( description = AMINO_ACID_INSERTIONS_ENDPOINT_DESCRIPTION, operationId = "getAminoAcidInsertionsAsTsv", - responses = [ApiResponse(responseCode = "200")], ) fun getAminoAcidInsertionsAsTsv( @PrimitiveFieldFilters @@ -1244,7 +1263,8 @@ class LapisController( @RequestParam dataFormat: String? = null, @RequestHeader httpHeaders: HttpHeaders, - ): String { + response: HttpServletResponse, + ) { val request = SequenceFiltersRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -1256,9 +1276,7 @@ class LapisController( offset, ) - requestContext.filter = request - - return getResponseAsCsv(request, httpHeaders.accept, TAB, siloQueryModel::getAminoAcidInsertions) + writeCsvToResponse(response, request, httpHeaders.accept, TAB, siloQueryModel::getAminoAcidInsertions) } @PostMapping(AMINO_ACID_INSERTIONS_ROUTE, produces = [MediaType.APPLICATION_JSON_VALUE]) @@ -1274,44 +1292,40 @@ class LapisController( requestContext.filter = request val result = siloQueryModel.getAminoAcidInsertions(request) - return LapisResponse(result) + return LapisResponse(result.toList()) } - @PostMapping(AMINO_ACID_INSERTIONS_ROUTE, produces = [TEXT_CSV]) - @Operation( + @PostMapping(AMINO_ACID_INSERTIONS_ROUTE, produces = [TEXT_CSV_VALUE]) + @StringResponseOperation( description = AMINO_ACID_INSERTIONS_ENDPOINT_DESCRIPTION, operationId = "postAminoAcidInsertionsAsCsv", - responses = [ApiResponse(responseCode = "200")], ) fun postAminoAcidInsertionsAsCsv( @Parameter(schema = Schema(ref = "#/components/schemas/$INSERTIONS_REQUEST_SCHEMA")) @RequestBody request: SequenceFiltersRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { - requestContext.filter = request - - return getResponseAsCsv(request, httpHeaders.accept, COMMA, siloQueryModel::getAminoAcidInsertions) + response: HttpServletResponse, + ) { + writeCsvToResponse(response, request, httpHeaders.accept, COMMA, siloQueryModel::getAminoAcidInsertions) } - @PostMapping(AMINO_ACID_INSERTIONS_ROUTE, produces = [TEXT_TSV]) - @Operation( + @PostMapping(AMINO_ACID_INSERTIONS_ROUTE, produces = [TEXT_TSV_VALUE]) + @StringResponseOperation( description = AMINO_ACID_INSERTIONS_ENDPOINT_DESCRIPTION, operationId = "postAminoAcidInsertionsAsTsv", - responses = [ApiResponse(responseCode = "200")], ) fun postAminoAcidInsertionsAsTsv( @Parameter(schema = Schema(ref = "#/components/schemas/$INSERTIONS_REQUEST_SCHEMA")) @RequestBody request: SequenceFiltersRequest, @RequestHeader httpHeaders: HttpHeaders, - ): String { - requestContext.filter = request - - return getResponseAsCsv(request, httpHeaders.accept, TAB, siloQueryModel::getAminoAcidInsertions) + response: HttpServletResponse, + ) { + writeCsvToResponse(response, request, httpHeaders.accept, TAB, siloQueryModel::getAminoAcidInsertions) } - @GetMapping("$ALIGNED_AMINO_ACID_SEQUENCES_ROUTE/{gene}", produces = [TEXT_X_FASTA]) + @GetMapping("$ALIGNED_AMINO_ACID_SEQUENCES_ROUTE/{gene}", produces = [TEXT_X_FASTA_VALUE]) @LapisAlignedAminoAcidSequenceResponse fun getAlignedAminoAcidSequence( @PathVariable(name = "gene", required = true) @@ -1341,7 +1355,8 @@ class LapisController( @Offset @RequestParam offset: Int? = null, - ): String { + response: HttpServletResponse, + ) { val request = SequenceFiltersRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -1356,9 +1371,10 @@ class LapisController( requestContext.filter = request return siloQueryModel.getGenomicSequence(request, SequenceType.ALIGNED, gene) + .writeFastaTo(response, dataVersion) } - @PostMapping("$ALIGNED_AMINO_ACID_SEQUENCES_ROUTE/{gene}", produces = [TEXT_X_FASTA]) + @PostMapping("$ALIGNED_AMINO_ACID_SEQUENCES_ROUTE/{gene}", produces = [TEXT_X_FASTA_VALUE]) @LapisAlignedAminoAcidSequenceResponse fun postAlignedAminoAcidSequence( @PathVariable(name = "gene", required = true) @@ -1367,45 +1383,57 @@ class LapisController( @Parameter(schema = Schema(ref = "#/components/schemas/$ALIGNED_AMINO_ACID_SEQUENCE_REQUEST_SCHEMA")) @RequestBody request: SequenceFiltersRequest, - ): String { + response: HttpServletResponse, + ) { requestContext.filter = request return siloQueryModel.getGenomicSequence(request, SequenceType.ALIGNED, gene) + .writeFastaTo(response, dataVersion) } - private fun getResponseAsCsv( + private fun writeCsvToResponse( + response: HttpServletResponse, request: Request, acceptHeader: List, delimiter: Delimiter, - getResponse: (request: Request) -> List, - ): String { + getData: (request: Request) -> Stream, + ) { requestContext.filter = request - val data = getResponse(request) + val data = getData(request) - if (data.isEmpty()) { - return "" + val targetMediaType = MediaType.valueOf( + when (delimiter) { + COMMA -> TEXT_CSV_VALUE + TAB -> TEXT_TSV_VALUE + }, + ) + val headersParameter = getHeadersParameter(targetMediaType, acceptHeader) + val includeHeaders = headersParameter != "false" + + val contentType = when { + !includeHeaders -> MediaType.TEXT_PLAIN + else -> MediaType(targetMediaType, Charset.defaultCharset()) } - val headersParameter = getHeadersParameter(delimiter, acceptHeader) - val dontIncludeHeaders = headersParameter == "false" + response.setHeader(LAPIS_DATA_VERSION, dataVersion.dataVersion) + if (response.contentType == null) { + response.contentType = contentType.toString() + } - val headers = when (dontIncludeHeaders) { - true -> null - false -> data[0].getHeader() + response.outputStream.use { + csvWriter.write( + appendable = it.writer(), + includeHeaders = includeHeaders, + data = data, + delimiter = delimiter, + ) } - return csvWriter.write(headers, data, delimiter) } private fun getHeadersParameter( - delimiter: Delimiter, + targetMediaType: MediaType, acceptHeader: List, ): String? { - val targetMediaType = MediaType.valueOf( - when (delimiter) { - COMMA -> TEXT_CSV - TAB -> TEXT_TSV - }, - ) return acceptHeader.find { it.includes(targetMediaType) } ?.parameters ?.get(HEADERS_ACCEPT_HEADER_PARAMETER) diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/MultiSegmentedSequenceController.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/MultiSegmentedSequenceController.kt index e5e18a1f0..280f25cb1 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/MultiSegmentedSequenceController.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/MultiSegmentedSequenceController.kt @@ -2,8 +2,9 @@ package org.genspectrum.lapis.controller import io.swagger.v3.oas.annotations.Parameter import io.swagger.v3.oas.annotations.media.Schema +import jakarta.servlet.http.HttpServletResponse import org.genspectrum.lapis.config.REFERENCE_GENOME_SEGMENTS_APPLICATION_ARG_PREFIX -import org.genspectrum.lapis.controller.LapisMediaType.TEXT_X_FASTA +import org.genspectrum.lapis.controller.LapisMediaType.TEXT_X_FASTA_VALUE import org.genspectrum.lapis.logging.RequestContext import org.genspectrum.lapis.model.SiloQueryModel import org.genspectrum.lapis.openApi.AminoAcidInsertions @@ -25,6 +26,8 @@ import org.genspectrum.lapis.request.NucleotideInsertion import org.genspectrum.lapis.request.NucleotideMutation import org.genspectrum.lapis.request.OrderByField import org.genspectrum.lapis.request.SequenceFiltersRequest +import org.genspectrum.lapis.response.writeFastaTo +import org.genspectrum.lapis.silo.DataVersion import org.genspectrum.lapis.silo.SequenceType import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.web.bind.annotation.GetMapping @@ -44,8 +47,9 @@ const val IS_MULTI_SEGMENT_SEQUENCE_EXPRESSION = class MultiSegmentedSequenceController( private val siloQueryModel: SiloQueryModel, private val requestContext: RequestContext, + private val dataVersion: DataVersion, ) { - @GetMapping("$ALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE/{segment}", produces = [TEXT_X_FASTA]) + @GetMapping("$ALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE/{segment}", produces = [TEXT_X_FASTA_VALUE]) @LapisAlignedMultiSegmentedNucleotideSequenceResponse fun getAlignedNucleotideSequence( @PathVariable(name = "segment", required = true) @@ -75,7 +79,8 @@ class MultiSegmentedSequenceController( @Offset @RequestParam offset: Int? = null, - ): String { + response: HttpServletResponse, + ) { val request = SequenceFiltersRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -94,9 +99,10 @@ class MultiSegmentedSequenceController( SequenceType.ALIGNED, segment, ) + .writeFastaTo(response, dataVersion) } - @PostMapping("$ALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE/{segment}", produces = [TEXT_X_FASTA]) + @PostMapping("$ALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE/{segment}", produces = [TEXT_X_FASTA_VALUE]) @LapisAlignedMultiSegmentedNucleotideSequenceResponse fun postAlignedNucleotideSequence( @PathVariable(name = "segment", required = true) @@ -105,7 +111,8 @@ class MultiSegmentedSequenceController( @Parameter(schema = Schema(ref = "#/components/schemas/$NUCLEOTIDE_SEQUENCE_REQUEST_SCHEMA")) @RequestBody request: SequenceFiltersRequest, - ): String { + response: HttpServletResponse, + ) { requestContext.filter = request return siloQueryModel.getGenomicSequence( @@ -113,9 +120,10 @@ class MultiSegmentedSequenceController( SequenceType.ALIGNED, segment, ) + .writeFastaTo(response, dataVersion) } - @GetMapping("$UNALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE/{segment}", produces = [TEXT_X_FASTA]) + @GetMapping("$UNALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE/{segment}", produces = [TEXT_X_FASTA_VALUE]) @LapisUnalignedMultiSegmentedNucleotideSequenceResponse fun getUnalignedNucleotideSequence( @PathVariable(name = "segment", required = true) @@ -145,7 +153,8 @@ class MultiSegmentedSequenceController( @Offset @RequestParam offset: Int? = null, - ): String { + response: HttpServletResponse, + ) { val request = SequenceFiltersRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -164,9 +173,10 @@ class MultiSegmentedSequenceController( SequenceType.UNALIGNED, segment, ) + .writeFastaTo(response, dataVersion) } - @PostMapping("$UNALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE/{segment}", produces = [TEXT_X_FASTA]) + @PostMapping("$UNALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE/{segment}", produces = [TEXT_X_FASTA_VALUE]) @LapisUnalignedMultiSegmentedNucleotideSequenceResponse fun postUnalignedNucleotideSequence( @PathVariable(name = "segment", required = true) @@ -175,7 +185,8 @@ class MultiSegmentedSequenceController( @Parameter(schema = Schema(ref = "#/components/schemas/$NUCLEOTIDE_SEQUENCE_REQUEST_SCHEMA")) @RequestBody request: SequenceFiltersRequest, - ): String { + response: HttpServletResponse, + ) { requestContext.filter = request return siloQueryModel.getGenomicSequence( @@ -183,5 +194,6 @@ class MultiSegmentedSequenceController( SequenceType.UNALIGNED, segment, ) + .writeFastaTo(response, dataVersion) } } diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/SingleSegmentedSequenceController.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/SingleSegmentedSequenceController.kt index 31215ca1e..ee7284447 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/SingleSegmentedSequenceController.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/SingleSegmentedSequenceController.kt @@ -2,9 +2,10 @@ package org.genspectrum.lapis.controller import io.swagger.v3.oas.annotations.Parameter import io.swagger.v3.oas.annotations.media.Schema +import jakarta.servlet.http.HttpServletResponse import org.genspectrum.lapis.config.REFERENCE_GENOME_SEGMENTS_APPLICATION_ARG_PREFIX import org.genspectrum.lapis.config.ReferenceGenomeSchema -import org.genspectrum.lapis.controller.LapisMediaType.TEXT_X_FASTA +import org.genspectrum.lapis.controller.LapisMediaType.TEXT_X_FASTA_VALUE import org.genspectrum.lapis.logging.RequestContext import org.genspectrum.lapis.model.SiloQueryModel import org.genspectrum.lapis.openApi.AminoAcidInsertions @@ -25,6 +26,8 @@ import org.genspectrum.lapis.request.NucleotideInsertion import org.genspectrum.lapis.request.NucleotideMutation import org.genspectrum.lapis.request.OrderByField import org.genspectrum.lapis.request.SequenceFiltersRequest +import org.genspectrum.lapis.response.writeFastaTo +import org.genspectrum.lapis.silo.DataVersion import org.genspectrum.lapis.silo.SequenceType import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression import org.springframework.web.bind.annotation.GetMapping @@ -44,8 +47,9 @@ class SingleSegmentedSequenceController( private val siloQueryModel: SiloQueryModel, private val requestContext: RequestContext, private val referenceGenomeSchema: ReferenceGenomeSchema, + private val dataVersion: DataVersion, ) { - @GetMapping(ALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE, produces = [TEXT_X_FASTA]) + @GetMapping(ALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE, produces = [TEXT_X_FASTA_VALUE]) @LapisAlignedSingleSegmentedNucleotideSequenceResponse fun getAlignedNucleotideSequences( @PrimitiveFieldFilters @@ -72,7 +76,8 @@ class SingleSegmentedSequenceController( @Offset @RequestParam offset: Int? = null, - ): String { + response: HttpServletResponse, + ) { val request = SequenceFiltersRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -86,20 +91,22 @@ class SingleSegmentedSequenceController( requestContext.filter = request - return siloQueryModel.getGenomicSequence( + val genomicSequence = siloQueryModel.getGenomicSequence( request, SequenceType.ALIGNED, referenceGenomeSchema.nucleotideSequences[0].name, ) + .writeFastaTo(response, dataVersion) } - @PostMapping(ALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE, produces = [TEXT_X_FASTA]) + @PostMapping(ALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE, produces = [TEXT_X_FASTA_VALUE]) @LapisAlignedSingleSegmentedNucleotideSequenceResponse fun postAlignedNucleotideSequence( @Parameter(schema = Schema(ref = "#/components/schemas/$NUCLEOTIDE_SEQUENCE_REQUEST_SCHEMA")) @RequestBody request: SequenceFiltersRequest, - ): String { + response: HttpServletResponse, + ) { requestContext.filter = request return siloQueryModel.getGenomicSequence( @@ -107,9 +114,10 @@ class SingleSegmentedSequenceController( SequenceType.ALIGNED, referenceGenomeSchema.nucleotideSequences[0].name, ) + .writeFastaTo(response, dataVersion) } - @GetMapping(UNALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE, produces = [TEXT_X_FASTA]) + @GetMapping(UNALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE, produces = [TEXT_X_FASTA_VALUE]) @LapisUnalignedSingleSegmentedNucleotideSequenceResponse fun getUnalignedNucleotideSequences( @PrimitiveFieldFilters @@ -136,7 +144,8 @@ class SingleSegmentedSequenceController( @Offset @RequestParam offset: Int? = null, - ): String { + response: HttpServletResponse, + ) { val request = SequenceFiltersRequest( sequenceFilters?.filter { !SPECIAL_REQUEST_PROPERTIES.contains(it.key) } ?: emptyMap(), nucleotideMutations ?: emptyList(), @@ -155,15 +164,17 @@ class SingleSegmentedSequenceController( SequenceType.UNALIGNED, referenceGenomeSchema.nucleotideSequences[0].name, ) + .writeFastaTo(response, dataVersion) } - @PostMapping(UNALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE, produces = [TEXT_X_FASTA]) + @PostMapping(UNALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE, produces = [TEXT_X_FASTA_VALUE]) @LapisUnalignedSingleSegmentedNucleotideSequenceResponse fun postUnalignedNucleotideSequence( @Parameter(schema = Schema(ref = "#/components/schemas/$NUCLEOTIDE_SEQUENCE_REQUEST_SCHEMA")) @RequestBody request: SequenceFiltersRequest, - ): String { + response: HttpServletResponse, + ) { requestContext.filter = request return siloQueryModel.getGenomicSequence( @@ -171,5 +182,6 @@ class SingleSegmentedSequenceController( SequenceType.UNALIGNED, referenceGenomeSchema.nucleotideSequences[0].name, ) + .writeFastaTo(response, dataVersion) } } diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/YamlHttpMessageConverter.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/YamlHttpMessageConverter.kt index 7242fdc80..555d5d52e 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/YamlHttpMessageConverter.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/controller/YamlHttpMessageConverter.kt @@ -9,5 +9,5 @@ import org.springframework.stereotype.Component class YamlHttpMessageConverter(yamlObjectMapper: YamlObjectMapper) : AbstractJackson2HttpMessageConverter( yamlObjectMapper.objectMapper, - MediaType.parseMediaType(LapisMediaType.APPLICATION_YAML), + MediaType.parseMediaType(LapisMediaType.APPLICATION_YAML_VALUE), ) diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/model/SiloQueryModel.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/model/SiloQueryModel.kt index 5991dbcca..08deb8b51 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/model/SiloQueryModel.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/model/SiloQueryModel.kt @@ -6,7 +6,6 @@ import org.genspectrum.lapis.request.SequenceFiltersRequest import org.genspectrum.lapis.request.SequenceFiltersRequestWithFields import org.genspectrum.lapis.response.AminoAcidInsertionResponse import org.genspectrum.lapis.response.AminoAcidMutationResponse -import org.genspectrum.lapis.response.DetailsData import org.genspectrum.lapis.response.InfoData import org.genspectrum.lapis.response.NucleotideInsertionResponse import org.genspectrum.lapis.response.NucleotideMutationResponse @@ -15,6 +14,7 @@ import org.genspectrum.lapis.silo.SiloAction import org.genspectrum.lapis.silo.SiloClient import org.genspectrum.lapis.silo.SiloQuery import org.springframework.stereotype.Component +import java.util.stream.Stream @Component class SiloQueryModel( @@ -37,7 +37,7 @@ class SiloQueryModel( fun computeNucleotideMutationProportions( sequenceFilters: MutationProportionsRequest, - ): List { + ): Stream { val data = siloClient.sendQuery( SiloQuery( SiloAction.mutations( @@ -73,7 +73,7 @@ class SiloQueryModel( fun computeAminoAcidMutationProportions( sequenceFilters: MutationProportionsRequest, - ): List { + ): Stream { val data = siloClient.sendQuery( SiloQuery( SiloAction.aminoAcidMutations( @@ -98,7 +98,7 @@ class SiloQueryModel( } } - fun getDetails(sequenceFilters: SequenceFiltersRequestWithFields): List = + fun getDetails(sequenceFilters: SequenceFiltersRequestWithFields) = siloClient.sendQuery( SiloQuery( SiloAction.details( @@ -111,7 +111,7 @@ class SiloQueryModel( ), ) - fun getNucleotideInsertions(sequenceFilters: SequenceFiltersRequest): List { + fun getNucleotideInsertions(sequenceFilters: SequenceFiltersRequest): Stream { val data = siloClient.sendQuery( SiloQuery( SiloAction.nucleotideInsertions( @@ -137,7 +137,7 @@ class SiloQueryModel( } } - fun getAminoAcidInsertions(sequenceFilters: SequenceFiltersRequest): List { + fun getAminoAcidInsertions(sequenceFilters: SequenceFiltersRequest): Stream { val data = siloClient.sendQuery( SiloQuery( SiloAction.aminoAcidInsertions( @@ -164,20 +164,18 @@ class SiloQueryModel( sequenceFilters: SequenceFiltersRequest, sequenceType: SequenceType, sequenceName: String, - ): String { - return siloClient.sendQuery( - SiloQuery( - SiloAction.genomicSequence( - sequenceType, - sequenceName, - sequenceFilters.orderByFields, - sequenceFilters.limit, - sequenceFilters.offset, - ), - siloFilterExpressionMapper.map(sequenceFilters), + ) = siloClient.sendQuery( + SiloQuery( + SiloAction.genomicSequence( + sequenceType, + sequenceName, + sequenceFilters.orderByFields, + sequenceFilters.limit, + sequenceFilters.offset, ), - ).joinToString("\n") { ">${it.sequenceKey}\n${it.sequence}" } - } + siloFilterExpressionMapper.map(sequenceFilters), + ), + ) fun getInfo(): InfoData = siloClient.callInfo() } diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/openApi/Schemas.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/openApi/Schemas.kt index 435097ec5..9f7ab54ef 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/openApi/Schemas.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/openApi/Schemas.kt @@ -171,6 +171,7 @@ annotation class LapisAminoAcidInsertionsResponse @Retention(AnnotationRetention.RUNTIME) @LapisResponseAnnotation( description = ALIGNED_AMINO_ACID_SEQUENCE_ENDPOINT_DESCRIPTION, + content = [Content(schema = Schema(type = "string"))], ) annotation class LapisAlignedAminoAcidSequenceResponse @@ -178,6 +179,7 @@ annotation class LapisAlignedAminoAcidSequenceResponse @Retention(AnnotationRetention.RUNTIME) @LapisResponseAnnotation( description = ALIGNED_SINGLE_SEGMENTED_NUCLEOTIDE_SEQUENCE_ENDPOINT_DESCRIPTION, + content = [Content(schema = Schema(type = "string"))], ) annotation class LapisAlignedSingleSegmentedNucleotideSequenceResponse @@ -185,6 +187,7 @@ annotation class LapisAlignedSingleSegmentedNucleotideSequenceResponse @Retention(AnnotationRetention.RUNTIME) @LapisResponseAnnotation( description = UNALIGNED_SINGLE_SEGMENTED_NUCLEOTIDE_SEQUENCE_ENDPOINT_DESCRIPTION, + content = [Content(schema = Schema(type = "string"))], ) annotation class LapisUnalignedSingleSegmentedNucleotideSequenceResponse @@ -192,6 +195,7 @@ annotation class LapisUnalignedSingleSegmentedNucleotideSequenceResponse @Retention(AnnotationRetention.RUNTIME) @LapisResponseAnnotation( description = ALIGNED_MULTI_SEGMENTED_NUCLEOTIDE_SEQUENCE_ENDPOINT_DESCRIPTION, + content = [Content(schema = Schema(type = "string"))], ) annotation class LapisAlignedMultiSegmentedNucleotideSequenceResponse @@ -199,14 +203,25 @@ annotation class LapisAlignedMultiSegmentedNucleotideSequenceResponse @Retention(AnnotationRetention.RUNTIME) @LapisResponseAnnotation( description = UNALIGNED_MULTI_SEGMENTED_NUCLEOTIDE_SEQUENCE_ENDPOINT_DESCRIPTION, + content = [Content(schema = Schema(type = "string"))], ) annotation class LapisUnalignedMultiSegmentedNucleotideSequenceResponse +@Target(AnnotationTarget.FUNCTION) +@Retention(AnnotationRetention.RUNTIME) +@Operation( + responses = [ApiResponse(responseCode = "200", content = [Content(schema = Schema(type = "string"))])], +) +annotation class StringResponseOperation( + @get:AliasFor(annotation = Operation::class, attribute = "description") val description: String = "", + @get:AliasFor(annotation = Operation::class, attribute = "operationId") val operationId: String, +) + @Target(AnnotationTarget.VALUE_PARAMETER) @Retention(AnnotationRetention.RUNTIME) @Parameter( description = - "Valid filters for sequence data. This may be empty. Only provide the fields that should be filtered by.", + "Valid filters for sequence data. This may be empty. Only provide the fields that should be filtered by.", schema = Schema(ref = "#/components/schemas/$PRIMITIVE_FIELD_FILTERS_SCHEMA"), explode = Explode.TRUE, style = ParameterStyle.FORM, diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/request/LapisInfo.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/request/LapisInfo.kt index 1a7ba57e3..75f49328e 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/request/LapisInfo.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/request/LapisInfo.kt @@ -62,7 +62,7 @@ class ResponseBodyAdviceDataVersion( request: ServerHttpRequest, response: ServerHttpResponse, ): Any? { - response.headers.add(LAPIS_DATA_VERSION, dataVersion.dataVersion) + response.headers.set(LAPIS_DATA_VERSION, dataVersion.dataVersion) val isDownload = response.headers.getFirst(HttpHeaders.CONTENT_DISPOSITION)?.startsWith("attachment") ?: false diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/response/LapisResponse.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/response/LapisResponse.kt index 73fdc97fe..9c06f129b 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/response/LapisResponse.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/response/LapisResponse.kt @@ -23,7 +23,7 @@ data class NucleotideMutationResponse( ) override fun getHeader() = - arrayOf( + listOf( "mutation", "count", "proportion", @@ -55,7 +55,7 @@ data class AminoAcidMutationResponse( ) override fun getHeader() = - arrayOf( + listOf( "mutation", "count", "proportion", @@ -83,7 +83,7 @@ data class NucleotideInsertionResponse( ) override fun getHeader() = - arrayOf( + listOf( "insertion", "count", "insertedSymbols", @@ -109,7 +109,7 @@ data class AminoAcidInsertionResponse( ) override fun getHeader() = - arrayOf( + listOf( "insertion", "count", "insertedSymbols", diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/response/SiloResponse.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/response/SiloResponse.kt index 988634b00..e82ce8084 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/response/SiloResponse.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/response/SiloResponse.kt @@ -10,9 +10,16 @@ import com.fasterxml.jackson.databind.JsonSerializer import com.fasterxml.jackson.databind.SerializerProvider import com.fasterxml.jackson.databind.node.NullNode import io.swagger.v3.oas.annotations.media.Schema +import jakarta.servlet.http.HttpServletResponse import org.genspectrum.lapis.config.DatabaseConfig import org.genspectrum.lapis.controller.CsvRecord +import org.genspectrum.lapis.controller.LapisHeaders.LAPIS_DATA_VERSION +import org.genspectrum.lapis.controller.LapisMediaType.TEXT_X_FASTA +import org.genspectrum.lapis.silo.DataVersion import org.springframework.boot.jackson.JsonComponent +import org.springframework.http.MediaType +import java.nio.charset.Charset +import java.util.stream.Stream const val COUNT_PROPERTY = "count" @@ -25,13 +32,13 @@ data class AggregationData( .map { it.toCsvValue() } .plus(count.toString()) - override fun getHeader() = fields.keys.plus(COUNT_PROPERTY).toTypedArray() + override fun getHeader() = fields.keys.plus(COUNT_PROPERTY) } data class DetailsData(val map: Map) : Map by map, CsvRecord { override fun getValuesList() = values.map { it.toCsvValue() } - override fun getHeader() = keys.toTypedArray() + override fun getHeader() = keys } private fun JsonNode.toCsvValue() = @@ -98,7 +105,24 @@ data class InsertionData( data class SequenceData( val sequenceKey: String, val sequence: String, -) +) { + fun writeAsString() = ">$sequenceKey\n$sequence" +} + +fun Stream.writeFastaTo( + response: HttpServletResponse, + dataVersion: DataVersion, +) { + response.setHeader(LAPIS_DATA_VERSION, dataVersion.dataVersion) + if (response.contentType == null) { + response.contentType = MediaType(TEXT_X_FASTA, Charset.defaultCharset()).toString() + } + response.outputStream.writer().use { + for (sequenceData in this) { + it.appendLine(sequenceData.writeAsString()) + } + } +} data class InfoData( val dataVersion: String, diff --git a/lapis2/src/main/kotlin/org/genspectrum/lapis/silo/SiloClient.kt b/lapis2/src/main/kotlin/org/genspectrum/lapis/silo/SiloClient.kt index 79681fbbd..aab8e07ca 100644 --- a/lapis2/src/main/kotlin/org/genspectrum/lapis/silo/SiloClient.kt +++ b/lapis2/src/main/kotlin/org/genspectrum/lapis/silo/SiloClient.kt @@ -19,6 +19,7 @@ import java.net.http.HttpClient import java.net.http.HttpRequest import java.net.http.HttpResponse import java.net.http.HttpResponse.BodyHandlers +import java.util.stream.Stream private val log = KotlinLogging.logger {} @@ -28,15 +29,21 @@ class SiloClient( private val dataVersion: DataVersion, private val requestContext: RequestContext, ) { - fun sendQuery(query: SiloQuery): List { - val result = cachedSiloClient.sendQuery(query) - dataVersion.dataVersion = result.dataVersion + fun sendQuery(query: SiloQuery): Stream { + val (currentDataVersion, queryResult) = when (query.action.cacheable) { + true -> cachedSiloClient.sendCachedQuery(query) + .let { it.dataVersion to it.queryResult.stream() } + + else -> cachedSiloClient.sendQuery(query).let { it.dataVersion to it.queryResult } + } + + dataVersion.dataVersion = currentDataVersion if (RequestContextHolder.getRequestAttributes() != null && requestContext.cached == null) { requestContext.cached = true } - return result.queryResult + return queryResult } fun callInfo(): InfoData { @@ -60,7 +67,12 @@ class CachedSiloClient( private val httpClient = HttpClient.newHttpClient() @Cacheable(SILO_QUERY_CACHE_NAME, condition = "#query.action.cacheable && !#query.action.randomize") - fun sendQuery(query: SiloQuery): WithDataVersion { + fun sendCachedQuery(query: SiloQuery): WithDataVersion> { + return sendQuery(query) + .let { WithDataVersion(it.dataVersion, it.queryResult.toList()) } + } + + fun sendQuery(query: SiloQuery): WithDataVersion> { if (RequestContextHolder.getRequestAttributes() != null) { requestContext.cached = false } @@ -89,8 +101,7 @@ class CachedSiloClient( exception::class.toString() + " " + exception.message throw RuntimeException(message, exception) } - } - .toList(), + }, dataVersion = getDataVersion(response), ) } @@ -172,7 +183,7 @@ class SiloUnavailableException(override val message: String, val retryAfter: Str data class WithDataVersion( val dataVersion: String, - val queryResult: List, + val queryResult: ResponseType, ) data class SiloErrorResponse(val error: String, val message: String) diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/auth/ProtectedDataAuthorizationTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/auth/ProtectedDataAuthorizationTest.kt index 43770e156..bb533c5da 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/auth/ProtectedDataAuthorizationTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/auth/ProtectedDataAuthorizationTest.kt @@ -26,6 +26,7 @@ import org.springframework.test.web.servlet.MockMvc import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get import org.springframework.test.web.servlet.result.MockMvcResultMatchers.content import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status +import java.util.stream.Stream private const val NOT_AUTHORIZED_TO_ACCESS_ENDPOINT_ERROR = """ { @@ -60,7 +61,7 @@ class ProtectedDataAuthorizationTest( @BeforeEach fun setUp() { - every { siloQueryModelMock.getAggregated(any()) } returns emptyList() + every { siloQueryModelMock.getAggregated(any()) } returns Stream.empty() every { lapisInfo.dataVersion @@ -231,7 +232,7 @@ class ProtectedDataAuthorizationTest( @Test fun `GIVEN aggregated accessKey in details request where fields only contains primaryKey THEN access is granted`() { - every { siloQueryModelMock.getDetails(any()) } returns emptyList() + every { siloQueryModelMock.getDetails(any()) } returns Stream.empty() mockMvc.perform( postSample(DETAILS_ROUTE) diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/InfoControllerTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/InfoControllerTest.kt index a99925edb..f63eff3bf 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/InfoControllerTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/InfoControllerTest.kt @@ -2,7 +2,7 @@ package org.genspectrum.lapis.controller import com.ninjasquad.springmockk.MockkBean import io.mockk.every -import org.genspectrum.lapis.controller.LapisMediaType.APPLICATION_YAML +import org.genspectrum.lapis.controller.LapisMediaType.APPLICATION_YAML_VALUE import org.genspectrum.lapis.model.SiloQueryModel import org.genspectrum.lapis.response.InfoData import org.hamcrest.Matchers.startsWith @@ -52,9 +52,9 @@ class InfoControllerTest( opennessLevel: "OPEN" """.trimIndent() - mockMvc.perform(getSample(DATABASE_CONFIG_ROUTE).accept(APPLICATION_YAML)) + mockMvc.perform(getSample(DATABASE_CONFIG_ROUTE).accept(APPLICATION_YAML_VALUE)) .andExpect(status().isOk) - .andExpect(content().contentType(APPLICATION_YAML)) + .andExpect(content().contentType(APPLICATION_YAML_VALUE)) .andExpect(content().string(startsWith(yamlStart))) } diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCommonFieldsTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCommonFieldsTest.kt index 75f8c6da5..95f4a8151 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCommonFieldsTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCommonFieldsTest.kt @@ -29,6 +29,7 @@ import org.springframework.http.MediaType.APPLICATION_JSON import org.springframework.test.web.servlet.MockMvc import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status +import java.util.stream.Stream @SpringBootTest @AutoConfigureMockMvc @@ -62,7 +63,7 @@ class LapisControllerCommonFieldsTest( listOf(OrderByField("country", Order.ASCENDING)), ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) mockMvc.perform(getSample("$AGGREGATED_ROUTE?orderBy=country")) .andExpect(status().isOk) @@ -88,7 +89,7 @@ class LapisControllerCommonFieldsTest( ), ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) val uppercaseField = FIELD_WITH_ONLY_LOWERCASE_LETTERS.uppercase() val lowercaseField = FIELD_WITH_UPPERCASE_LETTER.lowercase() @@ -116,7 +117,7 @@ class LapisControllerCommonFieldsTest( ), ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) val request = postSample(AGGREGATED_ROUTE) .content( @@ -155,7 +156,7 @@ class LapisControllerCommonFieldsTest( ), ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) val request = postSample(AGGREGATED_ROUTE) .content( @@ -207,7 +208,7 @@ class LapisControllerCommonFieldsTest( 100, ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) mockMvc.perform(getSample("$AGGREGATED_ROUTE?limit=100")) .andExpect(status().isOk) @@ -230,7 +231,7 @@ class LapisControllerCommonFieldsTest( 100, ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) val request = postSample(AGGREGATED_ROUTE) .content("""{"limit": 100}""") @@ -268,7 +269,7 @@ class LapisControllerCommonFieldsTest( 5, ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) mockMvc.perform(getSample("$AGGREGATED_ROUTE?offset=5")) .andExpect(status().isOk) @@ -292,7 +293,7 @@ class LapisControllerCommonFieldsTest( 5, ), ) - } returns listOf(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) + } returns Stream.of(AggregationData(0, mapOf("country" to TextNode("Switzerland")))) val request = postSample(AGGREGATED_ROUTE) .content("""{"offset": 5}""") @@ -327,7 +328,7 @@ class LapisControllerCommonFieldsTest( emptyList(), ), ) - } returns listOf(AggregationData(5, emptyMap())) + } returns Stream.of(AggregationData(5, emptyMap())) mockMvc.perform(getSample("$AGGREGATED_ROUTE?nucleotideInsertions=ins_123:ABC,ins_other_segment:124:DEF")) .andExpect(status().isOk) @@ -347,7 +348,7 @@ class LapisControllerCommonFieldsTest( emptyList(), ), ) - } returns listOf(AggregationData(5, emptyMap())) + } returns Stream.of(AggregationData(5, emptyMap())) mockMvc.perform(getSample("$AGGREGATED_ROUTE?aminoAcidInsertions=ins_gene1:123:ABC,ins_gene2:124:DEF")) .andExpect(status().isOk) diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCompressionTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCompressionTest.kt index 2868e6cb4..d0f9a5844 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCompressionTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCompressionTest.kt @@ -4,9 +4,9 @@ import com.github.luben.zstd.ZstdInputStream import com.jayway.jsonpath.JsonPath import com.ninjasquad.springmockk.MockkBean import io.mockk.every -import org.genspectrum.lapis.controller.LapisMediaType.TEXT_CSV -import org.genspectrum.lapis.controller.LapisMediaType.TEXT_TSV -import org.genspectrum.lapis.controller.LapisMediaType.TEXT_X_FASTA +import org.genspectrum.lapis.controller.LapisMediaType.TEXT_CSV_VALUE +import org.genspectrum.lapis.controller.LapisMediaType.TEXT_TSV_VALUE +import org.genspectrum.lapis.controller.LapisMediaType.TEXT_X_FASTA_VALUE import org.genspectrum.lapis.controller.MockDataCollection.DataFormat.CSV import org.genspectrum.lapis.controller.MockDataCollection.DataFormat.NESTED_JSON import org.genspectrum.lapis.controller.MockDataCollection.DataFormat.PLAIN_JSON @@ -306,7 +306,7 @@ private fun getFastaRequests( request = getSample("$endpoint?country=Switzerland") .header(ACCEPT_ENCODING, compressionFormat), compressionFormat = compressionFormat, - expectedContentType = "$TEXT_X_FASTA;charset=UTF-8", + expectedContentType = "$TEXT_X_FASTA_VALUE;charset=UTF-8", expectedContentEncoding = compressionFormat, ), RequestScenario( @@ -327,7 +327,7 @@ private fun getFastaRequests( .contentType(APPLICATION_JSON) .header(ACCEPT_ENCODING, compressionFormat), compressionFormat = compressionFormat, - expectedContentType = "$TEXT_X_FASTA;charset=UTF-8", + expectedContentType = "$TEXT_X_FASTA_VALUE;charset=UTF-8", expectedContentEncoding = compressionFormat, ), ) @@ -343,6 +343,6 @@ private fun getContentTypeForDataFormat(dataFormat: MockDataCollection.DataForma when (dataFormat) { PLAIN_JSON -> APPLICATION_JSON_VALUE NESTED_JSON -> APPLICATION_JSON_VALUE - CSV -> "$TEXT_CSV;charset=UTF-8" - TSV -> "$TEXT_TSV;charset=UTF-8" + CSV -> "$TEXT_CSV_VALUE;charset=UTF-8" + TSV -> "$TEXT_TSV_VALUE;charset=UTF-8" } diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCsvTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCsvTest.kt index a99426e6c..ae21739d1 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCsvTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerCsvTest.kt @@ -4,13 +4,14 @@ import com.fasterxml.jackson.databind.node.NullNode import com.fasterxml.jackson.databind.node.TextNode import com.ninjasquad.springmockk.MockkBean import io.mockk.every -import org.genspectrum.lapis.controller.LapisMediaType.TEXT_CSV -import org.genspectrum.lapis.controller.LapisMediaType.TEXT_CSV_WITHOUT_HEADERS -import org.genspectrum.lapis.controller.LapisMediaType.TEXT_TSV +import org.genspectrum.lapis.controller.LapisMediaType.TEXT_CSV_VALUE +import org.genspectrum.lapis.controller.LapisMediaType.TEXT_CSV_WITHOUT_HEADERS_VALUE +import org.genspectrum.lapis.controller.LapisMediaType.TEXT_TSV_VALUE import org.genspectrum.lapis.model.SiloQueryModel import org.genspectrum.lapis.request.LapisInfo import org.genspectrum.lapis.response.AggregationData import org.genspectrum.lapis.response.DetailsData +import org.hamcrest.Matchers.startsWith import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest @@ -26,6 +27,9 @@ import org.springframework.test.web.servlet.result.MockMvcResultMatchers.content import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status +import java.util.stream.Stream + +private const val DATA_VERSION = "1234" @SpringBootTest @AutoConfigureMockMvc @@ -40,9 +44,7 @@ class LapisControllerCsvTest( @BeforeEach fun setup() { - every { - lapisInfo.dataVersion - } returns "1234" + every { lapisInfo.dataVersion } returns DATA_VERSION } @ParameterizedTest(name = "GET {0} returns empty JSON") @@ -107,7 +109,7 @@ class LapisControllerCsvTest( mockMvc.perform(requestsScenario.request) .andExpect(status().isOk) .andExpect(header().string("Content-Type", "text/csv;charset=UTF-8")) - .andExpect(content().string(requestsScenario.mockDataCollection.expectedCsv)) + .andExpect(content().string(startsWith(requestsScenario.mockDataCollection.expectedCsv))) } @ParameterizedTest(name = "{0} returns data as CSV without headers") @@ -118,7 +120,7 @@ class LapisControllerCsvTest( mockMvc.perform(requestsScenario.request) .andExpect(status().isOk) .andExpect(header().string("Content-Type", "text/plain")) - .andExpect(content().string(returnedCsvWithoutHeadersData(requestsScenario.mockDataCollection))) + .andExpect(content().string(startsWith(returnedCsvWithoutHeadersData(requestsScenario.mockDataCollection)))) } @ParameterizedTest(name = "{0} returns data as TSV") @@ -129,7 +131,7 @@ class LapisControllerCsvTest( mockMvc.perform(requestsScenario.request) .andExpect(status().isOk) .andExpect(header().string("Content-Type", "text/tab-separated-values;charset=UTF-8")) - .andExpect(content().string(requestsScenario.mockDataCollection.expectedTsv)) + .andExpect(content().string(startsWith(requestsScenario.mockDataCollection.expectedTsv))) } fun returnedCsvWithoutHeadersData(mockDataCollection: MockDataCollection) = @@ -140,7 +142,7 @@ class LapisControllerCsvTest( @Test fun `GIVEN aggregated endpoint returns result with null values THEN CSV contains empty strings instead`() { - every { siloQueryModelMock.getAggregated(any()) } returns listOf( + every { siloQueryModelMock.getAggregated(any()) } returns Stream.of( AggregationData( 1, mapOf("firstKey" to TextNode("someValue"), "keyWithNullValue" to NullNode.instance), @@ -155,12 +157,12 @@ class LapisControllerCsvTest( mockMvc.perform(getSample("/aggregated?country=Switzerland").header(ACCEPT, "text/csv")) .andExpect(status().isOk) .andExpect(header().string("Content-Type", "text/csv;charset=UTF-8")) - .andExpect(content().string(expectedCsv)) + .andExpect(content().string(startsWith(expectedCsv))) } @Test fun `GIVEN details endpoint returns result with null values THEN CSV contains empty strings instead`() { - every { siloQueryModelMock.getDetails(any()) } returns listOf( + every { siloQueryModelMock.getDetails(any()) } returns Stream.of( DetailsData( mapOf( "firstKey" to TextNode("some first value"), @@ -178,7 +180,7 @@ class LapisControllerCsvTest( mockMvc.perform(getSample("/details?country=Switzerland").header(ACCEPT, "text/csv")) .andExpect(status().isOk) .andExpect(header().string("Content-Type", "text/csv;charset=UTF-8")) - .andExpect(content().string(expectedCsv)) + .andExpect(content().string(startsWith(expectedCsv))) } private companion object { @@ -220,9 +222,9 @@ class LapisControllerCsvTest( private fun getAcceptHeaderFor(dataFormat: String) = when (dataFormat) { - "csv" -> TEXT_CSV - "csv-without-headers" -> TEXT_CSV_WITHOUT_HEADERS - "tsv" -> TEXT_TSV + "csv" -> TEXT_CSV_VALUE + "csv-without-headers" -> TEXT_CSV_WITHOUT_HEADERS_VALUE + "tsv" -> TEXT_TSV_VALUE "json" -> MediaType.APPLICATION_JSON_VALUE else -> throw IllegalArgumentException("Unknown data format: $dataFormat") } diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerDownloadAsFileTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerDownloadAsFileTest.kt index 7b7493b17..ff16dc576 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerDownloadAsFileTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerDownloadAsFileTest.kt @@ -2,6 +2,7 @@ package org.genspectrum.lapis.controller import com.ninjasquad.springmockk.MockkBean import io.mockk.every +import org.awaitility.Awaitility.await import org.genspectrum.lapis.controller.MockDataCollection.DataFormat.PLAIN_JSON import org.genspectrum.lapis.controller.SampleRoute.AGGREGATED import org.genspectrum.lapis.controller.SampleRoute.ALIGNED_AMINO_ACID_SEQUENCES @@ -126,6 +127,11 @@ class LapisControllerDownloadAsFileTest( this.andExpect(header().string("Content-Disposition", attachmentWithFilename(expectedFilename))) .andReturn() .response + .also { response -> + await().until { + response.isCommitted + } + } .contentAsString .apply(assertFileContentMatches) } diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerTest.kt index 6bfa834e6..ffaa50795 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/LapisControllerTest.kt @@ -36,6 +36,7 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status +import java.util.stream.Stream @SpringBootTest @AutoConfigureMockMvc @@ -59,7 +60,7 @@ class LapisControllerTest( fun `GET aggregated`() { every { siloQueryModelMock.getAggregated(sequenceFiltersRequestWithFields(mapOf("country" to "Switzerland"))) - } returns listOf( + } returns Stream.of( AggregationData( 0, mapOf("country" to TextNode("Switzerland"), "age" to IntNode(42)), @@ -79,7 +80,7 @@ class LapisControllerTest( fun `POST aggregated`() { every { siloQueryModelMock.getAggregated(sequenceFiltersRequestWithFields(mapOf("country" to "Switzerland"))) - } returns listOf( + } returns Stream.of( AggregationData( 0, emptyMap(), @@ -105,7 +106,7 @@ class LapisControllerTest( listOf("country", "date"), ), ) - } returns listOf( + } returns Stream.of( AggregationData( 0, mapOf("country" to TextNode("Switzerland"), "date" to TextNode("a date")), @@ -131,7 +132,7 @@ class LapisControllerTest( mapOf("country" to listOf("Switzerland", "Germany")), ), ) - } returns listOf( + } returns Stream.of( AggregationData( 0, mapOf("country" to TextNode("Switzerland")), @@ -157,7 +158,7 @@ class LapisControllerTest( emptyList(), ), ) - } returns listOf(AggregationData(5, emptyMap())) + } returns Stream.of(AggregationData(5, emptyMap())) mockMvc.perform(getSample("$AGGREGATED_ROUTE?nucleotideMutations=123A,124B")) .andExpect(status().isOk) @@ -173,7 +174,7 @@ class LapisControllerTest( listOf("country", "date"), ), ) - } returns listOf( + } returns Stream.of( AggregationData( 0, mapOf("country" to TextNode("Switzerland"), "date" to TextNode("a date")), @@ -306,7 +307,7 @@ class LapisControllerTest( minProportion, ), ) - } returns listOf(someNucleotideMutationProportion()) + } returns Stream.of(someNucleotideMutationProportion()) } if (endpoint == "/aminoAcidMutations") { every { @@ -316,7 +317,7 @@ class LapisControllerTest( minProportion, ), ) - } returns listOf(someAminoAcidMutationProportion()) + } returns Stream.of(someAminoAcidMutationProportion()) } } @@ -379,13 +380,13 @@ class LapisControllerTest( siloQueryModelMock.getNucleotideInsertions( sequenceFiltersRequest(mapOf("country" to "Switzerland")), ) - } returns listOf(someNucleotideInsertion()) + } returns Stream.of(someNucleotideInsertion()) } AMINO_ACID_INSERTIONS_ROUTE -> { every { siloQueryModelMock.getAminoAcidInsertions(sequenceFiltersRequest(mapOf("country" to "Switzerland"))) - } returns listOf(someAminoAcidInsertion()) + } returns Stream.of(someAminoAcidInsertion()) } else -> throw IllegalArgumentException("Unknown endpoint: $endpoint") @@ -426,7 +427,7 @@ class LapisControllerTest( fun `GET details`() { every { siloQueryModelMock.getDetails(sequenceFiltersRequestWithFields(mapOf("country" to "Switzerland"))) - } returns listOf(DetailsData(mapOf("country" to TextNode("Switzerland"), "age" to IntNode(42)))) + } returns Stream.of(DetailsData(mapOf("country" to TextNode("Switzerland"), "age" to IntNode(42)))) mockMvc.perform(getSample("$DETAILS_ROUTE?country=Switzerland")) .andExpect(status().isOk) @@ -445,7 +446,7 @@ class LapisControllerTest( listOf("country", "date"), ), ) - } returns listOf(DetailsData(mapOf("country" to TextNode("Switzerland"), "date" to TextNode("a date")))) + } returns Stream.of(DetailsData(mapOf("country" to TextNode("Switzerland"), "date" to TextNode("a date")))) mockMvc.perform(getSample("$DETAILS_ROUTE?country=Switzerland&fields=country&fields=date")) .andExpect(status().isOk) @@ -457,7 +458,7 @@ class LapisControllerTest( fun `POST details`() { every { siloQueryModelMock.getDetails(sequenceFiltersRequestWithFields(mapOf("country" to "Switzerland"))) - } returns listOf(DetailsData(mapOf("country" to TextNode("Switzerland"), "age" to IntNode(42)))) + } returns Stream.of(DetailsData(mapOf("country" to TextNode("Switzerland"), "age" to IntNode(42)))) val request = postSample(DETAILS_ROUTE) .content("""{"country": "Switzerland"}""") @@ -480,7 +481,7 @@ class LapisControllerTest( listOf("country", "date"), ), ) - } returns listOf(DetailsData(mapOf("country" to TextNode("Switzerland"), "date" to TextNode("a date")))) + } returns Stream.of(DetailsData(mapOf("country" to TextNode("Switzerland"), "date" to TextNode("a date")))) val request = postSample(DETAILS_ROUTE) .content("""{"country": "Switzerland", "fields": ["country", "date"]}""") diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/MockData.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/MockData.kt index 594a3010d..1de724b43 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/MockData.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/MockData.kt @@ -6,8 +6,8 @@ import com.fasterxml.jackson.databind.node.NullNode import com.fasterxml.jackson.databind.node.TextNode import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import io.mockk.every -import org.genspectrum.lapis.controller.LapisMediaType.TEXT_CSV -import org.genspectrum.lapis.controller.LapisMediaType.TEXT_TSV +import org.genspectrum.lapis.controller.LapisMediaType.TEXT_CSV_VALUE +import org.genspectrum.lapis.controller.LapisMediaType.TEXT_TSV_VALUE import org.genspectrum.lapis.model.SiloQueryModel import org.genspectrum.lapis.response.AggregationData import org.genspectrum.lapis.response.AminoAcidInsertionResponse @@ -15,9 +15,11 @@ import org.genspectrum.lapis.response.AminoAcidMutationResponse import org.genspectrum.lapis.response.DetailsData import org.genspectrum.lapis.response.NucleotideInsertionResponse import org.genspectrum.lapis.response.NucleotideMutationResponse +import org.genspectrum.lapis.response.SequenceData import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.`is` import org.springframework.http.MediaType.APPLICATION_JSON_VALUE +import java.util.stream.Stream data class MockDataCollection( val mockToReturnEmptyData: (SiloQueryModel) -> Unit, @@ -29,20 +31,20 @@ data class MockDataCollection( enum class DataFormat(val fileFormat: String, val acceptHeader: String) { PLAIN_JSON("json", APPLICATION_JSON_VALUE), NESTED_JSON("json", APPLICATION_JSON_VALUE), - CSV("csv", TEXT_CSV), - TSV("tsv", TEXT_TSV), + CSV("csv", TEXT_CSV_VALUE), + TSV("tsv", TEXT_TSV_VALUE), } companion object { inline fun create( - crossinline siloQueryModelMockCall: (SiloQueryModel) -> (Arg) -> List, + crossinline siloQueryModelMockCall: (SiloQueryModel) -> (Arg) -> Stream, modelData: List, expectedJson: String, expectedCsv: String, expectedTsv: String, ) = MockDataCollection( - { modelMock -> every { siloQueryModelMockCall(modelMock)(any()) } returns emptyList() }, - { modelMock -> every { siloQueryModelMockCall(modelMock)(any()) } returns modelData }, + { modelMock -> every { siloQueryModelMockCall(modelMock)(any()) } returns Stream.empty() }, + { modelMock -> every { siloQueryModelMockCall(modelMock)(any()) } returns modelData.stream() }, expectedJson, expectedCsv, expectedTsv, @@ -99,12 +101,22 @@ data class MockData( ) companion object { - fun createForFastaEndpoint(fasta: String) = - MockData( - { modelMock -> every { modelMock.getGenomicSequence(any(), any(), any()) } returns "" }, - { modelMock -> every { modelMock.getGenomicSequence(any(), any(), any()) } returns fasta }, - fasta, - ) + fun createForFastaEndpoint( + sequenceData: List, + expectedFasta: String, + ) = MockData( + { modelMock -> every { modelMock.getGenomicSequence(any(), any(), any()) } returns Stream.empty() }, + { modelMock -> + every { + modelMock.getGenomicSequence( + any(), + any(), + any(), + ) + } returns sequenceData.stream() + }, + expectedFasta, + ) } } @@ -121,11 +133,16 @@ object MockDataForEndpoints { } val fastaMockData = MockData.createForFastaEndpoint( - """ + sequenceData = listOf( + SequenceData("sequence1", "CAGAA"), + SequenceData("sequence2", "CAGAA"), + ), + expectedFasta = """ >sequence1 CAGAA >sequence2 CAGAA + """.trimIndent(), ) @@ -149,10 +166,12 @@ object MockDataForEndpoints { expectedCsv = """ country,age,count Switzerland,42,0 + """.trimIndent(), expectedTsv = """ country age count Switzerland 42 0 + """.trimIndent(), ) @@ -192,11 +211,13 @@ object MockDataForEndpoints { country,age,floatValue Switzerland,42,3.14 Switzerland,43, + """.trimIndent(), expectedTsv = """ country age floatValue Switzerland 42 3.14 Switzerland 43 + """.trimIndent(), ) @@ -229,10 +250,12 @@ object MockDataForEndpoints { expectedCsv = """ mutation,count,proportion,sequenceName,mutationFrom,mutationTo,position sequenceName:A1234T,2345,0.987,sequenceName,A,T,1234 + """.trimIndent(), expectedTsv = """ mutation count proportion sequenceName mutationFrom mutationTo position sequenceName:A1234T 2345 0.987 sequenceName A T 1234 + """.trimIndent(), ) @@ -265,10 +288,12 @@ object MockDataForEndpoints { expectedCsv = """ mutation,count,proportion,sequenceName,mutationFrom,mutationTo,position sequenceName:A1234T,2345,0.987,sequenceName,A,T,1234 + """.trimIndent(), expectedTsv = """ mutation count proportion sequenceName mutationFrom mutationTo position sequenceName:A1234T 2345 0.987 sequenceName A T 1234 + """.trimIndent(), ) @@ -297,10 +322,12 @@ object MockDataForEndpoints { expectedCsv = """ insertion,count,insertedSymbols,position,sequenceName ins_1234:CAGAA,41,CAGAA,1234,sequenceName + """.trimIndent(), expectedTsv = """ insertion count insertedSymbols position sequenceName ins_1234:CAGAA 41 CAGAA 1234 sequenceName + """.trimIndent(), ) @@ -329,10 +356,12 @@ object MockDataForEndpoints { expectedCsv = """ insertion,count,insertedSymbols,position,sequenceName ins_ORF1a:1234:CAGAA,41,CAGAA,1234,ORF1a + """.trimIndent(), expectedTsv = """ insertion count insertedSymbols position sequenceName ins_ORF1a:1234:CAGAA 41 CAGAA 1234 ORF1a + """.trimIndent(), ) } diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/MultiSegmentedSequenceControllerTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/MultiSegmentedSequenceControllerTest.kt index cdf1634e0..8123f84f0 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/MultiSegmentedSequenceControllerTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/MultiSegmentedSequenceControllerTest.kt @@ -5,6 +5,7 @@ import io.mockk.every import org.genspectrum.lapis.config.REFERENCE_GENOME_GENES_APPLICATION_ARG_PREFIX import org.genspectrum.lapis.config.REFERENCE_GENOME_SEGMENTS_APPLICATION_ARG_PREFIX import org.genspectrum.lapis.model.SiloQueryModel +import org.genspectrum.lapis.response.SequenceData import org.genspectrum.lapis.silo.DataVersion import org.genspectrum.lapis.silo.SequenceType import org.junit.jupiter.api.BeforeEach @@ -17,6 +18,7 @@ import org.springframework.test.web.servlet.MockMvc import org.springframework.test.web.servlet.result.MockMvcResultMatchers.content import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status +import java.util.stream.Stream @SpringBootTest( properties = [ @@ -28,6 +30,13 @@ import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status class MultiSegmentedSequenceControllerTest( @Autowired val mockMvc: MockMvc, ) { + val returnedValue: Stream = Stream.of(SequenceData("sequenceKey", "theSequence")) + val expectedFasta = """ + >sequenceKey + theSequence + + """.trimIndent() + @MockkBean lateinit var siloQueryModelMock: SiloQueryModel @@ -43,7 +52,6 @@ class MultiSegmentedSequenceControllerTest( @Test fun `should GET alignedNucleotideSequences with empty filter`() { - val returnedValue = "TestSequenceContent" every { siloQueryModelMock.getGenomicSequence( sequenceFiltersRequest(emptyMap()), @@ -54,13 +62,12 @@ class MultiSegmentedSequenceControllerTest( mockMvc.perform(getSample("$ALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE/otherSegment")) .andExpect(status().isOk) - .andExpect(content().string(returnedValue)) + .andExpect(content().string(expectedFasta)) .andExpect(header().stringValues("Lapis-Data-Version", "1234")) } @Test fun `should GET alignedNucleotideSequences with filter`() { - val returnedValue = "TestSequenceContent" every { siloQueryModelMock.getGenomicSequence( sequenceFiltersRequest(mapOf("country" to "Switzerland")), @@ -71,13 +78,12 @@ class MultiSegmentedSequenceControllerTest( mockMvc.perform(getSample("$ALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE/otherSegment?country=Switzerland")) .andExpect(status().isOk) - .andExpect(content().string(returnedValue)) + .andExpect(content().string(expectedFasta)) .andExpect(header().stringValues("Lapis-Data-Version", "1234")) } @Test fun `should POST alignedNucleotideSequences with empty filter`() { - val returnedValue = "TestSequenceContent" every { siloQueryModelMock.getGenomicSequence( sequenceFiltersRequest(emptyMap()), @@ -92,13 +98,12 @@ class MultiSegmentedSequenceControllerTest( mockMvc.perform(request) .andExpect(status().isOk) - .andExpect(content().string(returnedValue)) + .andExpect(content().string(expectedFasta)) .andExpect(header().stringValues("Lapis-Data-Version", "1234")) } @Test fun `should POST alignedNucleotideSequences with filter`() { - val returnedValue = "TestSequenceContent" every { siloQueryModelMock.getGenomicSequence( sequenceFiltersRequest(mapOf("country" to "Switzerland")), @@ -113,13 +118,12 @@ class MultiSegmentedSequenceControllerTest( mockMvc.perform(request) .andExpect(status().isOk) - .andExpect(content().string(returnedValue)) + .andExpect(content().string(expectedFasta)) .andExpect(header().stringValues("Lapis-Data-Version", "1234")) } @Test fun `should not GET alignedNucleotideSequence without segment`() { - val returnedValue = "TestSequenceContent" every { siloQueryModelMock.getGenomicSequence( sequenceFiltersRequest(emptyMap()), @@ -134,7 +138,6 @@ class MultiSegmentedSequenceControllerTest( @Test fun `should not POST alignedNucleotideSequences without segment`() { - val returnedValue = "TestSequenceContent" every { siloQueryModelMock.getGenomicSequence( sequenceFiltersRequest(emptyMap()), @@ -153,7 +156,6 @@ class MultiSegmentedSequenceControllerTest( @Test fun `should GET unalignedNucleotideSequences with empty filter`() { - val returnedValue = "TestSequenceContent" every { siloQueryModelMock.getGenomicSequence( sequenceFiltersRequest(emptyMap()), @@ -164,13 +166,12 @@ class MultiSegmentedSequenceControllerTest( mockMvc.perform(getSample("$UNALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE/otherSegment")) .andExpect(status().isOk) - .andExpect(content().string(returnedValue)) + .andExpect(content().string(expectedFasta)) .andExpect(header().stringValues("Lapis-Data-Version", "1234")) } @Test fun `should GET unalignedNucleotideSequences with filter`() { - val returnedValue = "TestSequenceContent" every { siloQueryModelMock.getGenomicSequence( sequenceFiltersRequest(mapOf("country" to "Switzerland")), @@ -181,13 +182,12 @@ class MultiSegmentedSequenceControllerTest( mockMvc.perform(getSample("$UNALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE/otherSegment?country=Switzerland")) .andExpect(status().isOk) - .andExpect(content().string(returnedValue)) + .andExpect(content().string(expectedFasta)) .andExpect(header().stringValues("Lapis-Data-Version", "1234")) } @Test fun `should POST unalignedNucleotideSequences with empty filter`() { - val returnedValue = "TestSequenceContent" every { siloQueryModelMock.getGenomicSequence( sequenceFiltersRequest(emptyMap()), @@ -202,13 +202,12 @@ class MultiSegmentedSequenceControllerTest( mockMvc.perform(request) .andExpect(status().isOk) - .andExpect(content().string(returnedValue)) + .andExpect(content().string(expectedFasta)) .andExpect(header().stringValues("Lapis-Data-Version", "1234")) } @Test fun `should POST unalignedNucleotideSequences with filter`() { - val returnedValue = "TestSequenceContent" every { siloQueryModelMock.getGenomicSequence( sequenceFiltersRequest(mapOf("country" to "Switzerland")), @@ -223,7 +222,7 @@ class MultiSegmentedSequenceControllerTest( mockMvc.perform(request) .andExpect(status().isOk) - .andExpect(content().string(returnedValue)) + .andExpect(content().string(expectedFasta)) .andExpect(header().stringValues("Lapis-Data-Version", "1234")) } } diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/SingleSegmentedSequenceControllerTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/SingleSegmentedSequenceControllerTest.kt index 98125eb76..6a7a902fe 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/SingleSegmentedSequenceControllerTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/controller/SingleSegmentedSequenceControllerTest.kt @@ -5,6 +5,7 @@ import io.mockk.every import org.genspectrum.lapis.config.REFERENCE_GENOME_GENES_APPLICATION_ARG_PREFIX import org.genspectrum.lapis.config.REFERENCE_GENOME_SEGMENTS_APPLICATION_ARG_PREFIX import org.genspectrum.lapis.model.SiloQueryModel +import org.genspectrum.lapis.response.SequenceData import org.genspectrum.lapis.silo.DataVersion import org.genspectrum.lapis.silo.SequenceType import org.junit.jupiter.api.BeforeEach @@ -17,6 +18,7 @@ import org.springframework.test.web.servlet.MockMvc import org.springframework.test.web.servlet.result.MockMvcResultMatchers.content import org.springframework.test.web.servlet.result.MockMvcResultMatchers.header import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status +import java.util.stream.Stream @SpringBootTest( properties = [ @@ -28,6 +30,13 @@ import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status class SingleSegmentedSequenceControllerTest( @Autowired val mockMvc: MockMvc, ) { + val returnedValue: Stream = Stream.of(SequenceData("sequenceKey", "theSequence")) + val expectedFasta = """ + >sequenceKey + theSequence + + """.trimIndent() + @MockkBean lateinit var siloQueryModelMock: SiloQueryModel @@ -43,7 +52,6 @@ class SingleSegmentedSequenceControllerTest( @Test fun `should GET alignedNucleotideSequences with empty filter`() { - val returnedValue = "TestSequenceContent" every { siloQueryModelMock.getGenomicSequence( sequenceFiltersRequest(emptyMap()), @@ -54,13 +62,12 @@ class SingleSegmentedSequenceControllerTest( mockMvc.perform(getSample(ALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE)) .andExpect(status().isOk) - .andExpect(content().string(returnedValue)) + .andExpect(content().string(expectedFasta)) .andExpect(header().stringValues("Lapis-Data-Version", "1234")) } @Test fun `should GET alignedNucleotideSequences with filter`() { - val returnedValue = "TestSequenceContent" every { siloQueryModelMock.getGenomicSequence( sequenceFiltersRequest(mapOf("country" to "Switzerland")), @@ -71,13 +78,12 @@ class SingleSegmentedSequenceControllerTest( mockMvc.perform(getSample("$ALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE?country=Switzerland")) .andExpect(status().isOk) - .andExpect(content().string(returnedValue)) + .andExpect(content().string(expectedFasta)) .andExpect(header().stringValues("Lapis-Data-Version", "1234")) } @Test fun `should POST alignedNucleotideSequences with empty filter`() { - val returnedValue = "TestSequenceContent" every { siloQueryModelMock.getGenomicSequence( sequenceFiltersRequest(emptyMap()), @@ -92,13 +98,12 @@ class SingleSegmentedSequenceControllerTest( mockMvc.perform(request) .andExpect(status().isOk) - .andExpect(content().string(returnedValue)) + .andExpect(content().string(expectedFasta)) .andExpect(header().stringValues("Lapis-Data-Version", "1234")) } @Test fun `should POST alignedNucleotideSequences with filter`() { - val returnedValue = "TestSequenceContent" every { siloQueryModelMock.getGenomicSequence( sequenceFiltersRequest(mapOf("country" to "Switzerland")), @@ -107,19 +112,18 @@ class SingleSegmentedSequenceControllerTest( ) } returns returnedValue - val request = postSample(ALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE) + val request2 = postSample(ALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE) .content("""{"country": "Switzerland"}""") .contentType(MediaType.APPLICATION_JSON) - mockMvc.perform(request) + mockMvc.perform(request2) .andExpect(status().isOk) - .andExpect(content().string(returnedValue)) + .andExpect(content().string(expectedFasta)) .andExpect(header().stringValues("Lapis-Data-Version", "1234")) } @Test fun `should not GET alignedNucleotideSequence with segment`() { - val returnedValue = "TestSequenceContent" every { siloQueryModelMock.getGenomicSequence( sequenceFiltersRequest(emptyMap()), @@ -134,7 +138,6 @@ class SingleSegmentedSequenceControllerTest( @Test fun `should not POST alignedNucleotideSequences with segment`() { - val returnedValue = "TestSequenceContent" every { siloQueryModelMock.getGenomicSequence( sequenceFiltersRequest(emptyMap()), @@ -153,7 +156,6 @@ class SingleSegmentedSequenceControllerTest( @Test fun `should GET unalignedNucleotideSequence with empty filter`() { - val returnedValue = "TestSequenceContent" every { siloQueryModelMock.getGenomicSequence( sequenceFiltersRequest(emptyMap()), @@ -164,13 +166,12 @@ class SingleSegmentedSequenceControllerTest( mockMvc.perform(getSample(UNALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE)) .andExpect(status().isOk) - .andExpect(content().string(returnedValue)) + .andExpect(content().string(expectedFasta)) .andExpect(header().stringValues("Lapis-Data-Version", "1234")) } @Test fun `should GET unalignedNucleotideSequence with filter`() { - val returnedValue = "TestSequenceContent" every { siloQueryModelMock.getGenomicSequence( sequenceFiltersRequest(mapOf("country" to "Switzerland")), @@ -181,13 +182,12 @@ class SingleSegmentedSequenceControllerTest( mockMvc.perform(getSample("$UNALIGNED_NUCLEOTIDE_SEQUENCES_ROUTE?country=Switzerland")) .andExpect(status().isOk) - .andExpect(content().string(returnedValue)) + .andExpect(content().string(expectedFasta)) .andExpect(header().stringValues("Lapis-Data-Version", "1234")) } @Test fun `should POST unalignedNucleotideSequence with empty filter`() { - val returnedValue = "TestSequenceContent" every { siloQueryModelMock.getGenomicSequence( sequenceFiltersRequest(emptyMap()), @@ -202,13 +202,12 @@ class SingleSegmentedSequenceControllerTest( mockMvc.perform(request) .andExpect(status().isOk) - .andExpect(content().string(returnedValue)) + .andExpect(content().string(expectedFasta)) .andExpect(header().stringValues("Lapis-Data-Version", "1234")) } @Test fun `should POST unalignedNucleotideSequence with filter`() { - val returnedValue = "TestSequenceContent" every { siloQueryModelMock.getGenomicSequence( sequenceFiltersRequest(mapOf("country" to "Switzerland")), @@ -223,7 +222,7 @@ class SingleSegmentedSequenceControllerTest( mockMvc.perform(request) .andExpect(status().isOk) - .andExpect(content().string(returnedValue)) + .andExpect(content().string(expectedFasta)) .andExpect(header().stringValues("Lapis-Data-Version", "1234")) } } diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/model/SiloQueryModelTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/model/SiloQueryModelTest.kt index d53dfab1f..de9a54d1f 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/model/SiloQueryModelTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/model/SiloQueryModelTest.kt @@ -26,6 +26,7 @@ import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.Matchers.equalTo import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test +import java.util.stream.Stream private val someMutationData = MutationData( mutation = "A1234B", @@ -65,7 +66,7 @@ class SiloQueryModelTest { @Test fun `aggregate calls the SILO client with an aggregated action`() { - every { siloClientMock.sendQuery(any>>()) } returns emptyList() + every { siloClientMock.sendQuery(any>()) } returns Stream.empty() every { siloFilterExpressionMapperMock.map(any()) } returns True every { referenceGenomeSchemaMock.isSingleSegmented() } returns true @@ -89,7 +90,7 @@ class SiloQueryModelTest { @Test fun `computeNucleotideMutationProportions calls the SILO client with a mutations action`() { - every { siloClientMock.sendQuery(any>()) } returns emptyList() + every { siloClientMock.sendQuery(any>()) } returns Stream.empty() every { siloFilterExpressionMapperMock.map(any()) } returns True every { referenceGenomeSchemaMock.isSingleSegmented() } returns true @@ -106,13 +107,13 @@ class SiloQueryModelTest { @Test fun `computeNucleotideMutationProportions ignores the segmentName if singleSegmentedSequenceFeature is enabled`() { - every { siloClientMock.sendQuery(any>()) } returns listOf(someMutationData) + every { siloClientMock.sendQuery(any>()) } returns Stream.of(someMutationData) every { siloFilterExpressionMapperMock.map(any()) } returns True every { referenceGenomeSchemaMock.isSingleSegmented() } returns true val result = underTest.computeNucleotideMutationProportions( MutationProportionsRequest(emptyMap(), emptyList(), emptyList(), emptyList(), emptyList()), - ) + ).toList() val expectedMutation = NucleotideMutationResponse( mutation = "A1234B", @@ -128,13 +129,13 @@ class SiloQueryModelTest { @Test fun `computeNucleotideMutationProportions includes segmentName if singleSegmentedSequenceFeature is not enabled`() { - every { siloClientMock.sendQuery(any>()) } returns listOf(someMutationData) + every { siloClientMock.sendQuery(any>()) } returns Stream.of(someMutationData) every { siloFilterExpressionMapperMock.map(any()) } returns True every { referenceGenomeSchemaMock.isSingleSegmented() } returns false val result = underTest.computeNucleotideMutationProportions( MutationProportionsRequest(emptyMap(), emptyList(), emptyList(), emptyList(), emptyList()), - ) + ).toList() val expectedMutation = NucleotideMutationResponse( mutation = "sequenceName:A1234B", @@ -150,12 +151,12 @@ class SiloQueryModelTest { @Test fun `computeAminoAcidMutationsProportions returns the sequenceName with the position`() { - every { siloClientMock.sendQuery(any>()) } returns listOf(someMutationData) + every { siloClientMock.sendQuery(any>()) } returns Stream.of(someMutationData) every { siloFilterExpressionMapperMock.map(any()) } returns True val result = underTest.computeAminoAcidMutationProportions( MutationProportionsRequest(emptyMap(), emptyList(), emptyList(), emptyList(), emptyList()), - ) + ).toList() val expectedMutation = AminoAcidMutationResponse( mutation = "sequenceName:A1234B", @@ -171,7 +172,7 @@ class SiloQueryModelTest { @Test fun `getNucleotideInsertions ignores the field sequenceName if the nucleotide sequence has one segment`() { - every { siloClientMock.sendQuery(any>()) } returns listOf(someInsertionData) + every { siloClientMock.sendQuery(any>()) } returns Stream.of(someInsertionData) every { siloFilterExpressionMapperMock.map(any()) } returns True every { referenceGenomeSchemaMock.isSingleSegmented() } returns true @@ -184,7 +185,7 @@ class SiloQueryModelTest { emptyList(), emptyList(), ), - ) + ).toList() val expectedInsertion = NucleotideInsertionResponse( insertion = "ins_sequenceName:1234:ABCD", @@ -198,7 +199,7 @@ class SiloQueryModelTest { @Test fun `getAminoAcidInsertions returns the sequenceName with the position`() { - every { siloClientMock.sendQuery(any>()) } returns listOf(someInsertionData) + every { siloClientMock.sendQuery(any>()) } returns Stream.of(someInsertionData) every { siloFilterExpressionMapperMock.map(any()) } returns True val result = underTest.getAminoAcidInsertions( @@ -210,7 +211,7 @@ class SiloQueryModelTest { emptyList(), emptyList(), ), - ) + ).toList() val expectedInsertion = AminoAcidInsertionResponse( insertion = "ins_sequenceName:1234:ABCD", @@ -224,7 +225,7 @@ class SiloQueryModelTest { @Test fun `getGenomicSequence calls the SILO client with a sequence action`() { - every { siloClientMock.sendQuery(any>>()) } returns emptyList() + every { siloClientMock.sendQuery(any>()) } returns Stream.empty() every { siloFilterExpressionMapperMock.map(any()) } returns True underTest.getGenomicSequence( diff --git a/lapis2/src/test/kotlin/org/genspectrum/lapis/silo/SiloClientTest.kt b/lapis2/src/test/kotlin/org/genspectrum/lapis/silo/SiloClientTest.kt index 15e302feb..f223a2049 100644 --- a/lapis2/src/test/kotlin/org/genspectrum/lapis/silo/SiloClientTest.kt +++ b/lapis2/src/test/kotlin/org/genspectrum/lapis/silo/SiloClientTest.kt @@ -84,7 +84,7 @@ class SiloClientTest( ) val query = SiloQuery(SiloAction.aggregated(), StringEquals("theColumn", "theValue")) - val result = underTest.sendQuery(query) + val result = underTest.sendQuery(query).toList() assertThat( result, @@ -112,7 +112,7 @@ class SiloClientTest( ) val query = SiloQuery(action, StringEquals("theColumn", "theValue")) - val result = underTest.sendQuery(query) + val result = underTest.sendQuery(query).toList() assertThat(result, hasSize(2)) assertThat( @@ -157,7 +157,7 @@ class SiloClientTest( SiloAction.genomicSequence(SequenceType.ALIGNED, "someSequenceName"), StringEquals("theColumn", "theValue"), ) - val result = underTest.sendQuery(query) + val result = underTest.sendQuery(query).toList() assertThat(result, hasSize(2)) assertThat( @@ -183,7 +183,7 @@ class SiloClientTest( ) val query = SiloQuery(SiloAction.details(), StringEquals("theColumn", "theValue")) - val result = underTest.sendQuery(query) + val result = underTest.sendQuery(query).toList() assertThat(result, hasSize(2)) assertThat( @@ -214,7 +214,7 @@ class SiloClientTest( @ParameterizedTest @MethodSource("getInsertionActions") fun `GIVEN server returns insertions response THEN response can be deserialized`( - action: SiloAction>, + action: SiloAction, ) { expectQueryRequestAndRespondWith( response() @@ -228,7 +228,7 @@ class SiloClientTest( ) val query = SiloQuery(action, True) - val result = underTest.sendQuery(query) + val result = underTest.sendQuery(query).toList() assertThat(result, hasSize(2)) assertThat( @@ -261,7 +261,7 @@ class SiloClientTest( .withBody("""{"unexpectedKey": "some unexpected message"}"""), ) - val exception = assertThrows { underTest.sendQuery(someQuery) } + val exception = assertThrows { underTest.sendQuery(someQuery).toList() } assertThat(exception.statusCode, equalTo(500)) assertThat( @@ -279,7 +279,7 @@ class SiloClientTest( .withBody("""{"error": "Test Error", "message": "test message with details"}"""), ) - val exception = assertThrows { underTest.sendQuery(someQuery) } + val exception = assertThrows { underTest.sendQuery(someQuery).toList() } assertThat(exception.statusCode, equalTo(432)) assertThat(exception.message, equalTo("Error from SILO: test message with details")) } @@ -293,7 +293,7 @@ class SiloClientTest( .withBody("""{"unexpectedField": "some message"}"""), ) - val exception = assertThrows { underTest.sendQuery(someQuery) } + val exception = assertThrows { underTest.sendQuery(someQuery).toList() } assertThat(exception.message, containsString("Could not parse response from silo")) } @@ -309,7 +309,7 @@ class SiloClientTest( .withBody("""{"error": "Test Error", "message": "$errorMessage"}"""), ) - val exception = assertThrows { underTest.sendQuery(someQuery) } + val exception = assertThrows { underTest.sendQuery(someQuery).toList() } assertThat(exception.message, `is`("SILO is currently unavailable: $errorMessage")) assertThat(exception.retryAfter, `is`(retryAfterValue)) @@ -325,7 +325,7 @@ class SiloClientTest( .withBody("""{"error": "Test Error", "message": "$errorMessage"}"""), ) - val exception = assertThrows { underTest.sendQuery(someQuery) } + val exception = assertThrows { underTest.sendQuery(someQuery).toList() } assertThat(exception.message, `is`("SILO is currently unavailable: $errorMessage")) assertThat(exception.retryAfter, `is`(nullValue())) @@ -351,9 +351,9 @@ class SiloClientTest( Times.exactly(1), ) - underTest.sendQuery(query) + underTest.sendQuery(query).toList() - val exception = assertThrows { underTest.sendQuery(query) } + val exception = assertThrows { underTest.sendQuery(query).toList() } assertThat(exception.message, containsString(errorMessage)) } @@ -369,8 +369,8 @@ class SiloClientTest( Times.once(), ) - val result1 = underTest.sendQuery(query) - val result2 = underTest.sendQuery(query) + val result1 = underTest.sendQuery(query).toList() + val result2 = underTest.sendQuery(query).toList() assertThat(result1, `is`(result2)) } @@ -391,11 +391,11 @@ class SiloClientTest( val query = queriesThatShouldBeCached[0] assertThat(dataVersion.dataVersion, `is`(nullValue())) - underTest.sendQuery(query) + underTest.sendQuery(query).toList() assertThat(dataVersion.dataVersion, `is`(dataVersionValue)) dataVersion.dataVersion = null - underTest.sendQuery(query) + underTest.sendQuery(query).toList() assertThat(dataVersion.dataVersion, `is`(dataVersionValue)) } @@ -422,10 +422,10 @@ class SiloClientTest( val query = SiloQuery(SiloAction.mutations(orderByFields = listOf(orderByRandom)), True) assertThat(query.action.cacheable, `is`(true)) - val result = underTest.sendQuery(query) + val result = underTest.sendQuery(query).toList() assertThat(result, hasSize(0)) - val exception = assertThrows { underTest.sendQuery(query) } + val exception = assertThrows { underTest.sendQuery(query).toList() } assertThat(exception.message, containsString(errorMessage)) } @@ -506,8 +506,8 @@ class SiloClientAndCacheInvalidatorTest( Times.once(), ) - siloClient.sendQuery(someQuery) - siloClient.sendQuery(someQuery) + siloClient.sendQuery(someQuery).toList() + siloClient.sendQuery(someQuery).toList() assertThat(dataVersion.dataVersion, `is`(firstDataVersion)) } @@ -521,7 +521,7 @@ class SiloClientAndCacheInvalidatorTest( Times.once(), ) - val exception = assertThrows { siloClient.sendQuery(someQuery) } + val exception = assertThrows { siloClient.sendQuery(someQuery).toList() } assertThat(exception.message, containsString(errorMessage)) } } diff --git a/siloLapisTests/test/aggregated.spec.ts b/siloLapisTests/test/aggregated.spec.ts index af2876b2a..fde4155cb 100644 --- a/siloLapisTests/test/aggregated.spec.ts +++ b/siloLapisTests/test/aggregated.spec.ts @@ -167,7 +167,7 @@ age,country,count 57,Switzerland,10 58,Switzerland,9 59,Switzerland,9 - `.trim() + `.trim() + '\n' ); }); @@ -197,7 +197,7 @@ age country count 57 Switzerland 10 58 Switzerland 9 59 Switzerland 9 - `.trim() + `.trim() + '\n' ); }); }); diff --git a/siloLapisTests/test/common.spec.ts b/siloLapisTests/test/common.spec.ts index 34e03c6da..0cf1ca240 100644 --- a/siloLapisTests/test/common.spec.ts +++ b/siloLapisTests/test/common.spec.ts @@ -65,13 +65,29 @@ describe('All endpoints', () => { ); }); - it('should return the lapis data version in the response', async () => { + it('should return the lapis data version header', async () => { const response = await get(); expect(response.status).equals(200); expect(response.headers.get('lapis-data-version')).to.match(/\d{10}/); }); + if (!route.servesFasta) { + it('should return the lapis data version header for CSV data', async () => { + const response = await get(new URLSearchParams({ dataFormat: 'csv' })); + + expect(response.status).equals(200); + expect(response.headers.get('lapis-data-version')).to.match(/\d{10}/); + }); + + it('should return the lapis data version header for TSV data', async () => { + const response = await get(new URLSearchParams({ dataFormat: 'tsv' })); + + expect(response.status).equals(200); + expect(response.headers.get('lapis-data-version')).to.match(/\d{10}/); + }); + } + it('should return zstd compressed data when asking for compression', async () => { const urlParams = new URLSearchParams({ compression: 'zstd' }); diff --git a/siloLapisTests/test/common.ts b/siloLapisTests/test/common.ts index 34b3268e4..f90cf4064 100644 --- a/siloLapisTests/test/common.ts +++ b/siloLapisTests/test/common.ts @@ -46,7 +46,7 @@ export const lapisSingleSegmentedSequenceController = new SingleSegmentedSequenc ).withMiddleware(middleware); export function sequenceData(serverResponse: string) { - const lines = serverResponse.split('\n'); + const lines = serverResponse.split('\n').filter(line => line.length > 0); const primaryKeys = lines.filter(line => line.startsWith('>')); const sequences = lines.filter(line => !line.startsWith('>')); diff --git a/siloLapisTests/test/details.spec.ts b/siloLapisTests/test/details.spec.ts index 76351e15b..c4b23abf2 100644 --- a/siloLapisTests/test/details.spec.ts +++ b/siloLapisTests/test/details.spec.ts @@ -106,7 +106,7 @@ division,pangoLineage,primaryKey Vaud,B.1.177.44,key_1001493 Bern,B.1.177,key_1001920 Solothurn,B.1,key_1002052 - `.trim() + `.trim() + '\n' ); }); @@ -126,7 +126,7 @@ division pangoLineage primaryKey Vaud B.1.177.44 key_1001493 Bern B.1.177 key_1001920 Solothurn B.1 key_1002052 - `.trim() + `.trim() + '\n' ); }); @@ -190,7 +190,7 @@ Solothurn B.1 key_1002052 key_1001493 key_1001920 key_1002052 - `.trim() + `.trim() + '\n' ); }); diff --git a/siloLapisTests/test/unalignedNucleotideSequence.spec.ts b/siloLapisTests/test/unalignedNucleotideSequence.spec.ts index 944b23fcf..2193131d3 100644 --- a/siloLapisTests/test/unalignedNucleotideSequence.spec.ts +++ b/siloLapisTests/test/unalignedNucleotideSequence.spec.ts @@ -9,8 +9,8 @@ describe('The /unalignedNucleotideSequence endpoint', () => { const { primaryKeys, sequences } = sequenceData(result); - expect(primaryKeys).to.have.length(100); - expect(sequences).to.have.length(100); + expect(primaryKeys, 'primaryKeys').to.have.length(100); + expect(sequences, 'sequences').to.have.length(100); expect(primaryKeys[0]).to.equal('>key_3259931'); expect(sequences[0]).to.have.length(29903); });