diff --git a/.github/workflows/lapis.yml b/.github/workflows/lapis.yml index c7f193c38..9f8b6fd6d 100644 --- a/.github/workflows/lapis.yml +++ b/.github/workflows/lapis.yml @@ -91,6 +91,12 @@ jobs: arguments: generateOpenApiDocs -PopennessLevel=protected build-root-directory: lapis + - name: Build OpenAPI Spec Multi segmented + uses: gradle/actions/setup-gradle@v3 + with: + arguments: generateOpenApiDocs -Psegmented=true + build-root-directory: lapis + - name: Cache .npm uses: actions/cache@v4 with: @@ -109,6 +115,10 @@ jobs: run: npm run generateLapisClientProtected working-directory: lapis-e2e + - name: Generate Lapis Client Multi Segmented + run: npm run generateLapisClientMultiSegmented + working-directory: lapis-e2e + - name: Check Format run: npm run check-format working-directory: lapis-e2e @@ -152,6 +162,7 @@ jobs: docker compose logs silo > e2e-logs/silo.log docker compose logs lapisOpen > e2e-logs/lapisOpen.log docker compose logs lapisProtected > e2e-logs/lapisProtected.log + docker compose logs lapisMultiSegmented > e2e-logs/lapisMultiSegmented.log env: SILO_TAG: latest LAPIS_TAG: ${{ steps.lapisBranchTag.outputs.lapisTag }} diff --git a/.idea/runConfigurations/LapisOpen.xml b/.idea/runConfigurations/LapisOpen.xml new file mode 100644 index 000000000..9e0575f9d --- /dev/null +++ b/.idea/runConfigurations/LapisOpen.xml @@ -0,0 +1,13 @@ + + + + \ No newline at end of file diff --git a/.idea/runConfigurations/LapisProtected.xml b/.idea/runConfigurations/LapisProtected.xml new file mode 100644 index 000000000..2a4ed3b1f --- /dev/null +++ b/.idea/runConfigurations/LapisProtected.xml @@ -0,0 +1,13 @@ + + + + \ No newline at end of file diff --git a/.idea/runConfigurations/lapis_open.xml b/.idea/runConfigurations/lapis_open.xml deleted file mode 100644 index 829d85f77..000000000 --- a/.idea/runConfigurations/lapis_open.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - \ No newline at end of file diff --git a/.idea/runConfigurations/lapis_protected.xml b/.idea/runConfigurations/lapis_protected.xml deleted file mode 100644 index 9c5d9d44b..000000000 --- a/.idea/runConfigurations/lapis_protected.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - \ No newline at end of file diff --git a/lapis-docs/.env.example b/lapis-docs/.env.example index 953a6e400..6b05707d0 100644 --- a/lapis-docs/.env.example +++ b/lapis-docs/.env.example @@ -1,4 +1,4 @@ # Create a .env file with this input might NOT work. You might need to properly set CONFIG_FILE as an # environment variable. -CONFIG_FILE=../lapis-e2e/testData/testDatabaseConfig.yaml -REFERENCE_GENOMES_FILE=../lapis-e2e/testData/reference_genomes.json +CONFIG_FILE=../lapis-e2e/testData/singleSegmented/testDatabaseConfig.yaml +REFERENCE_GENOMES_FILE=../lapis-e2e/testData/singleSegmented/reference_genomes.json diff --git a/lapis-docs/package.json b/lapis-docs/package.json index 0ee868047..4a0d68f5f 100644 --- a/lapis-docs/package.json +++ b/lapis-docs/package.json @@ -10,7 +10,7 @@ "astro": "astro", "check-format": "prettier --check \"**/*.{ts,tsx,json,astro,md,mdx,mjs,cjs}\"", "format": "prettier --write \"**/*.{ts,tsx,json,astro,md,mdx,mjs,cjs}\"", - "check-types": "CONFIG_FILE=../lapis-e2e/testData/testDatabaseConfig.yaml astro sync && tsc --noEmit", + "check-types": "CONFIG_FILE=../lapis-e2e/testData/singleSegmented/testDatabaseConfig.yaml astro sync && tsc --noEmit", "e2e": "playwright test" }, "dependencies": { diff --git a/lapis-docs/test-docker-compose.yml b/lapis-docs/test-docker-compose.yml index 692c3a2fb..a7576a2e2 100644 --- a/lapis-docs/test-docker-compose.yml +++ b/lapis-docs/test-docker-compose.yml @@ -5,8 +5,8 @@ services: ports: - "4321:3000" volumes: - - ../lapis-e2e/testData/testDatabaseConfig.yaml:/config/database_config.yaml - - ../lapis-e2e/testData/reference_genomes.json:/config/reference_genomes.json + - ../lapis-e2e/testData/singleSegmented/testDatabaseConfig.yaml:/config/database_config.yaml + - ../lapis-e2e/testData/singleSegmented/reference_genomes.json:/config/reference_genomes.json environment: LAPIS_URL: http://localhost:8090 BASE_URL: /docs/ diff --git a/lapis-e2e/.gitignore b/lapis-e2e/.gitignore index c94e026c0..1ca49b7fe 100644 --- a/lapis-e2e/.gitignore +++ b/lapis-e2e/.gitignore @@ -3,3 +3,4 @@ node_modules/ **/output/ **/logs/ +**/temp/ diff --git a/lapis-e2e/.prettierignore b/lapis-e2e/.prettierignore index e0f97514e..df9332181 100644 --- a/lapis-e2e/.prettierignore +++ b/lapis-e2e/.prettierignore @@ -1,2 +1,3 @@ /test/lapisClient -/test/lapisClientProtected \ No newline at end of file +/test/lapisClientProtected +/test/lapisClientMultiSegmented \ No newline at end of file diff --git a/lapis-e2e/package.json b/lapis-e2e/package.json index c268d9473..b41814ceb 100644 --- a/lapis-e2e/package.json +++ b/lapis-e2e/package.json @@ -5,13 +5,17 @@ "scripts": { "test": "mocha --exit", "generateLapisClient": "npm run runOpenApiGenerator && npm run copyGeneratedFiles && npm run cleanUpGeneratedFiles", - "runOpenApiGenerator": "openapi-generator-cli generate -i ../lapis/lapis-openapi.json -g typescript-fetch -o generated-sources", + "runOpenApiGenerator": "openapi-generator-cli generate -i ../lapis/lapis-openapi-single-segmented.json -g typescript-fetch -o generated-sources", "copyGeneratedFiles": "mkdir -p test/lapisClient && cp generated-sources/index.ts generated-sources/runtime.ts test/lapisClient && cp -r generated-sources/apis generated-sources/models test/lapisClient", "generateLapisClientProtected": "npm run runOpenApiGeneratorProtected && npm run copyGeneratedFilesProtected && npm run cleanUpGeneratedFilesProtected", - "runOpenApiGeneratorProtected": "openapi-generator-cli generate -i ../lapis/lapis-openapi-protected.json -g typescript-fetch -o generated-sources-protected", + "runOpenApiGeneratorProtected": "openapi-generator-cli generate -i ../lapis/lapis-openapi-single-segmented-protected.json -g typescript-fetch -o generated-sources-protected", "copyGeneratedFilesProtected": "mkdir -p test/lapisClientProtected && cp generated-sources-protected/index.ts generated-sources-protected/runtime.ts test/lapisClientProtected && cp -r generated-sources-protected/apis generated-sources-protected/models test/lapisClientProtected", + "generateLapisClientMultiSegmented": "npm run runOpenApiGeneratorMultiSegmented && npm run copyGeneratedFilesMultiSegmented && npm run cleanUpGeneratedFilesMultiSegmented", + "runOpenApiGeneratorMultiSegmented": "openapi-generator-cli generate -i ../lapis/lapis-openapi-multi-segmented.json -g typescript-fetch -o generated-sources-multi-segmented", + "copyGeneratedFilesMultiSegmented": "mkdir -p test/lapisClientMultiSegmented && cp generated-sources-multi-segmented/index.ts generated-sources-multi-segmented/runtime.ts test/lapisClientMultiSegmented && cp -r generated-sources-multi-segmented/apis generated-sources-multi-segmented/models test/lapisClientMultiSegmented", "cleanUpGeneratedFiles": "rm -rf generated-sources", "cleanUpGeneratedFilesProtected": "rm -rf generated-sources-protected", + "cleanUpGeneratedFilesMultiSegmented": "rm -rf generated-sources-multi-segmented", "check-format": "prettier --check .", "format": "prettier --write ." }, diff --git a/lapis-e2e/test/aggregated.spec.ts b/lapis-e2e/test/aggregated.spec.ts index a58ce6033..107de5e82 100644 --- a/lapis-e2e/test/aggregated.spec.ts +++ b/lapis-e2e/test/aggregated.spec.ts @@ -1,5 +1,5 @@ import { expect } from 'chai'; -import { basePath, lapisClient } from './common'; +import { basePath, lapisClient, lapisClientMultiSegmented } from './common'; import fs from 'fs'; import { AggregatedPostRequest, AggregatedResponse } from './lapisClient'; @@ -46,6 +46,17 @@ describe('The /aggregated endpoint', () => { }) ); + it('should correcty handle aggregated request with multiple segments', async () => { + const result = await lapisClientMultiSegmented.postAggregated1({ + aggregatedPostRequest: { + nucleotideMutations: ['L:T1A', 'M:T1C'], + }, + }); + + expect(result.data).to.have.length(1); + expect(result.data[0]).to.have.property('count', 1); + }); + it('should correctly handle multiple mutation requests in GET requests', async () => { const urlParams = new URLSearchParams({ nucleotideMutations: 'T1-,A23062T', diff --git a/lapis-e2e/test/alignedNucleotideSequence.spec.ts b/lapis-e2e/test/alignedNucleotideSequence.spec.ts index 4116787e7..9595d6f95 100644 --- a/lapis-e2e/test/alignedNucleotideSequence.spec.ts +++ b/lapis-e2e/test/alignedNucleotideSequence.spec.ts @@ -2,6 +2,7 @@ import { expect } from 'chai'; import { basePath, expectIsZstdEncoded, + lapisMultiSegmentedSequenceController, lapisSingleSegmentedSequenceController, sequenceData, } from './common'; @@ -20,6 +21,20 @@ describe('The /alignedNucleotideSequence endpoint', () => { expect(sequences[0]).to.have.length(29903); }); + it('should return aligned nucleotide sequences for multi segmented sequences', async () => { + const result = await lapisMultiSegmentedSequenceController.postAlignedNucleotideSequence({ + nucleotideSequenceRequest: { country: 'Switzerland' }, + segment: 'M', + }); + + const { primaryKeys, sequences } = sequenceData(result); + + expect(primaryKeys).to.have.length(6); + expect(sequences).to.have.length(6); + expect(primaryKeys[0]).to.equal('>key_5'); + expect(sequences[0]).to.equal('TGGG'); + }); + it('should order ascending by specified fields', async () => { const result = await lapisSingleSegmentedSequenceController.postAlignedNucleotideSequence({ nucleotideSequenceRequest: { orderBy: [{ field: 'primaryKey', type: 'ascending' }] }, diff --git a/lapis-e2e/test/aminoAcidSequence.spec.ts b/lapis-e2e/test/aminoAcidSequence.spec.ts index 7502530ce..716c96ef0 100644 --- a/lapis-e2e/test/aminoAcidSequence.spec.ts +++ b/lapis-e2e/test/aminoAcidSequence.spec.ts @@ -1,5 +1,5 @@ import { expect } from 'chai'; -import { basePath, lapisClient, sequenceData } from './common'; +import { lapisClient, sequenceData } from './common'; describe('The /alignedAminoAcidSequence endpoint', () => { it('should return amino acid sequences for Switzerland', async () => { diff --git a/lapis-e2e/test/common.ts b/lapis-e2e/test/common.ts index f90cf4064..0c28a51c1 100644 --- a/lapis-e2e/test/common.ts +++ b/lapis-e2e/test/common.ts @@ -4,15 +4,18 @@ import { Middleware, SingleSegmentedSequenceControllerApi, } from './lapisClient'; +import { LapisControllerApi as LapisControllerApiMultiSegmented } from './lapisClientMultiSegmented'; import { expect } from 'chai'; import { LapisControllerApi as LapisControllerApiProtected, Configuration as ConfigurationProtected, } from './lapisClientProtected'; +import { MultiSegmentedSequenceControllerApi } from './lapisClientMultiSegmented'; export const basePath = 'http://localhost:8090'; export const basePathProtected = 'http://localhost:8092'; +export const basePathMultiSegmented = 'http://localhost:8094'; const middleware: Middleware = { onError: errorContext => { @@ -40,11 +43,18 @@ export const lapisClient = new LapisControllerApi(new Configuration({ basePath } export const lapisClientProtected = new LapisControllerApiProtected( new ConfigurationProtected({ basePath: basePathProtected }) ).withMiddleware(middleware); +export const lapisClientMultiSegmented = new LapisControllerApiMultiSegmented( + new Configuration({ basePath: basePathMultiSegmented }) +).withMiddleware(middleware); export const lapisSingleSegmentedSequenceController = new SingleSegmentedSequenceControllerApi( new Configuration({ basePath }) ).withMiddleware(middleware); +export const lapisMultiSegmentedSequenceController = new MultiSegmentedSequenceControllerApi( + new Configuration({ basePath: basePathMultiSegmented }) +).withMiddleware(middleware); + export function sequenceData(serverResponse: string) { const lines = serverResponse.split('\n').filter(line => line.length > 0); const primaryKeys = lines.filter(line => line.startsWith('>')); diff --git a/lapis-e2e/test/nucleotideInsertions.spec.ts b/lapis-e2e/test/nucleotideInsertions.spec.ts index eaaa157e9..e31318e48 100644 --- a/lapis-e2e/test/nucleotideInsertions.spec.ts +++ b/lapis-e2e/test/nucleotideInsertions.spec.ts @@ -1,5 +1,5 @@ import { expect } from 'chai'; -import { basePath, lapisClient } from './common'; +import { basePath, lapisClient, lapisClientMultiSegmented } from './common'; describe('The /nucleotideInsertions endpoint', () => { let someInsertion = 'ins_25701:CCC'; @@ -18,6 +18,20 @@ describe('The /nucleotideInsertions endpoint', () => { expect(specificInsertion?.sequenceName).to.be.undefined; }); + it('should return nucleotide insertions for multi segmented sequences', async () => { + const result = await lapisClientMultiSegmented.postNucleotideInsertions1({ + insertionsRequest: { country: 'Switzerland' }, + }); + + expect(result.data).to.have.length(2); + + const insertionsFirstSegment = result.data.find(mutationData => mutationData.insertion === 'ins_L:1:AB'); + expect(insertionsFirstSegment?.count).to.equal(2); + + const insertionsSecondSegment = result.data.find(mutationData => mutationData.insertion === 'ins_M:2:BC'); + expect(insertionsSecondSegment?.count).to.equal(1); + }); + it('should order by specified fields', async () => { const ascendingOrderedResult = await lapisClient.postNucleotideInsertions1({ insertionsRequest: { diff --git a/lapis-e2e/test/nucleotideMutations.spec.ts b/lapis-e2e/test/nucleotideMutations.spec.ts index 8d23168e9..722f63c07 100644 --- a/lapis-e2e/test/nucleotideMutations.spec.ts +++ b/lapis-e2e/test/nucleotideMutations.spec.ts @@ -1,5 +1,5 @@ import { expect } from 'chai'; -import { basePath, lapisClient } from './common'; +import { basePath, lapisClient, lapisClientMultiSegmented } from './common'; describe('The /nucleotideMutations endpoint', () => { let mutationWithLessThan10PercentProportion = 'C19220T'; @@ -29,6 +29,24 @@ describe('The /nucleotideMutations endpoint', () => { expect(commonMutationProportion?.position).to.be.equal(28280); }); + it('should return mutations proportions for multi segmented', async () => { + const result = await lapisClientMultiSegmented.postNucleotideMutations1({ + sequenceFiltersWithMinProportion: { country: 'Switzerland'}, + }); + + expect(result.data).to.have.length(2); + + const mutationProportionOnFirstSegment = result.data.find( + mutationData => mutationData.mutation === 'L:T1A' + ); + expect(mutationProportionOnFirstSegment?.count).to.equal(2); + + const mutationProportionOnSecondSegment = result.data.find( + mutationData => mutationData.mutation === 'M:T1C' + ); + expect(mutationProportionOnSecondSegment?.count).to.equal(1); + }); + it('should return mutation proportions for Switzerland with minProportion 0.5', async () => { const result = await lapisClient.postNucleotideMutations1({ sequenceFiltersWithMinProportion: { diff --git a/lapis-e2e/testData/multiSegmented/aa_insertions.tsv b/lapis-e2e/testData/multiSegmented/aa_insertions.tsv new file mode 100644 index 000000000..ead67f527 --- /dev/null +++ b/lapis-e2e/testData/multiSegmented/aa_insertions.tsv @@ -0,0 +1,7 @@ +primaryKey RdRp NP GPC +key_0 [] [] [] +key_1 [] [] [] +key_2 [] [] [] +key_3 [] [] [] +key_4 [] [] [] +key_5 [] [] [] diff --git a/lapis-e2e/testData/multiSegmented/gene_GPC.fasta b/lapis-e2e/testData/multiSegmented/gene_GPC.fasta new file mode 100644 index 000000000..17cdb7f92 --- /dev/null +++ b/lapis-e2e/testData/multiSegmented/gene_GPC.fasta @@ -0,0 +1,12 @@ +>key_0 +MS* +>key_1 +MS* +>key_2 +MS* +>key_3 +MS* +>key_4 +MS* +>key_5 +MS* \ No newline at end of file diff --git a/lapis-e2e/testData/multiSegmented/gene_NP.fasta b/lapis-e2e/testData/multiSegmented/gene_NP.fasta new file mode 100644 index 000000000..e68095f73 --- /dev/null +++ b/lapis-e2e/testData/multiSegmented/gene_NP.fasta @@ -0,0 +1,12 @@ +>key_0 +MEN* +>key_1 +MEN* +>key_2 +MEN* +>key_3 +MEN* +>key_4 +MEN* +>key_5 +MEN* \ No newline at end of file diff --git a/lapis-e2e/testData/multiSegmented/gene_RdRp.fasta b/lapis-e2e/testData/multiSegmented/gene_RdRp.fasta new file mode 100644 index 000000000..8d39d7b83 --- /dev/null +++ b/lapis-e2e/testData/multiSegmented/gene_RdRp.fasta @@ -0,0 +1,12 @@ +>key_0 +MDFL* +>key_1 +MDFL* +>key_2 +MDFL* +>key_3 +MDFL* +>key_4 +MDFL* +>key_5 +MDFL* \ No newline at end of file diff --git a/lapis-e2e/testData/multiSegmented/multi_segmented_metadata.tsv b/lapis-e2e/testData/multiSegmented/multi_segmented_metadata.tsv new file mode 100644 index 000000000..d4286cd5c --- /dev/null +++ b/lapis-e2e/testData/multiSegmented/multi_segmented_metadata.tsv @@ -0,0 +1,7 @@ +primaryKey date country +key_0 2021-03-18 Switzerland +key_1 2021-04-13 Switzerland +key_2 2021-04-25 Switzerland +key_3 2021-04-13 Switzerland +key_4 2021-03-19 Switzerland +key_5 Switzerland \ No newline at end of file diff --git a/lapis-e2e/testData/multiSegmented/nuc_L.fasta b/lapis-e2e/testData/multiSegmented/nuc_L.fasta new file mode 100644 index 000000000..b17d62d75 --- /dev/null +++ b/lapis-e2e/testData/multiSegmented/nuc_L.fasta @@ -0,0 +1,12 @@ +>key_0 +ACTCT +>key_1 +ACTCT +>key_2 +TCTCT +>key_3 +TCTCT +>key_4 +TCTCT +>key_5 +TCTCT \ No newline at end of file diff --git a/lapis-e2e/testData/multiSegmented/nuc_M.fasta b/lapis-e2e/testData/multiSegmented/nuc_M.fasta new file mode 100644 index 000000000..fdd2d8745 --- /dev/null +++ b/lapis-e2e/testData/multiSegmented/nuc_M.fasta @@ -0,0 +1,12 @@ +>key_0 +CGGG +>key_1 +TGGG +>key_2 +TGGG +>key_3 +TGGG +>key_4 +TGGG +>key_5 +TGGG \ No newline at end of file diff --git a/lapis-e2e/testData/multiSegmented/nuc_S.fasta b/lapis-e2e/testData/multiSegmented/nuc_S.fasta new file mode 100644 index 000000000..dcb4c9848 --- /dev/null +++ b/lapis-e2e/testData/multiSegmented/nuc_S.fasta @@ -0,0 +1,12 @@ +>key_0 +AAA +>key_1 +AAA +>key_2 +AAA +>key_3 +AAA +>key_4 +AAA +>key_5 +AAA \ No newline at end of file diff --git a/lapis-e2e/testData/multiSegmented/nuc_insertions.tsv b/lapis-e2e/testData/multiSegmented/nuc_insertions.tsv new file mode 100644 index 000000000..ec32bcf03 --- /dev/null +++ b/lapis-e2e/testData/multiSegmented/nuc_insertions.tsv @@ -0,0 +1,7 @@ +primaryKey L M S +key_0 [] [] [] +key_1 [1:AB] [] [] +key_2 [1:AB] [] [] +key_3 [] [2:BC] [] +key_4 [] [] [] +key_5 [] [] [] diff --git a/lapis-e2e/testData/multiSegmented/preprocessingConfig.yaml b/lapis-e2e/testData/multiSegmented/preprocessingConfig.yaml new file mode 100644 index 000000000..5b5dba652 --- /dev/null +++ b/lapis-e2e/testData/multiSegmented/preprocessingConfig.yaml @@ -0,0 +1,2 @@ +metadataFilename: multi_segmented_metadata.tsv +referenceGenomeFilename: reference_genomes.json diff --git a/lapis-e2e/testData/multiSegmented/reference_genomes.json b/lapis-e2e/testData/multiSegmented/reference_genomes.json new file mode 100644 index 000000000..b4f6753a6 --- /dev/null +++ b/lapis-e2e/testData/multiSegmented/reference_genomes.json @@ -0,0 +1,30 @@ +{ + "nucleotideSequences": [ + { + "name": "L", + "sequence": "TCTCT" + }, + { + "name": "M", + "sequence": "TGGG" + }, + { + "name": "S", + "sequence": "AAA" + } + ], + "genes": [ + { + "name": "RdRp", + "sequence": "MDFL*" + }, + { + "name": "GPC", + "sequence": "MS*" + }, + { + "name": "NP", + "sequence": "MEN*" + } + ] +} diff --git a/lapis-e2e/testData/multiSegmented/testDatabaseConfig.yaml b/lapis-e2e/testData/multiSegmented/testDatabaseConfig.yaml new file mode 100644 index 000000000..25e9f38df --- /dev/null +++ b/lapis-e2e/testData/multiSegmented/testDatabaseConfig.yaml @@ -0,0 +1,13 @@ +schema: + instanceName: crimean-congo-hemorrhagic-fever-virus-multi-segmented-test + opennessLevel: OPEN + metadata: + - name: primaryKey + type: string + - name: date + type: date + - name: country + type: string + generateIndex: true + primaryKey: primaryKey + dateToSortBy: date diff --git a/lapis-e2e/testData/multiSegmented/unaligned_L.fasta b/lapis-e2e/testData/multiSegmented/unaligned_L.fasta new file mode 100644 index 000000000..e69de29bb diff --git a/lapis-e2e/testData/multiSegmented/unaligned_M.fasta b/lapis-e2e/testData/multiSegmented/unaligned_M.fasta new file mode 100644 index 000000000..e69de29bb diff --git a/lapis-e2e/testData/multiSegmented/unaligned_S.fasta b/lapis-e2e/testData/multiSegmented/unaligned_S.fasta new file mode 100644 index 000000000..e69de29bb diff --git a/lapis-e2e/testData/aa_insertions.tsv b/lapis-e2e/testData/singleSegmented/aa_insertions.tsv similarity index 100% rename from lapis-e2e/testData/aa_insertions.tsv rename to lapis-e2e/testData/singleSegmented/aa_insertions.tsv diff --git a/lapis-e2e/testData/gene_E.fasta b/lapis-e2e/testData/singleSegmented/gene_E.fasta similarity index 100% rename from lapis-e2e/testData/gene_E.fasta rename to lapis-e2e/testData/singleSegmented/gene_E.fasta diff --git a/lapis-e2e/testData/gene_M.fasta b/lapis-e2e/testData/singleSegmented/gene_M.fasta similarity index 100% rename from lapis-e2e/testData/gene_M.fasta rename to lapis-e2e/testData/singleSegmented/gene_M.fasta diff --git a/lapis-e2e/testData/gene_N.fasta b/lapis-e2e/testData/singleSegmented/gene_N.fasta similarity index 100% rename from lapis-e2e/testData/gene_N.fasta rename to lapis-e2e/testData/singleSegmented/gene_N.fasta diff --git a/lapis-e2e/testData/gene_ORF1a.fasta b/lapis-e2e/testData/singleSegmented/gene_ORF1a.fasta similarity index 100% rename from lapis-e2e/testData/gene_ORF1a.fasta rename to lapis-e2e/testData/singleSegmented/gene_ORF1a.fasta diff --git a/lapis-e2e/testData/gene_ORF1b.fasta b/lapis-e2e/testData/singleSegmented/gene_ORF1b.fasta similarity index 100% rename from lapis-e2e/testData/gene_ORF1b.fasta rename to lapis-e2e/testData/singleSegmented/gene_ORF1b.fasta diff --git a/lapis-e2e/testData/gene_ORF3a.fasta b/lapis-e2e/testData/singleSegmented/gene_ORF3a.fasta similarity index 100% rename from lapis-e2e/testData/gene_ORF3a.fasta rename to lapis-e2e/testData/singleSegmented/gene_ORF3a.fasta diff --git a/lapis-e2e/testData/gene_ORF6.fasta b/lapis-e2e/testData/singleSegmented/gene_ORF6.fasta similarity index 100% rename from lapis-e2e/testData/gene_ORF6.fasta rename to lapis-e2e/testData/singleSegmented/gene_ORF6.fasta diff --git a/lapis-e2e/testData/gene_ORF7a.fasta b/lapis-e2e/testData/singleSegmented/gene_ORF7a.fasta similarity index 100% rename from lapis-e2e/testData/gene_ORF7a.fasta rename to lapis-e2e/testData/singleSegmented/gene_ORF7a.fasta diff --git a/lapis-e2e/testData/gene_ORF7b.fasta b/lapis-e2e/testData/singleSegmented/gene_ORF7b.fasta similarity index 100% rename from lapis-e2e/testData/gene_ORF7b.fasta rename to lapis-e2e/testData/singleSegmented/gene_ORF7b.fasta diff --git a/lapis-e2e/testData/gene_ORF8.fasta b/lapis-e2e/testData/singleSegmented/gene_ORF8.fasta similarity index 100% rename from lapis-e2e/testData/gene_ORF8.fasta rename to lapis-e2e/testData/singleSegmented/gene_ORF8.fasta diff --git a/lapis-e2e/testData/gene_ORF9b.fasta b/lapis-e2e/testData/singleSegmented/gene_ORF9b.fasta similarity index 100% rename from lapis-e2e/testData/gene_ORF9b.fasta rename to lapis-e2e/testData/singleSegmented/gene_ORF9b.fasta diff --git a/lapis-e2e/testData/gene_S.fasta b/lapis-e2e/testData/singleSegmented/gene_S.fasta similarity index 100% rename from lapis-e2e/testData/gene_S.fasta rename to lapis-e2e/testData/singleSegmented/gene_S.fasta diff --git a/lapis-e2e/testData/nuc_insertions.tsv b/lapis-e2e/testData/singleSegmented/nuc_insertions.tsv similarity index 100% rename from lapis-e2e/testData/nuc_insertions.tsv rename to lapis-e2e/testData/singleSegmented/nuc_insertions.tsv diff --git a/lapis-e2e/testData/nuc_main.fasta b/lapis-e2e/testData/singleSegmented/nuc_main.fasta similarity index 100% rename from lapis-e2e/testData/nuc_main.fasta rename to lapis-e2e/testData/singleSegmented/nuc_main.fasta diff --git a/lapis-e2e/testData/pangolineage_alias.json b/lapis-e2e/testData/singleSegmented/pangolineage_alias.json similarity index 100% rename from lapis-e2e/testData/pangolineage_alias.json rename to lapis-e2e/testData/singleSegmented/pangolineage_alias.json diff --git a/lapis-e2e/testData/preprocessingConfig.yaml b/lapis-e2e/testData/singleSegmented/preprocessingConfig.yaml similarity index 100% rename from lapis-e2e/testData/preprocessingConfig.yaml rename to lapis-e2e/testData/singleSegmented/preprocessingConfig.yaml diff --git a/lapis-e2e/testData/protectedTestDatabaseConfig.yaml b/lapis-e2e/testData/singleSegmented/protectedTestDatabaseConfig.yaml similarity index 100% rename from lapis-e2e/testData/protectedTestDatabaseConfig.yaml rename to lapis-e2e/testData/singleSegmented/protectedTestDatabaseConfig.yaml diff --git a/lapis-e2e/testData/reference_genomes.json b/lapis-e2e/testData/singleSegmented/reference_genomes.json similarity index 100% rename from lapis-e2e/testData/reference_genomes.json rename to lapis-e2e/testData/singleSegmented/reference_genomes.json diff --git a/lapis-e2e/testData/small_metadata_set.tsv b/lapis-e2e/testData/singleSegmented/small_metadata_set.tsv similarity index 100% rename from lapis-e2e/testData/small_metadata_set.tsv rename to lapis-e2e/testData/singleSegmented/small_metadata_set.tsv diff --git a/lapis-e2e/testData/testDatabaseConfig.yaml b/lapis-e2e/testData/singleSegmented/testDatabaseConfig.yaml similarity index 100% rename from lapis-e2e/testData/testDatabaseConfig.yaml rename to lapis-e2e/testData/singleSegmented/testDatabaseConfig.yaml diff --git a/lapis-e2e/testData/unaligned_main.fasta b/lapis-e2e/testData/singleSegmented/unaligned_main.fasta similarity index 100% rename from lapis-e2e/testData/unaligned_main.fasta rename to lapis-e2e/testData/singleSegmented/unaligned_main.fasta diff --git a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=0/data_0.parquet b/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=0/data_0.parquet deleted file mode 100644 index f67d7c71f..000000000 Binary files a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=0/data_0.parquet and /dev/null differ diff --git a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=1/data_0.parquet b/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=1/data_0.parquet deleted file mode 100644 index 15121e3a9..000000000 Binary files a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=1/data_0.parquet and /dev/null differ diff --git a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=10/data_0.parquet b/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=10/data_0.parquet deleted file mode 100644 index c3def95d4..000000000 Binary files a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=10/data_0.parquet and /dev/null differ diff --git a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=2/data_0.parquet b/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=2/data_0.parquet deleted file mode 100644 index b5823db2d..000000000 Binary files a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=2/data_0.parquet and /dev/null differ diff --git a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=3/data_0.parquet b/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=3/data_0.parquet deleted file mode 100644 index b953fd615..000000000 Binary files a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=3/data_0.parquet and /dev/null differ diff --git a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=4/data_0.parquet b/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=4/data_0.parquet deleted file mode 100644 index 764b458d9..000000000 Binary files a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=4/data_0.parquet and /dev/null differ diff --git a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=5/data_0.parquet b/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=5/data_0.parquet deleted file mode 100644 index f698c18d9..000000000 Binary files a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=5/data_0.parquet and /dev/null differ diff --git a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=6/data_0.parquet b/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=6/data_0.parquet deleted file mode 100644 index 448dffc21..000000000 Binary files a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=6/data_0.parquet and /dev/null differ diff --git a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=7/data_0.parquet b/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=7/data_0.parquet deleted file mode 100644 index 3724188b6..000000000 Binary files a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=7/data_0.parquet and /dev/null differ diff --git a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=8/data_0.parquet b/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=8/data_0.parquet deleted file mode 100644 index cdebbd3eb..000000000 Binary files a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=8/data_0.parquet and /dev/null differ diff --git a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=9/data_0.parquet b/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=9/data_0.parquet deleted file mode 100644 index 8d49e8078..000000000 Binary files a/lapis-e2e/testData/temp/unaligned_nuc_main/partition_id=9/data_0.parquet and /dev/null differ diff --git a/lapis/build.gradle b/lapis/build.gradle index e3dc3be3d..2953ba79a 100644 --- a/lapis/build.gradle +++ b/lapis/build.gradle @@ -80,9 +80,35 @@ openApi { outputDir.set(file("$rootDir")) def opennessLevel = project.hasProperty("opennessLevel") ? project.opennessLevel : "open" + def segmented = project.hasProperty("segmented") ? project.segmented : "false" + + def getCustomOutputFileName = { opennessLevel_, segmented_ -> + if (segmented_ == "true") { + return "lapis-openapi-multi-segmented.json" + } else { + return opennessLevel_ == "open" ? "lapis-openapi-single-segmented.json" : "lapis-openapi-single-segmented-protected.json" + } + } + + def getCustomLapisConfig = { opennessLevel_, segmented_ -> + if (segmented_ == "true") { + return "../lapis-e2e/testData/multiSegmented/testDatabaseConfig.yaml" + } else { + return opennessLevel_ == "open" ? "../lapis-e2e/testData/singleSegmented/testDatabaseConfig.yaml" : "../lapis-e2e/testData/singleSegmented/protectedTestDatabaseConfig.yaml" + } + } + + def getReferenceGenomeFilename = { segmented_ -> + if (segmented_ == "true") { + return "../lapis-e2e/testData/multiSegmented/reference_genomes.json" + } else { + return "../lapis-e2e/testData/singleSegmented/reference_genomes.json" + } + } - def customOutputFileName = opennessLevel == "open" ? "lapis-openapi.json" : "lapis-openapi-protected.json" - def customLapisConfig = opennessLevel == "open" ? "../lapis-e2e/testData/testDatabaseConfig.yaml" : "../lapis-e2e/testData/protectedTestDatabaseConfig.yaml" + def customOutputFileName = getCustomOutputFileName(opennessLevel, segmented) + def customLapisConfig = getCustomLapisConfig(opennessLevel, segmented) + def referenceGenomeFilename = getReferenceGenomeFilename(segmented) outputFileName.set(customOutputFileName) apiDocsUrl.set("http://localhost:8080/api-docs") @@ -91,7 +117,7 @@ openApi { args.set([ "--silo.url=does.not.matter.here", "--lapis.databaseConfig.path=$customLapisConfig", - "--referenceGenomeFilename=../lapis-e2e/testData/reference_genomes.json", + "--referenceGenomeFilename=$referenceGenomeFilename", "--lapis.accessKeys.path=./src/test/resources/config/testAccessKeys.yaml" ]) } diff --git a/lapis/docker-compose.yml b/lapis/docker-compose.yml index 22c921010..37616ba47 100644 --- a/lapis/docker-compose.yml +++ b/lapis/docker-compose.yml @@ -7,11 +7,11 @@ services: command: --silo.url=http://silo:8081 volumes: - type: bind - source: ../lapis-e2e/testData/testDatabaseConfig.yaml + source: ../lapis-e2e/testData/singleSegmented/testDatabaseConfig.yaml target: /workspace/database_config.yaml read_only: true - type: bind - source: ../lapis-e2e/testData/reference_genomes.json + source: ../lapis-e2e/testData/singleSegmented/reference_genomes.json target: /workspace/reference_genomes.json read_only: true @@ -21,7 +21,7 @@ services: - "8091:8081" command: --api volumes: - - ../lapis-e2e/testData/output:/data + - ../lapis-e2e/testData/singleSegmented/output:/data depends_on: siloPreprocessing: condition: service_completed_successfully @@ -30,10 +30,45 @@ services: image: ghcr.io/genspectrum/lapis-silo:${SILO_TAG} command: --preprocessing volumes: - - ../lapis-e2e/testData:/preprocessing/input - - ../lapis-e2e/testData/output:/preprocessing/output - - ../lapis-e2e/testData/preprocessingConfig.yaml:/app/preprocessing_config.yaml - - ../lapis-e2e/testData/testDatabaseConfig.yaml:/app/database_config.yaml + - ../lapis-e2e/testData/singleSegmented:/preprocessing/input + - ../lapis-e2e/testData/singleSegmented/output:/preprocessing/output + - ../lapis-e2e/testData/singleSegmented/preprocessingConfig.yaml:/app/preprocessing_config.yaml + - ../lapis-e2e/testData/singleSegmented/testDatabaseConfig.yaml:/app/database_config.yaml + + siloMultisegmented: + image: ghcr.io/genspectrum/lapis-silo:${SILO_TAG} + ports: + - "8093:8081" + command: --api + volumes: + - ../lapis-e2e/testData/multiSegmented/output:/data + depends_on: + siloPreprocessingMultisegmented: + condition: service_completed_successfully + + siloPreprocessingMultisegmented: + image: ghcr.io/genspectrum/lapis-silo:${SILO_TAG} + command: --preprocessing + volumes: + - ../lapis-e2e/testData/multiSegmented:/preprocessing/input + - ../lapis-e2e/testData/multiSegmented/output:/preprocessing/output + - ../lapis-e2e/testData/multiSegmented/preprocessingConfig.yaml:/app/preprocessing_config.yaml + - ../lapis-e2e/testData/multiSegmented/testDatabaseConfig.yaml:/app/database_config.yaml + + lapisMultiSegmented: + image: ghcr.io/genspectrum/lapis:${LAPIS_TAG} + ports: + - "8094:8080" + command: --silo.url=http://siloMultisegmented:8081 + volumes: + - type: bind + source: ../lapis-e2e/testData/multiSegmented/testDatabaseConfig.yaml + target: /workspace/database_config.yaml + read_only: true + - type: bind + source: ../lapis-e2e/testData/multiSegmented/reference_genomes.json + target: /workspace/reference_genomes.json + read_only: true lapisProtected: image: ghcr.io/genspectrum/lapis:${LAPIS_TAG} @@ -42,11 +77,11 @@ services: command: --silo.url=http://silo:8081 --lapis.accessKeys.path=/workspace/access_keys.yaml volumes: - type: bind - source: ../lapis-e2e/testData/protectedTestDatabaseConfig.yaml + source: ../lapis-e2e/testData/singleSegmented/protectedTestDatabaseConfig.yaml target: /workspace/database_config.yaml read_only: true - type: bind - source: ../lapis-e2e/testData/reference_genomes.json + source: ../lapis-e2e/testData/singleSegmented/reference_genomes.json target: /workspace/reference_genomes.json read_only: true - type: bind diff --git a/lapis/src/main/kotlin/org/genspectrum/lapis/model/SiloFilterExpressionMapper.kt b/lapis/src/main/kotlin/org/genspectrum/lapis/model/SiloFilterExpressionMapper.kt index 5a6a70bbb..6cccb1e5f 100644 --- a/lapis/src/main/kotlin/org/genspectrum/lapis/model/SiloFilterExpressionMapper.kt +++ b/lapis/src/main/kotlin/org/genspectrum/lapis/model/SiloFilterExpressionMapper.kt @@ -409,7 +409,11 @@ class SiloFilterExpressionMapper( } private fun toNucleotideInsertionFilter(nucleotideInsertion: NucleotideInsertion): NucleotideInsertionContains { - return NucleotideInsertionContains(nucleotideInsertion.position, nucleotideInsertion.insertions) + return NucleotideInsertionContains( + nucleotideInsertion.position, + nucleotideInsertion.insertions, + nucleotideInsertion.segment, + ) } private fun toAminoAcidInsertionFilter(aminoAcidInsertion: AminoAcidInsertion): AminoAcidInsertionContains { diff --git a/lapis/src/main/kotlin/org/genspectrum/lapis/model/VariantQueryCustomListener.kt b/lapis/src/main/kotlin/org/genspectrum/lapis/model/VariantQueryCustomListener.kt index 547dd18e5..b644cf0d2 100644 --- a/lapis/src/main/kotlin/org/genspectrum/lapis/model/VariantQueryCustomListener.kt +++ b/lapis/src/main/kotlin/org/genspectrum/lapis/model/VariantQueryCustomListener.kt @@ -131,6 +131,7 @@ class VariantQueryCustomListener(val referenceGenomeSchema: ReferenceGenomeSchem NucleotideInsertionContains( ctx.position().text.toInt(), value.uppercase(), + null, ), ) } diff --git a/lapis/src/main/kotlin/org/genspectrum/lapis/silo/SiloQuery.kt b/lapis/src/main/kotlin/org/genspectrum/lapis/silo/SiloQuery.kt index 76d4edf9a..d6f2a775a 100644 --- a/lapis/src/main/kotlin/org/genspectrum/lapis/silo/SiloQuery.kt +++ b/lapis/src/main/kotlin/org/genspectrum/lapis/silo/SiloQuery.kt @@ -238,7 +238,9 @@ data class HasAminoAcidMutation(val sequenceName: String, val position: Int) : data class DateBetween(val column: String, val from: LocalDate?, val to: LocalDate?) : SiloFilterExpression("DateBetween") -data class NucleotideInsertionContains(val position: Int, val value: String) : SiloFilterExpression("InsertionContains") +@JsonInclude(JsonInclude.Include.NON_NULL) +data class NucleotideInsertionContains(val position: Int, val value: String, val sequenceName: String?) : + SiloFilterExpression("InsertionContains") data class AminoAcidInsertionContains(val position: Int, val value: String, val sequenceName: String) : SiloFilterExpression( diff --git a/lapis/src/test/kotlin/org/genspectrum/lapis/model/SiloFilterExpressionMapperTest.kt b/lapis/src/test/kotlin/org/genspectrum/lapis/model/SiloFilterExpressionMapperTest.kt index a4407719c..91fa50328 100644 --- a/lapis/src/test/kotlin/org/genspectrum/lapis/model/SiloFilterExpressionMapperTest.kt +++ b/lapis/src/test/kotlin/org/genspectrum/lapis/model/SiloFilterExpressionMapperTest.kt @@ -370,7 +370,7 @@ class SiloFilterExpressionMapperTest { val result = underTest.map(filterParameter) val expected = - And(NucleotideInsertionContains(123, "ABCD"), NucleotideInsertionContains(999, "DEF")) + And(NucleotideInsertionContains(123, "ABCD", "segment"), NucleotideInsertionContains(999, "DEF", null)) assertThat(result, equalTo(expected)) } diff --git a/lapis/src/test/kotlin/org/genspectrum/lapis/model/VariantQueryFacadeTest.kt b/lapis/src/test/kotlin/org/genspectrum/lapis/model/VariantQueryFacadeTest.kt index 98b88af1d..c4ae5b08c 100644 --- a/lapis/src/test/kotlin/org/genspectrum/lapis/model/VariantQueryFacadeTest.kt +++ b/lapis/src/test/kotlin/org/genspectrum/lapis/model/VariantQueryFacadeTest.kt @@ -303,7 +303,15 @@ class VariantQueryFacadeTest { val result = underTest.map(variantQuery) - assertThat(result, equalTo(NucleotideInsertionContains(1234, "GAG"))) + assertThat(result, equalTo(NucleotideInsertionContains(1234, "GAG", null))) + } + + @Test + fun `given a variantQuery with a 'Insertion' expression with sequenceName throws`() { + // COVID variant queries do not support segment name in insertion queries + val variantQuery = "ins_sequence:1234:GAG" + + assertThrows { underTest.map(variantQuery) } } @Test @@ -312,7 +320,7 @@ class VariantQueryFacadeTest { val result = underTest.map(variantQuery) - assertThat(result, equalTo(NucleotideInsertionContains(1234, "GAG"))) + assertThat(result, equalTo(NucleotideInsertionContains(1234, "GAG", null))) } @Test @@ -321,7 +329,7 @@ class VariantQueryFacadeTest { val result = underTest.map(variantQuery) - assertThat(result, equalTo(NucleotideInsertionContains(1234, "GAG"))) + assertThat(result, equalTo(NucleotideInsertionContains(1234, "GAG", null))) } @Test @@ -330,7 +338,7 @@ class VariantQueryFacadeTest { val result = underTest.map(variantQuery) - assertThat(result, equalTo(NucleotideInsertionContains(1234, "G.*A.*G"))) + assertThat(result, equalTo(NucleotideInsertionContains(1234, "G.*A.*G", null))) } @Test diff --git a/lapis/src/test/kotlin/org/genspectrum/lapis/silo/SiloQueryTest.kt b/lapis/src/test/kotlin/org/genspectrum/lapis/silo/SiloQueryTest.kt index b0c056157..c1cdf06be 100644 --- a/lapis/src/test/kotlin/org/genspectrum/lapis/silo/SiloQueryTest.kt +++ b/lapis/src/test/kotlin/org/genspectrum/lapis/silo/SiloQueryTest.kt @@ -485,6 +485,38 @@ class SiloQueryTest { } """, ), + Arguments.of( + NucleotideInsertionContains(1234, "A", "segment"), + """ + { + "type": "InsertionContains", + "position": 1234, + "value": "A", + "sequenceName":"segment" + } + """, + ), + Arguments.of( + NucleotideInsertionContains(1234, "A", null), + """ + { + "type": "InsertionContains", + "position": 1234, + "value": "A" + } + """, + ), + Arguments.of( + AminoAcidInsertionContains(1234, "A", "someGene"), + """ + { + "type": "AminoAcidInsertionContains", + "position": 1234, + "value": "A", + "sequenceName":"someGene" + } + """, + ), Arguments.of( DateBetween("fieldName", LocalDate.of(2021, 3, 31), LocalDate.of(2022, 6, 3)), """