From b7fc0a6b4be5fb1499c49e0274e2716f0045b039 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Wed, 1 May 2024 22:49:54 +0100 Subject: [PATCH 01/16] start to move service account password to secrets --- ingest/config/config.yaml | 2 - .../loculus/templates/_config-processor.tpl | 26 ++++++++++++ .../templates/keycloak-config-map.yaml | 42 +++++++++---------- kubernetes/loculus/values.yaml | 7 ++++ 4 files changed, 54 insertions(+), 23 deletions(-) diff --git a/ingest/config/config.yaml b/ingest/config/config.yaml index e1b3418f2..47eff08ee 100644 --- a/ingest/config/config.yaml +++ b/ingest/config/config.yaml @@ -129,8 +129,6 @@ column_mapping: Virus Pangolin Classification: ncbi_virus_pangolin Virus Taxonomic ID: ncbi_virus_tax_id group_name: insdc_ingest_group -username: insdc_ingest_user -password: insdc_ingest_user keycloak_client_id: backend-client backend_url: https://backend-main.loculus.org/ keycloak_token_url: https://authentication-main.loculus.org/realms/loculus/protocol/openid-connect/token diff --git a/kubernetes/loculus/templates/_config-processor.tpl b/kubernetes/loculus/templates/_config-processor.tpl index bd2263978..d56e7aec3 100644 --- a/kubernetes/loculus/templates/_config-processor.tpl +++ b/kubernetes/loculus/templates/_config-processor.tpl @@ -15,6 +15,32 @@ secretKeyRef: name: smtp-password key: secretKey + - name: LOCULUSSUB_insdcIngestUserPassword + valueFrom: + secretKeyRef: + name: serviceAccounts + key: insdcIngestUserPassword + - name: LOCULUSSUB_dummyPreprocessingPipelinePassword + valueFrom: + secretKeyRef: + name: serviceAccounts + key: dummyPreprocessingPipelinePassword + - name: LOCULUSSUB_siloImportJobPassword + valueFrom: + secretKeyRef: + name: serviceAccounts + key: siloImportJobPassword + - name: LOCULUSSUB_backendUserPassword + valueFrom: + secretKeyRef: + name: serviceAccounts + key: backendUserPassword + - name: LOCULUSSUB_backendKeycloakClientSecret + valueFrom: + secretKeyRef: + name: backend-keycloak-client-secret + key: backendKeycloakClientSecret + {{- end }} diff --git a/kubernetes/loculus/templates/keycloak-config-map.yaml b/kubernetes/loculus/templates/keycloak-config-map.yaml index 66424b433..e491338c3 100644 --- a/kubernetes/loculus/templates/keycloak-config-map.yaml +++ b/kubernetes/loculus/templates/keycloak-config-map.yaml @@ -88,20 +88,20 @@ data: } }, { - "username": "insdc_ingest_user", + "username": "superuser", "enabled": true, - "email": "insdc_ingest_user@void.o", + "email": "superuser@void.o", "emailVerified" : true, - "firstName": "INSDC Ingest", - "lastName": "User", + "firstName": "Dummy", + "lastName": "SuperUser", "credentials": [ { "type": "password", - "value": "insdc_ingest_user" + "value": "superuser" } ], "realmRoles": [ - "user", + "super_user", "offline_access" ], "attributes": { @@ -112,22 +112,23 @@ data: "manage-account" ] } - }, + }, + {{ end }} { - "username": "dummy_preprocessing_pipeline", + "username": "insdc_ingest_user", "enabled": true, - "email": "dummy_preprocessing_pipeline@void.o", + "email": "insdc_ingest_user@void.o", "emailVerified" : true, - "firstName": "Dummy", - "lastName": "Preprocessing", + "firstName": "INSDC Ingest", + "lastName": "User", "credentials": [ { "type": "password", - "value": "dummy_preprocessing_pipeline" + "value": "[[insdcIngestUserPassword]]" } ], "realmRoles": [ - "preprocessing_pipeline", + "user", "offline_access" ], "attributes": { @@ -140,20 +141,20 @@ data: } }, { - "username": "superuser", + "username": "dummy_preprocessing_pipeline", "enabled": true, - "email": "superuser@void.o", + "email": "dummy_preprocessing_pipeline@void.o", "emailVerified" : true, "firstName": "Dummy", - "lastName": "SuperUser", + "lastName": "Preprocessing", "credentials": [ { "type": "password", - "value": "superuser" + "value": "[[dummyPreprocessingPipelinePassword]]" } ], "realmRoles": [ - "super_user", + "preprocessing_pipeline", "offline_access" ], "attributes": { @@ -165,7 +166,6 @@ data: ] } }, - {{ end }} { "username": "silo_import_job", "enabled": true, @@ -176,7 +176,7 @@ data: "credentials": [ { "type": "password", - "value": "silo_import_job" + "value": "[[siloImportJobPassword]]" } ], "realmRoles": [ @@ -205,7 +205,7 @@ data: "credentials": [ { "type": "password", - "value": "backend" + "value": "[[backendUserPassword]]" } ], "clientRoles": { diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index f6834eaac..5c4348758 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -911,6 +911,13 @@ secrets: username: "dummy" testPassword: "dummy" livePassword: "dummy" + serviceAccounts: + type: raw + data: + insdcIngestUserPassword: "insdc_ingest_user" + dummyPreprocessingPipelinePassword: "dummy_preprocessing_pipeline" + siloImportJobPassword: "silo_import_job" + backendUserPassword: "backend" enableCrossRefCredentials: true runDevelopmentKeycloakDatabase: true runDevelopmentMainDatabase: true \ No newline at end of file From 562fa6b51f3771ee1dc634f2362a19cbfa744c21 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Wed, 1 May 2024 23:00:26 +0100 Subject: [PATCH 02/16] fix --- kubernetes/loculus/templates/_config-processor.tpl | 8 ++++---- kubernetes/loculus/values.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kubernetes/loculus/templates/_config-processor.tpl b/kubernetes/loculus/templates/_config-processor.tpl index d56e7aec3..c6a8e5e23 100644 --- a/kubernetes/loculus/templates/_config-processor.tpl +++ b/kubernetes/loculus/templates/_config-processor.tpl @@ -18,22 +18,22 @@ - name: LOCULUSSUB_insdcIngestUserPassword valueFrom: secretKeyRef: - name: serviceAccounts + name: service-accounts key: insdcIngestUserPassword - name: LOCULUSSUB_dummyPreprocessingPipelinePassword valueFrom: secretKeyRef: - name: serviceAccounts + name: service-accounts key: dummyPreprocessingPipelinePassword - name: LOCULUSSUB_siloImportJobPassword valueFrom: secretKeyRef: - name: serviceAccounts + name: service-accounts key: siloImportJobPassword - name: LOCULUSSUB_backendUserPassword valueFrom: secretKeyRef: - name: serviceAccounts + name: service-accounts key: backendUserPassword - name: LOCULUSSUB_backendKeycloakClientSecret valueFrom: diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index 5c4348758..30c95e8c0 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -911,7 +911,7 @@ secrets: username: "dummy" testPassword: "dummy" livePassword: "dummy" - serviceAccounts: + service-accounts: type: raw data: insdcIngestUserPassword: "insdc_ingest_user" From d6a899cf694d1116749b0d27c967cc232c5322a1 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Wed, 1 May 2024 23:09:03 +0100 Subject: [PATCH 03/16] handle backend and silo import --- .../loculus/templates/lapis-silo-import-cronjob.yaml | 5 ++++- kubernetes/loculus/templates/loculus-backend.yaml | 7 ++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/kubernetes/loculus/templates/lapis-silo-import-cronjob.yaml b/kubernetes/loculus/templates/lapis-silo-import-cronjob.yaml index 5af90f0a8..d69b55214 100644 --- a/kubernetes/loculus/templates/lapis-silo-import-cronjob.yaml +++ b/kubernetes/loculus/templates/lapis-silo-import-cronjob.yaml @@ -44,7 +44,10 @@ spec: - name: IMPORT_JOB_USER value: "silo_import_job" - name: IMPORT_JOB_USER_PASSWORD - value: "silo_import_job" + valueFrom: + secretKeyRef: + name: service-accounts + key: siloImportJobPassword volumeMounts: - name: lapis-silo-database-config-processed mountPath: /preprocessing/input/reference_genomes.json diff --git a/kubernetes/loculus/templates/loculus-backend.yaml b/kubernetes/loculus/templates/loculus-backend.yaml index d78f5390f..f5acd7bc4 100644 --- a/kubernetes/loculus/templates/loculus-backend.yaml +++ b/kubernetes/loculus/templates/loculus-backend.yaml @@ -43,7 +43,7 @@ spec: - "--crossref.endpoint=$(CROSSREF_TEST_ENDPOINT)" - "--crossref.username=$(CROSSREF_USERNAME)" - "--crossref.password=$(CROSSREF_TEST_PASSWORD)" - - "--keycloak.password=backend" + - "--keycloak.password=$(BACKEND_KEYCLOAK_PASSWORD)" - "--keycloak.realm=loculus" - "--keycloak.client=backend-client" - "--keycloak.url=http://loculus-keycloak-service:8083" @@ -70,6 +70,11 @@ spec: name: crossref key: livePassword {{- end }} + - name: BACKEND_KEYCLOAK_PASSWORD + valueFrom: + secretKeyRef: + name: service-accounts + key: backendUserPassword - name: DB_URL valueFrom: secretKeyRef: From 99c1495914a313aacc832b5b1a2264deb7afb549 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Wed, 1 May 2024 23:14:14 +0100 Subject: [PATCH 04/16] update --- kubernetes/loculus/templates/loculus-backend.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kubernetes/loculus/templates/loculus-backend.yaml b/kubernetes/loculus/templates/loculus-backend.yaml index f5acd7bc4..7c93362c7 100644 --- a/kubernetes/loculus/templates/loculus-backend.yaml +++ b/kubernetes/loculus/templates/loculus-backend.yaml @@ -70,11 +70,11 @@ spec: name: crossref key: livePassword {{- end }} - - name: BACKEND_KEYCLOAK_PASSWORD - valueFrom: - secretKeyRef: - name: service-accounts - key: backendUserPassword + - name: BACKEND_KEYCLOAK_PASSWORD + valueFrom: + secretKeyRef: + name: service-accounts + key: backendUserPassword - name: DB_URL valueFrom: secretKeyRef: From 967dca807039cea9b05aa08b8158b0be61805221 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Wed, 1 May 2024 23:19:11 +0100 Subject: [PATCH 05/16] add preprocessing --- .../templates/loculus-preprocessing-deployment.yaml | 7 +++++++ preprocessing/dummy/main.py | 2 +- .../nextclade/src/loculus_preprocessing/config.py | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/kubernetes/loculus/templates/loculus-preprocessing-deployment.yaml b/kubernetes/loculus/templates/loculus-preprocessing-deployment.yaml index fb2ce0a17..385c24b4c 100644 --- a/kubernetes/loculus/templates/loculus-preprocessing-deployment.yaml +++ b/kubernetes/loculus/templates/loculus-preprocessing-deployment.yaml @@ -41,8 +41,15 @@ spec: - "--backend-host={{ $backendHost }}/{{ $organism }}" - "--keycloak-host={{ $organismcloakHost }}" - "--pipeline-version={{ $processingConfig.version }}" + - "--keycloak-password=$(KEYCLOAK_PASSWORD)" {{- if $processingConfig.configFile }} - "--config=/etc/config/preprocessing-config.yaml" + env: + - name: KEYCLOAK_PASSWORD + valueFrom: + secretKeyRef: + name: service-accounts + key: dummyPreprocessingPipelinePassword volumeMounts: - name: preprocessing-config-volume-{{ $organism }}-v{{ $processingConfig.version }}-{{ $processingIndex }} mountPath: /etc/config diff --git a/preprocessing/dummy/main.py b/preprocessing/dummy/main.py index 9f6956631..39b9d2d86 100644 --- a/preprocessing/dummy/main.py +++ b/preprocessing/dummy/main.py @@ -19,7 +19,7 @@ parser.add_argument("--keycloak-host", type=str, default="http://172.0.0.1:8083", help="Host address of Keycloak") parser.add_argument("--keycloak-user", type=str, default="dummy_preprocessing_pipeline", help="Keycloak user to use for authentication") -parser.add_argument("--keycloak-password", type=str, default="dummy_preprocessing_pipeline", +parser.add_argument("--keycloak-password", type=str, default="", help="Keycloak password to use for authentication") parser.add_argument("--keycloak-token-path", type=str, default="/realms/loculus/protocol/openid-connect/token", help="Path to Keycloak token endpoint") parser.add_argument("--pipeline-version", type=int, default=1) diff --git a/preprocessing/nextclade/src/loculus_preprocessing/config.py b/preprocessing/nextclade/src/loculus_preprocessing/config.py index 6eb9fe102..1afb581ae 100644 --- a/preprocessing/nextclade/src/loculus_preprocessing/config.py +++ b/preprocessing/nextclade/src/loculus_preprocessing/config.py @@ -20,7 +20,7 @@ class Config: backend_host: str = "http://127.0.0.1:8079" keycloak_host: str = "http://172.0.0.1:8083" keycloak_user: str = "dummy_preprocessing_pipeline" - keycloak_password: str = "dummy_preprocessing_pipeline" + keycloak_password: str = "" keycloak_token_path: str = "realms/loculus/protocol/openid-connect/token" nextclade_dataset_name: str = "nextstrain/mpox/all-clades" nextclade_dataset_tag: str | None = None From 0918158b89e5e1ee64782bb0e45a4169115c3a26 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Wed, 1 May 2024 23:26:13 +0100 Subject: [PATCH 06/16] add ingest --- ingest/scripts/submit_to_loculus.py | 5 ++++- kubernetes/loculus/templates/loculus-ingest-deployment.yaml | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/ingest/scripts/submit_to_loculus.py b/ingest/scripts/submit_to_loculus.py index e4fbe98ca..17551254c 100644 --- a/ingest/scripts/submit_to_loculus.py +++ b/ingest/scripts/submit_to_loculus.py @@ -5,6 +5,7 @@ import click import requests import yaml +import os logging.basicConfig(level=logging.DEBUG) @@ -33,9 +34,11 @@ def get_jwt(config: Config) -> str: Get a JWT token for the given username and password """ + keycloak_ingest_password = os.getenv("KEYCLOAK_INGEST_PASSWORD") + data = { "username": config.username, - "password": config.password, + "password": keycloak_ingest_password, "grant_type": "password", "client_id": config.keycloak_client_id, } diff --git a/kubernetes/loculus/templates/loculus-ingest-deployment.yaml b/kubernetes/loculus/templates/loculus-ingest-deployment.yaml index 76c755fce..532bd1e7b 100644 --- a/kubernetes/loculus/templates/loculus-ingest-deployment.yaml +++ b/kubernetes/loculus/templates/loculus-ingest-deployment.yaml @@ -26,6 +26,12 @@ spec: - name: ingest-{{ $key }} image: {{ $value.ingest.image}}:{{ $dockerTag }} imagePullPolicy: Always + env: + KEYCLOAK_INGEST_PASSWORD: + valueFrom: + secretKeyRef: + name: service-accounts + key: insdcIngestUserPassword args: {{- range $arg := $value.ingest.args }} - "{{ $arg }}" From 95c4348978b82da7cb823d944101a3cf28a8c63e Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Wed, 1 May 2024 23:31:52 +0100 Subject: [PATCH 07/16] update --- kubernetes/loculus/templates/loculus-ingest-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/loculus/templates/loculus-ingest-deployment.yaml b/kubernetes/loculus/templates/loculus-ingest-deployment.yaml index 532bd1e7b..d289bdbe0 100644 --- a/kubernetes/loculus/templates/loculus-ingest-deployment.yaml +++ b/kubernetes/loculus/templates/loculus-ingest-deployment.yaml @@ -27,7 +27,7 @@ spec: image: {{ $value.ingest.image}}:{{ $dockerTag }} imagePullPolicy: Always env: - KEYCLOAK_INGEST_PASSWORD: + - name: KEYCLOAK_INGEST_PASSWORD valueFrom: secretKeyRef: name: service-accounts From 5ac4d2e83e4bd61fb8ef37292a4f62e3049ec248 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Wed, 1 May 2024 23:40:53 +0100 Subject: [PATCH 08/16] autogenpw --- kubernetes/loculus/values.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kubernetes/loculus/values.yaml b/kubernetes/loculus/values.yaml index 30c95e8c0..506e7c42a 100644 --- a/kubernetes/loculus/values.yaml +++ b/kubernetes/loculus/values.yaml @@ -912,12 +912,12 @@ secrets: testPassword: "dummy" livePassword: "dummy" service-accounts: - type: raw + type: autogen data: - insdcIngestUserPassword: "insdc_ingest_user" - dummyPreprocessingPipelinePassword: "dummy_preprocessing_pipeline" - siloImportJobPassword: "silo_import_job" - backendUserPassword: "backend" + insdcIngestUserPassword: "" + dummyPreprocessingPipelinePassword: "" + siloImportJobPassword: "" + backendUserPassword: "" enableCrossRefCredentials: true runDevelopmentKeycloakDatabase: true runDevelopmentMainDatabase: true \ No newline at end of file From 6ab79364aa35eb43140cc21a103b334489ef0475 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Thu, 2 May 2024 00:04:01 +0100 Subject: [PATCH 09/16] update --- website/tests/e2e.fixture.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/website/tests/e2e.fixture.ts b/website/tests/e2e.fixture.ts index 858bf8e15..dce65d939 100644 --- a/website/tests/e2e.fixture.ts +++ b/website/tests/e2e.fixture.ts @@ -107,6 +107,9 @@ export async function getToken(username: string, password: string) { } } + + e2eLogger.info(`Getting token for user ${username}, password ${password}`); + // eslint-disable-next-line const { access_token, refresh_token } = await client.grant({ grant_type: 'password', From c3b204d0e812044cefb2644b7f7b1cfb7855a6ec Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Thu, 2 May 2024 00:25:19 +0100 Subject: [PATCH 10/16] special e2e vals --- .github/workflows/e2e-k3d.yml | 4 ++-- deploy.py | 7 ++++++- kubernetes/loculus/values_e2e_test.yaml | 8 ++++++++ website/tests/e2e.fixture.ts | 3 +-- 4 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 kubernetes/loculus/values_e2e_test.yaml diff --git a/.github/workflows/e2e-k3d.yml b/.github/workflows/e2e-k3d.yml index 422fa6214..b46101acc 100644 --- a/.github/workflows/e2e-k3d.yml +++ b/.github/workflows/e2e-k3d.yml @@ -46,7 +46,7 @@ jobs: - name: Template with helm uses: WyriHaximus/github-action-helm3@v4 with: - exec: ./deploy.py --verbose helm --branch ${{ github.ref_name }} --sha ${{ github.sha }} --template > /tmp/helm_template.yaml + exec: ./deploy.py --verbose helm --branch ${{ github.ref_name }} --sha ${{ github.sha }} --for-e2e --template > /tmp/helm_template.yaml - name: Upload default helm template uses: actions/upload-artifact@v4 @@ -55,7 +55,7 @@ jobs: path: /tmp/helm_template.yaml - name: Deploy with helm run: | - ./deploy.py --verbose helm --branch ${{ github.ref_name }} --sha ${{ github.sha }} + ./deploy.py --verbose helm --branch ${{ github.ref_name }} --sha ${{ github.sha }} --for-e2e - uses: actions/setup-node@v4 with: diff --git a/deploy.py b/deploy.py index 066f93f39..87751b112 100755 --- a/deploy.py +++ b/deploy.py @@ -50,6 +50,7 @@ default=HELM_VALUES_FILE) helm_parser.add_argument('--template', help='Just template and print out the YAML produced', action='store_true') +helm_parser.add_argument('--for-e2e', action='store_true', help='Use the E2E values file') upgrade_parser = subparsers.add_parser('upgrade', help='Upgrade helm installation') @@ -141,6 +142,8 @@ def handle_helm(): branch = args.branch else: branch = 'latest' + + parameters = [ 'helm', 'template' if args.template else 'install', HELM_RELEASE_NAME, HELM_CHART_DIR, @@ -148,7 +151,9 @@ def handle_helm(): '--set', "environment=local", '--set', f"branch={branch}", ] - + + if args.for_e2e: + parameters += ['-f', 'kubernetes/loculus/values_e2e_test.yaml'] if args.sha: parameters += ['--set', f"sha={args.sha[:7]}"] diff --git a/kubernetes/loculus/values_e2e_test.yaml b/kubernetes/loculus/values_e2e_test.yaml new file mode 100644 index 000000000..146a92464 --- /dev/null +++ b/kubernetes/loculus/values_e2e_test.yaml @@ -0,0 +1,8 @@ +secrets: + service-accounts: + type: raw + data: + insdcIngestUserPassword: "insdc_ingest" + dummyPreprocessingPipelinePassword: "dummy_preprocessing_pipeline" + siloImportJobPassword: "silo_import_job" + backendUserPassword: "backend" \ No newline at end of file diff --git a/website/tests/e2e.fixture.ts b/website/tests/e2e.fixture.ts index dce65d939..413955162 100644 --- a/website/tests/e2e.fixture.ts +++ b/website/tests/e2e.fixture.ts @@ -107,9 +107,8 @@ export async function getToken(username: string, password: string) { } } - e2eLogger.info(`Getting token for user ${username}, password ${password}`); - + // eslint-disable-next-line const { access_token, refresh_token } = await client.grant({ grant_type: 'password', From 656a23aa33a914aac61f01ad0b271b4db1f3e3b4 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Thu, 2 May 2024 11:40:42 +0100 Subject: [PATCH 11/16] update --- .../loculus/templates/loculus-database-standin.yaml | 1 + .../templates/loculus-preprocessing-deployment.yaml | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/kubernetes/loculus/templates/loculus-database-standin.yaml b/kubernetes/loculus/templates/loculus-database-standin.yaml index 9ae6b4bf2..1e83a3a3c 100644 --- a/kubernetes/loculus/templates/loculus-database-standin.yaml +++ b/kubernetes/loculus/templates/loculus-database-standin.yaml @@ -1,3 +1,4 @@ +{{- $dockerTag := include "loculus.dockerTag" .Values }} {{- if not .Values.externalDatabaseUrl }} apiVersion: apps/v1 kind: Deployment diff --git a/kubernetes/loculus/templates/loculus-preprocessing-deployment.yaml b/kubernetes/loculus/templates/loculus-preprocessing-deployment.yaml index 385c24b4c..ef60a608c 100644 --- a/kubernetes/loculus/templates/loculus-preprocessing-deployment.yaml +++ b/kubernetes/loculus/templates/loculus-preprocessing-deployment.yaml @@ -34,6 +34,12 @@ spec: - name: preprocessing-{{ $organism }} image: {{ $processingConfig.image}}:{{ $dockerTag }} imagePullPolicy: Always + env: + - name: KEYCLOAK_PASSWORD + valueFrom: + secretKeyRef: + name: service-accounts + key: dummyPreprocessingPipelinePassword args: {{- range $arg := $processingConfig.args }} - "{{ $arg }}" @@ -44,12 +50,6 @@ spec: - "--keycloak-password=$(KEYCLOAK_PASSWORD)" {{- if $processingConfig.configFile }} - "--config=/etc/config/preprocessing-config.yaml" - env: - - name: KEYCLOAK_PASSWORD - valueFrom: - secretKeyRef: - name: service-accounts - key: dummyPreprocessingPipelinePassword volumeMounts: - name: preprocessing-config-volume-{{ $organism }}-v{{ $processingConfig.version }}-{{ $processingIndex }} mountPath: /etc/config From 9af31c1145eac77e2afc841dd3ce8b3699491198 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Thu, 2 May 2024 11:46:39 +0100 Subject: [PATCH 12/16] Update main.py --- preprocessing/dummy/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/preprocessing/dummy/main.py b/preprocessing/dummy/main.py index 39b9d2d86..fe91c2c2f 100644 --- a/preprocessing/dummy/main.py +++ b/preprocessing/dummy/main.py @@ -19,7 +19,7 @@ parser.add_argument("--keycloak-host", type=str, default="http://172.0.0.1:8083", help="Host address of Keycloak") parser.add_argument("--keycloak-user", type=str, default="dummy_preprocessing_pipeline", help="Keycloak user to use for authentication") -parser.add_argument("--keycloak-password", type=str, default="", +parser.add_argument("--keycloak-password", type=str, help="Keycloak password to use for authentication") parser.add_argument("--keycloak-token-path", type=str, default="/realms/loculus/protocol/openid-connect/token", help="Path to Keycloak token endpoint") parser.add_argument("--pipeline-version", type=int, default=1) From 2602063a45a884fdfbc807c4aeba43f678036a79 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Thu, 2 May 2024 11:47:09 +0100 Subject: [PATCH 13/16] Update e2e.fixture.ts --- website/tests/e2e.fixture.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/website/tests/e2e.fixture.ts b/website/tests/e2e.fixture.ts index 413955162..858bf8e15 100644 --- a/website/tests/e2e.fixture.ts +++ b/website/tests/e2e.fixture.ts @@ -107,8 +107,6 @@ export async function getToken(username: string, password: string) { } } - e2eLogger.info(`Getting token for user ${username}, password ${password}`); - // eslint-disable-next-line const { access_token, refresh_token } = await client.grant({ grant_type: 'password', From f782b39ad71ec2471adc4e3493effaf72fb4bbe4 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Thu, 2 May 2024 14:04:27 +0100 Subject: [PATCH 14/16] Update submit_to_loculus.py --- ingest/scripts/submit_to_loculus.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ingest/scripts/submit_to_loculus.py b/ingest/scripts/submit_to_loculus.py index 17551254c..69c6f79db 100644 --- a/ingest/scripts/submit_to_loculus.py +++ b/ingest/scripts/submit_to_loculus.py @@ -35,6 +35,8 @@ def get_jwt(config: Config) -> str: """ keycloak_ingest_password = os.getenv("KEYCLOAK_INGEST_PASSWORD") + if not keycloak_ingest_password: + keycloak_ingest_password = config.password data = { "username": config.username, From 7768cb168b62d68be498a4e89100d6c39cabeec9 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Thu, 2 May 2024 14:12:40 +0100 Subject: [PATCH 15/16] rename dummy_preprocessing_pipeline to preprocessing pipeline --- kubernetes/loculus/templates/keycloak-config-map.yaml | 4 ++-- kubernetes/loculus/values_e2e_test.yaml | 2 +- preprocessing/dummy/main.py | 2 +- preprocessing/nextclade/src/loculus_preprocessing/config.py | 2 +- website/tests/util/preprocessingPipeline.ts | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/kubernetes/loculus/templates/keycloak-config-map.yaml b/kubernetes/loculus/templates/keycloak-config-map.yaml index e491338c3..3a430ff02 100644 --- a/kubernetes/loculus/templates/keycloak-config-map.yaml +++ b/kubernetes/loculus/templates/keycloak-config-map.yaml @@ -141,9 +141,9 @@ data: } }, { - "username": "dummy_preprocessing_pipeline", + "username": "preprocessing_pipeline", "enabled": true, - "email": "dummy_preprocessing_pipeline@void.o", + "email": "preprocessing_pipeline@void.o", "emailVerified" : true, "firstName": "Dummy", "lastName": "Preprocessing", diff --git a/kubernetes/loculus/values_e2e_test.yaml b/kubernetes/loculus/values_e2e_test.yaml index 146a92464..bd47ebdab 100644 --- a/kubernetes/loculus/values_e2e_test.yaml +++ b/kubernetes/loculus/values_e2e_test.yaml @@ -3,6 +3,6 @@ secrets: type: raw data: insdcIngestUserPassword: "insdc_ingest" - dummyPreprocessingPipelinePassword: "dummy_preprocessing_pipeline" + dummyPreprocessingPipelinePassword: "preprocessing_pipeline" siloImportJobPassword: "silo_import_job" backendUserPassword: "backend" \ No newline at end of file diff --git a/preprocessing/dummy/main.py b/preprocessing/dummy/main.py index fe91c2c2f..b15a4ed72 100644 --- a/preprocessing/dummy/main.py +++ b/preprocessing/dummy/main.py @@ -17,7 +17,7 @@ parser.add_argument("--randomWarnError", action="store_true", help="Make errors and warnings occur stochastically") parser.add_argument("--maxSequences", type=int, help="Max number of sequence entry versions to process.") parser.add_argument("--keycloak-host", type=str, default="http://172.0.0.1:8083", help="Host address of Keycloak") -parser.add_argument("--keycloak-user", type=str, default="dummy_preprocessing_pipeline", +parser.add_argument("--keycloak-user", type=str, default="preprocessing_pipeline", help="Keycloak user to use for authentication") parser.add_argument("--keycloak-password", type=str, help="Keycloak password to use for authentication") diff --git a/preprocessing/nextclade/src/loculus_preprocessing/config.py b/preprocessing/nextclade/src/loculus_preprocessing/config.py index 1afb581ae..33a319dd7 100644 --- a/preprocessing/nextclade/src/loculus_preprocessing/config.py +++ b/preprocessing/nextclade/src/loculus_preprocessing/config.py @@ -19,7 +19,7 @@ class Config: backend_host: str = "http://127.0.0.1:8079" keycloak_host: str = "http://172.0.0.1:8083" - keycloak_user: str = "dummy_preprocessing_pipeline" + keycloak_user: str = "preprocessing_pipeline" keycloak_password: str = "" keycloak_token_path: str = "realms/loculus/protocol/openid-connect/token" nextclade_dataset_name: str = "nextstrain/mpox/all-clades" diff --git a/website/tests/util/preprocessingPipeline.ts b/website/tests/util/preprocessingPipeline.ts index 6eaab8fff..ee9e92f13 100644 --- a/website/tests/util/preprocessingPipeline.ts +++ b/website/tests/util/preprocessingPipeline.ts @@ -56,8 +56,8 @@ async function submit(preprocessingOptions: PreprocessingOptions[]) { } async function getJwtTokenForPreprocessingPipeline( - username: string = 'dummy_preprocessing_pipeline', - password: string = 'dummy_preprocessing_pipeline', + username: string = 'preprocessing_pipeline', + password: string = 'preprocessing_pipeline', ): Promise { const token = await getToken(username, password); From b41706720cb278fda1c6e24df83ef380658b7d60 Mon Sep 17 00:00:00 2001 From: Theo Sanderson Date: Thu, 2 May 2024 15:25:40 +0100 Subject: [PATCH 16/16] use hardcoded passwords for dev deployments too --- deploy.py | 4 ++-- .../loculus/{values_e2e_test.yaml => values_e2e_and_dev.yaml} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename kubernetes/loculus/{values_e2e_test.yaml => values_e2e_and_dev.yaml} (100%) diff --git a/deploy.py b/deploy.py index 87751b112..2920a474b 100755 --- a/deploy.py +++ b/deploy.py @@ -152,8 +152,8 @@ def handle_helm(): '--set', f"branch={branch}", ] - if args.for_e2e: - parameters += ['-f', 'kubernetes/loculus/values_e2e_test.yaml'] + if args.for_e2e or args.dev: + parameters += ['-f', 'kubernetes/loculus/values_e2e_and_dev.yaml'] if args.sha: parameters += ['--set', f"sha={args.sha[:7]}"] diff --git a/kubernetes/loculus/values_e2e_test.yaml b/kubernetes/loculus/values_e2e_and_dev.yaml similarity index 100% rename from kubernetes/loculus/values_e2e_test.yaml rename to kubernetes/loculus/values_e2e_and_dev.yaml