From fc85f093be06fe2ec23a53d34d97bc6f496df5ca Mon Sep 17 00:00:00 2001 From: acies312 Date: Wed, 6 Mar 2024 16:20:42 +0300 Subject: [PATCH 1/9] fixes for kubernetes deployment --- .../application-kubernetes.properties | 1 + cosv-backend/build.gradle.kts | 2 + .../application-kubernetes.properties | 18 +++++---- .../src/main/resources/application.properties | 1 + .../save-cloud/templates/backend-cosv.yaml | 8 +++- .../save-cloud/templates/backend.yaml | 26 +++++++------ .../src/main/resources/img/codehub.svg | 37 ------------------- .../views/index/IndexLogoButtons.kt | 2 +- 8 files changed, 37 insertions(+), 58 deletions(-) delete mode 100644 save-cosv-frontend/src/main/resources/img/codehub.svg diff --git a/api-gateway/src/main/resources/application-kubernetes.properties b/api-gateway/src/main/resources/application-kubernetes.properties index aa0aac44db..29736dd0ac 100644 --- a/api-gateway/src/main/resources/application-kubernetes.properties +++ b/api-gateway/src/main/resources/application-kubernetes.properties @@ -1,6 +1,7 @@ gateway.backend.url=http://backend gateway.frontend.url=http://frontend gateway.demo-cpg.url=http://demo-cpg +gateway.backend-cosv.url=http://backend-cosv gateway.frontend-cosv.url=http://frontend-cosv gateway.demo.url=http://demo server.shutdown=graceful diff --git a/cosv-backend/build.gradle.kts b/cosv-backend/build.gradle.kts index c1ef2e8ea0..80aca29362 100644 --- a/cosv-backend/build.gradle.kts +++ b/cosv-backend/build.gradle.kts @@ -14,4 +14,6 @@ dependencies { implementation(libs.spring.security.core) implementation(libs.spring.data.jpa) implementation(libs.hibernate.jpa21.api) +// needed for kubernetes configs and secrets to be added as additional configurations + implementation(libs.spring.cloud.starter.kubernetes.client.config) } diff --git a/cosv-backend/src/main/resources/application-kubernetes.properties b/cosv-backend/src/main/resources/application-kubernetes.properties index cc12d5135c..88ac0c0d74 100644 --- a/cosv-backend/src/main/resources/application-kubernetes.properties +++ b/cosv-backend/src/main/resources/application-kubernetes.properties @@ -1,13 +1,17 @@ -backend.preprocessor-url=http://preprocessor -backend.orchestrator-url=http://orchestrator -backend.demo-url=http://demo -backend.loki.url=http://loki:3100 -backend.loki.labels.agent-container-name=pod -backend.loki.labels.application-name=app -backend.agent-settings.backend-cosv-url=http://backend-cosv +cosv.preprocessor-url=http://preprocessor +cosv.orchestrator-url=http://orchestrator +cosv.demo-url=http://demo +cosv.loki.url=http://loki:3100 +cosv.loki.labels.agent-container-name=pod +cosv.loki.labels.application-name=app +cosv.agent-settings.backend-cosv-url=http://backend-cosv +cosv.gatewayUrl=http://gateway server.shutdown=graceful management.endpoints.web.exposure.include=* management.server.port=5701 spring.datasource.url=${spring.datasource.backend-cosv-url} +spring.second-datasource.jdbc-url = ${spring.datasource.backend-url} +spring.second-datasource.username = ${spring.datasource.username} +spring.second-datasource.password = ${spring.datasource.password} logging.level.org.springframework=DEBUG logging.level.com.saveourtool=DEBUG \ No newline at end of file diff --git a/cosv-backend/src/main/resources/application.properties b/cosv-backend/src/main/resources/application.properties index 923ae3aed8..7b208c6e24 100644 --- a/cosv-backend/src/main/resources/application.properties +++ b/cosv-backend/src/main/resources/application.properties @@ -16,3 +16,4 @@ cosv.s3-storage.bucketName=${s3-storage.bucketName} cosv.s3-storage.prefix=cnb/files cosv.s3-storage.credentials.accessKeyId=${s3-storage.credentials.accessKeyId} cosv.s3-storage.credentials.secretAccessKey=${s3-storage.credentials.secretAccessKey} +cosv.working-dir=/home/cnb/cosv-working-dir diff --git a/save-cloud-charts/save-cloud/templates/backend-cosv.yaml b/save-cloud-charts/save-cloud/templates/backend-cosv.yaml index 0f1154e422..fd30a76d51 100644 --- a/save-cloud-charts/save-cloud/templates/backend-cosv.yaml +++ b/save-cloud-charts/save-cloud/templates/backend-cosv.yaml @@ -77,7 +77,7 @@ spec: runAsGroup: 1001 args: - --url=$(DB_URL)?createDatabaseIfNotExist=true - - --changeLogFile=db/db.changelog-master.xml + - --changeLogFile=cosv-backend/db/db.changelog-cosv.xml - --username=$(DB_USERNAME) - --password=$(DB_PASSWORD) - --log-level=info @@ -96,7 +96,7 @@ spec: valueFrom: secretKeyRef: name: db-secrets - key: spring.datasource.backend-url + key: spring.datasource.backend-cosv-url - name: DB_USERNAME valueFrom: secretKeyRef: @@ -117,10 +117,12 @@ spec: name: migrations-data - mountPath: {{ .Values.mysql.dbPasswordFile }} name: database-secret + {{ if .Values.gateway.initCerts.enabled }} - name: ca-pemstore mountPath: /etc/ssl/certs/ca-certificates.crt subPath: ca-certificates.crt readOnly: false + {{ end }} volumes: - {{ include "spring-boot.config-volume" (dict "service" .Values.backend_cosv) | indent 10 | trim }} - name: database-secret @@ -131,9 +133,11 @@ spec: secretName: s3-secrets - name: migrations-data emptyDir: {} + {{ if .Values.gateway.initCerts.enabled }} - name: ca-pemstore configMap: name: ca-pemstore + {{ if .Values.gateway.initCerts.enabled }} --- apiVersion: v1 kind: ConfigMap diff --git a/save-cloud-charts/save-cloud/templates/backend.yaml b/save-cloud-charts/save-cloud/templates/backend.yaml index d159fa88e7..1c098d7f0c 100644 --- a/save-cloud-charts/save-cloud/templates/backend.yaml +++ b/save-cloud-charts/save-cloud/templates/backend.yaml @@ -117,10 +117,12 @@ spec: name: migrations-data - mountPath: {{ .Values.mysql.dbPasswordFile }} name: database-secret + {{ if .Values.gateway.initCerts.enabled }} - name: ca-pemstore mountPath: /etc/ssl/certs/ca-certificates.crt subPath: ca-certificates.crt readOnly: false + {{ end }} volumes: - {{ include "spring-boot.config-volume" (dict "service" .Values.backend) | indent 10 | trim }} - name: database-secret @@ -131,9 +133,21 @@ spec: secretName: s3-secrets - name: migrations-data emptyDir: {} + {{ if .Values.gateway.initCerts.enabled }} - name: ca-pemstore configMap: name: ca-pemstore + {{ end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.backend.name }}-config +data: + application.properties: | + {{ if .Values.backend.applicationProperties }} + {{- .Values.backend.applicationProperties | nindent 4 }} + {{ end }} --- apiVersion: v1 kind: Service @@ -146,14 +160,4 @@ spec: ports: {{- include "service.common.ports" (dict "service" .Values.backend) | nindent 4 }} selector: - {{- include "service.common.selectors" (dict "service" .Values.backend) | nindent 4 }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Values.backend.name }}-config -data: - application.properties: | - {{ if .Values.backend.applicationProperties }} - {{- .Values.backend.applicationProperties | nindent 4 }} - {{ end }} + {{- include "service.common.selectors" (dict "service" .Values.backend) | nindent 4 }} \ No newline at end of file diff --git a/save-cosv-frontend/src/main/resources/img/codehub.svg b/save-cosv-frontend/src/main/resources/img/codehub.svg deleted file mode 100644 index 6153995a75..0000000000 --- a/save-cosv-frontend/src/main/resources/img/codehub.svg +++ /dev/null @@ -1,37 +0,0 @@ - - logo - - - - - - - - - - - - - - - - - - - - - - - - Layer 1 - - - \ No newline at end of file diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/index/IndexLogoButtons.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/index/IndexLogoButtons.kt index 74f11e13f6..4082af06d3 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/index/IndexLogoButtons.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/index/IndexLogoButtons.kt @@ -41,7 +41,7 @@ val logoButtons: FC = FC { logo( "VULN", - "https://cosv.gitlink.org.cn/${FrontendCosvRoutes.VULN}", + "https://cosv.gitlink.org.cn", "Archive of 1-Day Vulnerabilities Aggregated from Various Sources".t(), "/img/vuln-logo-bg.png", From 153b28af30c7d328ca19e6d28a1ae2767dc76c62 Mon Sep 17 00:00:00 2001 From: acies312 Date: Wed, 6 Mar 2024 16:24:39 +0300 Subject: [PATCH 2/9] fixes for kubernetes deployment --- save-cloud-charts/save-cloud/templates/backend-cosv.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/save-cloud-charts/save-cloud/templates/backend-cosv.yaml b/save-cloud-charts/save-cloud/templates/backend-cosv.yaml index fd30a76d51..36c05f47f3 100644 --- a/save-cloud-charts/save-cloud/templates/backend-cosv.yaml +++ b/save-cloud-charts/save-cloud/templates/backend-cosv.yaml @@ -137,7 +137,7 @@ spec: - name: ca-pemstore configMap: name: ca-pemstore - {{ if .Values.gateway.initCerts.enabled }} + {{ end }} --- apiVersion: v1 kind: ConfigMap @@ -160,4 +160,4 @@ spec: ports: {{- include "service.common.ports" (dict "service" .Values.backend_cosv) | nindent 4 }} selector: - {{- include "service.common.selectors" (dict "service" .Values.backend_cosv) | nindent 4 }} + {{- include "service.common.selectors" (dict "service" .Values.backend_cosv) | nindent 4 }} \ No newline at end of file From 19c3005810f77849df6616fda4606e9689158b87 Mon Sep 17 00:00:00 2001 From: acies312 <47986068+acies312@users.noreply.github.com> Date: Wed, 6 Mar 2024 16:38:24 +0300 Subject: [PATCH 3/9] fix comment --- cosv-backend/build.gradle.kts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cosv-backend/build.gradle.kts b/cosv-backend/build.gradle.kts index 80aca29362..5f77fab6dd 100644 --- a/cosv-backend/build.gradle.kts +++ b/cosv-backend/build.gradle.kts @@ -14,6 +14,8 @@ dependencies { implementation(libs.spring.security.core) implementation(libs.spring.data.jpa) implementation(libs.hibernate.jpa21.api) -// needed for kubernetes configs and secrets to be added as additional configurations + + //needed for kubernetes configs and secrets to be added as additional configurations + implementation(libs.spring.cloud.starter.kubernetes.client.config) } From 7948635a2e79763dd56904876000c3e0fe6a1b7e Mon Sep 17 00:00:00 2001 From: acies312 <47986068+acies312@users.noreply.github.com> Date: Wed, 6 Mar 2024 16:44:09 +0300 Subject: [PATCH 4/9] fix comment --- cosv-backend/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cosv-backend/build.gradle.kts b/cosv-backend/build.gradle.kts index 5f77fab6dd..d5afe22582 100644 --- a/cosv-backend/build.gradle.kts +++ b/cosv-backend/build.gradle.kts @@ -15,7 +15,7 @@ dependencies { implementation(libs.spring.data.jpa) implementation(libs.hibernate.jpa21.api) - //needed for kubernetes configs and secrets to be added as additional configurations + // needed for kubernetes configs and secrets to be added as additional configurations implementation(libs.spring.cloud.starter.kubernetes.client.config) } From f7a7bdb21c57d54e548332a79557b34ce3813772 Mon Sep 17 00:00:00 2001 From: acies312 <47986068+acies312@users.noreply.github.com> Date: Wed, 6 Mar 2024 16:55:18 +0300 Subject: [PATCH 5/9] Update cosv-backend/build.gradle.kts Co-authored-by: Nariman Abdullin --- cosv-backend/build.gradle.kts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cosv-backend/build.gradle.kts b/cosv-backend/build.gradle.kts index d5afe22582..5a45785979 100644 --- a/cosv-backend/build.gradle.kts +++ b/cosv-backend/build.gradle.kts @@ -17,5 +17,7 @@ dependencies { // needed for kubernetes configs and secrets to be added as additional configurations - implementation(libs.spring.cloud.starter.kubernetes.client.config) + implementation(libs.spring.cloud.starter.kubernetes.client.config) { + because("needed for kubernetes configs and secrets to be added as additional configurations") + } } From 4de71af7d132cfe7920b9f10f54c73c3087c3505 Mon Sep 17 00:00:00 2001 From: acies312 Date: Wed, 6 Mar 2024 17:06:10 +0300 Subject: [PATCH 6/9] fixes for kubernetes deployment --- save-cloud-charts/save-cloud/values.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/save-cloud-charts/save-cloud/values.yaml b/save-cloud-charts/save-cloud/values.yaml index 85145181dd..e29764789a 100644 --- a/save-cloud-charts/save-cloud/values.yaml +++ b/save-cloud-charts/save-cloud/values.yaml @@ -2,6 +2,9 @@ imageRegistry: ghcr.io/saveourtool # dockerTag: # this should be set explicitly during chart installation env: staging +# Additional certs installation for internal CA case +initCerts: + enabled: false # Unless overridden, this will be set as Spring profile of a component. profile: dev pullPolicy: IfNotPresent @@ -57,9 +60,6 @@ preprocessor: # Additional properties for `application.properties` file that will be mounted as an optional property source. applicationProperties: gateway: - # Additional certs installation for internal CA case - initCerts: - enabled: false name: gateway profile: dev,kubernetes imageName: api-gateway From cb4150ab1003d65863ec38e17c36a16a80e152b6 Mon Sep 17 00:00:00 2001 From: acies312 Date: Wed, 6 Mar 2024 17:11:56 +0300 Subject: [PATCH 7/9] fixes for kubernetes deployment --- api-gateway/build.gradle.kts | 4 +++- .../src/main/resources/application-kubernetes.properties | 6 +++--- save-backend/build.gradle.kts | 4 +++- save-cloud-charts/save-cloud/templates/backend-cosv.yaml | 4 ++-- save-cloud-charts/save-cloud/templates/backend.yaml | 4 ++-- save-cloud-charts/save-cloud/templates/gateway.yaml | 2 +- save-demo/build.gradle.kts | 4 +++- save-orchestrator/build.gradle.kts | 4 +++- 8 files changed, 20 insertions(+), 12 deletions(-) diff --git a/api-gateway/build.gradle.kts b/api-gateway/build.gradle.kts index f27f70ccf9..1f45f56987 100644 --- a/api-gateway/build.gradle.kts +++ b/api-gateway/build.gradle.kts @@ -9,7 +9,9 @@ dependencies { implementation(libs.spring.cloud.starter.gateway) implementation(libs.spring.boot.starter.security) implementation(libs.spring.boot.starter.oauth2.client) - implementation(libs.spring.cloud.starter.kubernetes.client.config) + implementation(libs.spring.cloud.starter.kubernetes.client.config) { + because("needed for kubernetes configs and secrets to be added as additional configurations") + } implementation(libs.spring.security.core) implementation(projects.authenticationService) diff --git a/cosv-backend/src/main/resources/application-kubernetes.properties b/cosv-backend/src/main/resources/application-kubernetes.properties index 88ac0c0d74..84f75140a0 100644 --- a/cosv-backend/src/main/resources/application-kubernetes.properties +++ b/cosv-backend/src/main/resources/application-kubernetes.properties @@ -10,8 +10,8 @@ server.shutdown=graceful management.endpoints.web.exposure.include=* management.server.port=5701 spring.datasource.url=${spring.datasource.backend-cosv-url} -spring.second-datasource.jdbc-url = ${spring.datasource.backend-url} -spring.second-datasource.username = ${spring.datasource.username} -spring.second-datasource.password = ${spring.datasource.password} +spring.second-datasource.jdbc-url=${spring.datasource.backend-url} +spring.second-datasource.username=${spring.datasource.username} +spring.second-datasource.password=${spring.datasource.password} logging.level.org.springframework=DEBUG logging.level.com.saveourtool=DEBUG \ No newline at end of file diff --git a/save-backend/build.gradle.kts b/save-backend/build.gradle.kts index a907a6e49f..e103a99b84 100644 --- a/save-backend/build.gradle.kts +++ b/save-backend/build.gradle.kts @@ -53,7 +53,9 @@ dependencies { implementation(libs.spring.boot.starter.security) implementation(libs.spring.security.core) implementation(libs.hibernate.micrometer) - implementation(libs.spring.cloud.starter.kubernetes.client.config) + implementation(libs.spring.cloud.starter.kubernetes.client.config) { + because("needed for kubernetes configs and secrets to be added as additional configurations") + } implementation(libs.reactor.extra) implementation(libs.arrow.kt.core) implementation(project.dependencies.platform(libs.aws.sdk.bom)) diff --git a/save-cloud-charts/save-cloud/templates/backend-cosv.yaml b/save-cloud-charts/save-cloud/templates/backend-cosv.yaml index 36c05f47f3..06f43c1194 100644 --- a/save-cloud-charts/save-cloud/templates/backend-cosv.yaml +++ b/save-cloud-charts/save-cloud/templates/backend-cosv.yaml @@ -117,7 +117,7 @@ spec: name: migrations-data - mountPath: {{ .Values.mysql.dbPasswordFile }} name: database-secret - {{ if .Values.gateway.initCerts.enabled }} + {{ if .Values.initCerts.enabled }} - name: ca-pemstore mountPath: /etc/ssl/certs/ca-certificates.crt subPath: ca-certificates.crt @@ -133,7 +133,7 @@ spec: secretName: s3-secrets - name: migrations-data emptyDir: {} - {{ if .Values.gateway.initCerts.enabled }} + {{ if .Values.initCerts.enabled }} - name: ca-pemstore configMap: name: ca-pemstore diff --git a/save-cloud-charts/save-cloud/templates/backend.yaml b/save-cloud-charts/save-cloud/templates/backend.yaml index 1c098d7f0c..adcee66f70 100644 --- a/save-cloud-charts/save-cloud/templates/backend.yaml +++ b/save-cloud-charts/save-cloud/templates/backend.yaml @@ -117,7 +117,7 @@ spec: name: migrations-data - mountPath: {{ .Values.mysql.dbPasswordFile }} name: database-secret - {{ if .Values.gateway.initCerts.enabled }} + {{ if .Values.initCerts.enabled }} - name: ca-pemstore mountPath: /etc/ssl/certs/ca-certificates.crt subPath: ca-certificates.crt @@ -133,7 +133,7 @@ spec: secretName: s3-secrets - name: migrations-data emptyDir: {} - {{ if .Values.gateway.initCerts.enabled }} + {{ if .Values.initCerts.enabled }} - name: ca-pemstore configMap: name: ca-pemstore diff --git a/save-cloud-charts/save-cloud/templates/gateway.yaml b/save-cloud-charts/save-cloud/templates/gateway.yaml index 95f3e1fa49..da9cdffd67 100644 --- a/save-cloud-charts/save-cloud/templates/gateway.yaml +++ b/save-cloud-charts/save-cloud/templates/gateway.yaml @@ -39,7 +39,7 @@ spec: name: cacerts - mountPath: /home/cnb/secrets/oauth name: oauth-credentials - {{ if .Values.gateway.initCerts.enabled }} + {{ if .Values.initCerts.enabled }} initContainers: - name: init-cacerts image: ghcr.io/saveourtool/api-gateway:0.4.0-alpha.0.408-9b56d61 diff --git a/save-demo/build.gradle.kts b/save-demo/build.gradle.kts index 7d04e38b0c..73f5b1fb4c 100644 --- a/save-demo/build.gradle.kts +++ b/save-demo/build.gradle.kts @@ -30,7 +30,9 @@ dependencies { api(projects.saveCloudCommon) implementation(libs.save.common.jvm) - implementation(libs.spring.cloud.starter.kubernetes.client.config) + implementation(libs.spring.cloud.starter.kubernetes.client.config) { + because("needed for kubernetes configs and secrets to be added as additional configurations") + } api(libs.ktor.client.auth) implementation(libs.ktor.client.core) diff --git a/save-orchestrator/build.gradle.kts b/save-orchestrator/build.gradle.kts index 4e0f623a86..67660860c9 100644 --- a/save-orchestrator/build.gradle.kts +++ b/save-orchestrator/build.gradle.kts @@ -22,7 +22,9 @@ dependencies { implementation(libs.commons.compress) implementation(libs.kotlinx.datetime) implementation(libs.zip4j) - implementation(libs.spring.cloud.starter.kubernetes.client.config) + implementation(libs.spring.cloud.starter.kubernetes.client.config) { + because("needed for kubernetes configs and secrets to be added as additional configurations") + } implementation(libs.fabric8.kubernetes.client) { exclude("org.slf4j", "slf4j-api") } From 7d2d94b5742c7875cbb7bc784f1a5e1348501515 Mon Sep 17 00:00:00 2001 From: acies312 Date: Wed, 6 Mar 2024 17:13:38 +0300 Subject: [PATCH 8/9] fixes for kubernetes deployment --- cosv-backend/build.gradle.kts | 3 --- 1 file changed, 3 deletions(-) diff --git a/cosv-backend/build.gradle.kts b/cosv-backend/build.gradle.kts index 5a45785979..250fb6d124 100644 --- a/cosv-backend/build.gradle.kts +++ b/cosv-backend/build.gradle.kts @@ -14,9 +14,6 @@ dependencies { implementation(libs.spring.security.core) implementation(libs.spring.data.jpa) implementation(libs.hibernate.jpa21.api) - - // needed for kubernetes configs and secrets to be added as additional configurations - implementation(libs.spring.cloud.starter.kubernetes.client.config) { because("needed for kubernetes configs and secrets to be added as additional configurations") } From d0e1e291abff0afbf26af8f5cab615454055ed02 Mon Sep 17 00:00:00 2001 From: acies312 Date: Wed, 6 Mar 2024 17:28:09 +0300 Subject: [PATCH 9/9] fixes for kubernetes deployment --- .../save/frontend/components/views/index/IndexLogoButtons.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/index/IndexLogoButtons.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/index/IndexLogoButtons.kt index 4082af06d3..f493a03e60 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/index/IndexLogoButtons.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/index/IndexLogoButtons.kt @@ -5,7 +5,6 @@ package com.saveourtool.save.frontend.components.views.index import com.saveourtool.save.frontend.externals.i18next.useTranslation -import com.saveourtool.save.validation.FrontendCosvRoutes import com.saveourtool.save.validation.FrontendRoutes import js.core.jso import react.ChildrenBuilder