Separate save and cosv services (#2908)

* save and cosv divide in gateway

* added cosv deployment templates for helm chart

* Update values.yaml

* modified agent service templates

api-gateway/src/main/resources/application-dev.yml

@@ -1,13 +1,23 @@
 gateway:
   backend:
     url: http://localhost:5800
+  backend-cosv:
+    url: http://localhost:5700
   frontend:
     # In the "dev" environment, the front-end uses TCP port 8080 when run using `webpack-dev-server` (i.e. `browserDevelopmentRun` or `run` Gradle task).
     url: http://localhost:8080
+  frontend-cosv:
+    # In the "dev" environment, the frontend-cosv uses TCP port 8081 when run using `webpack-dev-server` (i.e. `browserDevelopmentRun` or `run` Gradle task).
+    url: http://localhost:8081
   demo:
     url: http://localhost:5421
   demo-cpg:
     url: http://localhost:5500
+#  for local test add additional lines in hosts file in your OS
+#  then you can access application using <domain>:5300
+hosts:
+  save: save.local.test
+  cosv: cosv.local.test
 spring:
   security:
     oauth2:

api-gateway/src/main/resources/application.yml

@@ -5,8 +5,12 @@ server:
 gateway:
   backend:
     url: http://backend:5800
+  backend-cosv:
+    url: http://backend-cosv:5700
   frontend:
     url: http://frontend:5810
+  frontend-cosv:
+    url: http://frontend-cosv:5820
   demo:
     url: http://demo:5421
   demo-cpg:
@@ -43,6 +47,16 @@ spring:
         - id: api_route
           uri: ${gateway.backend.url}
           predicates:
+            - Host=**.${hosts.save}:5300
+            - Path=/api/**
+          filters:
+            # If SESSION cookie is passed to downstream, it is then removed, because downstream discards it
+            - RemoveRequestHeader=Cookie
+            - AuthorizationHeaders=
+        - id: cosv_api_route
+          uri: ${gateway.backend-cosv.url}
+          predicates:
+            - Host=**.${hosts.cosv}:5300
             - Path=/api/**
           filters:
             # If SESSION cookie is passed to downstream, it is then removed, because downstream discards it
@@ -57,13 +71,31 @@ spring:
         - id: resource_route
           uri: ${gateway.frontend.url}
           predicates:
+            - Host=**.${hosts.save}:5300
             - Path=/*.html,/*.js*,/*.css,/img/**,/*.ico,/*.png
           filters:
             # If SESSION cookie is passed to downstream, it is then removed, because downstream discards it
             - RemoveRequestHeader=Cookie
         - id: index_route
           uri: ${gateway.frontend.url}/index.html
           predicates:
+            - Host=**.${hosts.save}:5300
+            - Path=/**
+          filters:
+            # If SESSION cookie is passed to downstream, it is then removed, because downstream discards it
+            - RemoveRequestHeader=Cookie
+        - id: resource_route_cosv
+          uri: ${gateway.frontend-cosv.url}
+          predicates:
+            - Host=**.${hosts.cosv}:5300
+            - Path=/*.html,/*.js*,/*.css,/img/**,/*.ico,/*.png
+          filters:
+            # If SESSION cookie is passed to downstream, it is then removed, because downstream discards it
+            - RemoveRequestHeader=Cookie
+        - id: index_route_cosv
+          uri: ${gateway.frontend-cosv.url}/index.html
+          predicates:
+            - Host=**.${hosts.cosv}:5300
             - Path=/**
           filters:
             # If SESSION cookie is passed to downstream, it is then removed, because downstream discards it

save-cloud-charts/save-cloud/templates/agent-backend-cosv-service.yaml

@@ -0,0 +1,14 @@
+{{ if .Values.agentNamespace }}
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Values.backend_cosv.name }}
+  namespace: {{ .Values.agentNamespace }}
+spec:
+  type: ExternalName
+  externalName: {{ .Values.backend_cosv.name }}.{{ .Values.namespace }}.svc.cluster.local
+  ports:
+    - port: {{ .Values.backend_cosv.containerPort }}
+
+{{ end }}

save-cloud-charts/save-cloud/templates/agent-network-policy-general.yaml

@@ -15,6 +15,7 @@ spec:
             cidr: 0.0.0.0/0
             # Forbid private IP ranges effectively allowing only egress to the Internet
             except:
+              # - 12.0.0.0/8
               - 10.0.0.0/8
               - 172.16.0.0/12
               - 192.168.0.0/16

save-cloud-charts/save-cloud/templates/agent-network-policy-orchestrator.yaml

@@ -35,3 +35,14 @@ spec:
       ports:
         - protocol: TCP
           port: {{ .Values.backend.containerPort }}
+    - to:
+        # Allow traffic to save-backend-cosv
+        - namespaceSelector:
+            matchLabels:
+              name: {{ .Values.namespace }}
+        - podSelector:
+            matchLabels:
+              io.kompose.service: backend-cosv
+      ports:
+        - protocol: TCP
+          port: {{ .Values.backend_cosv.containerPort }}

save-cloud-charts/save-cloud/templates/backend-cosv-configmap.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Values.backend_cosv.name }}-config
+data:
+  application.properties: |
+    backend.preprocessor-url=http://preprocessor
+    backend.orchestrator-url=http://orchestrator
+    backend.demo-url=http://demo
+    backend.loki.url=http://loki:3100
+    backend.loki.labels.agent-container-name=pod
+    backend.loki.labels.application-name=app
+    backend.agent-settings.backend-url=http://{{ .Values.backend_cosv.name }}
+    server.shutdown=graceful
+    management.endpoints.web.exposure.include=*
+    management.server.port={{ .Values.backend_cosv.managementPort }}
+    spring.datasource.url=${spring.datasource.backend-cosv-url}
+    logging.level.org.springframework=DEBUG
+    logging.level.com.saveourtool=DEBUG

save-cloud-charts/save-cloud/templates/backend-cosv-deployment.yaml

@@ -0,0 +1,138 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: backend-cosv
+  labels:
+    {{- include "common.labels" (merge (dict "service" .Values.backend_cosv) .) | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      io.kompose.service: backend-cosv
+  strategy:
+    # Because of shared volume with multi-attach problem
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        {{- include "pod.common.labels" (merge (dict "service" .Values.backend_cosv ) .) | nindent 8 }}
+      annotations:
+        {{- include "pod.common.annotations" (dict "service" .Values.backend_cosv ) | nindent 8 }}
+    spec:
+      restartPolicy: Always
+      {{- include "cnb.securityContext" . | nindent 6 }}
+      containers:
+        - name: backend-cosv
+          {{- include "spring-boot.common" (merge (dict "service" .Values.backend_cosv) .) | nindent 10 }}
+          env:
+            {{- include "spring-boot.common.env" (merge (dict "service" .Values.backend_cosv) .) | nindent 12 }}
+            - name: DATABASE_SECRETS_PATH
+              value: {{ .Values.mysql.dbPasswordFile }}
+            - name: S3_SECRETS_PATH
+              value: {{ .Values.s3.secretFile }}
+            - name: JAVA_TOOL_OPTIONS
+              value: -XX:ReservedCodeCacheSize=48M
+          volumeMounts:
+            - {{ include "spring-boot.config-volume-mount" . | indent 14 | trim }}
+            - name: database-secret
+              mountPath: {{ .Values.mysql.dbPasswordFile }}
+            - name: s3-secrets
+              mountPath: {{ .Values.s3.secretFile }}
+          {{- include "spring-boot.management" .Values.backend_cosv | nindent 10 }}
+          resources:
+            limits:
+              memory: 2G
+            requests:
+              memory: 1G
+      {{ if .Values.mysql.migrations.enabled }}
+      initContainers:
+        - name: git-cloner
+          image: alpine/git
+          env:
+            {{ if .Values.proxy.enabled }}
+            {{- with .Values.proxy.extraEnv }}
+              {{- toYaml . | nindent 12 }}
+            {{- end }}
+            {{ end }}
+          args:
+            - clone
+            - --progress
+            - --verbose
+            {{ if .Values.proxy.enabled }}
+            {{- with .Values.proxy.extraArgs }}
+              {{- toYaml . | nindent 12 }}
+            {{- end }}
+            {{ end }}
+            - --single-branch
+            - --branch
+            - {{ .Values.mysql.migrations.branch | default "master" }}
+            - --
+            - https://github.com/saveourtool/save-cloud.git
+            - /data
+          volumeMounts:
+            - mountPath: /data
+              name: migrations-data
+        - name: liquibase-runner
+          image: liquibase/liquibase:4.20
+          securityContext:
+            runAsUser: 1001
+            runAsGroup: 1001
+          args:
+            - --url=$(DB_URL)?createDatabaseIfNotExist=true
+            - --changeLogFile=db/db.changelog-master.xml
+            - --username=$(DB_USERNAME)
+            - --password=$(DB_PASSWORD)
+            - --log-level=info
+            - --contexts={{ .Values.profile }}
+            - update
+          resources:
+            requests:
+              memory: 100M
+            limits:
+              memory: 300M
+          env:
+            # See https://hub.docker.com/r/liquibase/liquibase, section 'Notice for MySQL Users'
+            - name: INSTALL_MYSQL
+              value: 'true'
+            - name: DB_URL
+              valueFrom:
+                secretKeyRef:
+                  name: db-secrets
+                  key: spring.datasource.backend-url
+            - name: DB_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: db-secrets
+                  key: spring.datasource.username
+            - name: DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: db-secrets
+                  key: spring.datasource.password
+            {{ if .Values.proxy.enabled }}
+            {{- with .Values.proxy.extraEnv }}
+              {{- toYaml . | nindent 12 }}
+            {{- end }}
+            {{ end }}
+          volumeMounts:
+            - mountPath: /liquibase/changelog
+              name: migrations-data
+            - mountPath: {{ .Values.mysql.dbPasswordFile }}
+              name: database-secret
+            - mountPath: /etc/ssl/certs/ca-certificates.crt
+              name: ca-certs
+              subPath: cert-check.crt
+              readOnly: false
+      {{ end }}
+      volumes:
+        - {{ include "spring-boot.config-volume" (dict "service" .Values.backend_cosv) | indent 10 | trim }}
+        - name: database-secret
+          secret:
+            secretName: db-secrets
+        - name: s3-secrets
+          secret:
+            secretName: s3-secrets
+        - name: migrations-data
+          emptyDir: {}
+        - name: ca-certs
+          configMap:
+            name: ca-certs

save-cloud-charts/save-cloud/templates/backend-cosv-service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+{{- include "service.common.metadata" (dict "service" .Values.backend_cosv) | nindent 2 }}
+spec:
+  {{ if .Values.backend_cosv.clusterIP }}
+  clusterIP: {{ .Values.backend_cosv.clusterIP }}
+  {{ end }}
+  ports:
+    {{- include "service.common.ports" (dict "service" .Values.backend_cosv) | nindent 4 }}
+  selector:
+    {{- include "service.common.selectors" (dict "service" .Values.backend_cosv) | nindent 4 }}

save-cloud-charts/save-cloud/templates/frontend-cosv-deployment.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: frontend-cosv
+  labels:
+    {{- include "common.labels" (merge (dict "service" .Values.frontend_cosv) .) | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      io.kompose.service: frontend-cosv
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        {{- include "pod.common.labels" (merge (dict "service" .Values.frontend_cosv ) .) | nindent 8 }}
+    spec:
+      restartPolicy: Always
+      {{- include "cnb.securityContext" . | nindent 6 }}
+      containers:
+        - name: frontend-cosv
+          {{- include "spring-boot.common" (merge (dict "service" .Values.frontend_cosv) .) | nindent 10 }}
+          env:
+            - name: PORT
+              value: '{{ .Values.frontend_cosv.containerPort | toString }}'
+          resources:
+            limits:
+              memory: 200M
+            requests:
+              memory: 100M

save-cloud-charts/save-cloud/templates/frontend-cosv-service.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Service
+metadata:
+{{- include "service.common.metadata" (dict "service" .Values.frontend_cosv) | nindent 2 }}
+spec:
+  ports:
+    {{- include "service.common.ports" (dict "service" .Values.frontend_cosv) | nindent 4 }}
+  selector:
+    {{- include "service.common.selectors" (dict "service" .Values.frontend_cosv) | nindent 4 }}

save-cloud-charts/save-cloud/values-minikube.yaml

@@ -5,6 +5,8 @@ ingress:
   enabled: false
 backend:
   profile: dev,secure,kubernetes
+backend-cosv:
+  profile: dev,secure,kubernetes
 gateway:
   profile: dev,kubernetes
   applicationProperties: |+

save-cloud-charts/save-cloud/values.yaml

@@ -18,10 +18,21 @@ backend:
   # Fixed ClusterIP can be assigned to make it easier to query backend from services outside Kubernetes.
   # Should be chosen depending on cluster's network configuration: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address.
   clusterIP: null
+backend_cosv:
+  name: backend-cosv
+  profile: dev,secure,kubernetes
+  imageName: save-cosv
+  containerPort: 5700
+  managementPort: 5701
+  clusterIP: null
 frontend:
   name: frontend
   imageName: save-frontend
   containerPort: 5810
+frontend_cosv:
+  name: frontend-cosv
+  imageName: save-cosv-frontend
+  containerPort: 5820
 orchestrator:
   name: orchestrator
   imageName: save-orchestrator
@@ -38,8 +49,9 @@ preprocessor:
   # Fixed ClusterIP can be assigned to make it easier to query preprocessor from services outside Kubernetes
   clusterIP: null
 gateway:
+  # Additional certs installation for internal CA case
   initCerts:
-    enabled: true
+    enabled: false
   name: gateway
   profile: dev,kubernetes
   imageName: api-gateway

save-cosv-frontend/webpack.config.d/dev-server.js

@@ -2,6 +2,7 @@ config.devServer = Object.assign(
     {},
     config.devServer || {},
     {
+        port: 8081,
         setupMiddlewares: (middlewares, devServer) => {
             devServer.app.get("/sec/oauth-providers", (req, res) => { return res.send([]); });
             return middlewares;