analysis_system_example.yaml

#  redis
#  redis-service
apiVersion: v1
kind: Service
metadata:
  name: redis
  
  labels:
    app: redis
    tier: backend
spec:
  ports:
  - port: 6379
    protocol: TCP
    targetPort: 6379
  selector:
    app: redis
    tier: backend
  
  type: NodePort
  
---
#  redis-secret
apiVersion: v1
kind: Secret
metadata:
  name: redis-secret
  
data:
  password: dGVzdA==

type: Opaque
---
#  redis-persistent-volume-claim
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  labels:
    type: claim
  name: redis-pv-claim
  
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 150Mi
status:
  accessModes:
  - ReadWriteOnce
  capacity:
    storage: 1000M
---
#  redis config-map
apiVersion: v1
kind: ConfigMap
metadata:
  name: redis-config
  
data:
  redis.conf: >
    # disable the config command
    rename-command CONFIG ""

    # use redis like a cache - once the memory limit is reached, results / keys are deleted
    maxmemory 100mb
    maxmemory-policy allkeys-lru

    # backup the data every 60 sec if at least 1 key changed
    save 60 1
# 
---
#  redis deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: redis
  
spec:
  progressDeadlineSeconds: 600
  replicas: 1
  selector:
    matchLabels:
      app: redis
      tier: backend
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: redis
        tier: backend
    spec:
      containers:
      - name: redis
        image: redis:5.0.3
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 6379
          protocol: TCP
        command:
          - /bin/bash
          - -c
          - |
            REDIS_PASSWORD=`cat /etc/redis/secret/redis.password`
            ARGS=("--requirepass" "${REDIS_PASSWORD}")
            ARGS+=("--port" "6379")
            ARGS+=("--include" "/etc/redis/config/redis.conf")
            redis-server "${ARGS[@]}"
        resources:
          requests:
            cpu: 100m
            memory: 100Mi
          limits:
            cpu: 200m
            memory: 200Mi
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /etc/redis/config
          name: redis-config
          readOnly: true
        - mountPath: /etc/redis/secret
          name: redis-password-file
          readOnly: true
        - mountPath: /data
          name: redis-persistent-storage
      volumes:
      - name: redis-config
        configMap:
          name: redis-config
      - name: redis-password-file
        secret:
          secretName: redis-secret
          items:
          - key: password
            path: redis.password
      - name: redis-persistent-storage
        persistentVolumeClaim:
          claimName: redis-pv-claim
---
#  rabbit-mq service
apiVersion: v1
kind: Service
metadata:
  name: rabbit-mq
  
  labels:
    app: rabbit-mq
    tier: backend
spec:
  ports:
  - port: 5672
    protocol: TCP
    targetPort: 5672
  selector:
    app: rabbit-mq
    tier: backend
  
  type: NodePort
  
---
#  rabbit-mq config-map
apiVersion: v1
kind: ConfigMap
metadata:
  name: rabbit-mq-config
  
data:
  11-custom.conf: >
    # set the high memory watermark
    vm_memory_high_watermark.absolute = 200Mib
# 
---
#  rabbit-mq monitoring service
apiVersion: v1
kind: Service
metadata:
  name: rabbit-mq-monitor
  
  labels:
    app: rabbit-mq
    tier: backend
spec:
  ports:
  - port: 15672
    protocol: TCP
    targetPort: 15672
  selector:
    app: rabbit-mq
    tier: backend
  type: NodePort
---
#  rabbit-mq secret
apiVersion: v1
kind: Secret
metadata:
  name: rabbitmq-secret
  
data:
  password: dGVzdA==

  user: dGVzdA==

type: Opaque
---
#  rabbit-mq deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  annotations:
  name: rabbit-mq
  
spec:
  replicas: 1
  selector:
    matchLabels:
      app: rabbit-mq
      tier: backend
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: rabbit-mq
        tier: backend
    spec:
      containers:
      - name: rabbit-mq
        image: rabbitmq:3.11-management-alpine
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 5672
          protocol: TCP
        - containerPort: 15672
          protocol: TCP
        env:
        - name: RABBIT_DELIVERY_LIMIT
          value: "3"
        - name: RABBIT_MAX_QUEUE_LENGTH
          value: "10"
        - name: RABBITMQ_ERLANG_COOKIE
          value: VUR6VzVYTkJxY3ZnOEt0Y1N3aElIaHNTeTZHS2sxY2V0cVhNN0JybEprb1V1OWtEZTc=

        - name: RABBITMQ_DEFAULT_USER
          valueFrom:
            secretKeyRef:
              key: user
              name: rabbitmq-secret
        - name: RABBITMQ_DEFAULT_PASS
          valueFrom:
            secretKeyRef:
              key: password
              name: rabbitmq-secret
        volumeMounts:
        - mountPath: /etc/rabbitmq/conf.d
          name: rabbit-mq-config
          readOnly: true
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        resources:
          requests:
            cpu: 100m
            memory: 300Mi
          limits:
            cpu: 200m
            memory: 500Mi
      volumes:
      - name: rabbit-mq-config
        configMap:
          name: rabbit-mq-config
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30
---
#  reactome service
apiVersion: v1
kind: Service
metadata:
  name: reactome
  
  labels:
    tier: external
spec:
  ports:
  - port: 443
    protocol: TCP
    targetPort: 443
---
#  reactome endpoint
apiVersion: v1
kind: Endpoints
metadata:
  name: reactome
  
subsets:
- addresses:
  - ip: 100.25.71.177
  ports:
  - port: 443
    protocol: TCP
---
# REACTOME API service
apiVersion: v1
kind: Service
metadata:
  name: reactome-analysis-api
  
  labels:
    app: reactome-analysis-api
    tier: frontend
spec:
  ports:
  - name: http
    protocol: TCP
    port: 8080
    targetPort: 8080
  selector:
    app: reactome-analysis-api
    tier: frontend
  type: NodePort
---
# Config Map for all REACTOME pods
apiVersion: v1
kind: ConfigMap
metadata:
  name: reactome-global-config
  
data:
  # number of tries to post a RABBIT message
  MAX_MESSAGE_TRIES: "3"
  RABBIT_HOST: "rabbit-mq"
  RABBIT_PORT: "5672"
  REDIS_HOST: "redis"
  REDIS_PORT: "6379"
  REDIS_DATABASE: "0"
  USE_REDIS_CLUSTER: "False"
  RABBIT_MAX_QUEUE_LENGTH: "10"
  MAIL_ERROR_ADDRESS: "notify@me.com"
---
# Config Map for the e-mail settings
apiVersion: v1
kind: ConfigMap
metadata:
  name: reactome-email-config
  
data:
  SMTP_SERVER: ""
  SMTP_PORT: "587"
  FROM_ADDRESS: ""
  BASE_URL: "http://192.168.99.100:30966"
---
#  mail-secret
apiVersion: v1
kind: Secret
metadata:
  name: mail-secret
  
data:
  user: dGVzdAo=
  password: dGVzdAo=
type: Opaque
---
# REACTOME API deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: reactome-analysis-api
  
spec:
  progressDeadlineSeconds: 600
  replicas: 1
  selector:
    matchLabels:
      app: reactome-analysis-api
      tier: frontend
  template:
    metadata:
      labels:
        app: reactome-analysis-api
        tier: frontend
    spec:
      containers:
      - name: reactome-analysis-api
        image: jgriss/reactome-analysis_public-api:latest
        ports:
        - containerPort: 8080
          protocol: TCP
        envFrom:
        - configMapRef:
            name: reactome-global-config
        - configMapRef:
            name: reactome-email-config
        env:
          - name: REDIS_PASSWORD_FILE
            value: "/etc/redis/redis.password"
          - name: RABBIT_USER_SECRETS_FILE
            value: "/etc/rabbit/user"
          - name: RABBIT_PASSWORD_SECRETS_FILE
            value: "/etc/rabbit/password"
          - name: EMAIL_USER_FILE
            value: "/etc/mail/user"
          - name: EMAIL_PASSWORD_FILE
            value: "/etc/mail/password"
        volumeMounts:
        - mountPath: /etc/redis/
          name: redis-secret-volume
          readOnly: true
        - mountPath: /etc/rabbit/
          name: rabbit-secret-volume
          readOnly: true
        - mountPath: /etc/mail/
          name: mail-secret-volume
          readOnly: true
        resources:
          requests:
            cpu: 100m
            memory: 200Mi
          limits:
            cpu: 250m
            memory: 500Mi
        readinessProbe:
          httpGet:
            path: /0.1/ui/
            port: 8080
      volumes:
      - name: redis-secret-volume
        secret:
          secretName: redis-secret
          items:
          - key: password
            path: redis.password
      - name: rabbit-secret-volume
        secret:
          secretName: rabbitmq-secret
          items:
          - key: user
            path: user
          - key: password
            path: password
      - name: mail-secret-volume
        secret:
          secretName: mail-secret
          items:
          - key: user
            path: user
          - key: password
            path: password
---
# REACTOME api autoscaler
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: reactome-analysis-api
  
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: reactome-analysis-api
  minReplicas: 1
  maxReplicas: 3
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 75
---
# REACTOME worker deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: reactome-analysis-worker
  
spec:
  progressDeadlineSeconds: 600
  replicas: 1
  selector:
    matchLabels:
      app: reactome-analysis-worker
      tier: backend
  template:
    metadata:
      labels:
        app: reactome-analysis-worker
        tier: backend
    spec:
      containers:
      - name: reactome-analysis-worker
        image: jgriss/reactome-analysis_worker:latest
        
        imagePullPolicy: "Always"
        
        livenessProbe:
          exec:
            command:
            - cat
            - /tmp/healthy
          initialDelaySeconds: 30
          periodSeconds: 10
        ports:
        - containerPort: 9090
          name: "prometheus"
          protocol: TCP
        envFrom:
        - configMapRef:
            name: reactome-global-config
        - configMapRef:
            name: reactome-email-config
        env:
          - name: PROMETHEUS_PORT
            value: "9090"
          - name: REDIS_PASSWORD_FILE
            value: "/etc/redis/redis.password"
          - name: RABBIT_USER_SECRETS_FILE
            value: "/etc/rabbit/user"
          - name: RABBIT_PASSWORD_SECRETS_FILE
            value: "/etc/rabbit/password"
          - name: EMAIL_USER_FILE
            value: "/etc/mail/user"
          - name: EMAIL_PASSWORD_FILE
            value: "/etc/mail/password"
          - name: MAX_WORKER_TIMEOUT
            value: "60"
        volumeMounts:
        - mountPath: /etc/redis/
          name: redis-secret-volume
          readOnly: true
        - mountPath: /etc/rabbit/
          name: rabbit-secret-volume
          readOnly: true
        - mountPath: /etc/mail/
          name: mail-secret-volume
          readOnly: true
        resources:
          requests:
            cpu: 0.5
            memory: 500Mi
          limits:
            cpu: 1
            memory: 1000Mi
      volumes:
      - name: redis-secret-volume
        secret:
          secretName: redis-secret
          items:
          - key: password
            path: redis.password
      - name: rabbit-secret-volume
        secret:
          secretName: rabbitmq-secret
          items:
          - key: user
            path: user
          - key: password
            path: password
      - name: mail-secret-volume
        secret:
          secretName: mail-secret
          items:
          - key: user
            path: user
          - key: password
            path: password
      tolerations:
      - key: "stability"
        operator: "Equal"
        value: "pre-emptible"
        effect: "NoSchedule"
---
# REACTOME worker autoscaler
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: reactome-analysis-worker
  
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: reactome-analysis-worker
  minReplicas: 1
  maxReplicas: 1
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 75
---
# REACTOME report deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: reactome-analysis-report
  
spec:
  progressDeadlineSeconds: 600
  replicas: 1
  selector:
    matchLabels:
      app: reactome-analysis-report
      tier: backend
  template:
    metadata:
      labels:
        app: reactome-analysis-report
        tier: backend
    spec:
      containers:
      - name: reactome-analysis-report
        image: jgriss/reactome-analysis_report:latest
        
        imagePullPolicy: "Always"
        
        ports:
        - containerPort: 9090
          name: "prometheus"
          protocol: TCP
        envFrom:
        - configMapRef:
            name: reactome-global-config
        - configMapRef:
            name: reactome-email-config
        env:
          - name: PROMETHEUS_PORT
            value: "9090"
          - name: REDIS_PASSWORD_FILE
            value: "/etc/redis/redis.password"
          - name: RABBIT_USER_SECRETS_FILE
            value: "/etc/rabbit/user"
          - name: RABBIT_PASSWORD_SECRETS_FILE
            value: "/etc/rabbit/password"
          - name: EMAIL_USER_FILE
            value: "/etc/mail/user"
          - name: EMAIL_PASSWORD_FILE
            value: "/etc/mail/password"
          - name: MEM_LIMIT
            valueFrom:
              resourceFieldRef:
                containerName: reactome-analysis-report
                resource: limits.memory
        volumeMounts:
        - mountPath: /etc/redis/
          name: redis-secret-volume
          readOnly: true
        - mountPath: /etc/rabbit/
          name: rabbit-secret-volume
          readOnly: true
        - mountPath: /etc/mail/
          name: mail-secret-volume
          readOnly: true
        resources:
          requests:
            cpu: 0.3
            memory: 250Mi
          limits:
            cpu: 1
            memory: 1000Mi
            ephemeral-storage: 1Gi
      volumes:
      - name: redis-secret-volume
        secret:
          secretName: redis-secret
          items:
          - key: password
            path: redis.password
      - name: rabbit-secret-volume
        secret:
          secretName: rabbitmq-secret
          items:
          - key: user
            path: user
          - key: password
            path: password
      - name: mail-secret-volume
        secret:
          secretName: mail-secret
          items:
          - key: user
            path: user
          - key: password
            path: password
      tolerations:
      - key: "stability"
        operator: "Equal"
        value: "pre-emptible"
        effect: "NoSchedule"
---
# REACTOME report autoscaler
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: reactome-analysis-report
  
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: reactome-analysis-report
  minReplicas: 1
  maxReplicas: 1
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 75
---
# REACTOME datasets deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: reactome-analysis-datasets
  
spec:
  progressDeadlineSeconds: 600
  replicas: 1
  selector:
    matchLabels:
      app: reactome-analysis-datasets
      tier: backend
  template:
    metadata:
      labels:
        app: reactome-analysis-datasets
        tier: backend
    spec:
      containers:
      - name: reactome-analysis-datasets
        image: jgriss/reactome-analysis_datasets:latest
        
        imagePullPolicy: "Always"
        
        ports:
        - containerPort: 9090
          name: "prometheus"
          protocol: TCP
        envFrom:
        - configMapRef:
            name: reactome-global-config  
        - configMapRef:
            name: reactome-email-config
        env:
          - name: PROMETHEUS_PORT
            value: "9090"
          - name: REDIS_PASSWORD_FILE
            value: "/etc/redis/redis.password"
          - name: RABBIT_USER_SECRETS_FILE
            value: "/etc/rabbit/user"
          - name: RABBIT_PASSWORD_SECRETS_FILE
            value: "/etc/rabbit/password"
          - name: EMAIL_USER_FILE
            value: "/etc/mail/user"
          - name: EMAIL_PASSWORD_FILE
            value: "/etc/mail/password"
          - name: LOADING_MAX_TIMEOUT
            value: "60"
          - name: USE_GREIN_PROXY
            value: "false"
        volumeMounts:
        - mountPath: /etc/redis/
          name: redis-secret-volume
          readOnly: true
        - mountPath: /etc/rabbit/
          name: rabbit-secret-volume
          readOnly: true
        - mountPath: /etc/mail/
          name: mail-secret-volume
          readOnly: true
        resources:
          requests:
            cpu: 0.1
            memory: 100Mi
            ephemeral-storage: 500Mi
          limits:
            cpu: 0.3
            memory: 200Mi
            ephemeral-storage: 1Gi
      volumes:
      - name: redis-secret-volume
        secret:
          secretName: redis-secret
          items:
          - key: password
            path: redis.password
      - name: rabbit-secret-volume
        secret:
          secretName: rabbitmq-secret
          items:
          - key: user
            path: user
          - key: password
            path: password
      - name: mail-secret-volume
        secret:
          secretName: mail-secret
          items:
          - key: user
            path: user
          - key: password
            path: password
      tolerations:
      - key: "stability"
        operator: "Equal"
        value: "pre-emptible"
        effect: "NoSchedule"
---
# REACTOME datasets autoscaler
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: reactome-analysis-datasets
  
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: reactome-analysis-datasets
  minReplicas: 1
  maxReplicas: 1
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 75