diff --git a/compose/planet.yml b/compose/planet.yml index c16f9990..daa89902 100644 --- a/compose/planet.yml +++ b/compose/planet.yml @@ -1,8 +1,22 @@ version: '3' services: - ###################################################### - ### OSM minute replication files section - ###################################################### + osm-processor: + image: osmseed-osm-processor:v1 + build: + context: ../images/osm-processor + dockerfile: Dockerfile + volumes: + - ../data/osm-processor-data:/mnt/data + command: /start.sh + env_file: + - ../envs/.env.db + - ../envs/.env.db-utils + - ../envs/.env.cloudprovider + restart: always + + ##################################################### + ## OSM minute replication files section + ##################################################### replication-job: image: osmseed-replication-job:v1 build: diff --git a/images/full-history/Dockerfile b/images/full-history/Dockerfile index 3f40d16a..ef89f12c 100644 --- a/images/full-history/Dockerfile +++ b/images/full-history/Dockerfile @@ -1,4 +1,4 @@ -FROM developmentseed/osmseed-osm-processor:0.1.0-n795.h1c8d32f +FROM developmentseed/osmseed-osm-processor:0.1.0-n802.h0d9f574 VOLUME /mnt/data COPY ./start.sh / diff --git a/images/osm-processor/Dockerfile b/images/osm-processor/Dockerfile index 88186d49..b2392ff3 100644 --- a/images/osm-processor/Dockerfile +++ b/images/osm-processor/Dockerfile @@ -2,20 +2,22 @@ FROM debian:bookworm-slim ENV workdir /mnt/data WORKDIR $workdir -# Installs osmosis v0.48.3 & osmium-tool v1.15.0 +# Installs osmosis v0.48.3, osmium-tool v1.15.0, and PostgreSQL client RUN set -ex \ - && apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install \ - -y --no-install-recommends \ + && apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install \ + -y --no-install-recommends \ "osmosis" \ "osmium-tool" \ # Cloud provider CLIs "awscli" \ "gsutil" \ "azure-cli" \ + # PostgreSQL client + "postgresql-client" \ # Other useful packages "rsync" \ "pyosmium" \ "tmux" \ "zsh" \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ No newline at end of file diff --git a/images/planet-dump/Dockerfile b/images/planet-dump/Dockerfile index 3f40d16a..ef89f12c 100644 --- a/images/planet-dump/Dockerfile +++ b/images/planet-dump/Dockerfile @@ -1,4 +1,4 @@ -FROM developmentseed/osmseed-osm-processor:0.1.0-n795.h1c8d32f +FROM developmentseed/osmseed-osm-processor:0.1.0-n802.h0d9f574 VOLUME /mnt/data COPY ./start.sh / diff --git a/images/populate-apidb/Dockerfile b/images/populate-apidb/Dockerfile index 68f0fd56..8ca3d071 100644 --- a/images/populate-apidb/Dockerfile +++ b/images/populate-apidb/Dockerfile @@ -1,4 +1,4 @@ -FROM developmentseed/osmseed-osm-processor:0.1.0-n795.h1c8d32f +FROM developmentseed/osmseed-osm-processor:0.1.0-n802.h0d9f574 VOLUME /mnt/data COPY ./start.sh / diff --git a/images/replication-job/Dockerfile b/images/replication-job/Dockerfile index 5594a3f7..992421f8 100644 --- a/images/replication-job/Dockerfile +++ b/images/replication-job/Dockerfile @@ -1,7 +1,11 @@ -FROM developmentseed/osmseed-osm-processor:0.1.0-n795.h1c8d32f +FROM developmentseed/osmseed-osm-processor:0.1.0-n802.h0d9f574 RUN apt-get update && \ - apt-get install -y nginx python3-pip python3-venv && \ + apt-get install -y \ + nginx \ + python3-pip \ + python3-venv \ + procps && \ rm -rf /var/lib/apt/lists/* RUN python3 -m venv /opt/venv @@ -13,4 +17,4 @@ COPY monitoring.py / WORKDIR /mnt/data -CMD /start.sh \ No newline at end of file +CMD /start.sh diff --git a/images/web/liveness.sh b/images/web/liveness.sh index 5d30dfd1..331df5b9 100755 --- a/images/web/liveness.sh +++ b/images/web/liveness.sh @@ -1,9 +1,36 @@ #!/usr/bin/env bash -# This is a script for the complex evaluation of whether Apache or other processes are running in the container. -if [ $(ps -ef | grep -E 'httpd|apache2' | grep -v grep | wc -l) -ge 1 ]; then - echo "Apache is running." - exit 0 +# This is a script for evaluating if openstreetmap-cgimap, apache2, and PostgreSQL are running in the container. +check_process() { + if ps aux | grep "$1" | grep -v grep > /dev/null; then + return 0 + else + return 1 + fi +} + +# Check for openstreetmap-cgimap process +check_process "/openstreetmap-cgimap/build/openstreetmap-cgimap" +cgimap_status=$? + +# Check for apache2 process +check_process "apache2" +apache_status=$? + +# Check PostgreSQL connection +check_postgres() { + PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -U $POSTGRES_USER -d $POSTGRES_DB -c "SELECT 1;" > /dev/null 2>&1 + return $? +} + +check_postgres +postgres_status=$? + +if [ $cgimap_status -eq 0 ] && [ $apache_status -eq 0 ] && [ $postgres_status -eq 0 ]; then + echo "All services (openstreetmap-cgimap, apache2, PostgreSQL) are running." + exit 0 else - echo "Apache is not running!" 1>&2 - exit 1 -fi + [ $cgimap_status -ne 0 ] && echo "openstreetmap-cgimap is not running!" 1>&2 + [ $apache_status -ne 0 ] && echo "apache2 is not running!" 1>&2 + [ $postgres_status -ne 0 ] && echo "Failed to connect to PostgreSQL!" 1>&2 + exit 1 +fi \ No newline at end of file diff --git a/osm-seed/templates/id-editor/id-editor-deployment.yaml b/osm-seed/templates/id-editor/id-editor-deployment.yaml deleted file mode 100644 index e16f6cc4..00000000 --- a/osm-seed/templates/id-editor/id-editor-deployment.yaml +++ /dev/null @@ -1,54 +0,0 @@ -{{- if .Values.idEditor.enabled -}} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "osm-seed.fullname" . }}-id-editor - labels: - app: {{ template "osm-seed.name" . }} - component: id-editor-deployment - environment: {{ .Values.environment }} - release: {{ .Release.Name }} -spec: - replicas: {{ .Values.idEditor.replicaCount }} - selector: - matchLabels: - app: {{ template "osm-seed.name" . }} - release: {{ .Release.Name }} - run: {{ .Release.Name }}-id-editor - template: - metadata: - labels: - app: {{ template "osm-seed.name" . }} - release: {{ .Release.Name }} - run: {{ .Release.Name }}-id-editor - spec: - containers: - - name: {{ .Chart.Name }}-id-editor - image: "{{ .Values.idEditor.image }}:{{ .Values.osmSeedVersion }}" - ports: - - name: http - containerPort: {{ .Values.idEditor.env.ID_EDITOR_PORT }} - protocol: TCP - {{- if .Values.idEditor.resources.enabled }} - resources: - requests: - memory: {{ .Values.idEditor.resources.requests.memory }} - cpu: {{ .Values.idEditor.resources.requests.cpu }} - limits: - memory: {{ .Values.idEditor.resources.limits.memory }} - cpu: {{ .Values.idEditor.resources.limits.cpu }} - {{- end }} - env: - - name: OAUTH_CONSUMER_KEY - value: {{ .Values.idEditor.env.OAUTH_CONSUMER_KEY }} - - name: OAUTH_SECRET - value: {{ .Values.idEditor.env.OAUTH_SECRET }} - - name: SERVER_URL - value: {{ .Values.idEditor.env.OSM_API_DOMAIN }} - - name: SERVER_PROTOCOL - value : {{ .Values.idEditor.env.OSM_API_PROTOCOL }} - {{- if .Values.idEditor.nodeSelector.enabled }} - nodeSelector: - {{ .Values.idEditor.nodeSelector.label_key }} : {{ .Values.idEditor.nodeSelector.label_value }} - {{- end }} -{{- end }} diff --git a/osm-seed/templates/id-editor/id-editor-service.yaml b/osm-seed/templates/id-editor/id-editor-service.yaml deleted file mode 100644 index 18d6ce76..00000000 --- a/osm-seed/templates/id-editor/id-editor-service.yaml +++ /dev/null @@ -1,37 +0,0 @@ -{{- if .Values.idEditor.enabled -}} -apiVersion: v1 -kind: Service -metadata: - name: {{ template "osm-seed.fullname" . }}-id-editor - labels: - app: {{ template "osm-seed.name" . }} - component: id-editor-service - environment: {{ .Values.environment }} - release: {{ .Release.Name }} - {{- if eq .Values.cloudProvider "aws" }} - annotations: - service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "300" - {{- end }} -spec: - # In case cloudProvider=aws - {{- if eq .Values.cloudProvider "aws" }} - type: LoadBalancer - {{- end }} - # In case cloudProvider=gcp - {{- if eq .Values.cloudProvider "gcp" }} - type: LoadBalancer - {{- end }} - # In case cloudProvider=minikube - {{ if eq .Values.cloudProvider "minikube" }} - type: NodePort - {{- end }} - ports: - - port: {{ .Values.idEditor.env.ID_EDITOR_PORT }} - targetPort: 80 - protocol: TCP - name: http - selector: - app: {{ template "osm-seed.name" . }} - release: {{ .Release.Name }} - run: {{ .Release.Name }}-id-editor -{{- end }} \ No newline at end of file diff --git a/osm-seed/templates/tiler-visor/tiler-visor-deployment.yaml b/osm-seed/templates/tiler-visor/tiler-visor-deployment.yaml deleted file mode 100644 index 08398918..00000000 --- a/osm-seed/templates/tiler-visor/tiler-visor-deployment.yaml +++ /dev/null @@ -1,52 +0,0 @@ -{{- if .Values.tilerVisor.enabled -}} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "osm-seed.fullname" . }}-tiler-visor - labels: - app: {{ template "osm-seed.name" . }} - component: tiler-visor-deployment - environment: {{ .Values.environment }} - release: {{ .Release.Name }} -spec: - replicas: {{ .Values.tilerVisor.replicaCount }} - selector: - matchLabels: - app: {{ template "osm-seed.name" . }} - release: {{ .Release.Name }} - run: {{ .Release.Name }}-tiler-visor - template: - metadata: - labels: - app: {{ template "osm-seed.name" . }} - release: {{ .Release.Name }} - run: {{ .Release.Name }}-tiler-visor - spec: - containers: - - name: {{ .Chart.Name }}-tiler-visor - image: "{{ .Values.tilerVisor.image.name }}:{{ .Values.tilerVisor.image.tag }}" - ports: - - name: http - containerPort: {{ .Values.tilerVisor.env.TILER_VISOR_PORT }} - protocol: TCP - {{- if .Values.tilerVisor.resources.enabled }} - resources: - requests: - memory: {{ .Values.tilerVisor.resources.requests.memory }} - cpu: {{ .Values.tilerVisor.resources.requests.cpu }} - limits: - memory: {{ .Values.tilerVisor.resources.limits.memory }} - cpu: {{ .Values.tilerVisor.resources.limits.cpu }} - {{- end }} - env: - - name: TILER_SERVER_HOST - value: {{ .Values.tilerVisor.staticIp }} - - name: TILER_SERVER_PORT - value: {{ .Values.tilerServer.env.TILER_SERVER_PORT | quote }} - - name: TILER_VISOR_PROTOCOL - value: {{ .Values.tilerVisor.env.TILER_VISOR_PROTOCOL }} - - name: TILER_VISOR_HOST - value: {{ .Values.tilerVisor.staticIp }} - - name: TILER_VISOR_PORT - value: {{ .Values.tilerVisor.env.TILER_VISOR_PORT | quote }} -{{- end }} diff --git a/osm-seed/templates/tiler-visor/tiler-visor-ingress.yaml b/osm-seed/templates/tiler-visor/tiler-visor-ingress.yaml deleted file mode 100644 index 836ca7da..00000000 --- a/osm-seed/templates/tiler-visor/tiler-visor-ingress.yaml +++ /dev/null @@ -1,26 +0,0 @@ -{{- if and .Values.tilerVisor.enabled (eq .Values.serviceType "ClusterIP") }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ template "osm-seed.fullname" . }}-ingress-tiler-visor - annotations: - kubernetes.io/ingress.class: nginx - cert-manager.io/cluster-issuer: letsencrypt-prod-issuer -spec: - tls: - - hosts: - - tiler-visor.{{ .Values.domain }} - secretName: {{ template "osm-seed.fullname" . }}-secret-tiler-visor - - rules: - - host: tiler-visor.{{ .Values.domain }} - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: {{ template "osm-seed.fullname" . }}-tiler-visor - port: - number: 80 -{{- end }} \ No newline at end of file diff --git a/osm-seed/templates/tiler-visor/tiler-visor-service.yaml b/osm-seed/templates/tiler-visor/tiler-visor-service.yaml deleted file mode 100644 index d880f362..00000000 --- a/osm-seed/templates/tiler-visor/tiler-visor-service.yaml +++ /dev/null @@ -1,43 +0,0 @@ -{{- if .Values.tilerVisor.enabled -}} -apiVersion: v1 -kind: Service -metadata: - name: {{ template "osm-seed.fullname" . }}-tiler-visor - labels: - app: {{ template "osm-seed.name" . }} - component: tiler-visor-service - environment: {{ .Values.environment }} - release: {{ .Release.Name }} - annotations: - {{- if and (eq .Values.serviceType "LoadBalancer") .Values.AWS_SSL_ARN }} - service.beta.kubernetes.io/aws-load-balancer-ssl-cert: {{ .Values.AWS_SSL_ARN }} - service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http - service.beta.kubernetes.io/aws-load-balancer-ssl-ports: https - {{- end }} - {{- if eq .Values.serviceType "ClusterIP" }} - kubernetes.io/ingress.class: nginx - cert-manager.io/cluster-issuer: letsencrypt-prod-issuer - {{- else }} - fake.annotation: fake - {{- end }} - {{- with .Values.tilerVisor.serviceAnnotations }} - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - type: {{ .Values.serviceType }} - ports: - - port: 80 - targetPort: http - protocol: TCP - name: http - {{- if and (eq .Values.serviceType "LoadBalancer") .Values.AWS_SSL_ARN }} - - port: 443 - targetPort: http - protocol: TCP - name: https - {{- end }} - selector: - app: {{ template "osm-seed.name" . }} - release: {{ .Release.Name }} - run: {{ .Release.Name }}-tiler-visor -{{- end }} \ No newline at end of file diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml index 9743490c..b30717bf 100644 --- a/osm-seed/values.yaml +++ b/osm-seed/values.yaml @@ -186,11 +186,6 @@ memcached: memory: "8Gi" cpu: "2" # ==================================================================================================== -# Variables for id-editor -# ==================================================================================================== -idEditor: - enabled: false -# ==================================================================================================== # Variables for full-history container # ==================================================================================================== fullHistory: @@ -529,31 +524,6 @@ tilerServerCacheCleaner: maxReplicas: 2 cpuUtilization: 60 # ==================================================================================================== -# Variables for tiler-visor -# ==================================================================================================== -tilerVisor: - enabled: false - image: - name: "" - tag: "" - serviceAnnotations: {} - replicaCount: 1 - env: - TILER_VISOR_PROTOCOL: http - TILER_VISOR_PORT: 8081 - resources: - enabled: false - requests: - memory: "1Gi" - cpu: "2" - limits: - memory: "2Gi" - cpu: "2" - nodeSelector: - enabled: false - label_key: nodegroup_type - label_value: tiler -# ==================================================================================================== # Variables for Tasking Manager DB # ==================================================================================================== tmDb: