TB-124 Add pgpool connection manager

.env.cluster

@@ -21,9 +21,10 @@ OPENHIM_MONGO_ATNAURL=mongodb://mongo-1:27017,mongo-2:27017,mongo-3:27017/openhi
 HAPI_FHIR_INSTANCES=3
 REPMGR_PARTNER_NODES=postgres-1,postgres-2,postgres-3
 POSTGRES_REPLICA_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432
+HAPI_DB_SET=pgpool-1:5432,pgpool-2:5432,pgpool-3:5432
 
 # Sante Datastore - Sante MPI
-REPMGR_PARTNER_NODES=santempi-psql-1,santempi-psql-2,santempi-psql-3
+SANTEMPI_REPMGR_PARTNER_NODES=santempi-psql-1,santempi-psql-2,santempi-psql-3
 
 # Reverse Proxy - Nginx
 REVERSE_PROXY_INSTANCES=3

.env.remote

@@ -7,7 +7,6 @@ OPENHIM_TRANSACTION_API_PORT=<port>
 OPENHIM_MEDIATOR_API_PORT=<OPENHIM_MEDIATOR_API_PORT>
 MOCK_SERVER_HOST=<mock_service_hostname>
 MOCK_SERVER_PORT=<port>
-POSTGRES_REPLICA_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432
 
 # Analytics Datastore - Elastic Search
 ES_HOSTS="\"analytics-datastore-elastic-search-01:9200","analytics-datastore-elastic-search-02:9200","analytics-datastore-elastic-search-03:9200\""
@@ -18,6 +17,7 @@ CLICKHOUSE_HOST=analytics-datastore-clickhouse-01
 # FHIR Datastore - HAPI FHIR
 REPMGR_PARTNER_NODES=postgres-1,postgres-2,postgres-3
 POSTGRES_REPLICA_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432
+HAPI_DB_SET=postgres-1:5432,postgres-2:5432,postgres-3:5432
 
 # Interoperability Layer - OpenHIM
 MONGO_SET_COUNT=3

.github/workflows/ci.yml

@@ -95,7 +95,6 @@ jobs:
       - run: sed -i 's/OPENHIM_MEDIATOR_API_PORT=8080/OPENHIM_MEDIATOR_API_PORT=443/g' .env.local
       - run: sed -i 's/INSECURE=true/INSECURE=false/g' .env.local
       - run: sed -i 's/domain/${{ github.run_id }}.jembi.cloud/g' .env.local
-
       - name: Install dependencies
         working-directory: ./test/cucumber
         run: yarn

analytics-datastore-elastic-search/swarm.sh

@@ -78,7 +78,7 @@ function create_certs() {
 
 function add_docker_configs() {
   local -r TIMESTAMP="$(date "+%Y%m%d%H%M%S")"
-  local -r path_config_certs="/usr/share/elasticsearch/config/certs/"
+  local -r path_config_certs="/usr/share/elasticsearch/config/certs"
 
   log info "Creating configs..."
 

config.yaml

@@ -1,5 +1,5 @@
 projectName: platform
-image: jembi/platform:2.3.3
+image: jembi/platform:2.4.0
 logPath: /tmp/logs
 
 packages:

fhir-datastore-hapi-fhir/config/custom_pgpool.conf

@@ -0,0 +1,29 @@
+failover_on_backend_error='on'
+detach_false_primary='on'
+auto_failback='on'
+health_check_period=25
+health_check_max_retries=3
+health_check_user='postgres'
+reserved_connections=1
+max_pool=6
+client_idle_limit=300
+connection_life_time=300
+child_max_connections=25
+
+use_watchdog='on'
+hostname0='pgpool-1'
+wd_port0=9000
+pgpool_port0=9999
+hostname1='pgpool-2'
+wd_port1=9000
+pgpool_port1=9999
+hostname2='pgpool-3'
+wd_port2=9000
+pgpool_port2=9999
+heartbeat_hostname0='pgpool-1'
+heartbeat_port0=9694
+heartbeat_hostname1='pgpool-2'
+heartbeat_port1=9694
+heartbeat_hostname2='pgpool-3'
+heartbeat_port2=9694
+trusted_servers='pgpool-1,pgpool-2,pgpool-3'

fhir-datastore-hapi-fhir/config/pgpool_node_id0

@@ -0,0 +1 @@
+0

fhir-datastore-hapi-fhir/config/pgpool_node_id1

@@ -0,0 +1 @@
+1

fhir-datastore-hapi-fhir/config/pgpool_node_id2

@@ -0,0 +1 @@
+2

fhir-datastore-hapi-fhir/docker-compose-postgres.cluster.yml

@@ -21,6 +21,7 @@ services:
       REPMGR_NODE_NAME: postgres-2
       REPMGR_PRIMARY_HOST: ${REPMGR_PRIMARY_HOST}
       REPMGR_PARTNER_NODES: ${REPMGR_PARTNER_NODES}
+      POSTGRESQL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD}
     volumes:
       - 'hapi-postgres-2-data:/bitnami/postgresql'
     deploy:
@@ -37,9 +38,7 @@ services:
           memory: ${HF_POSTGRES_MEMORY_RESERVE}
     networks:
       default:
-      hapi-fhir-postgres:
-      pg_backup_net: {}
-
+      pg_backup_net:
 
   postgres-3:
     image: bitnami/postgresql-repmgr:14
@@ -53,6 +52,7 @@ services:
       REPMGR_NODE_NAME: postgres-3
       REPMGR_PRIMARY_HOST: ${REPMGR_PRIMARY_HOST}
       REPMGR_PARTNER_NODES: ${REPMGR_PARTNER_NODES}
+      POSTGRESQL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD}
     volumes:
       - 'hapi-postgres-3-data:/bitnami/postgresql'
     deploy:
@@ -69,9 +69,131 @@ services:
           memory: ${HF_POSTGRES_MEMORY_RESERVE}
     networks:
       default:
-      hapi-fhir-postgres:
-      pg_backup_net: {}
+      pg_backup_net:
+
+  pgpool-1:
+    image: bitnami/pgpool:4.4.3
+    deploy:
+      placement:
+        max_replicas_per_node: 1
+      resources:
+        limits:
+          cpus: ${HF_PGPOOL_CPU_LIMIT}
+          memory: ${HF_PGPOOL_MEMORY_LIMIT}
+        reservations:
+          cpus: ${HF_PGPOOL_CPU_RESERVE}
+          memory: ${HF_PGPOOL_MEMORY_RESERVE}
+    configs:
+      - target: /config/custom_pgpool.conf
+        source: pgpool.conf
+      - target: /opt/bitnami/pgpool/conf/pgpool_node_id
+        source: pgpool_node_id0.conf
+    environment:
+      PGPOOL_BACKEND_NODES: ${HF_PGPOOL_NODES}
+      PGPOOL_BACKEND_APPLICATION_NAMES: ${REPMGR_PARTNER_NODES}
+      PGPOOL_SR_CHECK_USER: postgres
+      PGPOOL_SR_CHECK_PASSWORD: ${HAPI_POSTGRES_PASSWORD}
+      PGPOOL_POSTGRES_USERNAME: postgres
+      PGPOOL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD}
+      PGPOOL_ADMIN_USERNAME: ${HF_PGPOOL_USERNAME}
+      PGPOOL_ADMIN_PASSWORD: ${HF_PGPOOL_PASSWORD}
+      PGPOOL_POSTGRES_CUSTOM_USERS: ${HF_POSTGRESQL_USERNAME}
+      PGPOOL_POSTGRES_CUSTOM_PASSWORDS: ${HF_POSTGRESQL_PASSWORD}
+      PGPOOL_ENABLE_LOAD_BALANCING: "no"
+      PGPOOL_AUTO_FAILBACK: "yes"
+      PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf"
+    networks:
+      default:
+
+  pgpool-2:
+    image: bitnami/pgpool:4.4.3
+    deploy:
+      placement:
+        max_replicas_per_node: 1
+      resources:
+        limits:
+          cpus: ${HF_PGPOOL_CPU_LIMIT}
+          memory: ${HF_PGPOOL_MEMORY_LIMIT}
+        reservations:
+          cpus: ${HF_PGPOOL_CPU_RESERVE}
+          memory: ${HF_PGPOOL_MEMORY_RESERVE}
+    configs:
+      - target: /config/custom_pgpool.conf
+        source: pgpool.conf
+      - target: /opt/bitnami/pgpool/conf/pgpool_node_id
+        source: pgpool_node_id1.conf
+    environment:
+      PGPOOL_BACKEND_NODES: ${HF_PGPOOL_NODES}
+      PGPOOL_BACKEND_APPLICATION_NAMES: ${REPMGR_PARTNER_NODES}
+      PGPOOL_SR_CHECK_USER: postgres
+      PGPOOL_SR_CHECK_PASSWORD: ${HAPI_POSTGRES_PASSWORD}
+      PGPOOL_POSTGRES_USERNAME: postgres
+      PGPOOL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD}
+      PGPOOL_ADMIN_USERNAME: ${HF_PGPOOL_USERNAME}
+      PGPOOL_ADMIN_PASSWORD: ${HF_PGPOOL_PASSWORD}
+      PGPOOL_POSTGRES_CUSTOM_USERS: ${HF_POSTGRESQL_USERNAME}
+      PGPOOL_POSTGRES_CUSTOM_PASSWORDS: ${HF_POSTGRESQL_PASSWORD}
+      PGPOOL_ENABLE_LOAD_BALANCING: "no"
+      PGPOOL_AUTO_FAILBACK: "yes"
+      PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf"
+    networks:
+      default:
+
+  pgpool-3:
+    image: bitnami/pgpool:4.4.3
+    deploy:
+      placement:
+        max_replicas_per_node: 1
+      resources:
+        limits:
+          cpus: ${HF_PGPOOL_CPU_LIMIT}
+          memory: ${HF_PGPOOL_MEMORY_LIMIT}
+        reservations:
+          cpus: ${HF_PGPOOL_CPU_RESERVE}
+          memory: ${HF_PGPOOL_MEMORY_RESERVE}
+    configs:
+      - target: /config/custom_pgpool.conf
+        source: pgpool.conf
+      - target: /opt/bitnami/pgpool/conf/pgpool_node_id
+        source: pgpool_node_id2.conf
+    environment:
+      PGPOOL_BACKEND_NODES: ${HF_PGPOOL_NODES}
+      PGPOOL_BACKEND_APPLICATION_NAMES: ${REPMGR_PARTNER_NODES}
+      PGPOOL_SR_CHECK_USER: postgres
+      PGPOOL_SR_CHECK_PASSWORD: ${HAPI_POSTGRES_PASSWORD}
+      PGPOOL_POSTGRES_USERNAME: postgres
+      PGPOOL_POSTGRES_PASSWORD: ${HAPI_POSTGRES_PASSWORD}
+      PGPOOL_ADMIN_USERNAME: ${HF_PGPOOL_USERNAME}
+      PGPOOL_ADMIN_PASSWORD: ${HF_PGPOOL_PASSWORD}
+      PGPOOL_POSTGRES_CUSTOM_USERS: ${HF_POSTGRESQL_USERNAME}
+      PGPOOL_POSTGRES_CUSTOM_PASSWORDS: ${HF_POSTGRESQL_PASSWORD}
+      PGPOOL_ENABLE_LOAD_BALANCING: "no"
+      PGPOOL_AUTO_FAILBACK: "yes"
+      PGPOOL_USER_CONF_FILE: "/config/custom_pgpool.conf"
+    networks:
+      default:
 
+configs:
+  pgpool.conf:
+    name: pgpool_conf-${pgpool_conf_DIGEST:?err}
+    file: ./config/custom_pgpool.conf
+    labels:
+      name: hapi-fhir
+  pgpool_node_id0.conf:
+    name: pgpool_node_id0-${pgpool_node_id0_DIGEST:?err}
+    file: ./config/pgpool_node_id0
+    labels:
+      name: hapi-fhir
+  pgpool_node_id1.conf:
+    name: pgpool_node_id1-${pgpool_node_id1_DIGEST:?err}
+    file: ./config/pgpool_node_id1
+    labels:
+      name: hapi-fhir
+  pgpool_node_id2.conf:
+    name: pgpool_node_id2-${pgpool_node_id2_DIGEST:?err}
+    file: ./config/pgpool_node_id2
+    labels:
+      name: hapi-fhir
 
 volumes:
   hapi-postgres-2-data:

fhir-datastore-hapi-fhir/docker-compose.yml

@@ -8,20 +8,24 @@ services:
       public:
       default:
     environment:
-      - spring.datasource.url=jdbc:postgresql://${POSTGRES_REPLICA_SET}/hapi?targetServerType=primary
-      - spring.datasource.username=admin
-      - spring.datasource.password=instant101
+      - spring.datasource.url=jdbc:postgresql://${HAPI_DB_SET}/hapi?targetServerType=primary
+      - spring.datasource.username=${HF_POSTGRESQL_USERNAME}
+      - spring.datasource.password=${HF_POSTGRESQL_PASSWORD}
       - spring.datasource.driverClassName=org.postgresql.Driver
       - spring.jpa.properties.hibernate.dialect=org.hibernate.dialect.PostgreSQL95Dialect
+      - spring.datasource.hikari.maximumPoolSize=250
+      - spring.datasource.hikari.minimumIdle=20
+      - spring.datasource.hikari.connectionTimeout=60000
+      - spring.datasource.hikari.idleTimeout=600000
       - hapi.fhir.allow_external_references=true
       - hapi.fhir.bulk_export_enabled=true
       - hapi.fhir.enable_repository_validating_interceptor=true
-      - JAVA_TOOL_OPTIONS=-Xmx2g
-      - CATALINA_OPTS=-Xmx2g
+      - JAVA_TOOL_OPTIONS=${HF_JAVA_OPTS}
+      - CATALINA_OPTS=${HF_JAVA_OPTS}
     deploy:
       replicas: ${HAPI_FHIR_INSTANCES}
       placement:
-        max_replicas_per_node: 1
+        max_replicas_per_node: ${HAPI_FHIR_MAX_REPLICAS_PER_NODE}
       resources:
         limits:
           cpus: ${HAPI_FHIR_CPU_LIMIT}
@@ -30,10 +34,10 @@ services:
           cpus: ${HAPI_FHIR_CPU_RESERVE}
           memory: ${HAPI_FHIR_MEMORY_RESERVE}
     healthcheck:
-      test: /bin/wget --no-verbose --tries=1 --spider http://localhost:8080
+      test: /bin/wget --no-verbose --tries=1 --spider http://localhost:8080/fhir/Organization?identifier=urn:healthcheck
       interval: 30s
-      timeout: 5s
-      retries: 5
+      timeout: 30s
+      retries: 2
       start_period: 2m
 
 networks:

fhir-datastore-hapi-fhir/package-metadata.json

@@ -10,18 +10,28 @@
     "REPMGR_PARTNER_NODES": "postgres-1",
     "REPMGR_PASSWORD": "instant101",
     "HAPI_FHIR_INSTANCES": "1",
+    "HAPI_FHIR_MAX_REPLICAS_PER_NODE": "1",
     "HAPI_FHIR_CPU_LIMIT": "0",
     "HAPI_FHIR_CPU_RESERVE": "0.05",
     "HAPI_FHIR_MEMORY_LIMIT": "3G",
     "HAPI_FHIR_MEMORY_RESERVE": "500M",
     "POSTGRES_REPLICA_SET": "postgres-1:5432",
+    "HAPI_DB_SET": "postgres-1:5432",
     "HAPI_POSTGRES_PASSWORD": "postgres",
     "HF_POSTGRES_CPU_LIMIT": "0",
     "HF_POSTGRES_CPU_RESERVE": "0.05",
     "HF_POSTGRES_MEMORY_LIMIT": "3G",
     "HF_POSTGRES_MEMORY_RESERVE": "500M",
     "HF_POSTGRESQL_PASSWORD": "instant101",
     "HF_POSTGRESQL_USERNAME": "admin",
-    "HF_POSTGRESQL_DATABASE": "hapi"
+    "HF_POSTGRESQL_DATABASE": "hapi",
+    "HF_PGPOOL_NODES": "0:postgres-1:5432,1:postgres-2:5432,2:postgres-3:5432",
+    "HF_PGPOOL_USERNAME": "pgadmin",
+    "HF_PGPOOL_PASSWORD": "dev_password_only",
+    "HF_PGPOOL_CPU_LIMIT": "0",
+    "HF_PGPOOL_MEMORY_LIMIT": "500M",
+    "HF_PGPOOL_CPU_RESERVE": "0.05",
+    "HF_PGPOOL_MEMORY_RESERVE": "50M",
+    "HF_JAVA_OPTS": "-Xmx2g"
   }
 }

message-bus-helper-hapi-proxy/docker-compose.yml

@@ -8,6 +8,7 @@ services:
       HAPI_SERVER_URL: ${HAPI_SERVER_URL}
       KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_HOSTS}
       HAPI_SERVER_VALIDATE_FORMAT: ${HAPI_SERVER_VALIDATE_FORMAT}
+      JAVA_OPTS: "-XX:MaxRAMPercentage=90"
     deploy:
       placement:
         max_replicas_per_node: 1

test/cucumber/features/cluster-mode/hapi-fhir.cluster.feature

@@ -9,21 +9,28 @@ Feature: Fhir Datastore HAPI-FHIR?
       | hapi-fhir_postgres_public | hapi-fhir_default | pg_backup |
     And The service "postgres-2" should be started with 1 replica
     And The service "postgres-2" should be connected to the networks
-      | hapi-fhir_postgres_public | hapi-fhir_default | pg_backup |
+      | hapi-fhir_default | pg_backup |
     And The service "postgres-3" should be started with 1 replica
     And The service "postgres-3" should be connected to the networks
-      | hapi-fhir_postgres_public | hapi-fhir_default | pg_backup |
+      | hapi-fhir_default | pg_backup |
+        And The service "postgres-3" should be started with 1 replica
+    And The service "pgpool-1" should be connected to the networks
+      | hapi-fhir_default |
+    And The service "pgpool-2" should be connected to the networks
+      | hapi-fhir_default |
+    And The service "pgpool-3" should be connected to the networks
+      | hapi-fhir_default |
     And The service "hapi-fhir" should be started with 3 replicas
     And The service "hapi-fhir" should be connected to the networks
       | mpi_public | hapi-fhir_public | hapi-fhir_default |
-    And There should be 4 services
+    And There should be 7 services
     And The service "hapi-fhir" should have healthy containers
 
   Scenario: Init Message Bus Helper Hapi Proxy
     Given I use parameters "package init -n=message-bus-helper-hapi-proxy --only --dev --env-file=.env.cluster"
     When I launch the platform with params
     Then The service "hapi-proxy" should be started with 3 replicas
-    And There should be 5 services
+    And There should be 8 services
     And The service "hapi-proxy" should be connected to the networks
       | hapi-fhir_public | kafka_public | openhim_public |
 
@@ -33,6 +40,9 @@ Feature: Fhir Datastore HAPI-FHIR?
     Then The service "postgres-1" should be removed
     And The service "postgres-2" should be removed
     And The service "postgres-3" should be removed
+    And The service "pgpool-1" should be removed
+    And The service "pgpool-2" should be removed
+    And The service "pgpool-3" should be removed
     And The service "hapi-fhir" should be removed
     And The service "hapi-proxy" should be removed
     And There should be 0 service