[HYD-890] Upgrade Spilo image to 3.2-p2 (#263)

* Update Spilo config files for 3.0-p1

* Remove pgq_ticker.ini fixes

This has been fixed in 3.0-p1: https://github.com/zalando/spilo/blob/3.0-p1/postgres-appliance/pgq_ticker.ini

* Have to use our fork with the PGVERSION fixes

* Bump Spilo to 3.2-p2

Dockerfile.spilo

@@ -25,7 +25,6 @@ COPY --from=columnar_15 /pg_ext /
 
 # configuration
 COPY files/spilo/postgres-appliance/scripts /scripts/
-COPY files/spilo/postgres-appliance/pgq_ticker.ini /home/postgres/
 
 ARG POSTGRES_BASE_VERSION
 # Default envs
@@ -35,5 +34,5 @@ ENV PGVERSION=${POSTGRES_BASE_VERSION} SPILO_PROVIDER=local PGUSER_SUPERUSER=pos
 # Always force rebuild of this layer
 ARG TIMESTAMP=1
 COPY third-party/pgxman /tmp/pgxman/
-RUN curl -sfL https://install.pgx.sh | sh -s -- /tmp/pgxman/pgxman_13.yaml /tmp/pgxman/pgxman_14_spilo.yaml && \
+RUN curl -sfL https://install.pgx.sh | sh -s -- /tmp/pgxman/pgxman_13_spilo.yaml /tmp/pgxman/pgxman_14_spilo.yaml && \
   rm -rf /tmp/pgxman

docker-bake.hcl

@@ -7,7 +7,7 @@ variable "SPILO_REPO" {
 }
 
 variable "SPILO_VERSION" {
-  default = "hydra-2.1-p9"
+  default = "3.2-p2"
 }
 
 variable "POSTGRES_BASE_VERSION" {
@@ -80,7 +80,7 @@ target "spilo" {
 target "spilo_base" {
   inherits = ["shared"]
 
-  context = "https://github.com/hydradatabase/spilo.git#${SPILO_VERSION}:postgres-appliance"
+  context = "https://github.com/zalando/spilo.git#${SPILO_VERSION}:postgres-appliance"
 
   args = {
     TIMESCALEDB = ""

files/spilo/postgres-appliance/scripts/configure_spilo.py

@@ -580,8 +580,10 @@ def get_placeholders(provider):
     placeholders.setdefault('CLONE_TARGET_TIME', '')
     placeholders.setdefault('CLONE_TARGET_INCLUSIVE', True)
 
+    placeholders.setdefault('LOG_GROUP_BY_DATE', False)
     placeholders.setdefault('LOG_SHIP_SCHEDULE', '1 0 * * *')
     placeholders.setdefault('LOG_S3_BUCKET', '')
+    placeholders.setdefault('LOG_S3_ENDPOINT', '')
     placeholders.setdefault('LOG_TMPDIR', os.path.abspath(os.path.join(placeholders['PGROOT'], '../tmp')))
     placeholders.setdefault('LOG_BUCKET_SCOPE_SUFFIX', '')
 
@@ -725,7 +727,9 @@ def get_dcs_config(config, placeholders):
         config['kubernetes']['labels'] = kubernetes_labels
 
         if not config['kubernetes'].pop('use_configmaps'):
-            config['kubernetes'].update({'use_endpoints': True, 'ports': [{'port': 5432, 'name': 'postgresql'}]})
+            config['kubernetes'].update({'use_endpoints': True,
+                                         'pod_ip': placeholders['instance_data']['ip'],
+                                         'ports': [{'port': 5432, 'name': 'postgresql'}]})
         if str(config['kubernetes'].pop('bypass_api_service', None)).lower() == 'true':
             config['kubernetes']['bypass_api_service'] = True
     else:
@@ -749,9 +753,12 @@ def write_log_environment(placeholders):
     aws_region = log_env.get('AWS_REGION')
     if not aws_region:
         aws_region = placeholders['instance_data']['zone'][:-1]
-    log_env['LOG_AWS_HOST'] = 's3.{}.amazonaws.com'.format(aws_region)
+
+    log_env['LOG_AWS_REGION'] = aws_region
 
     log_s3_key = 'spilo/{LOG_BUCKET_SCOPE_PREFIX}{SCOPE}{LOG_BUCKET_SCOPE_SUFFIX}/log/'.format(**log_env)
+    if os.getenv('LOG_GROUP_BY_DATE'):
+        log_s3_key += '{DATE}/'
     log_s3_key += placeholders['instance_data']['id']
     log_env['LOG_S3_KEY'] = log_s3_key
 
@@ -762,7 +769,7 @@ def write_log_environment(placeholders):
     if not os.path.exists(log_env['LOG_ENV_DIR']):
         os.makedirs(log_env['LOG_ENV_DIR'])
 
-    for var in ('LOG_TMPDIR', 'LOG_AWS_HOST', 'LOG_S3_KEY', 'LOG_S3_BUCKET', 'PGLOG'):
+    for var in ('LOG_TMPDIR', 'LOG_AWS_REGION', 'LOG_S3_ENDPOINT', 'LOG_S3_KEY', 'LOG_S3_BUCKET', 'PGLOG'):
         write_file(log_env[var], os.path.join(log_env['LOG_ENV_DIR'], var), True)
 
 
@@ -995,7 +1002,7 @@ def write_crontab(placeholders, overwrite):
         lines += [('{LOG_SHIP_SCHEDULE} nice -n 5 envdir "{LOG_ENV_DIR}"' +
                    ' /scripts/upload_pg_log_to_s3.py').format(**placeholders)]
 
-    lines += yaml.load(placeholders['CRONTAB'])
+    lines += yaml.safe_load(placeholders['CRONTAB'])
 
     if len(lines) > 1 or root_lines:
         setup_runit_cron(placeholders)
@@ -1051,10 +1058,11 @@ def main():
     placeholders = get_placeholders(provider)
     logging.info('Looks like you are running %s', provider)
 
-    config = yaml.load(pystache_render(TEMPLATE, placeholders))
+    config = yaml.safe_load(pystache_render(TEMPLATE, placeholders))
     config.update(get_dcs_config(config, placeholders))
 
-    user_config = yaml.load(os.environ.get('SPILO_CONFIGURATION', os.environ.get('PATRONI_CONFIGURATION', ''))) or {}
+    user_config = yaml.safe_load(os.environ.get('SPILO_CONFIGURATION',
+                                                os.environ.get('PATRONI_CONFIGURATION', ''))) or {}
     if not isinstance(user_config, dict):
         config_var_name = 'SPILO_CONFIGURATION' if 'SPILO_CONFIGURATION' in os.environ else 'PATRONI_CONFIGURATION'
         raise ValueError('{0} should contain a dict, yet it is a {1}'.format(config_var_name, type(user_config)))

files/spilo/postgres-appliance/scripts/post_init.sh

@@ -2,10 +2,13 @@
 
 cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1
 
+export PGOPTIONS="-c synchronous_commit=local -c search_path=pg_catalog"
+
 PGVER=$(psql -d "$2" -XtAc "SELECT pg_catalog.current_setting('server_version_num')::int/10000")
 if [ "$PGVER" -ge 12 ]; then RESET_ARGS="oid, oid, bigint"; fi
 
-(echo "DO \$\$
+(echo "\set ON_ERROR_STOP on"
+echo "DO \$\$
 BEGIN
     PERFORM * FROM pg_catalog.pg_authid WHERE rolname = 'admin';
     IF FOUND THEN
@@ -48,7 +51,7 @@ CREATE EXTENSION IF NOT EXISTS pg_auth_mon SCHEMA public;
 ALTER EXTENSION pg_auth_mon UPDATE;
 GRANT SELECT ON TABLE public.pg_auth_mon TO robot_zmon;
 
-CREATE EXTENSION IF NOT EXISTS pg_cron SCHEMA public;
+CREATE EXTENSION IF NOT EXISTS pg_cron SCHEMA pg_catalog;
 DO \$\$
 BEGIN
     PERFORM 1 FROM pg_catalog.pg_proc WHERE pronamespace = 'cron'::pg_catalog.regnamespace AND proname = 'schedule' AND proargnames = '{p_schedule,p_database,p_command}';
@@ -92,20 +95,10 @@ BEGIN
     RETURN l_jobid;
 END;
 \$function\$;
-REVOKE EXECUTE ON FUNCTION cron.alter_job(bigint, text, text, text, text, boolean) FROM admin, public;
-GRANT EXECUTE ON FUNCTION cron.alter_job(bigint, text, text, text, text, boolean) TO cron_admin;
-REVOKE EXECUTE ON FUNCTION cron.schedule(text, text) FROM admin, public;
-GRANT EXECUTE ON FUNCTION cron.schedule(text, text) TO cron_admin;
-REVOKE EXECUTE ON FUNCTION cron.schedule(text, text, text) FROM admin, public;
-GRANT EXECUTE ON FUNCTION cron.schedule(text, text, text) TO cron_admin;
-REVOKE EXECUTE ON FUNCTION cron.schedule_in_database(text, text, text) FROM admin, public;
-GRANT EXECUTE ON FUNCTION cron.schedule_in_database(text, text, text) TO cron_admin;
-REVOKE EXECUTE ON FUNCTION cron.schedule_in_database(text, text, text, text, text, boolean) FROM admin, public;
-GRANT EXECUTE ON FUNCTION cron.schedule_in_database(text, text, text, text, text, boolean) TO cron_admin;
-REVOKE EXECUTE ON FUNCTION cron.unschedule(bigint) FROM admin, public;
-GRANT EXECUTE ON FUNCTION cron.unschedule(bigint) TO cron_admin;
-REVOKE EXECUTE ON FUNCTION cron.unschedule(name) FROM admin, public;
-GRANT EXECUTE ON FUNCTION cron.unschedule(name) TO cron_admin;
+
+REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cron FROM admin, public;
+GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cron TO cron_admin;
+
 REVOKE USAGE ON SCHEMA cron FROM admin;
 GRANT USAGE ON SCHEMA cron TO cron_admin;
 
@@ -179,6 +172,10 @@ while IFS= read -r db_name; do
     if [ "$UPGRADE_TIMESCALEDB" = "t" ]; then
         echo "ALTER EXTENSION timescaledb UPDATE;"
     fi
+    UPGRADE_TIMESCALEDB_TOOLKIT=$(echo -e "SELECT NULL;\nSELECT default_version != installed_version FROM pg_catalog.pg_available_extensions WHERE name = 'timescaledb_toolkit'" | psql -tAX -d "${db_name}" 2> /dev/null | tail -n 1)
+    if [ "$UPGRADE_TIMESCALEDB_TOOLKIT" = "t" ]; then
+        echo "ALTER EXTENSION timescaledb_toolkit UPDATE;"
+    fi
     UPGRADE_POSTGIS=$(echo -e "SELECT COUNT(*) FROM pg_catalog.pg_extension WHERE extname = 'postgis'" | psql -tAX -d "${db_name}" 2> /dev/null | tail -n 1)
     if [ "$UPGRADE_POSTGIS" = "1" ]; then
         # public.postgis_lib_version() is available only if postgis extension is created
@@ -207,4 +204,4 @@ GRANT EXECUTE ON FUNCTION public.pg_stat_statements_reset($RESET_ARGS) TO admin;
     if [ "$ENABLE_PG_MON" = "true" ] && [ "$PGVER" -ge 11 ]; then echo "CREATE EXTENSION IF NOT EXISTS pg_mon SCHEMA public;"; fi
     cat metric_helpers.sql
 done < <(psql -d "$2" -tAc 'select pg_catalog.quote_ident(datname) from pg_catalog.pg_database where datallowconn')
-) | PGOPTIONS="-c synchronous_commit=local" psql -Xd "$2"
+) | psql -Xd "$2"

files/spilo/postgres-appliance/scripts/spilo_commons.py

@@ -13,16 +13,16 @@
 # (min_version, max_version, shared_preload_libraries, extwlist.extensions)
 extensions = {
     'columnar': (13, 16, True,  True),
-    #'timescaledb':    (9.6, 14, True,  True),
+    #'timescaledb':    (9.6, 16, True,  True),
     'pg_cron':        (9.5, 15, True,  True),
-    'pg_stat_kcache': (9.4, 15, True,  False),
-    'pg_partman':     (9.4, 15, False, True),
+    'pg_stat_kcache': (9.4, 16, True,  False),
+    'pg_partman':     (9.4, 16, False, True),
     'wrappers':       (14, 16, False, True),
     'pgsodium':       (14, 16, False, True),
     'supabase_vault': (14, 16, False, True)
 }
 if os.environ.get('ENABLE_PG_MON') == 'true':
-    extensions['pg_mon'] = (11,  15, True,  False)
+    extensions['pg_mon'] = (11,  16, True,  False)
 
 
 def adjust_extensions(old, version, extwlist=False):

third-party/pgxman/pgxman_13_spilo.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+postgres:
+  version: "13"
+extensions:
+  - name: "multicorn"
+    version: "2.4.0+b68b75c"
+  - name: "mysql_fdw"
+    version: "2.9.1"
+  - name: "parquet_s3_fdw"
+    version: "1.0.0+5298b7f"
+  - name: "pg_ivm"
+    version: "1.7.0"
+  - name: "pgvector"
+    version: "0.5.1"
+    overwrite: true
+  - name: "pg_hint_plan"
+    version: "1.3.9"

third-party/pgxman/pgxman_14_spilo.yaml

@@ -14,6 +14,7 @@ extensions:
     version: "1.7.0"
   - name: "pgvector"
     version: "0.5.1"
+    overwrite: true
   - name: "pg_hint_plan"
     version: "1.4.2"
   - name: "wrappers"