Merge pull request #494 from OpenHistoricalMap/staging

Optimizing Vector Tiles: Merging Duplicated Boundary Lines

Author: Rub21

.github/workflows/chartpress.yaml

@@ -5,7 +5,7 @@ on:
       - 'main'
       - 'staging'
       - 'development'
-      - 'refactor/web_docker'
+      - 'admin_lines'
 jobs:
   build:
     runs-on: ubuntu-20.04

.github/workflows/frontend-nominatim.yaml

@@ -5,7 +5,6 @@ on:
     branches:
       - 'main'
       - 'staging'
-      - 'nominatim-ui'
 
 jobs:
   nominatim:
@@ -17,10 +16,10 @@ jobs:
         with:
           access_token: ${{ github.token }}
 
-      - name: Set environment variables - Staging
+      - name: Set environment variables - Staging & Nominatim-UI-2
         if: github.ref == 'refs/heads/staging'
         run: |
-          echo "NOMINATIM_API=https://nominatim-api.staging.openhistoricalmap.org/" >> $GITHUB_ENV
+          echo "NOMINATIM_API=https://nominatim-api.openhistoricalmap.org/" >> $GITHUB_ENV
           echo "NOMINATIM_BUCKET=nominatim-staging.openhistoricalmap.org" >> $GITHUB_ENV
           echo "CLOUDFRONT_DISTRIBUTION_ID=${{ secrets.STAGING_NOMINATIM_CLOUDFRONT_ID }}" >> $GITHUB_ENV
 
@@ -38,10 +37,15 @@ jobs:
           ref: c14d033389acedf4dcbaabd9d88e14bc40212279
           fetch-depth: 0
 
+      - name: Install Dependencies with Legacy Peer Deps
+        run: |
+          npm install --legacy-peer-deps
+          npm install rollup-plugin-svelte@latest --save-dev --legacy-peer-deps
+
       - name: Build nominatim-ui frontend
         run: |
           echo "Nominatim_Config.Nominatim_API_Endpoint = '${NOMINATIM_API}';" >> dist/theme/config.theme.js
-          # npm run build
+          npm run build
 
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -55,7 +59,7 @@ jobs:
 
       - name: Deploy to S3 and Invalidate CloudFront
         run: |
-          aws s3 sync dist/ s3://${NOMINATIM_BUCKET}/ --delete
+          aws s3 sync dist/ s3://${NOMINATIM_BUCKET}/
           aws cloudfront create-invalidation --distribution-id=${CLOUDFRONT_DISTRIBUTION_ID} --paths "/*"
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

.github/workflows/frontend-overpass.yaml

@@ -44,7 +44,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: OpenHistoricalMap/overpass-turbo
-          ref: ac9201e6b9a28cf6883335d4a956b3120509c972
+          ref: 1a57c9348fb9b23a9ecb1109b54f3fc311331213
           path: overpass-turbo
 
       - name: Enable Corepack

hetzner/config/postgresql.production.conf

@@ -1,77 +1,77 @@
 #------------------------------------------------------------------------------
 # CONNECTIONS AND AUTHENTICATION
 #------------------------------------------------------------------------------
-
-listen_addresses = '*'                  # Allow connections from all addresses
-max_connections = 200                     # Increase connections for a larger server
-superuser_reserved_connections = 5       # Reserve for superusers
+listen_addresses = '*'                    # Allow connections from any network interface
+max_connections = 200                      # Increase if you expect more concurrent connections
+superuser_reserved_connections = 5         # Reserve connections for superusers
 
 #------------------------------------------------------------------------------
 # RESOURCE USAGE
 #------------------------------------------------------------------------------
 
 # - Memory Configuration -
-shared_buffers = 14GB                    # Allocate ~25% of total memory
-work_mem = 512MB                         # Increase per-query memory
-maintenance_work_mem = 4GB               # Allow more memory for VACUUM/ALTER operations
-effective_cache_size = 41GB              # 75% of total memory for caching
+shared_buffers = 14GB                      # ~25% of total 55GB; adjust if needed
+work_mem = 256MB                           # Memory for each sort/hash operation; be cautious with many parallel queries
+maintenance_work_mem = 4GB                 # Larger memory for VACUUM / CREATE INDEX / ALTER
+effective_cache_size = 36GB                # ~60-70% of total memory to inform the planner
 
-# - Disk Optimization for SSD -
-random_page_cost = 1.0                   # Optimize for SSD storage
-seq_page_cost = 1.0                      # Default for sequential scans
+# - Disk Optimization for SSD (if using SSD) -
+random_page_cost = 1.0                     # Lower cost for random I/O on SSD
+seq_page_cost = 1.0                        # Default cost for sequential scans
 
 #------------------------------------------------------------------------------
 # WRITE-AHEAD LOG (WAL)
 #------------------------------------------------------------------------------
-
-wal_level = replica                      # Required for replication if needed
-checkpoint_timeout = 15min               # Increase checkpoint frequency
-max_wal_size = 4GB                       # Allow more WAL before forcing a checkpoint
-min_wal_size = 1GB                       # Set a lower boundary for WAL size
-wal_compression = lz4                    # Enable LZ4 compression for WAL files (default in PG 17)
-synchronous_commit = off                 # Improve performance by reducing commit overhead
+wal_level = replica                        # Required for replication; 'minimal' if no replication is used
+checkpoint_timeout = 15min                 # Interval between automatic WAL checkpoints
+max_wal_size = 4GB                         # Amount of WAL to accumulate before forcing a checkpoint
+min_wal_size = 1GB                         # Minimum size to keep WAL files
+wal_compression = lz4                      # Compress WAL segments to reduce disk usage (PG17 default)
+synchronous_commit = off                   # Improves write performance, risk of data loss if crash
 
 #------------------------------------------------------------------------------
 # AUTOVACUUM SETTINGS
 #------------------------------------------------------------------------------
-
-autovacuum_max_workers = 6               # More workers due to higher CPU count
-autovacuum_naptime = 30s                 # Run autovacuum frequently
-autovacuum_vacuum_cost_limit = -1        # Allow PostgreSQL to auto-adjust vacuum cost
+autovacuum_max_workers = 6                 # More parallel vacuum workers for busy systems
+autovacuum_naptime = 30s                   # How often the autovacuum daemon checks for work
+autovacuum_vacuum_cost_limit = -1          # Let PostgreSQL adjust vacuum cost dynamically
 
 #------------------------------------------------------------------------------
 # QUERY TUNING
 #------------------------------------------------------------------------------
-
-effective_io_concurrency = 300           # Optimize IO operations for SSDs
-parallel_tuple_cost = 0.1                # Encourage parallel execution
-parallel_setup_cost = 500                # Lower parallel setup cost
-max_worker_processes = 28                # Utilize all CPU cores
-max_parallel_workers_per_gather = 8      # Allow more parallel workers per query
-max_parallel_workers = 28                # Utilize all available parallel workers
+effective_io_concurrency = 300             # For SSD; helps the planner estimate IO concurrency
+parallel_tuple_cost = 0.1                  # Lower cost to encourage parallelization
+parallel_setup_cost = 500                  # Lower to encourage more parallel plans
+max_worker_processes = 28                  # Allow up to 28 worker processes
+max_parallel_workers_per_gather = 8        # Max workers that can help a single query
+max_parallel_workers = 28                  # Total number of parallel workers across all queries
 
 #------------------------------------------------------------------------------
 # LOGGING
 #------------------------------------------------------------------------------
-
-logging_collector = off                  # Disable logging to improve performance
-log_statement = 'none'                   # Disable all SQL statement logging
-log_min_duration_statement = -1          # Do not log slow queries
-log_duration = off                       # Disable duration logging
-log_error_verbosity = terse              # Show minimal error messages
-
-; #------------------------------------------------------------------------------
-; # CLIENT CONNECTION DEFAULTS
-; #------------------------------------------------------------------------------
-
-; statement_timeout = 600000               # Terminate queries running longer than 10 min
-; lock_timeout = 600000                    # Timeout for acquiring locks
-; idle_in_transaction_session_timeout = 600000 # Close idle transactions after 10 min
+logging_collector = off                   # Disable log collection
+log_statement = 'none'                     # Do not log any statements
+log_duration = off                         # Disable logging query duration
+log_min_duration_statement = -1            # Disable logging slow queries
+log_error_verbosity = terse                # Minimal error messages
+log_autovacuum_min_duration = -1           # Do not log autovacuum runs
+log_connections = on                      # Do not log new connections
+log_disconnections = on                   # Do not log disconnections
+log_lock_waits = off                       # Do not log lock waits
+log_temp_files = -1                        # Do not log temporary file creation
+log_checkpoints = off                      # Do not log checkpoints
+log_replication_commands = off             # Do not log replication-related commands
+log_directory = '/dev/null'                # Redirect logs to /dev/null (no storage)
+#------------------------------------------------------------------------------
+# CLIENT CONNECTION DEFAULTS
+#------------------------------------------------------------------------------
+statement_timeout = 600000                 # Kill any query running longer than 10 minutes
+lock_timeout = 600000                      # Timeout for acquiring locks
+idle_in_transaction_session_timeout = 600000 # Close transactions idle for more than 10 minutes
 
 #------------------------------------------------------------------------------
 # LOCALE AND TIMING
 #------------------------------------------------------------------------------
-
 datestyle = 'iso, mdy'
 timezone = 'Etc/UTC'
 lc_messages = 'en_US.utf8'

hetzner/seed.sh

@@ -11,7 +11,7 @@ seed_global() {
       --map=osm \
       --min-zoom=0 \
       --max-zoom=5 \
-      --concurrency=32 \
+      --concurrency=4 \
       --overwrite=true
 
     # Seed land areas separately (zoom 6-7)
@@ -24,11 +24,11 @@ seed_global() {
         --map=osm \
         --min-zoom="$zoom" \
         --max-zoom="$zoom" \
-        --concurrency=32 \
+        --concurrency=4 \
         --overwrite=true
     done
-    echo "Global seeding completed. Sleeping for 1 hour..."
-    sleep 3600
+    echo "Global seeding completed. Sleeping for 3 hour..."
+    sleep 10800
   done
 }
 
@@ -38,13 +38,18 @@ seed_coverage() {
     wget -O /opt/tile-list.tiles "https://s3.amazonaws.com/planet.openhistoricalmap.org/tile_coverage/tiles_14.list"
     pkill -f "tegola" && sleep 5
 
-    tegola cache seed tile-list /opt/tile-list.tiles \
-      --config=/opt/tegola_config/config.toml \
-      --map=osm \
-      --min-zoom=0 \
-      --max-zoom=14 \
-      --concurrency=32 \
-      --overwrite=false
+    for zoom in $(seq 8 14); do
+      echo "Downloading tile list for zoom level $zoom..."
+      wget -O /opt/tile-list.tiles "https://s3.amazonaws.com/planet.openhistoricalmap.org/tile_coverage/tiles_boundary_$zoom.list"
+
+      tegola cache seed tile-list /opt/tile-list.tiles \
+        --config=/opt/tegola_config/config.toml \
+        --map=osm \
+        --min-zoom=$zoom \
+        --max-zoom=$zoom \
+        --concurrency=4 \
+        --overwrite=false
+    done
     sleep 300
   done
 }

hetzner/tiler.production.yml

@@ -24,7 +24,7 @@ services:
 
   # imposm3
   imposm_production:
-    image: ghcr.io/openhistoricalmap/tiler-imposm:0.0.1-0.dev.git.2191.hdcb5877
+    image: ghcr.io/openhistoricalmap/tiler-imposm:0.0.1-0.dev.git.2296.h6936774
     volumes:
       - /data/tiler_production_imposm_20250211:/mnt/data
     command:
@@ -48,14 +48,14 @@ services:
       - tiler_network
   # Tiler server
   tiler_production:
-      image: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.2248.he042bfb
-      ports:
-        - "9090:9090"
-      env_file:
-        - .env.production
-      restart: always
-      networks:
-        - tiler_network
+    image: ghcr.io/openhistoricalmap/tiler-cache:0.0.1-0.dev.git.2238.h67c76c8
+    ports:
+      - "9090:9090"
+    env_file:
+      - .env.production
+    restart: always
+    networks:
+      - tiler_network
   # Tiler cache
   cache_production:
     image: ghcr.io/openhistoricalmap/tiler-cache:0.0.1-0.dev.git.2238.h67c76c8

hetzner/tiler.staging.yml

@@ -24,7 +24,7 @@ services:
           memory: 2G
 
   imposm:
-    image: ghcr.io/openhistoricalmap/tiler-imposm:0.0.1-0.dev.git.2191.hdcb5877
+    image: ghcr.io/openhistoricalmap/tiler-imposm:0.0.1-0.dev.git.2296.h6936774
     volumes:
       - /data/tiler_production_imposm_20250225_v7:/mnt/data
     command:
@@ -43,7 +43,7 @@ services:
           cpus: '1'
           memory: 1G
   tiler:
-    image: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.2248.he042bfb
+    image: ghcr.io/openhistoricalmap/tiler-cache:0.0.1-0.dev.git.2238.h67c76c8
     ports:
       - "9091:9090"
     env_file:

images/tiler-imposm/queries/date_utils.sql

@@ -0,0 +1,63 @@
+--This is fuction to convert decimal date to iso date, in that it handles BCE dates properly, and it also returns NULL if any conversion error happens.
+--It is used in images/tiler-imposm/queries/mviews_admin_boundaries_merged.sql
+--This script differs from https://github.com/OpenHistoricalMap/DateFunctions-plpgsql/blob/master/datefunctions.sql
+CREATE OR REPLACE FUNCTION convert_decimal_to_iso_date(decimal_date NUMERIC)
+RETURNS VARCHAR AS $$
+DECLARE
+    year_part INT;
+    decimal_part NUMERIC;
+    days_in_year INT;
+    day_of_year INT;
+    final_date DATE;
+    formatted_year VARCHAR;
+BEGIN
+    IF decimal_date IS NULL THEN
+        RETURN NULL;
+    END IF;
+
+    year_part := FLOOR(decimal_date);
+    decimal_part := decimal_date - year_part;
+
+    -- Determine number of days in the year (leap year check)
+    days_in_year := CASE 
+        WHEN (year_part % 4 = 0 AND year_part % 100 <> 0) OR (year_part % 400 = 0) 
+        THEN 366  -- Leap year
+        ELSE 365  -- Regular year
+    END;
+
+    -- Convert decimal fraction to day of the year
+    day_of_year := CEIL(decimal_part * days_in_year);
+
+    -- Format the year properly for BCE cases
+    IF year_part < 1 THEN
+        formatted_year := LPAD(ABS(year_part - 1)::TEXT, 4, '0');  -- Convert BCE format
+        formatted_year := '-' || formatted_year;  -- Add negative sign for BCE
+    ELSE
+        formatted_year := LPAD(year_part::TEXT, 4, '0');
+    END IF;
+
+    -- Try converting year + day-of-year into full ISO date
+    BEGIN
+        final_date := TO_DATE(formatted_year || '-' || day_of_year, 'YYYY-DDD');
+    EXCEPTION
+        WHEN others THEN
+            RETURN NULL;  -- If any conversion error happens, return NULL
+    END;
+
+    RETURN TO_CHAR(final_date, 'YYYY-MM-DD');
+END;
+$$
+LANGUAGE plpgsql;
+
+
+--  This function converts a date in ISO format to a decimal date. usually user in triggers
+CREATE OR REPLACE FUNCTION convert_dates_to_decimal ()
+RETURNS TRIGGER AS
+$$
+BEGIN
+    NEW.start_decdate := isodatetodecimaldate(pad_date(NEW.start_date::TEXT, 'start')::TEXT, FALSE);
+    NEW.end_decdate := isodatetodecimaldate(pad_date(NEW.end_date::TEXT, 'end')::TEXT, FALSE);
+    RETURN NEW;
+END;
+$$
+LANGUAGE plpgsql;

images/tiler-imposm/queries/mviews_admin_boundaries.sql

@@ -52,6 +52,7 @@ BEGIN
             FROM %s;
         $sql$, zoom, table_name);
 
+        -- NOTE: Do not create admin lines since we are merging in mviews_admin_boundaries_merged.sql
         EXECUTE sql_create_lines;
         RAISE NOTICE 'Created materialized view: mview_admin_boundaries_lines_%s', zoom;