diff --git a/hetzner/README.md b/hetzner/README.md index 29c4e959..580ac46d 100644 --- a/hetzner/README.md +++ b/hetzner/README.md @@ -11,6 +11,12 @@ Ensure you are using the correct Docker images for production deployment, https: ```sh docker compose -f hetzner/tiler.production.yml up -d +# docker compose -f hetzner/tiler.production.yml up tiler_production -d +# docker compose -f hetzner/tiler.production.yml up imposm_production -d +# docker compose -f hetzner/tiler.production.yml up cache_production -d +# docker compose -f hetzner/tiler.production.yml up global_seeding_production -d +# docker compose -f hetzner/tiler.production.yml up tile_coverage_seeding_production -d + ``` 🛠Deploying to Staging @@ -30,3 +36,4 @@ docker compose -f hetzner/tiler.staging.yml up tile_coverage_seeding -d • Ensure that you are using the correct Docker images for each environment. • Manually update the images before deploying production services. • For troubleshooting, check logs using: + diff --git a/hetzner/seed.sh b/hetzner/seed.sh new file mode 100644 index 00000000..70d40f8d --- /dev/null +++ b/hetzner/seed.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +seed_global() { + while true; do + echo "Starting global seeding..." + pkill -f "tegola" && sleep 5 + + # Seed all areas (min 0 to max 5) + tegola cache seed tile-name "0/0/0" \ + --config=/opt/tegola_config/config.toml \ + --map=osm \ + --min-zoom=0 \ + --max-zoom=5 \ + --concurrency=32 \ + --overwrite=true + + # Seed land areas separately (zoom 6-7) + for zoom in $(seq 6 7); do + echo "Downloading tile list for zoom level $zoom..." + wget -O /opt/tile-list.tiles "https://s3.amazonaws.com/planet.openhistoricalmap.org/tile_coverage/tiles_boundary_$zoom.list" + + tegola cache seed tile-list /opt/tile-list.tiles \ + --config=/opt/tegola_config/config.toml \ + --map=osm \ + --min-zoom="$zoom" \ + --max-zoom="$zoom" \ + --concurrency=32 \ + --overwrite=true + done + echo "Global seeding completed. Sleeping for 1 hour..." + sleep 3600 + done +} + +seed_coverage() { + while true; do + echo "Starting coverage seeding..." + wget -O /opt/tile-list.tiles "https://s3.amazonaws.com/planet.openhistoricalmap.org/tile_coverage/tiles_14.list" + pkill -f "tegola" && sleep 5 + + tegola cache seed tile-list /opt/tile-list.tiles \ + --config=/opt/tegola_config/config.toml \ + --map=osm \ + --min-zoom=0 \ + --max-zoom=14 \ + --concurrency=32 \ + --overwrite=false + sleep 300 + done +} + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 <global|coverage>" + exit 1 +fi + +case "$1" in + global) + seed_global + ;; + coverage) + seed_coverage + ;; + *) + echo "Invalid option. Use 'global' or 'coverage'." + exit 1 + ;; +esac diff --git a/hetzner/tiler.production.yml b/hetzner/tiler.production.yml index eff41afc..f6772fe5 100644 --- a/hetzner/tiler.production.yml +++ b/hetzner/tiler.production.yml @@ -48,7 +48,7 @@ services: - tiler_network # Tiler server tiler_production: - image: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.2241.hba0f0a5 + image: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.2248.he042bfb ports: - "9090:9090" env_file: @@ -75,23 +75,13 @@ services: image: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.2234.hfa6976e env_file: - .env.production + volumes: + - ./seed.sh:/opt/seed.sh entrypoint: - /bin/bash - "-c" - | - set -x - while true; do - pkill -f "tegola" && sleep 5 - echo "0/0/0" > /opt/tile-list.tiles - tegola cache seed tile-list /opt/tile-list.tiles \ - --config=/opt/tegola_config/config.toml \ - --map=osm \ - --min-zoom=0 \ - --max-zoom=7\ - --concurrency=32 \ - --overwrite=true - sleep 3600 - done + /opt/seed.sh global deploy: resources: limits: @@ -104,23 +94,13 @@ services: image: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.2234.hfa6976e env_file: - .env.production + volumes: + - ./seed.sh:/opt/seed.sh entrypoint: - /bin/bash - "-c" - | - set -x - while true; do - wget -O tile-list.tiles https://s3.amazonaws.com/planet.openhistoricalmap.org/tile_coverage/tiles.list - pkill -f "tegola" && sleep 5 - tegola cache seed tile-list tile-list.tiles \ - --config=/opt/tegola_config/config.toml \ - --map=osm \ - --min-zoom=8 \ - --max-zoom=14 \ - --concurrency=32 \ - --overwrite=false - sleep 3600 - done + /opt/seed.sh coverage deploy: resources: limits: diff --git a/hetzner/tiler.staging.yml b/hetzner/tiler.staging.yml index aa595561..37c154a8 100644 --- a/hetzner/tiler.staging.yml +++ b/hetzner/tiler.staging.yml @@ -43,7 +43,7 @@ services: cpus: '1' memory: 1G tiler: - image: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.2234.hfa6976e + image: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.2248.he042bfb ports: - "9091:9090" env_file: diff --git a/images/tiler-server/Dockerfile b/images/tiler-server/Dockerfile index 4c2700e4..7ecbf907 100644 --- a/images/tiler-server/Dockerfile +++ b/images/tiler-server/Dockerfile @@ -32,8 +32,7 @@ RUN mkdir /opt/tegola_config RUN cd /opt/ && python build_config.py \ --output=/opt/tegola_config/config.toml \ --provider_names \ - "land,\ - admin_boundaries_lines,\ + "admin_boundaries_lines,\ admin_boundaries.centroids,\ admin_boundaries_maritime,\ place_areas,\ diff --git a/images/tiler-server/config/providers/land.toml b/images/tiler-server/config/providers/land.toml deleted file mode 100644 index 60f69a65..00000000 --- a/images/tiler-server/config/providers/land.toml +++ /dev/null @@ -1,92 +0,0 @@ -[[providers.layers]] -name = "land_0-2" -geometry_fieldname = "geometry" -geometry_type = "multipolygon" -id_fieldname = "ogc_fid" -sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_z0_2 WHERE geometry && !BBOX!" - -[[providers.layers]] -name = "land_3-5" -geometry_fieldname = "geometry" -geometry_type = "multipolygon" -id_fieldname = "ogc_fid" -sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_z3_5 WHERE geometry && !BBOX!" - -[[providers.layers]] -name = "land_6-7" -geometry_fieldname = "geometry" -geometry_type = "multipolygon" -id_fieldname = "ogc_fid" -sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_z6_7 WHERE geometry && !BBOX!" - -[[providers.layers]] -name = "land_8-9" -geometry_fieldname = "geometry" -geometry_type = "multipolygon" -id_fieldname = "ogc_fid" -sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_z8_9 WHERE geometry && !BBOX!" - -[[providers.layers]] -name = "land_10-12" -geometry_fieldname = "geometry" -geometry_type = "multipolygon" -id_fieldname = "ogc_fid" -sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_z10_12 WHERE geometry && !BBOX!" - -[[providers.layers]] -name = "land_13-15" -geometry_fieldname = "geometry" -geometry_type = "multipolygon" -id_fieldname = "ogc_fid" -sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_z13_15 WHERE geometry && !BBOX!" - -[[providers.layers]] -name = "land_16-20" -geometry_fieldname = "geometry" -geometry_type = "multipolygon" -id_fieldname = "ogc_fid" -sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_z16_20 WHERE geometry && !BBOX!" - -#######Maps - -[[maps.layers]] -name = "land" -provider_layer = "osm.land_0-2" -min_zoom = 0 -max_zoom = 2 - -[[maps.layers]] -name = "land" -provider_layer = "osm.land_3-5" -min_zoom = 3 -max_zoom = 5 - -[[maps.layers]] -name = "land" -provider_layer = "osm.land_6-7" -min_zoom = 6 -max_zoom = 7 - -[[maps.layers]] -name = "land" -provider_layer = "osm.land_8-9" -min_zoom = 8 -max_zoom = 9 - -[[maps.layers]] -name = "land" -provider_layer = "osm.land_10-12" -min_zoom = 10 -max_zoom = 12 - -[[maps.layers]] -name = "land" -provider_layer = "osm.land_13-15" -min_zoom = 13 -max_zoom = 15 - -[[maps.layers]] -name = "land" -provider_layer = "osm.land_16-20" -min_zoom = 16 -max_zoom = 20 diff --git a/images/web/Dockerfile b/images/web/Dockerfile index 68b005f0..595faf78 100644 --- a/images/web/Dockerfile +++ b/images/web/Dockerfile @@ -86,7 +86,7 @@ RUN apache2ctl configtest RUN chown -R www-data: $workdir # Clone Map-styles -ENV OPENHISTORICALMAP_MAP_STYLES_GITSHA=6de6e329569ab6561d1fa6ab2cebb42df7fd17c7 +ENV OPENHISTORICALMAP_MAP_STYLES_GITSHA=7a70bbef491db6928f28256a3a68d3158a6f9710 RUN git clone --branch staging --depth 1 https://github.com/OpenHistoricalMap/map-styles.git $workdir/public/map-styles RUN cd $workdir/public/map-styles && git fetch --depth 1 origin $OPENHISTORICALMAP_MAP_STYLES_GITSHA && git checkout $OPENHISTORICALMAP_MAP_STYLES_GITSHA RUN rm -rf $workdir/public/map-styles/.git diff --git a/values.production.template.yaml b/values.production.template.yaml index 693cfc75..99bfd4af 100644 --- a/values.production.template.yaml +++ b/values.production.template.yaml @@ -983,7 +983,7 @@ ohm: label_value: web_large tilerCacheSeedCoverage: - enabled: true + enabled: false schedule: '0 * * * *' env: CONCURRENCY: 128 diff --git a/values.staging.template.yaml b/values.staging.template.yaml index 3b7f20e3..46406774 100644 --- a/values.staging.template.yaml +++ b/values.staging.template.yaml @@ -68,7 +68,7 @@ osm-seed: nodeSelector: enabled: true label_key: nodegroup_type - label_value: db_large + label_value: web_medium env: POSTGRES_DB: {{STAGING_DB}} POSTGRES_USER: {{STAGING_DB_USER}} @@ -137,7 +137,7 @@ osm-seed: nodeSelector: enabled: true label_key: nodegroup_type - label_value: db_large + label_value: web_medium replicaCount: 1 # Set staticIp, if you are using cloudProvider=gcp staticIp: c @@ -249,7 +249,7 @@ osm-seed: nodeSelector: enabled: true label_key: nodegroup_type - label_value: db_large + label_value: web_medium env: ENABLE_SEND_SLACK_MESSAGE: "true" SLACK_WEBHOOK_URL: {{OHM_SLACK_WEBHOOK_URL}}