Release 2024-02-19 (#93)

.github/workflows/deployment.production.yml

@@ -10,7 +10,7 @@ jobs:
     uses: ./.github/workflows/deployment.yml
     with:
       ENVIRONMENT: production
-      URL: tiles.radverkehrsatlas.de
+      TILES_URL: tiles.radverkehrsatlas.de
     secrets:
       SERVICE_NAME: ${{ secrets.SERVICE_NAME }}
       DATABASE_NAME: ${{ secrets.DATABASE_NAME }}
@@ -24,4 +24,4 @@ jobs:
       SSH_USERNAME: ${{ secrets.SSH_USERNAME }}
       SSH_PASSWORD: ${{ secrets.SSH_PASSWORD }}
       SYNOLOGY_LOG_TOKEN: ${{ secrets.SYNOLOGY_LOG_TOKEN }}
-      SYNOLOGY_ERROR_LOG_TOKEN: ${{ secrets.SYNOLOGY_ERROR_LOG_TOKEN }}
+      SYNOLOGY_ERROR_LOG_TOKEN: ${{ secrets.SYNOLOGY_ERROR_LOG_TOKEN }}

.github/workflows/deployment.staging.yml

@@ -10,7 +10,7 @@ jobs:
     uses: ./.github/workflows/deployment.yml
     with:
       ENVIRONMENT: staging
-      URL: staging-tiles.radverkehrsatlas.de
+      TILES_URL: staging-tiles.radverkehrsatlas.de
     secrets:
       SERVICE_NAME: ${{ secrets.SERVICE_NAME }}
       DATABASE_NAME: ${{ secrets.DATABASE_NAME }}

.github/workflows/deployment.yml

@@ -33,7 +33,7 @@ on:
       ENVIRONMENT:
         type: string
         required: true
-      URL:
+      TILES_URL:
         type: string
         required: true
 
@@ -42,11 +42,40 @@ jobs:
     runs-on: ubuntu-latest
     environment:
       name: ${{ inputs.ENVIRONMENT }}
-      url: https://${{ inputs.URL }}
+      url: https://${{ inputs.TILES_URL }}
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 6
+
+      - name: Get last run commit SHA
+        id: last_run
+        run: |
+          LAST_RUN_SHA=$(curl --request GET \
+          --url https://api.github.com/repos/${{ github.repository }}/actions/runs \
+          --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
+          | jq -r '.workflow_runs[1].head_sha')
+          echo "::set-output name=sha::$LAST_RUN_SHA"
+        shell: bash
+
+      - name: Check if LAST_RUN_SHA is one of the last 6 commits and check changes
+        id: git_changes
+        run: |
+          LAST_6_COMMITS=$(git log -n 6 --pretty=format:"%H")
+          if echo "$LAST_6_COMMITS" | grep -q "${{ steps.last_run.outputs.sha }}"; then
+            if git diff --quiet ${{ steps.last_run.outputs.sha }} HEAD -- app/ helpers/ app.Dockerfile docker-compose.yml; then
+              echo "::set-output name=changes::false"
+            else
+              echo "::set-output name=changes::true"
+            fi
+          else
+            echo "::set-output name=changes::true"
+          fi
+        shell: bash
+
 
       - name: Upgrade AWS CLI version and setup lightsailctl
+        if: steps.git_changes.outputs.changes == 'true'
         run: |
           aws --version
           curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
@@ -58,16 +87,19 @@ jobs:
           sudo chmod +x /usr/local/bin/lightsailctl
 
       - name: Configure AWS credentials
+        if: steps.git_changes.outputs.changes == 'true'
         uses: aws-actions/configure-aws-credentials@v2
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: ${{ secrets.AWS_REGION }}
 
       - name: Setup buildx
+        if: steps.git_changes.outputs.changes == 'true'
         uses: docker/setup-buildx-action@v3
 
       - name: Build app image
+        if: steps.git_changes.outputs.changes == 'true'
         uses: docker/build-push-action@v5
         with:
           context: .
@@ -79,6 +111,7 @@ jobs:
           tags: public.ecr.aws/n0p8j4k5/atlas/app:${{ github.sha }}
 
       - name: Push the app image
+        if: steps.git_changes.outputs.changes == 'true'
         run: |
           aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/n0p8j4k5/
           docker push public.ecr.aws/n0p8j4k5/atlas/app:${{ github.sha }}
@@ -93,6 +126,17 @@ jobs:
           target: "/srv/processing/"
           overwrite: true
 
+      - name: Update GITHUB_SHA
+        if: steps.git_changes.outputs.changes == 'true'
+        uses: appleboy/ssh-action@master
+        with:
+          host: ${{ secrets.SSH_HOST }}
+          username: ${{ secrets.SSH_USERNAME }}
+          password: ${{ secrets.SSH_PASSWORD }}
+          script: |
+            cd /srv/processing/
+            sed -i "s|^GITHUB_SHA=.*$|GITHUB_SHA='${{ github.sha }}'|" .env
+
       - name: Stop & Start containers on VPS
         uses: appleboy/ssh-action@master
         with:
@@ -101,20 +145,18 @@ jobs:
           password: ${{ secrets.SSH_PASSWORD }}
           script: |
             cd /srv/processing/
-            echo "Updating '.env'"
-            rm .env
-            touch .env
-            echo PGHOST='${{ vars.DATABASE_HOST }}' >> .env
-            echo ENVIRONMENT='${{ inputs.ENVIRONMENT }}' >> .env
-            echo SYNOLOGY_LOG_TOKEN='${{ secrets.SYNOLOGY_LOG_TOKEN }}' >> .env
-            echo SYNOLOGY_ERROR_LOG_TOKEN='${{ secrets.SYNOLOGY_ERROR_LOG_TOKEN }}' >> .env
-            echo PGUSER='${{ secrets.DATABASE_USER }}' >> .env
-            echo PGPASSWORD='${{ secrets.DATABASE_PASSWORD }}' >> .env
-            echo PGDATABASE='${{ secrets.DATABASE_NAME }}' >> .env
-            echo OSM_DOWNLOAD_URL='${{ vars.OSM_DOWNLOAD_URL }}' >> .env
-            echo URL='${{ inputs.URL }}' >> .env
-            echo GITHUB_SHA='${{ github.sha }}' >> .env
+            sed -i \
+              -e "s|^PGHOST=.*$|PGHOST='${{ vars.DATABASE_HOST }}'|" \
+              -e "s|^ENVIRONMENT=.*$|ENVIRONMENT='${{ inputs.ENVIRONMENT }}'|" \
+              -e "s|^SYNOLOGY_LOG_TOKEN=.*$|SYNOLOGY_LOG_TOKEN='${{ secrets.SYNOLOGY_LOG_TOKEN }}'|" \
+              -e "s|^SYNOLOGY_ERROR_LOG_TOKEN=.*$|SYNOLOGY_ERROR_LOG_TOKEN='${{ secrets.SYNOLOGY_ERROR_LOG_TOKEN }}'|" \
+              -e "s|^PGUSER=.*$|PGUSER='${{ secrets.DATABASE_USER }}'|" \
+              -e "s|^PGPASSWORD=.*$|PGPASSWORD='${{ secrets.DATABASE_PASSWORD }}'|" \
+              -e "s|^PGDATABASE=.*$|PGDATABASE='${{ secrets.DATABASE_NAME }}'|" \
+              -e "s|^OSM_DOWNLOAD_URL=.*$|OSM_DOWNLOAD_URL='${{ vars.OSM_DOWNLOAD_URL }}'|" \
+              -e "s|^TILES_URL=.*$|TILES_URL='${{ inputs.TILES_URL }}'|" \
+              .env
             echo "Reload containers"
-            docker compose -f docker-compose.traefik.yml up -d
+            docker compose pull
             docker compose up -d
             docker image prune -fa

app/process/barriers/barriers.lua

@@ -1,6 +1,5 @@
 package.path = package.path .. ";/app/process/helper/?.lua;/app/process/shared/?.lua"
 require("Set")
-require("FilterTags")
 require("Metadata")
 require("HighwayClasses")
 
@@ -14,16 +13,6 @@ local lineBarriers = osm2pgsql.define_table({
   }
 })
 
--- local excludedLineBarriers = osm2pgsql.define_table({
---   name = 'barrierLines_excluded',
---   ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' },
---   columns = {
---     { column = 'tags', type = 'jsonb' },
---     { column = 'meta', type = 'jsonb' },
---     { column = 'geom', type = 'linestring' },
---   }
--- })
-
 local areaBarriers = osm2pgsql.define_table({
   name = 'barrierAreas',
   ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' },
@@ -34,18 +23,7 @@ local areaBarriers = osm2pgsql.define_table({
   }
 })
 
--- local excludedAreaBarriers = osm2pgsql.define_table({
---   name = 'barrierAreas_excluded',
---   ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' },
---   columns = {
---     { column = 'tags', type = 'jsonb' },
---     { column = 'meta', type = 'jsonb' },
---     { column = 'geom', type = 'multipolygon' },
---   }
--- })
-
-
-local allowedTags = Set({
+local tags_cc = {
   'tunnel',
   'waterway',
   'aerodrome',
@@ -57,7 +35,7 @@ local allowedTags = Set({
   'area',
   'highway',
   'bridge',
-})
+}
 
 local function isAreaBarrier(object)
   local tags = object.tags
@@ -88,63 +66,44 @@ end
 function osm2pgsql.process_way(object)
   if object.is_closed then -- process as polygon
     if isAreaBarrier(object) then
-      FilterTags(object.tags, allowedTags)
       areaBarriers:insert({
-        tags = object.tags,
-        meta=Metadata(object),
-        geom=object:as_multipolygon()
+        tags = CopyTags({}, object.tags, tags_cc),
+        meta = Metadata(object),
+        geom = object:as_multipolygon()
       })
       return
     end
-    -- excludedAreaBarriers:insert({
-    --   tags=object.tags,
-    --   meta=Metadata(object),
-    --   geom=object:as_multipolygon()
-    -- })
   else --process as linestring
     local tags = object.tags
-    -- if tags.tunnel =='yes' then return end -- we don't consider tunnels as barriers
-
     local isBarrier = HighwayClasses[tags.highway]
 
-    -- only need for low zoom levels
-    local waterBarriers = Set({"river", "canal"})
+    -- waterways as lines are used for low zoom levels
+    local waterBarriers = Set({ "river", "canal" })
     isBarrier = isBarrier or waterBarriers[tags.waterway]
 
-    local trainBarriers = Set({"main", "branch"})
+    local trainBarriers = Set({ "main", "branch" })
     if (tags.railway == 'rail' or tags.railway == 'light_rail') then
       isBarrier = isBarrier or trainBarriers[tags.usage]
     end
+
     if isBarrier then
-      FilterTags(object.tags, allowedTags)
       lineBarriers:insert({
-        tags = object.tags,
-        meta=Metadata(object),
-        geom=object:as_linestring(),
+        tags = CopyTags({}, object.tags, tags_cc),
+        meta = Metadata(object),
+        geom = object:as_linestring(),
       })
       return
     end
-    -- excludedLineBarriers:insert({
-    --   tags=object.tags,
-    --   meta=Metadata(object),
-    --   geom=object:as_linestring()
-    -- })
   end
 end
 
 function osm2pgsql.process_relation(object)
   if isAreaBarrier(object) then
-    FilterTags(object.tags, allowedTags)
     areaBarriers:insert({
-      tags = object.tags,
-      meta=Metadata(object),
-      geom=object:as_multipolygon()
+      tags = CopyTags({}, object.tags, tags_cc),
+      meta = Metadata(object),
+      geom = object:as_multipolygon()
     })
     return
   end
-  -- excludedAreaBarriers:insert({
-  --   tags=object.tags,
-  --   meta=Metadata(object),
-  --   geom=object:as_multipolygon()
-  -- })
 end

app/process/bicycleParking/bicycleParking.lua

@@ -53,13 +53,13 @@ end
 local function processTags(tags)
   -- this is the list of tags found in the wiki: https://wiki.openstreetmap.org/wiki/Tag:amenity%3Dbicycle_parking
   -- also https://wiki.openstreetmap.org/wiki/Berlin/Verkehrswende/Fahrradparkpl%C3%A4tze
-  local processedTags = capacityNormalization(tags)
+  local results = capacityNormalization(tags)
   local binary = Set({ "yes", "no" })
-  processedTags.access = Sanitize(tags.access, Set({ "yes", "private", "permissive", "customers" }))
-  processedTags.covered = Sanitize(tags.covered, binary, "implicit_no")
-  processedTags.fee = Sanitize(tags.fee, binary, "implicit_no")
-  processedTags.access_cargo_bike = Sanitize(tags.cargo_bike, binary, "implicit_no")
-  processedTags.bicycle_parking = Sanitize(
+  results.access = Sanitize(tags.access, Set({ "yes", "private", "permissive", "customers" }))
+  results.covered = Sanitize(tags.covered, binary, "implicit_no")
+  results.fee = Sanitize(tags.fee, binary, "implicit_no")
+  results.access_cargo_bike = Sanitize(tags.cargo_bike, binary, "implicit_no")
+  results.bicycle_parking = Sanitize(
     tags.bicycle_parking,
     Set({ "stands", "wide_stands", "bollard", "wall_loops", "shed", "two-tier", "lockers" })
   )
@@ -78,14 +78,14 @@ local function processTags(tags)
     "mapillary",
     "description",
   }
-  CopyTags(processedTags, tags, allowed_tags)
-  CopyTags(processedTags, tags, tags_cc, "osm_")
+  CopyTags(results, tags, allowed_tags)
+  CopyTags(results, tags, tags_cc, "osm_")
 
   local checkDateTag = "check_date"
   if tags[checkDateTag] then
-    processedTags.age = AgeInDays(ParseDate(tags[checkDateTag]))
+    results.age = AgeInDays(ParseDate(tags[checkDateTag]))
   end
-  return processedTags
+  return results
 end
 
 function osm2pgsql.process_node(object)

app/process/boundaries/boundaries.lua

@@ -1,6 +1,5 @@
 package.path = package.path .. ";/app/process/helper/?.lua;/app/process/shared/?.lua"
 require("Set")
-require("FilterTags")
 require("Metadata")
 require("CopyTags")
 
@@ -41,51 +40,51 @@ function osm2pgsql.process_relation(object)
     return
   end
 
-  local results_tags = {}
-  results_tags.admin_level = tonumber(tags.admin_level)
-  results_tags.population = tonumber(tags.population)
+  local results = {}
+  results.admin_level = tonumber(tags.admin_level)
+  results.population = tonumber(tags.population)
   -- Categories:
-  if (results_tags.admin_level == 8) then
-    results_tags.category_municipality = "Gemeinde"
+  if (results.admin_level == 8) then
+    results.category_municipality = "Gemeinde"
   end
-  if (results_tags.admin_level == 6) then
-    results_tags.category_district = "Landkreis"
+  if (results.admin_level == 6) then
+    results.category_district = "Landkreis"
     if (tags.place == "city" or tags["name:prefix"] == "Kreisfreie Stadt") then
-      results_tags.category_municipality = "Kreisfreie Stadt"
-      results_tags.category_district = "Kreisfreie Stadt"
+      results.category_municipality = "Kreisfreie Stadt"
+      results.category_district = "Kreisfreie Stadt"
     end
   end
-  if (results_tags.admin_level == 4 and tags.place == "city") then
-    results_tags.category_municipality = "Stadtstaat"
-    results_tags.category_district = "Stadtstaat"
+  if (results.admin_level == 4 and tags.place == "city") then
+    results.category_municipality = "Stadtstaat"
+    results.category_district = "Stadtstaat"
   end
   -- these tags are copied (Eigennamen)
   local allowed_tags = { "name", "name:prefix" }
-  CopyTags(results_tags, tags, allowed_tags)
+  CopyTags(results, tags, allowed_tags)
   -- these tags are copied and prefixed with `osm_`
   local tags_cc = { "de:regionalschluessel", "population:date", "wikidata", "wikipedia" }
-  CopyTags(results_tags, tags, tags_cc, "osm_")
+  CopyTags(results, tags, tags_cc, "osm_")
 
   -- Make sure we only include boundaries with a geometry
   -- https://osm2pgsql.org/doc/manual.html#processing-callbacks
   -- https://osm2pgsql.org/doc/manual.html#geometry-objects-in-lua
   if object:as_multipolygon():is_null() then return end
 
   table:insert({
-    tags = results_tags,
+    tags = results,
     meta = Metadata(object),
     geom = object:as_multipolygon()
   })
   labelTable:insert({
-    tags = results_tags,
+    tags = results,
     meta = Metadata(object),
     geom = object:as_multipolygon():centroid()
   })
 
   local admin_levels = Set({ "4", "6", "7", "8" })
   if admin_levels[tags.admin_level] then
     statsTable:insert({
-      tags = results_tags,
+      tags = results,
       meta = Metadata(object),
       geom = object:as_multipolygon()
     })