Release 2024-01-15 (#80)

.github/workflows/deployment.yml

@@ -111,7 +111,7 @@ jobs:
           host: ${{ secrets.SSH_HOST }}
           username: ${{ secrets.SSH_USERNAME }}
           password: ${{ secrets.SSH_PASSWORD }}
-          source: "docker-compose.yml,docker-compose.traefik.yml,postgres/postgres.conf"
+          source: "docker-compose.yml,docker-compose.traefik.yml,postgres/postgres.conf,configs/nginx.conf"
           target: "/srv/processing/"
           overwrite: true
 
@@ -138,10 +138,7 @@ jobs:
             echo API_URL='${{ inputs.API_URL }}' >> .env
             echo GITHUB_SHA='${{ github.sha }}' >> .env
             echo API_SECRET=$(echo $RANDOM | md5sum | head -c 20) >> .env
-            echo "Stop containers on VPS"
-            docker compose down
-            docker compose -f docker-compose.traefik.yml down
-            echo "Start container on VPS"
+            echo "Reload containers"
             docker compose -f docker-compose.traefik.yml up -d
             docker compose up -d
             docker image prune -fa

.gitignore

@@ -12,3 +12,7 @@ __pycache__
 .idea
 
 node_modules
+
+# Nginx
+cache/
+logs/

app/process/bicycleParking/bicycleParking.lua

@@ -2,7 +2,7 @@ package.path = package.path .. ";/app/process/helper/?.lua;/app/process/shared/?
 require("CopyTags")
 require("Metadata")
 require("Set")
-require("Sanatize")
+require("Sanitize")
 
 local nodeTable = osm2pgsql.define_table({
   name = 'bicycleParking_points',
@@ -59,15 +59,26 @@ local function processTags(tags)
   processedTags.covered = Sanitize(tags.covered, binary, "implicit_no")
   processedTags.fee = Sanitize(tags.fee, binary, "implicit_no")
   processedTags.access_cargo_bike = Sanitize(tags.cargo_bike, binary, "implicit_no")
-  processedTags.bicycle_parking = Sanitize(tags.bicycle_parking,
-    Set({ "stands", "wide_stands", "bollard", "wave", "handlebar_holder", "streetpod", "rack", "wall_loops", "safe_loops",
-      "building", "shed", "two-tier", "lockers", "tree", "ground_slots", "crossbar", "rope", "floor", "informal",
-      "arcadia",
-      "anchors", "lean_and_stick" }))
+  processedTags.bicycle_parking = Sanitize(
+    tags.bicycle_parking,
+    Set({ "stands", "wide_stands", "bollard", "wall_loops", "shed", "two-tier", "lockers" })
+  )
 
-  local tags_cc = { "area", "operator", "operator:type", "covered", "indoor", "access", "cargo_bike", "capacity",
-    "capacity:cargo_bike", "fee", "lit", "surface", "bicycle_parking", "mapillary", "maxstay", "surveillance",
-    "bicycle_parking:count", "bicycle_parking:position", "traffic_sign", "description" }
+  -- these tags are copied (Eigennamen)
+  local allowed_tags = {
+    "name",
+    "operator",
+  }
+  -- these tags are copied and prefixed with `osm_`
+  -- we need to sanatize them at some point
+  local tags_cc = {
+    "area", "operator:type", "covered", "indoor", "access", "cargo_bike", "capacity",
+    "capacity:cargo_bike", "fee", "lit", "surface", "bicycle_parking", "maxstay", "surveillance",
+    "bicycle_parking:count", "bicycle_parking:position", "traffic_sign", "description",
+    "mapillary",
+    "description",
+  }
+  CopyTags(processedTags, tags, allowed_tags)
   CopyTags(processedTags, tags, tags_cc, "osm_")
 
   local checkDateTag = "check_date"

app/process/helper/CopyTags.lua

@@ -1,4 +1,5 @@
 function CopyTags(dst, src, tags, prefix)
+  prefix = prefix or ''
   for _, val in pairs(tags) do
     dst[prefix .. val] = src[val]
   end

app/process/helper/Sanitize.lua

@@ -0,0 +1,11 @@
+-- makes sure that `value` is in the set `allowed`. Returns `default` if value is nil
+function Sanitize(value, allowed, default)
+  if value == nil then
+    return default
+  end
+  if allowed[value] then
+    return value
+  end
+  -- maybe add to TODO list or smth.
+  return nil
+end

app/process/publicTransport.lua

@@ -1,10 +1,9 @@
 package.path = package.path .. ";/app/process/helper/?.lua;/app/process/shared/?.lua"
 require("Set")
-require("FilterTags")
+require("CopyTags")
 require("MergeArray")
 require("Metadata")
 
-
 local table = osm2pgsql.define_table({
   name = 'publicTransport',
   ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' },
@@ -16,55 +15,90 @@ local table = osm2pgsql.define_table({
 })
 
 local function ExitProcessing(object)
-  if not object.tags.public_transport then
+  if not (object.tags.railway or object.tags.amenity == "ferry_terminal") then
     return true
   end
 
-  local shouldExit = false
-
   -- ["operator"!= "Berliner Parkeisenbahn"] - a smalll train in a park that we cannot propery exclude by other means
   if object.tags.operator == "Berliner Parkeisenbahn" then
-    shouldExit = true
-  end
-
-  -- "station" includes "railway=yes" and "ferry=yes", "stop_position" includes "bus=yes"
-  -- TODO for now we exclude "stop_position". We could include those, but need to clean the data so it's easy to filter by transportation mode (bus, train, ferry) and also for train, only include the station, not the stop_position.
-  local allowed_values = Set({ "station" })
-  if not allowed_values[object.tags.public_transport] then
-    shouldExit = true
+    return true
   end
 
   -- ["disused"!="yes"] - ignore all that are not in use
   if object.tags.disused == "yes" then
-    shouldExit = true
+    return true
   end
 
-  return shouldExit
+  return false
 end
 
 local function processTags(tags)
-  local allowed_tags = Set({
+  local category
+
+  -- Ferry
+  -- https://wiki.openstreetmap.org/wiki/DE:Tag:amenity%3Dferry_terminal
+  -- https://wiki.openstreetmap.org/wiki/DE:Key:ferry
+  if (tags.amenity == "ferry_terminal") then
+    category = "ferry_station"
+  end
+
+  -- U-Bahn
+  -- https://wiki.openstreetmap.org/wiki/DE:Tag:station%3Dsubway
+  if (tags.railway == "subway") then
+    category = "subway_station"
+  end
+
+  -- S-Bahn
+  -- https://wiki.openstreetmap.org/wiki/DE:Tag:railway%3Dlight_rail
+  if (tags.railway == "light_rail") then
+    category = "light_rail_station"
+  end
+
+  -- Straßenbahn
+  -- https://wiki.openstreetmap.org/wiki/DE:Tag:railway%3Dtram_stop
+  if (tags.railway == "tram_stop") then
+    category = "tram_station"
+  end
+
+  -- Bahn
+  -- https://wiki.openstreetmap.org/wiki/DE:Tag:railway%3Dstation
+  -- https://wiki.openstreetmap.org/wiki/DE:Tag:railway%3Dhalt
+  if (tags.railway == "station" or tags.railway == "halt") then
+    category = "railway_station"
+  end
+
+  -- Bus:
+  -- We don't handle bus stops ATM because they are not too relevant for our use cases.
+  -- We might add them later…
+
+  if (category == nil) then
+    -- We need to check those and either filter them or fix the categories
+    category = "undefined"
+  end
+
+  -- these tags are copied (Eigennamen)
+  local allowed_tags = {
     "name",
+    "operator",
+  }
+  -- these tags are copied and prefixed with `osm_`
+  -- we need to sanatize them at some point
+  local tags_cc = {
     "network",
     "network:short",
-    "operator",
-    "public_transport",
-    "railway",
-    "light_rail",
-    "bus",
-    "ferry",
-    "station",
-  })
-  FilterTags(tags, allowed_tags)
+  }
+  local result = { category = category }
+  CopyTags(result, tags, tags_cc, 'osm_')
+  CopyTags(result, tags, allowed_tags)
+
+  return result
 end
 
 function osm2pgsql.process_node(object)
   if ExitProcessing(object) then return end
 
-  processTags(object.tags)
-
   table:insert({
-    tags = object.tags,
+    tags = processTags(object.tags),
     meta = Metadata(object),
     geom = object:as_point()
   })
@@ -74,10 +108,8 @@ function osm2pgsql.process_way(object)
   if ExitProcessing(object) then return end
   if not object.is_closed then return end
 
-  processTags(object.tags)
-
   table:insert({
-    tags = object.tags,
+    tags = processTags(object.tags),
     meta = Metadata(object),
     geom = object:as_polygon():centroid()
   })
@@ -87,10 +119,8 @@ function osm2pgsql.process_relation(object)
   if ExitProcessing(object) then return end
   if not object.tags.type == 'multipolygon' then return end
 
-  processTags(object.tags)
-
   table:insert({
-    tags = object.tags,
+    tags = processTags(object.tags),
     meta = Metadata(object),
     geom = object:as_multipolygon():centroid()
   })

app/process/roads_bikelanes/bikelanes/Bikelanes.lua

@@ -20,9 +20,9 @@ local bikelanesTable = osm2pgsql.define_table({
   name = '_bikelanes_temp',
   ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' },
   columns = {
-    { column = 'tags',     type = 'jsonb' },
-    { column = 'meta',     type = 'jsonb' },
-    { column = 'geom',     type = 'linestring' },
+    { column = 'tags', type = 'jsonb' },
+    { column = 'meta', type = 'jsonb' },
+    { column = 'geom', type = 'linestring' },
   }
 })
 
@@ -37,35 +37,32 @@ local excludeTable = osm2pgsql.define_table({
   }
 })
 
--- whitelist of tags we want to insert intro the DB
+-- these tags are copied (Eigennamen)
+local allowed_tags = {
+  "name",
+}
+-- these tags are copied and prefixed with `osm_`
 local tags_cc = {
-  'access',
-  'bicycle_road',
-  'bicycle',
-  'conditional',
   'cycleway',
   'cycleway:lane', -- 'advisory', 'exclusive'
+  'lane',          -- 'cycleway:SIDE:lane'
   'dual_carriageway',
-  'foot',
-  'footway',
   'highway',
-  'is_sidepath',
-  'name',
   'oneway', -- we use oneway:bicycle=no (which is transformed to oneway=no) to add a notice in the UI about two way cycleways in one geometry
   'prefix',
-  'segregated',
   'side',
-  'smoothness',
   'surface:colour',
-  'surface',
   'traffic_sign',
-  'width',
-  'bicycle:lanes',
-  'cycleway:lanes',
+  'traffic_sign:forward',
+  'traffic_sign:backward',
   'separation',
   'separation:left',
   'separation:right',
-  'lane', -- 'cycleway:SIDE:lane'
+  'traffic_mode',
+  'traffic_mode:left',
+  'traffic_mode:right',
+  "mapillary",
+  "description",
 }
 
 local sides = { LEFT_SIGN, CENTER_SIGN, RIGHT_SIGN }
@@ -118,7 +115,10 @@ function Bikelanes(object, road)
         if sign == CENTER_SIGN then
           -- if we're dealing with the original object (center line) then prefix only keep all the tags from the `tags_cc` list and prefix them
           -- due to that it's important that the precceding operations happen before
-          cycleway = CopyTags({}, tags, tags_cc, 'osm_')
+          cycleway = {}
+          cycleway = CopyTags(cycleway, tags, allowed_tags)
+          cycleway = CopyTags(cycleway, tags, tags_cc, 'osm_')
+          cycleway.width = ParseLength(tags.width)
         else
           freshTag = "check_date:" .. cycleway.prefix
         end

app/process/roads_bikelanes/bikelanes/categories.lua

@@ -153,8 +153,11 @@ local function cyclewaySeparatedCases(tags)
   -- traffic_sign=DE:237, "Radweg", https://wiki.openstreetmap.org/wiki/DE:Tag:traffic%20sign=DE:237
   -- cycleway=track, https://wiki.openstreetmap.org/wiki/DE:Tag:cycleway%3Dtrack
   -- cycleway=opposite_track, https://wiki.openstreetmap.org/wiki/DE:Tag:cycleway%3Dopposite_track
-  if tags.traffic_sign == "DE:237"
-      or (tags.highway == "cycleway" and (tags.cycleway == "track" or tags.cycleway == "opposite_track" or tags.is_sidepath)) then
+  local taggedWithAccessTagging = tags.highway == "cycleway" and
+      (tags.cycleway == "track" or tags.cycleway == "opposite_track" or tags.is_sidepath)
+  -- Testcase: The "not 'lane'" part is needed for places like https://www.openstreetmap.org/way/964589554 which have the traffic sign but are not separated.
+  local taggedWithTrafficsign = osm2pgsql.has_prefix(tags.traffic_sign, "DE:237") and not tags.cycleway == "lane"
+  if taggedWithAccessTagging or taggedWithTrafficsign then
     -- `_parent_highway` indicates that this way was split of the centerline; in this case, we consider it a sidepath.
     if tags.is_sidepath == "yes" or tags._parent_highway then
       -- This could be PBLs "Protected Bike Lanes"

app/process/roads_bikelanes/roadClassification/RoadClassification.lua

@@ -1,6 +1,7 @@
 package.path = package.path .. ";/app/process/helper/?.lua;/app/process/shared/?.lua"
 require("Set")
 require("CopyTags")
+require("Sanitize")
 
 function RoadClassification(object)
   -- Values that we would allow, but skip here:
@@ -82,20 +83,23 @@ function RoadClassification(object)
     roadClassification['road_oneway:bicycle'] = tags['oneway:bicycle']
   end
 
-  local tags_cc = {
+  -- these tags are copied (Eigennamen)
+  local allowed_tags = {
     "name",
-    "highway",
-    "footway",
-    "access",
-    "service",
-    "is_sidepath",
-    "maxspeed",
-    "surface",
-    "smoothness",
-    "oneway",
-    "oneway:bicycle",
   }
+  -- these tags are copied and prefixed with `osm_`
+  -- we need to sanatize them at some point
+  local tags_cc = {
+    'traffic_sign',
+    'traffic_sign:forward',
+    'traffic_sign:backward',
+    "mapillary",
+    "description",
+  }
+  CopyTags(roadClassification, tags, allowed_tags)
   CopyTags(roadClassification, tags, tags_cc, "osm_")
+  roadClassification.width = ParseLength(tags.width)
+  roadClassification.oneway = Sanitize(tags.oneway, Set({ "yes", "no" }))
 
   return roadClassification
 end

app/run-5-analysis.sh → app/run-7-analysis.sh

@@ -5,4 +5,4 @@ echo -e "\e[1m\e[7m Analysis – START \e[27m\e[21m\e[0m"
 
 psql -q -f "./analysis/boundaryStats.sql"
 
-echo -e "\e[1m\e[7m Analysis – END \e[27m\e[21m\e[0m"
+echo -e "\e[1m\e[7m Analysis – END \e[27m\e[21m\e[0m"