Sanitize traffic sign (#136)

I checked the database for Germany and wrote transformations for the
most common tagging errors.
The helper allows tags that can be transformed to conform the pattern.

Everything else is removed.

processing/topics/bicycleParking/bicycleParking.lua

@@ -4,6 +4,7 @@ require("Metadata")
 require("Set")
 require("Sanitize")
 require("DefaultId")
+require("SanitizeTrafficSign")
 
 local nodeTable = osm2pgsql.define_table({
   name = 'bicycleParking_points',
@@ -87,12 +88,13 @@ local function processTags(tags)
   local tags_cc = {
     "area", "operator:type", "covered", "indoor", "access", "cargo_bike", "capacity",
     "capacity:cargo_bike", "fee", "lit", "surface", "bicycle_parking", "maxstay", "surveillance",
-    "bicycle_parking:count", "bicycle_parking:position", "traffic_sign", "description",
+    "bicycle_parking:count", "bicycle_parking:position", "description",
     "mapillary",
     "description",
   }
   CopyTags(results, tags, allowed_tags)
   CopyTags(results, tags, tags_cc, "osm_")
+  results.traffic_sign = SanitizeTrafficSign(tags.traffic_sign)
 
   results.age = AgeInDays(ParseCheckDate(tags["check_date"]))
   return results

processing/topics/helper/DeriveTrafficSigns.lua

@@ -0,0 +1,12 @@
+package.path = package.path .. ";/processing/topics/helper/?.lua"
+
+require("SanitizeTrafficSign")
+
+function DeriveTrafficSigns(tags)
+  local results = {
+    ['traffic_sign'] = SanitizeTrafficSign(tags.traffic_sign) or SanitizeTrafficSign(tags['traffic_sign:both']),
+    ['traffic_sign:forward'] = SanitizeTrafficSign(tags['traffic_sign:forward']),
+    ['traffic_sign:backward'] = SanitizeTrafficSign(tags['traffic_sign:backward'])
+  }
+  return results
+end

processing/topics/helper/SanitizeTrafficSign.lua

@@ -0,0 +1,52 @@
+-- Remove all whitespaces after delimeters
+local function stripWhitespaces(traffic_sign)
+  local stripped = string.gsub(traffic_sign, ', ', ',')
+  stripped = string.gsub(stripped, '; ', ';')
+  return stripped
+end
+
+--- Cleanup the `traffic_sign=*` tag
+--- @param traffic_sign string
+--- @returns string
+function SanitizeTrafficSign(traffic_sign)
+  if traffic_sign == nil then
+    return nil
+  end
+  if traffic_sign == "no" or traffic_sign == 'none' then
+    return "none"
+  end
+
+  -- Docs: patterns with "^" target beginning of string
+
+  -- This is the correct tagging, all traffic signs should start with DE:
+  if string.find(traffic_sign, '^DE:%S') then
+    return stripWhitespaces(traffic_sign)
+  end
+
+  local substitutions = {
+    ['^DE: '] = 'DE:',
+    ['^DE.'] = 'DE:',
+    ['^D:'] = 'DE:',
+    ['^D%.'] = 'DE:',
+    ['^de:'] = 'DE:',
+    ['^DE1'] = 'DE:1',
+    ['^DE2'] = 'DE:2',
+    ['^2'] = 'DE:2',
+    ['^1'] = 'DE:1',
+    -- These patterns could handle all the above in a more generalized way
+    -- ['^DE?[:.]%s?'] = 'DE:',
+    -- ['^de?[:.]%s?'] = 'DE:'
+    -- ['^DE(%d)'] = 'DE:%1'
+    -- ['^(%d)'] = 'DE:%1'
+  }
+  for pattern, substitude in pairs(substitutions) do
+    local val, n = string.gsub(traffic_sign, pattern, substitude)
+    if n > 0 then
+      -- TODO: add to todo list
+      return stripWhitespaces(val)
+    end
+  end
+
+  -- Discard everything else
+  return nil
+end

processing/topics/helper/__tests__/SanitizeTrafficSign.test.lua

@@ -0,0 +1,73 @@
+require("osm2pgsql")
+
+describe("SanitizeTrafficSign", function()
+  package.path = package.path .. ";/processing/topics/helper/?.lua"
+  require("SanitizeTrafficSign")
+
+  -- Cleanup
+  it('renames `no` to `none`', function()
+    assert.are.same(SanitizeTrafficSign("no"), "none")
+  end)
+
+  it('renames `DE:SPACE` to `DE:`', function()
+    assert.are.same(SanitizeTrafficSign("DE: 123"), "DE:123")
+  end)
+
+  it('renames `DE234` to `DE:234`', function()
+    assert.are.same(SanitizeTrafficSign("DE234"), "DE:234")
+  end)
+
+  it('renames `DE1010` to `DE:1010`', function()
+    assert.are.same(SanitizeTrafficSign("DE1010"), "DE:1010")
+  end)
+
+  it('renames `D:234` to `DE:234`', function()
+    assert.are.same(SanitizeTrafficSign("D:234"), "DE:234")
+  end)
+
+  it('renames `de:234` to `DE:234`', function()
+    assert.are.same(SanitizeTrafficSign("de:234"), "DE:234")
+  end)
+
+  it('renames `234` to `DE:234`', function()
+    assert.are.same(SanitizeTrafficSign("234"), "DE:234")
+  end)
+
+  it('renames `1010` to `DE:1010`', function()
+    assert.are.same(SanitizeTrafficSign("1010"), "DE:1010")
+  end)
+
+  it('renames `DE.234` to `DE:234`', function()
+    assert.are.same(SanitizeTrafficSign("DE.234"), "DE:234")
+  end)
+
+  it('cleans spaces `DE:123, 1010; 234` to `DE:123,1010;234`', function()
+    assert.are.same(SanitizeTrafficSign("DE:123, 1010; 234"), "DE:123,1010;234")
+  end)
+
+  -- Allow
+  it('allows `DE:234`', function()
+    assert.are.same(SanitizeTrafficSign("DE:234"), "DE:234")
+  end)
+
+  it('allows `DE:1010`', function()
+    assert.are.same(SanitizeTrafficSign("DE:1010"), "DE:1010")
+  end)
+
+  it('allows `none` as value', function()
+    assert.are.same(SanitizeTrafficSign("none"), "none")
+  end)
+
+  -- Disallow
+  it('handles nil', function()
+    assert.are.same(SanitizeTrafficSign(nil), nil)
+  end)
+  it('disallows everything else', function()
+    assert.are.same(SanitizeTrafficSign("foobar"), nil)
+    assert.are.same(SanitizeTrafficSign("yes"), nil)
+    assert.are.same(SanitizeTrafficSign("unkown"), nil)
+    assert.are.same(SanitizeTrafficSign("AT:foobar"), nil)
+    assert.are.same(SanitizeTrafficSign("pictogram"), nil)
+    assert.are.same(SanitizeTrafficSign("(comment)"), nil)
+  end)
+end)

processing/topics/roads_bikelanes/bikelanes/Bikelanes.lua

@@ -12,16 +12,14 @@ require("BikelanesTodos")
 require("Sanitize")
 require("DeriveOneway")
 require("DefaultId")
+require("DeriveTrafficSigns")
 
 local tags_copied = {
   "mapillary",
   "description",
 }
 local tags_prefixed = {
   'surface:colour',
-  'traffic_sign',
-  'traffic_sign:forward',
-  'traffic_sign:backward',
   'separation',
   'separation:left',
   'separation:right',
@@ -80,15 +78,16 @@ function Bikelanes(object)
           bridge = Sanitize(tags.bridge, { "yes" }),
           tunnel = Sanitize(tags.tunnel, { "yes" }),
         }
-
+
+        MergeTable(result_tags, DeriveTrafficSigns(transformedTags))
         MergeTable(result_tags, DeriveSmoothness(transformedTags))
         MergeTable(result_tags, DeriveSurface(transformedTags))
         CopyTags(result_tags, transformedTags, tags_prefixed, 'osm_')
 
         -- copy original tags
         CopyTags(result_tags, tags, tags_copied)
 
-         -- these keys are different for projected geometries
+        -- these keys are different for projected geometries
         if transformedTags._side ~= "self" then
           result_tags._id = DefaultId(object) .. '/' .. transformedTags._prefix .. '/' .. transformedTags._side
           result_tags._parent_highway = transformedTags._parent_highway

processing/topics/roads_bikelanes/bikelanes/categories.lua

@@ -1,7 +1,9 @@
 package.path = package.path .. ";/processing/topics/roads_bikelanes/bikelanes/categories/?.lua"
+package.path = package.path .. ";/processing/topics/helper/?.lua"
 require("IsTermInString")
 require("IsSidepath")
 require("AddAdjoiningOrIsolated")
+require("SanitizeTrafficSign")
 
 -- this category is for the explicit absence of bike infrastrucute
 local function dataNo(tags)
@@ -60,11 +62,12 @@ end
 -- https://wiki.openstreetmap.org/wiki/DE:Key:bicycle%20road
 -- traffic_sign=DE:244, https://wiki.openstreetmap.org/wiki/DE:Tag:traffic_sign%3DDE:244
 local function bicycleRoad(tags)
+  local trafficSign = SanitizeTrafficSign(tags.traffic_sign)
   if tags.bicycle_road == "yes"
-      or osm2pgsql.has_prefix(tags.traffic_sign, 'DE:244') then
+      or osm2pgsql.has_prefix(trafficSign, 'DE:244') then
     -- Subcategory when bicycle road allows vehicle traffic
-    if tags.traffic_sign == 'DE:244.1,1020-30'
-        or tags.traffic_sign == 'DE:244,1020-30'
+    if trafficSign == 'DE:244.1,1020-30'
+        or trafficSign == 'DE:244,1020-30'
         or tags.vehicle == 'destination'
         or tags.motor_vehicle == 'destination' then
       return "bicycleRoad_vehicleDestination"
@@ -78,8 +81,9 @@ end
 -- traffic_sign=DE:240, https://wiki.openstreetmap.org/wiki/DE:Tag:traffic_sign%3DDE:240
 local function footAndCyclewaySharedCases(tags)
   if tags.area == "yes" then return end
+  local trafficSign = SanitizeTrafficSign(tags.traffic_sign)
   local taggedWithAccessTagging = tags.bicycle == "designated" and tags.foot == "designated" and tags.segregated == "no"
-  local taggedWithTrafficsign = osm2pgsql.has_prefix(tags.traffic_sign, "DE:240")
+  local taggedWithTrafficsign = osm2pgsql.has_prefix(trafficSign, "DE:240")
   if taggedWithAccessTagging or taggedWithTrafficsign then
     -- isolated:
     -- Eg https://www.openstreetmap.org/way/440072364 highway=service
@@ -92,8 +96,9 @@ end
 -- traffic_sign=DE:241-31, https://wiki.openstreetmap.org/wiki/DE:Tag:traffic_sign%3DDE:241-31
 local function footAndCyclewaySegregatedCases(tags)
   if tags.area == "yes" then return end
+  local trafficSign = SanitizeTrafficSign(tags.traffic_sign)
   local taggedWithAccessTagging = tags.bicycle == "designated" and tags.foot == "designated" and tags.segregated == "yes"
-  local taggedWithTrafficsign = osm2pgsql.has_prefix(tags.traffic_sign, "DE:241")
+  local taggedWithTrafficsign = osm2pgsql.has_prefix(trafficSign, "DE:241")
   if taggedWithAccessTagging or taggedWithTrafficsign then
     return AddAdjoiningOrIsolated("footAndCyclewaySegregated", tags)
   end
@@ -102,6 +107,8 @@ end
 -- Case: "Gehweg, Fahrrad frei"
 -- traffic_sign=DE:1022-10 "Fahrrad frei", https://wiki.openstreetmap.org/wiki/DE:Tag:traffic_sign%3DDE:239
 local function footwayBicycleYesCases(tags)
+  local trafficSign = SanitizeTrafficSign(tags.traffic_sign)
+
   -- mtb:scale=* is a strong indicator for path' that we do not want to show, so we skip them;
   --    This will likely need a better solution in the future.
   --    Eg https://www.openstreetmap.org/way/23366687
@@ -110,7 +117,7 @@ local function footwayBicycleYesCases(tags)
 
   if tags.highway == "footway" or tags.highway == "path" then
     local taggedWithAccessTagging = tags.bicycle == "yes"
-    local taggedWithTrafficsign = IsTermInString("1022-10", tags.traffic_sign)
+    local taggedWithTrafficsign = IsTermInString("1022-10", trafficSign)
     if taggedWithAccessTagging or taggedWithTrafficsign then
       return AddAdjoiningOrIsolated("footwayBicycleYes", tags)
     end
@@ -121,14 +128,16 @@ end
 -- The sub-tagging specifies if the cycleway is part of a road or a separate way.
 -- This part relies heavly on the `is_sidepath` tagging.
 local function cyclewaySeparatedCases(tags)
+  local trafficSign = SanitizeTrafficSign(tags.traffic_sign)
+
   -- Case "Hochbordradwege"
   -- Detailled tagging to separate this case from `footAndCyclewaySegregatedCases`
   if tags.highway == "path"
       and (tags.foot == "yes" or tags.foot == "designated")
       and (tags.bicycle == "yes" or tags.bicycle == "designated")
       and tags.segregated == "yes"
       and tags.is_sidepath == "yes"
-      and not IsTermInString("241", tags.traffic_sign) then
+      and not IsTermInString("241", trafficSign) then
     return "cycleway_adjoining"
   end
 
@@ -138,7 +147,7 @@ local function cyclewaySeparatedCases(tags)
   local taggedWithAccessTagging = tags.highway == "cycleway" and
       (tags.cycleway == "track" or tags.cycleway == "opposite_track" or tags.is_sidepath)
   -- Testcase: The "not 'lane'" part is needed for places like https://www.openstreetmap.org/way/964589554 which have the traffic sign but are not separated.
-  local taggedWithTrafficsign = osm2pgsql.has_prefix(tags.traffic_sign, "DE:237") and not tags.cycleway == "lane"
+  local taggedWithTrafficsign = osm2pgsql.has_prefix(trafficSign, "DE:237") and not tags.cycleway == "lane"
   if taggedWithAccessTagging or taggedWithTrafficsign then
     -- adjoining:
     -- This could be PBLs "Protected Bike Lanes"
@@ -223,10 +232,11 @@ end
 -- "Fahrrad & Mofa frei" traffic_sign=DE:245,1022-14
 -- (History: Until 2023-03-2: cyclewayAlone)
 local function sharedBusLaneBusWithBike(tags)
+  local trafficSign = SanitizeTrafficSign(tags.traffic_sign)
   local taggedWithAccessTagging = tags.highway == "cycleway" and
       (tags.cycleway == "share_busway" or tags.cycleway == "opposite_share_busway")
-  local taggedWithTrafficsign = osm2pgsql.has_prefix(tags.traffic_sign, "DE:245") and
-      (IsTermInString("1022-10", tags.traffic_sign) or IsTermInString("1022-14", tags.traffic_sign))
+  local taggedWithTrafficsign = osm2pgsql.has_prefix(trafficSign, "DE:245") and
+      (IsTermInString("1022-10", trafficSign) or IsTermInString("1022-14", trafficSign))
   if taggedWithAccessTagging or taggedWithTrafficsign then
     -- Note: Was `sharedBusLane` until 2024-05-02 when we introduced `sharedBusLaneBikeWithBus`
     return "sharedBusLaneBusWithBike"
@@ -242,9 +252,10 @@ end
 --   - 87 overall https://taginfo.openstreetmap.org/tags/cycleway%3Aright%3Alane=share_busway#overview
 --   - 1 overall for left https://taginfo.openstreetmap.org/tags/cycleway%3Aleft%3Alane=share_busway#overview
 local function sharedBusLaneBikeWithBus(tags)
+  local trafficSign = SanitizeTrafficSign(tags.traffic_sign)
   local taggedWithAccessTagging = tags.highway == "cycleway" and tags.lane == "share_busway"
   local taggedWithTrafficsign =
-      osm2pgsql.has_prefix(tags.traffic_sign, "DE:237") and IsTermInString("1024-14", tags.traffic_sign)
+      osm2pgsql.has_prefix(trafficSign, "DE:237") and IsTermInString("1024-14", trafficSign)
   if taggedWithAccessTagging or taggedWithTrafficsign then
     return "sharedBusLaneBikeWithBus"
   end

processing/topics/roads_bikelanes/roads/RoadClassification.lua

@@ -2,16 +2,13 @@ package.path = package.path .. ";/processing/topics/helper/?.lua"
 require("Set")
 require("CopyTags")
 require("Sanitize")
+require("DeriveTrafficSigns")
 
 local tags_copied = {
   "mapillary",
   "description",
 }
-local tags_prefixed = {
-  'traffic_sign',
-  'traffic_sign:forward',
-  'traffic_sign:backward',
-}
+local tags_prefixed = {}
 
 
 function RoadClassification(object)
@@ -104,6 +101,7 @@ function RoadClassification(object)
   result_tags.oneway = Sanitize(tags.oneway, { "yes", "no" })
   result_tags.bridge = Sanitize(tags.bridge, { "yes" })
   result_tags.tunnel = Sanitize(tags.tunnel, { "yes" })
+  MergeTable(result_tags, DeriveTrafficSigns(tags))
 
   return result_tags
 end

processing/topics/roads_bikelanes/roads_bikelanes.lua

@@ -121,7 +121,7 @@ function osm2pgsql.process_way(object)
       publicTags.road = results.road
       publicTags._parent_highway = cycleway._parent_highway
 
-      cycleway.segregated = nil            -- no idea why that is present in the inspector frontend for way 9717355
+      cycleway.segregated = nil -- no idea why that is present in the inspector frontend for way 9717355
       bikelanesTable:insert({
         id = cycleway._id,
         tags = publicTags,