Release 2024-05-21 (#141)

.env.example

@@ -16,7 +16,7 @@ SKIP_WARM_CACHE=1 # skips cache warming
 WAIT_FOR_FRESH_DATA=0 # waits till the given file is from today
 ID_FILTER='' # if not empty only the objects with these ids are processed. Ids need to be prefixed with the oms_type, see https://docs.osmcode.org/osmium/latest/osmium-getid.html
 COMPUTE_DIFFS=1 # whether to create diff tables or not
-FREEZE_DATA=0 # whether to surpress updating the tables. Works only in combination with `COMPUTE_DIFFS=1`
+FREEZE_DATA=0 # whether to surpress updating the tables.  Works only in combination with `COMPUTE_DIFFS=1`. `0` creates a new reference point, `1` uses the last reference point to diff against.
 
 # Token for Synology log. Leave blank to disable logging
 SYNOLOGY_LOG_TOKEN=

.github/workflows/ci.yml

@@ -0,0 +1,27 @@
+name: Check PR
+on: [pull_request]
+jobs:
+  run-ci:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build Image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./processing.Dockerfile
+          tags: test_img
+          target: lua
+          push: false
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Run Tests
+        run: |
+          docker run -v ./processing:/processing test_img busted /processing/topics/ -p %.test%.lua$

.github/workflows/deployment.yml

@@ -92,7 +92,7 @@ jobs:
 
       - name: Configure AWS credentials
         if: steps.git_changes.outputs.changes == 'true'
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.gitignore

@@ -7,10 +7,11 @@
 # IDE
 .idea
 
-node_modules
-
 # Nginx
 logs/
 
 # warm-cache
 processing/warm-cache
+
+# run helper
+processing/topics/helper/._trigger-full-run-helper.lua

README.md

@@ -75,7 +75,9 @@ If any helper in (`topics/helper`)[processing/topics/helper] changed, we rerun e
 
 Whenever we talk about `hash`es in this code, this feature is referenced.
 
-**Force rerun:** Whenever you need to force a rerun, open [any lua helper](./processing/topics/helper/Set.lua) and add a temporary code comment, save and restart the processing.
+#### Force rerun
+
+Whenever you need to force a rerun, open [any lua helper](./processing/topics/helper/Set.lua) and add a temporary code comment, save and restart the processing. Use the helper `run-full.sh` to do this automatically.
 
 ### Processing: Inspect changes
 
@@ -85,7 +87,17 @@ It will compare the `tags` column to the previous run.
 
 Whenever we talk about `diff`s in this code, this feature is referenced.
 
-With `FREEZE_DATA=1` the system will **not** update the tables. This is usefull during development if one only wants to see the changes to a certain version. This flag will be ignored if `COMPUTE_DIFFS=0`.
+#### Reference
+
+- With `FREEZE_DATA=0` you see the changes to the last run on every run
+- With `FREEZE_DATA=1` you see the changes to the last reference-run, allowing you to compare your changes to a certain version of your data. The reference will be the last time you ran with `FREEZE_DATA=0`. In this case the system will **not** update the `<tablename>_diff` tables. This flag will be ignored if `COMPUTE_DIFFS=0`.
+
+Use `run-full.sh` to toggle `FREEZE_DATA` and force a full rerun for a fresh reference.
+
+#### `age` diffs
+
+If `age` diffs show up, you need to create a fresh reference run of all the data.
+You may use `run-full.sh` to set `FREEZE_DATA=0` and modify the helper folder to trigger a full rerun.
 
 ### Process only a single object
 

processing.Dockerfile

@@ -1,4 +1,11 @@
-FROM ubuntu:mantic
+FROM ubuntu:mantic as lua
+
+# Install Lua and "luarocks" (Lua package manager) – https://luarocks.org/, https://packages.ubuntu.com/luarocks
+RUN apt update && apt install -y lua5.3 liblua5.3-dev luarocks
+
+RUN luarocks install busted
+
+FROM lua as processing
 
 ARG DEBIAN_FRONTEND=noninteractive
 ENV TZ=Europe/Berlin
@@ -10,21 +17,9 @@ LABEL maintainer="FixMyCity - https://fixmycity.de"
 #   - /var/run/docker.sock:/var/run/docker.sock
 COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
 
-# - "luarocks" – Lua package manager – https://luarocks.org/, https://packages.ubuntu.com/luarocks
 RUN apt update && \
-  apt install -y osm2pgsql osmium-tool postgresql-client-15 tzdata wget libpq-dev curl lua5.3 liblua5.3-dev luarocks && \
+  apt install -y osm2pgsql osmium-tool postgresql-client-15 tzdata wget curl && \
   apt upgrade -y
-# LUA Libaries:
-# - "dkjson" used by parking.lua to write JSON for debugging
-# RUN luarocks install dkjson
-# TODO: We need to find a way to use those packages locally; otherwise using them does not make much sense
-# - "inspect" https://github.com/kikito/inspect.lua
-#   (recommended in https://github.com/openstreetmap/osm2pgsql/blob/master/flex-config/README.md#dependencies)
-# RUN luarocks install inspect
-# - "date" https://luarocks.org/modules/tieske/date, https://github.com/Tieske/date
-
-# RUN luarocks install date
-RUN luarocks install busted
 
 # install node
 RUN apt-get install -y nodejs npm

processing/run-8-analysis.sh

@@ -4,6 +4,6 @@ set -e
 source /processing/utils/logging.sh
 log_start "$0"
 
-psql -q -f "./analysis/boundaryStats.sql"
+psql -q -f "./analysis/presenceStats.sql"
 
 log_end "$0"

processing/topics/bicycleParking/bicycleParking.lua

@@ -4,6 +4,7 @@ require("Metadata")
 require("Set")
 require("Sanitize")
 require("DefaultId")
+require("SanitizeTrafficSign")
 
 local nodeTable = osm2pgsql.define_table({
   name = 'bicycleParking_points',
@@ -87,12 +88,13 @@ local function processTags(tags)
   local tags_cc = {
     "area", "operator:type", "covered", "indoor", "access", "cargo_bike", "capacity",
     "capacity:cargo_bike", "fee", "lit", "surface", "bicycle_parking", "maxstay", "surveillance",
-    "bicycle_parking:count", "bicycle_parking:position", "traffic_sign", "description",
+    "bicycle_parking:count", "bicycle_parking:position", "description",
     "mapillary",
     "description",
   }
   CopyTags(results, tags, allowed_tags)
   CopyTags(results, tags, tags_cc, "osm_")
+  results.traffic_sign = SanitizeTrafficSign(tags.traffic_sign)
 
   results.age = AgeInDays(ParseCheckDate(tags["check_date"]))
   return results

processing/topics/helper/DeriveTrafficSigns.lua

@@ -0,0 +1,12 @@
+package.path = package.path .. ";/processing/topics/helper/?.lua"
+
+require("SanitizeTrafficSign")
+
+function DeriveTrafficSigns(tags)
+  local results = {
+    ['traffic_sign'] = SanitizeTrafficSign(tags.traffic_sign) or SanitizeTrafficSign(tags['traffic_sign:both']),
+    ['traffic_sign:forward'] = SanitizeTrafficSign(tags['traffic_sign:forward']),
+    ['traffic_sign:backward'] = SanitizeTrafficSign(tags['traffic_sign:backward'])
+  }
+  return results
+end

processing/topics/helper/RemoveNilValues.lua

@@ -0,0 +1,12 @@
+--- Remove table entries where the value is `nil`
+---@param inputTable table
+---@return table
+function RemoveNilValues(inputTable)
+  local cleanTable = {}
+  for _, v in ipairs(inputTable) do
+    if v ~= nil then
+      cleanTable[#cleanTable + 1] = v
+    end
+  end
+  return cleanTable
+end

processing/topics/helper/SanitizeTrafficSign.lua

@@ -0,0 +1,52 @@
+-- Remove all whitespaces after delimeters
+local function stripWhitespaces(traffic_sign)
+  local stripped = string.gsub(traffic_sign, ', ', ',')
+  stripped = string.gsub(stripped, '; ', ';')
+  return stripped
+end
+
+--- Cleanup the `traffic_sign=*` tag
+--- @param traffic_sign string
+--- @returns string
+function SanitizeTrafficSign(traffic_sign)
+  if traffic_sign == nil then
+    return nil
+  end
+  if traffic_sign == "no" or traffic_sign == 'none' then
+    return "none"
+  end
+
+  -- Docs: patterns with "^" target beginning of string
+
+  -- This is the correct tagging, all traffic signs should start with DE:
+  if string.find(traffic_sign, '^DE:%S') then
+    return stripWhitespaces(traffic_sign)
+  end
+
+  local substitutions = {
+    ['^DE: '] = 'DE:',
+    ['^DE%.'] = 'DE:',
+    ['^D:'] = 'DE:',
+    ['^D%.'] = 'DE:',
+    ['^de:'] = 'DE:',
+    ['^DE1'] = 'DE:1',
+    ['^DE2'] = 'DE:2',
+    ['^2'] = 'DE:2',
+    ['^1'] = 'DE:1',
+    -- These patterns could handle all the above in a more generalized way
+    -- ['^DE?[:.]%s?'] = 'DE:',
+    -- ['^de?[:.]%s?'] = 'DE:'
+    -- ['^DE(%d)'] = 'DE:%1'
+    -- ['^(%d)'] = 'DE:%1'
+  }
+  for pattern, substitude in pairs(substitutions) do
+    local val, n = string.gsub(traffic_sign, pattern, substitude)
+    if n > 0 then
+      -- TODO: add to todo list
+      return stripWhitespaces(val)
+    end
+  end
+
+  -- Discard everything else
+  return nil
+end

processing/topics/helper/__tests__/SanitizeTrafficSign.test.lua

@@ -0,0 +1,73 @@
+require("osm2pgsql")
+
+describe("SanitizeTrafficSign", function()
+  package.path = package.path .. ";/processing/topics/helper/?.lua"
+  require("SanitizeTrafficSign")
+
+  -- Cleanup
+  it('renames `no` to `none`', function()
+    assert.are.same(SanitizeTrafficSign("no"), "none")
+  end)
+
+  it('renames `DE:SPACE` to `DE:`', function()
+    assert.are.same(SanitizeTrafficSign("DE: 123"), "DE:123")
+  end)
+
+  it('renames `DE234` to `DE:234`', function()
+    assert.are.same(SanitizeTrafficSign("DE234"), "DE:234")
+  end)
+
+  it('renames `DE1010` to `DE:1010`', function()
+    assert.are.same(SanitizeTrafficSign("DE1010"), "DE:1010")
+  end)
+
+  it('renames `D:234` to `DE:234`', function()
+    assert.are.same(SanitizeTrafficSign("D:234"), "DE:234")
+  end)
+
+  it('renames `de:234` to `DE:234`', function()
+    assert.are.same(SanitizeTrafficSign("de:234"), "DE:234")
+  end)
+
+  it('renames `234` to `DE:234`', function()
+    assert.are.same(SanitizeTrafficSign("234"), "DE:234")
+  end)
+
+  it('renames `1010` to `DE:1010`', function()
+    assert.are.same(SanitizeTrafficSign("1010"), "DE:1010")
+  end)
+
+  it('renames `DE.234` to `DE:234`', function()
+    assert.are.same(SanitizeTrafficSign("DE.234"), "DE:234")
+  end)
+
+  it('cleans spaces `DE:123, 1010; 234` to `DE:123,1010;234`', function()
+    assert.are.same(SanitizeTrafficSign("DE:123, 1010; 234"), "DE:123,1010;234")
+  end)
+
+  -- Allow
+  it('allows `DE:234`', function()
+    assert.are.same(SanitizeTrafficSign("DE:234"), "DE:234")
+  end)
+
+  it('allows `DE:1010`', function()
+    assert.are.same(SanitizeTrafficSign("DE:1010"), "DE:1010")
+  end)
+
+  it('allows `none` as value', function()
+    assert.are.same(SanitizeTrafficSign("none"), "none")
+  end)
+
+  -- Disallow
+  it('handles nil', function()
+    assert.are.same(SanitizeTrafficSign(nil), nil)
+  end)
+  it('disallows everything else', function()
+    assert.are.same(SanitizeTrafficSign("foobar"), nil)
+    assert.are.same(SanitizeTrafficSign("yes"), nil)
+    assert.are.same(SanitizeTrafficSign("unkown"), nil)
+    assert.are.same(SanitizeTrafficSign("AT:foobar"), nil)
+    assert.are.same(SanitizeTrafficSign("pictogram"), nil)
+    assert.are.same(SanitizeTrafficSign("(comment)"), nil)
+  end)
+end)