diff --git a/.github/workflows/deployment.production.yml b/.github/workflows/deployment.production.yml index d4fc00d4..cb777e32 100644 --- a/.github/workflows/deployment.production.yml +++ b/.github/workflows/deployment.production.yml @@ -10,7 +10,7 @@ jobs: uses: ./.github/workflows/deployment.yml with: ENVIRONMENT: production - URL: tiles.radverkehrsatlas.de + TILES_URL: tiles.radverkehrsatlas.de secrets: SERVICE_NAME: ${{ secrets.SERVICE_NAME }} DATABASE_NAME: ${{ secrets.DATABASE_NAME }} @@ -24,4 +24,4 @@ jobs: SSH_USERNAME: ${{ secrets.SSH_USERNAME }} SSH_PASSWORD: ${{ secrets.SSH_PASSWORD }} SYNOLOGY_LOG_TOKEN: ${{ secrets.SYNOLOGY_LOG_TOKEN }} - SYNOLOGY_ERROR_LOG_TOKEN: ${{ secrets.SYNOLOGY_ERROR_LOG_TOKEN }} \ No newline at end of file + SYNOLOGY_ERROR_LOG_TOKEN: ${{ secrets.SYNOLOGY_ERROR_LOG_TOKEN }} diff --git a/.github/workflows/deployment.staging.yml b/.github/workflows/deployment.staging.yml index 17dffafa..2c579bed 100644 --- a/.github/workflows/deployment.staging.yml +++ b/.github/workflows/deployment.staging.yml @@ -10,7 +10,7 @@ jobs: uses: ./.github/workflows/deployment.yml with: ENVIRONMENT: staging - URL: staging-tiles.radverkehrsatlas.de + TILES_URL: staging-tiles.radverkehrsatlas.de secrets: SERVICE_NAME: ${{ secrets.SERVICE_NAME }} DATABASE_NAME: ${{ secrets.DATABASE_NAME }} diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index bef2fca0..ed76ce20 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -33,7 +33,7 @@ on: ENVIRONMENT: type: string required: true - URL: + TILES_URL: type: string required: true @@ -42,11 +42,40 @@ jobs: runs-on: ubuntu-latest environment: name: ${{ inputs.ENVIRONMENT }} - url: https://${{ inputs.URL }} + url: https://${{ inputs.TILES_URL }} steps: - uses: actions/checkout@v4 + with: + fetch-depth: 6 + + - name: Get last run commit SHA + id: last_run + run: | + LAST_RUN_SHA=$(curl --request GET \ + --url https://api.github.com/repos/${{ github.repository }}/actions/runs \ + --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ + | jq -r '.workflow_runs[1].head_sha') + echo "::set-output name=sha::$LAST_RUN_SHA" + shell: bash + + - name: Check if LAST_RUN_SHA is one of the last 6 commits and check changes + id: git_changes + run: | + LAST_6_COMMITS=$(git log -n 6 --pretty=format:"%H") + if echo "$LAST_6_COMMITS" | grep -q "${{ steps.last_run.outputs.sha }}"; then + if git diff --quiet ${{ steps.last_run.outputs.sha }} HEAD -- app/ helpers/ app.Dockerfile docker-compose.yml; then + echo "::set-output name=changes::false" + else + echo "::set-output name=changes::true" + fi + else + echo "::set-output name=changes::true" + fi + shell: bash + - name: Upgrade AWS CLI version and setup lightsailctl + if: steps.git_changes.outputs.changes == 'true' run: | aws --version curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" @@ -58,6 +87,7 @@ jobs: sudo chmod +x /usr/local/bin/lightsailctl - name: Configure AWS credentials + if: steps.git_changes.outputs.changes == 'true' uses: aws-actions/configure-aws-credentials@v2 with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -65,9 +95,11 @@ jobs: aws-region: ${{ secrets.AWS_REGION }} - name: Setup buildx + if: steps.git_changes.outputs.changes == 'true' uses: docker/setup-buildx-action@v3 - name: Build app image + if: steps.git_changes.outputs.changes == 'true' uses: docker/build-push-action@v5 with: context: . @@ -79,6 +111,7 @@ jobs: tags: public.ecr.aws/n0p8j4k5/atlas/app:${{ github.sha }} - name: Push the app image + if: steps.git_changes.outputs.changes == 'true' run: | aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/n0p8j4k5/ docker push public.ecr.aws/n0p8j4k5/atlas/app:${{ github.sha }} @@ -93,6 +126,17 @@ jobs: target: "/srv/processing/" overwrite: true + - name: Update GITHUB_SHA + if: steps.git_changes.outputs.changes == 'true' + uses: appleboy/ssh-action@master + with: + host: ${{ secrets.SSH_HOST }} + username: ${{ secrets.SSH_USERNAME }} + password: ${{ secrets.SSH_PASSWORD }} + script: | + cd /srv/processing/ + sed -i "s|^GITHUB_SHA=.*$|GITHUB_SHA='${{ github.sha }}'|" .env + - name: Stop & Start containers on VPS uses: appleboy/ssh-action@master with: @@ -101,20 +145,18 @@ jobs: password: ${{ secrets.SSH_PASSWORD }} script: | cd /srv/processing/ - echo "Updating '.env'" - rm .env - touch .env - echo PGHOST='${{ vars.DATABASE_HOST }}' >> .env - echo ENVIRONMENT='${{ inputs.ENVIRONMENT }}' >> .env - echo SYNOLOGY_LOG_TOKEN='${{ secrets.SYNOLOGY_LOG_TOKEN }}' >> .env - echo SYNOLOGY_ERROR_LOG_TOKEN='${{ secrets.SYNOLOGY_ERROR_LOG_TOKEN }}' >> .env - echo PGUSER='${{ secrets.DATABASE_USER }}' >> .env - echo PGPASSWORD='${{ secrets.DATABASE_PASSWORD }}' >> .env - echo PGDATABASE='${{ secrets.DATABASE_NAME }}' >> .env - echo OSM_DOWNLOAD_URL='${{ vars.OSM_DOWNLOAD_URL }}' >> .env - echo URL='${{ inputs.URL }}' >> .env - echo GITHUB_SHA='${{ github.sha }}' >> .env + sed -i \ + -e "s|^PGHOST=.*$|PGHOST='${{ vars.DATABASE_HOST }}'|" \ + -e "s|^ENVIRONMENT=.*$|ENVIRONMENT='${{ inputs.ENVIRONMENT }}'|" \ + -e "s|^SYNOLOGY_LOG_TOKEN=.*$|SYNOLOGY_LOG_TOKEN='${{ secrets.SYNOLOGY_LOG_TOKEN }}'|" \ + -e "s|^SYNOLOGY_ERROR_LOG_TOKEN=.*$|SYNOLOGY_ERROR_LOG_TOKEN='${{ secrets.SYNOLOGY_ERROR_LOG_TOKEN }}'|" \ + -e "s|^PGUSER=.*$|PGUSER='${{ secrets.DATABASE_USER }}'|" \ + -e "s|^PGPASSWORD=.*$|PGPASSWORD='${{ secrets.DATABASE_PASSWORD }}'|" \ + -e "s|^PGDATABASE=.*$|PGDATABASE='${{ secrets.DATABASE_NAME }}'|" \ + -e "s|^OSM_DOWNLOAD_URL=.*$|OSM_DOWNLOAD_URL='${{ vars.OSM_DOWNLOAD_URL }}'|" \ + -e "s|^TILES_URL=.*$|TILES_URL='${{ inputs.TILES_URL }}'|" \ + .env echo "Reload containers" - docker compose -f docker-compose.traefik.yml up -d + docker compose pull docker compose up -d docker image prune -fa diff --git a/app/process/barriers/barriers.lua b/app/process/barriers/barriers.lua index 8d429952..b4438912 100644 --- a/app/process/barriers/barriers.lua +++ b/app/process/barriers/barriers.lua @@ -14,16 +14,6 @@ local lineBarriers = osm2pgsql.define_table({ } }) --- local excludedLineBarriers = osm2pgsql.define_table({ --- name = 'barrierLines_excluded', --- ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, --- columns = { --- { column = 'tags', type = 'jsonb' }, --- { column = 'meta', type = 'jsonb' }, --- { column = 'geom', type = 'linestring' }, --- } --- }) - local areaBarriers = osm2pgsql.define_table({ name = 'barrierAreas', ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, @@ -34,17 +24,6 @@ local areaBarriers = osm2pgsql.define_table({ } }) --- local excludedAreaBarriers = osm2pgsql.define_table({ --- name = 'barrierAreas_excluded', --- ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, --- columns = { --- { column = 'tags', type = 'jsonb' }, --- { column = 'meta', type = 'jsonb' }, --- { column = 'geom', type = 'multipolygon' }, --- } --- }) - - local allowedTags = Set({ 'tunnel', 'waterway', @@ -91,44 +70,33 @@ function osm2pgsql.process_way(object) FilterTags(object.tags, allowedTags) areaBarriers:insert({ tags = object.tags, - meta=Metadata(object), - geom=object:as_multipolygon() + meta = Metadata(object), + geom = object:as_multipolygon() }) return end - -- excludedAreaBarriers:insert({ - -- tags=object.tags, - -- meta=Metadata(object), - -- geom=object:as_multipolygon() - -- }) else --process as linestring local tags = object.tags - -- if tags.tunnel =='yes' then return end -- we don't consider tunnels as barriers - local isBarrier = HighwayClasses[tags.highway] - -- only need for low zoom levels - local waterBarriers = Set({"river", "canal"}) + -- waterways as lines are used for low zoom levels + local waterBarriers = Set({ "river", "canal" }) isBarrier = isBarrier or waterBarriers[tags.waterway] - local trainBarriers = Set({"main", "branch"}) + local trainBarriers = Set({ "main", "branch" }) if (tags.railway == 'rail' or tags.railway == 'light_rail') then isBarrier = isBarrier or trainBarriers[tags.usage] end + if isBarrier then FilterTags(object.tags, allowedTags) lineBarriers:insert({ tags = object.tags, - meta=Metadata(object), - geom=object:as_linestring(), + meta = Metadata(object), + geom = object:as_linestring(), }) return end - -- excludedLineBarriers:insert({ - -- tags=object.tags, - -- meta=Metadata(object), - -- geom=object:as_linestring() - -- }) end end @@ -137,14 +105,9 @@ function osm2pgsql.process_relation(object) FilterTags(object.tags, allowedTags) areaBarriers:insert({ tags = object.tags, - meta=Metadata(object), - geom=object:as_multipolygon() + meta = Metadata(object), + geom = object:as_multipolygon() }) return end - -- excludedAreaBarriers:insert({ - -- tags=object.tags, - -- meta=Metadata(object), - -- geom=object:as_multipolygon() - -- }) end diff --git a/app/process/roads_bikelanes/bikelanes/categories.lua b/app/process/roads_bikelanes/bikelanes/categories.lua index 5f108b9e..cb27fc93 100644 --- a/app/process/roads_bikelanes/bikelanes/categories.lua +++ b/app/process/roads_bikelanes/bikelanes/categories.lua @@ -89,7 +89,8 @@ local function footAndCyclewaySharedCases(tags) local taggedWithAccessTagging = tags.bicycle == "designated" and tags.foot == "designated" and tags.segregated == "no" local taggedWithTrafficsign = osm2pgsql.has_prefix(tags.traffic_sign, "DE:240") if taggedWithAccessTagging or taggedWithTrafficsign then - if tags.is_sidepath == "yes" or tags.footway == "sidewalk" then + -- `_parent_highway` indicates that this way was split of the centerline; in this case, we consider it a sidepath. + if tags.is_sidepath == "yes" or tags._parent_highway or tags.footway == "sidewalk" then return "footAndCyclewayShared_adjoining" end -- Eg https://www.openstreetmap.org/way/440072364 highway=service @@ -108,7 +109,8 @@ local function footAndCyclewaySegregatedCases(tags) local taggedWithAccessTagging = tags.bicycle == "designated" and tags.foot == "designated" and tags.segregated == "yes" local taggedWithTrafficsign = osm2pgsql.has_prefix(tags.traffic_sign, "DE:241") if taggedWithAccessTagging or taggedWithTrafficsign then - if tags.is_sidepath == "yes" or tags.footway == "sidewalk" then + -- `_parent_highway` indicates that this way was split of the centerline; in this case, we consider it a sidepath. + if tags.is_sidepath == "yes" or tags._parent_highway or tags.footway == "sidewalk" then return "footAndCyclewaySegregated_adjoining" end if tags.is_sidepath == "no" then @@ -129,12 +131,13 @@ local function footwayBicycleYesCases(tags) if tags.highway == "footway" or tags.highway == "path" then if tags.bicycle == "yes" or IsTermInString("1022-10", tags.traffic_sign) then - -- https://www.openstreetmap.org/way/946438663 - if tags.is_sidepath == "yes" or tags.footway == "sidewalk" then - return "footwayBicycleYes_isolated" + -- `_parent_highway` indicates that this way was split of the centerline; in this case, we consider it a sidepath. + if tags.is_sidepath == "yes" or tags._parent_highway or tags.footway == "sidewalk" then + return "footwayBicycleYes_adjoining" end + -- https://www.openstreetmap.org/way/946438663 if tags.is_sidepath == "no" then - return "footwayBicycleYes_adjoining" + return "footwayBicycleYes_isolated" end return "footwayBicycleYes_adjoiningOrIsolated" end diff --git a/app/run.sh b/app/run.sh index 476bbe21..078829b5 100755 --- a/app/run.sh +++ b/app/run.sh @@ -26,33 +26,33 @@ alert() { curl -X POST $url -d "payload=$payload" --silent --output "/dev/null" } -if ! ./run-1-download.sh; then - alert '*ERROR*: #run-1-download exited with non-zero status code' -fi - -if ! ./run-2-filter.sh; then - alert '*ERROR*: #run-2-filter exited with non-zero status code' -fi - -if ! ./run-3-migration.sh; then - alert '*ERROR*: #run-3-migration exited with non-zero status code' -fi - -process_start_time=$(date +%s) -if ! ./run-4-process.sh; then - alert '*ERROR*: #run-4-process exited with non-zero status code' -fi -process_end_time=$(date +%s) -export PROCESS_RUN_TIME_DIFF=$((process_end_time - process_start_time)) # used by metadata.sh - -if ! ./run-5-postprocess.sh; then - alert '*ERROR*: #run-5-postprocess exited with non-zero status code' -fi - -if ! ./run-6-analysis.sh; then - alert '*ERROR*: #run-6-analysis exited with non-zero status code' -fi - -if ! ./run-7-metadata.sh; then - alert '*ERROR*: #run-7-metadata exited with non-zero status code' -fi +# if ! ./run-1-download.sh; then +# alert '*ERROR*: #run-1-download exited with non-zero status code' +# fi + +# if ! ./run-2-filter.sh; then +# alert '*ERROR*: #run-2-filter exited with non-zero status code' +# fi + +# if ! ./run-3-migration.sh; then +# alert '*ERROR*: #run-3-migration exited with non-zero status code' +# fi + +# process_start_time=$(date +%s) +# if ! ./run-4-process.sh; then +# alert '*ERROR*: #run-4-process exited with non-zero status code' +# fi +# process_end_time=$(date +%s) +# export PROCESS_RUN_TIME_DIFF=$((process_end_time - process_start_time)) # used by metadata.sh + +# if ! ./run-5-postprocess.sh; then +# alert '*ERROR*: #run-5-postprocess exited with non-zero status code' +# fi + +# if ! ./run-6-analysis.sh; then +# alert '*ERROR*: #run-6-analysis exited with non-zero status code' +# fi + +# if ! ./run-7-metadata.sh; then +# alert '*ERROR*: #run-7-metadata exited with non-zero status code' +# fi diff --git a/configs/nginx.conf b/configs/nginx.conf index 76af1598..084fa879 100644 --- a/configs/nginx.conf +++ b/configs/nginx.conf @@ -35,7 +35,7 @@ http { proxy_cache_path /var/cache/nginx/ levels=1:2 - max_size=15g + max_size=50g inactive=60m use_temp_path=off keys_zone=backend_cache:240m; @@ -61,7 +61,7 @@ http { proxy_cache backend_cache; proxy_cache_lock on; proxy_cache_revalidate on; - proxy_cache_valid 200 204 302 1d; + proxy_cache_valid 200 204 302 12h; proxy_cache_valid 404 1m; proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504; add_header X-Cache-Status $upstream_cache_status; diff --git a/docker-compose.traefik.yml b/docker-compose.traefik.yml deleted file mode 100644 index e47bc227..00000000 --- a/docker-compose.traefik.yml +++ /dev/null @@ -1,31 +0,0 @@ -version: "3" -services: - traefik: - image: traefik:v2.10 - container_name: traefik - # See https://techoverflow.net/2021/06/11/how-to-fix-traefik-gateway-timeout-for-docker-services/ for network_mode - network_mode: host - restart: always - command: - - "--log.level=WARN" - - "--api.insecure=false" - - "--api.dashboard=false" - - "--providers.docker=true" - # Do not expose containers unless explicitly told so - - "--providers.docker.exposedbydefault=false" - - "--entrypoints.web.address=:80" - - "--entrypoints.websecure.address=:443" - - "--entrypoints.web.http.redirections.entryPoint.to=websecure" - - "--entrypoints.web.http.redirections.entryPoint.scheme=https" - - "--certificatesresolvers.letsencrypt.acme.httpchallenge=true" - - "--certificatesresolvers.letsencrypt.acme.httpchallenge.entrypoint=web" - - "--certificatesresolvers.letsencrypt.acme.email=dev-team@fixmycity.de" - - "--certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json" - - "--serversTransport.forwardingTimeouts.dialTimeout=100s" - ports: - - "80:80" - - "443:443" - - "8080:8080" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - "./letsencrypt:/letsencrypt" diff --git a/docker-compose.yml b/docker-compose.yml index ad1dfd34..dad72754 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,35 @@ version: "3" services: + traefik: + image: traefik:v2.10 + container_name: traefik + # Networking: See https://techoverflow.net/2021/06/11/how-to-fix-traefik-gateway-timeout-for-docker-services/ for network_mode + restart: unless-stopped + command: + - "--log.level=WARN" + - "--api.insecure=false" + - "--api.dashboard=false" + - "--providers.docker=true" + # Do not expose containers unless explicitly told so + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.web.address=:80" + - "--entrypoints.websecure.address=:443" + - "--entrypoints.web.http.redirections.entryPoint.to=websecure" + - "--entrypoints.web.http.redirections.entryPoint.scheme=https" + - "--certificatesresolvers.letsencrypt.acme.httpchallenge=true" + - "--certificatesresolvers.letsencrypt.acme.httpchallenge.entrypoint=web" + - "--certificatesresolvers.letsencrypt.acme.email=dev-team@fixmycity.de" + - "--certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json" + - "--serversTransport.forwardingTimeouts.dialTimeout=100s" + ports: + - "80:80" + - "443:443" + - "8080:8080" + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - "./letsencrypt:/letsencrypt" + networks: + - app_bridge app: image: public.ecr.aws/n0p8j4k5/atlas/app:${GITHUB_SHA} entrypoint: /app/run.sh @@ -35,17 +65,35 @@ services: command: ["--auto-bounds=calc"] ports: - 3333:3000 + labels: + - "traefik.enable=false" + networks: + - app_bridge + depends_on: + db: + condition: service_healthy + proxy: + image: nginx:alpine + restart: unless-stopped + container_name: tiles_proxy labels: - "traefik.enable=true" - - "traefik.http.routers.tiles.rule=Host(`${URL}`)" + - "traefik.http.routers.tiles.rule=Host(`${TILES_URL}`)" - "traefik.http.routers.tiles.entrypoints=websecure" - "traefik.http.routers.tiles.tls.certresolver=letsencrypt" - "traefik.http.routers.tiles.tls=true" networks: - app_bridge - depends_on: - db: - condition: service_healthy + ports: + - "4444:80" + volumes: + - ./cache:/var/cache/nginx + - ./logs:/var/log/nginx + - ./configs/nginx.conf:/etc/nginx/nginx.conf:ro + # Needs healty check for tiles server + # depends_on: + # tiles: + # condition: service_healthy db: image: postgis/postgis:14-3.3-alpine container_name: db