From e4f7b0889bfff397f82c0cf24bea269e9105de43 Mon Sep 17 00:00:00 2001 From: Holger Bruch Date: Tue, 17 Mar 2020 00:26:09 +0100 Subject: [PATCH] First version --- .dockerignore | 1 + .gitignore | 1 + Dockerfile | 27 ++ README.md | 77 ++++ config/gtfs-feeds.csv | 41 ++ config/gtfs-rules/VGC.rule | 33 ++ config/gtfs-rules/VGF.rule | 33 ++ config/gtfs-rules/VRN.rule | 20 + config/gtfs-rules/VVS.rule | 4 + config/gtfs-rules/naldo.rule | 16 + config/osm/bw_buffered.poly | 32 ++ config/osm/park_ride_transform.xml | 711 +++++++++++++++++++++++++++++ makefile | 29 ++ update_all.sh | 13 + update_gtfs.sh | 133 ++++++ update_osm.sh | 36 ++ 16 files changed, 1207 insertions(+) create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 config/gtfs-feeds.csv create mode 100644 config/gtfs-rules/VGC.rule create mode 100644 config/gtfs-rules/VGF.rule create mode 100644 config/gtfs-rules/VRN.rule create mode 100644 config/gtfs-rules/VVS.rule create mode 100644 config/gtfs-rules/naldo.rule create mode 100644 config/osm/bw_buffered.poly create mode 100644 config/osm/park_ride_transform.xml create mode 100644 makefile create mode 100644 update_all.sh create mode 100755 update_gtfs.sh create mode 100755 update_osm.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..adbb97d --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +data/ \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..adbb97d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +data/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..54670a2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM alpine +MAINTAINER MITFAHR|DE|ZENTRALE + +RUN apk add --update --no-cache \ + make \ + bash \ + curl \ + zip \ + && rm -rf /var/cache/apk/* + +WORKDIR /opt/gtfs-hub + +RUN curl -O https://download.docker.com/linux/static/stable/x86_64/docker-18.06.1-ce.tgz && \ + tar xzf docker-18.06.1-ce.tgz && \ + cp docker/docker /usr/bin/docker && \ + rm -rf docker* + +VOLUME /var/data + +ADD update_gtfs.sh . +ADD update_osm.sh . +ADD update_all.sh . +ADD makefile . + +ADD config/ ./config/ + +CMD bash ./update_all.sh diff --git a/README.md b/README.md new file mode 100644 index 0000000..bbe86ee --- /dev/null +++ b/README.md @@ -0,0 +1,77 @@ +# GTFS-Hub + +This project aims at providing community tested, probably quality enhanced, partially merged GTFS-feeds of (currently) German transport agencies. + +In contrast to a Germany-wide GTFS feed distributed via Delfi (the national access point), GTFS-hub collects and enhances feeds provided by local authorities. + +## Motivation +Currently, the national access point provides timetable data in NeTEx format. Via [gtfs.de](http://gtfs.de) derived GTFS publications are available. + +However, for upstreaming data to the national data access point seems not to be lossfree or regularly. And there is no feedback channel to fix quality issues in a timely manner + +While we hope, that all these problems are overcome soon, we currently still see a need to work with locally published GTFS data. + +## Inner workings + +### Updating, checking, enhancing GTFS +GTFS-Hub regularly checks on a list of well known GTFS-feeds, if the were updated. + +If yes, they are + +* downloaded, +* optionally enhanced with shapes using OSM data and the pfaedle tool +* quality checked via google's transitfeed feedvalidator +* optionally transformed with onebusaway transformer tool (fed with a feed specific rule file) +* and optionally merged into larger aggregated GTFS feeds + +### Updating and preparing OSM data +Before GFTS data is updated, the OSM data which is used to generate GFTFS shapes is updated. +To avoid daily downloading large pbf datasets (GTFS-Hub downloads DACH (Germany, Austria, Switzerland) and Alsace (France)) from scratch, we only download the original datases once +and update these afterwards via pyosmium and prepares some region clipped extracts (namely Baden-Württemberg including a buffer of some kilometers around the border). + +As this extract will serve as input to OpenTripPlanner as well, we do some additionally data processing on it to enhance some infos, e.g. + +* Set some parkings to park_ride=yes for parkings close to stations but no park_ride tag yet +* Set some well known parkings to park_ride=hov + +### Publishing +After updating OSM and GTFS data, you'll find the datasets in data/www, ready to publish e.g. via a web serve serving this directory. + +### External references +This project uses a couple of other dockerized applications: + +* mfdz/pfaedle: a dockerized version of Patrick Brosi's tool pfaedle the enhance GTFS feeds by shapes map matched using OSM data +* mfdz/pyosmium: a dockerized version of (py)osmium to quickly update / cut / merge OSM data +* mfdz/osmosis: a dockerized version of osmosis to enhance OSM data with a declarative instruction set +* mfdz/transitfeed: a dockerized version of google's transitfeed feedvalidator +* mfdz/otp-data-tools: a dockerized version of onebusaway's GTFS transform and merge tools + +Thanks to everybody contributing to these tools, the OSM community and Geofabrik and the transit agencies providing the data download services. + +## How to start gtfs-hub + +### Prerequisites + +You'll need to have docker installed. + +### Running GTFS-Hub +Start the download/transform process chain. Note the necessary HOST_DATA environment variable, which requires to be set to an absolute path to the data directory, as we use a Docker in Docker setup +where data is shared via host-relative volumes. + +```sh +docker run -e HOST_DATA=$(PWD)/data -v $(PWD)/data:/var/data -v /var/run/docker.sock:/var/run/docker.sock mfdz/gtfs-hub +``` + +If you want to use your own config instead, you may mount your own config directory, which + +```sh +docker run -e HOST_DATA=$(PWD)/data -v $(PWD)/data:/var/data -v /var/run/docker.sock:/var/run/docker.sock -v $(PWD)/config:/opt/gtfs-hub/config mfdz/gtfs-hub +``` + +### Building the docker image +To build you own docker image, just do: + +```sh +docker build -t mfdz/gtfs-hub . +``` + diff --git a/config/gtfs-feeds.csv b/config/gtfs-feeds.csv new file mode 100644 index 0000000..8b5830f --- /dev/null +++ b/config/gtfs-feeds.csv @@ -0,0 +1,41 @@ +shortname;license;attribution;permanentUrl?;downloadUrl;infoUrl;infoEmail;enhanceShapes +AVV;cc-zero;;Ja;http://opendata.avv.de/current_GTFS/AVV_GTFS_mit_SPNV.zip;https://avv.de/de/fahrplaene/angebote-fuer-webentwickler;;Nein +bodo;dl-de/by-2.0 / Shapes ODbL;Datensatz der NVBW GmbH, Shapes (C) OpenStreetMap Mitwirkende;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_mit_Liniennetz/bodo.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/fahrplandaten-mit-liniennetz/;opendata@nvbw.de;Nein +DING;dl-de/by-2.0 / Shapes ODbL;Datensatz der NVBW GmbH, Shapes (C) OpenStreetMap Mitwirkende;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_mit_Liniennetz/ding.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/fahrplandaten-mit-liniennetz/;opendata@nvbw.de;Nein +Filsland;dl-de/by-2.0 / Shapes ODbL;Datensatz der NVBW GmbH, Shapes (C) OpenStreetMap Mitwirkende;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_mit_Liniennetz/filsland.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/fahrplandaten-mit-liniennetz/;opendata@nvbw.de;Nein +FlixBus;?;?;ja;http://data.ndovloket.nl/flixbus/flixbus-eu.zip;http://data.ndovloket.nl/flixbus/;Nein +HNV;dl-de/by-2.0;Datensatz der NVBW GmbH;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_ohne_Liniennetz/hnv.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/fahrplandaten-ohne-liniennetz/;opendata@nvbw.de;Ja +HVG;dl-de/by-2.0 / Shapes ODbL;Datensatz der NVBW GmbH, Shapes (C) OpenStreetMap Mitwirkende;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_mit_Liniennetz/hvg.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/fahrplandaten-mit-liniennetz/;opendata@nvbw.de;Nein +HVV;dl-de/by-2.0;Hamburger Verkehrsverbund GmbH;Nein;http://daten.transparenz.hamburg.de/Dataport.HmbTG.ZS.Webservice.GetRessource100/GetRessource100.svc/63302429-4b7d-46c7-aba2-c6e8e090f286/Upload__HVV_Rohdaten_GTFS_Fpl_20200312.zip;http://suche.transparenz.hamburg.de/?q=gtfs;;Nein +KVSH;dl-de/by-2.0 / Shapes ODbL;Datensatz der NVBW GmbH, Shapes (C) OpenStreetMap Mitwirkende;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_mit_Liniennetz/kvsh.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/fahrplandaten-mit-liniennetz/;opendata@nvbw.de;Nein +KVV;cc-zero;;Ja;https://projekte.kvv-efa.de/GTFS/google_transit.zip;https://www.kvv.de/fahrt-planen/fahrplaene/open-data.html;;Ja +LVB;dl-de/by-2.0;Leipziger Verkehrsbetriebe (LVB) GmbH;Ja;https://opendata.leipzig.de/dataset/8803f612-2ce1-4643-82d1-213434889200/resource/b38955c4-431c-4e8b-a4ef-9964a3a2c95d/download/gtfsmdvlvb.zip;https://opendata.leipzig.de/dataset/lvb-fahrplandaten;verkehrsbetriebe@L.de;Nein +MVV;cc-by;Münchner Verkehrs- und Tarifverbund GmbH;Nein;http://cms.opendata-oepnv.de/dataset/c231ec57-efb5-44fd-8bd6-7deb86cfcc60/resource/f443bf5d-ae07-42e3-85b8-da757159620b/download/gtfs-mvv-01-2020.zip;https://www.opendata-oepnv.de/dataset/soll-fahrplandaten-mvv;;Nein +naldo;dl-de/by-2.0 / Shapes ODbL;Datensatz der NVBW GmbH, Shapes (C) OpenStreetMap Mitwirkende;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_mit_Liniennetz/naldo.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/fahrplandaten-mit-liniennetz/;opendata@nvbw.de;Nein +NWL;CC BY 4.0;NWL;Nein;https://cms.opendata-oepnv.de/dataset/89892705-b6e7-4ffc-977c-2ba9b86dde46/resource/44f7d39c-02d7-4c2e-8efe-1046e5e959a2/download/gtfs-nwl-20200302.zip;https://www.opendata-oepnv.de/ht/de/organisation/verkehrsverbuende/nwl/startseite?tx_vrrkit_view%5Bdataset_name%5D=soll-fahrplandaten-nwl&tx_vrrkit_view%5Bdataset_formats%5D%5B0%5D=ZIP&tx_vrrkit_view%5Baction%5D=details&tx_vrrkit_view%5Bcontroller%5D=View;;Ja +OstalbMobil;dl-de/by-2.0 / Shapes ODbL;Datensatz der NVBW GmbH, Shapes (C) OpenStreetMap Mitwirkende;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_mit_Liniennetz/oam.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/fahrplandaten-mit-liniennetz/;opendata@nvbw.de;Nein +RAB;dl-de/by-2.0 / Shapes ODbL;Datensatz der NVBW GmbH, Shapes (C) OpenStreetMap Mitwirkende;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_mit_Liniennetz/rab.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/fahrplandaten-mit-liniennetz/;opendata@nvbw.de;Nein +RBS;dl-de/by-2.0;Datensatz der NVBW GmbH;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_ohne_Liniennetz/rbs.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/fahrplandaten-ohne-liniennetz/;opendata@nvbw.de;Ja +Rexer;dl-de/by-2.0 / Shapes ODbL;Datensatz der NVBW GmbH, Shapes (C) OpenStreetMap Mitwirkende;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_mit_Liniennetz/rxr.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/fahrplandaten-mit-liniennetz/;opendata@nvbw.de;Nein +RNV;dl-de/by-2.0;Rhein-Neckar-Verkehr GmbH (rnv);Ja;https://opendata.rnv-online.de/sites/default/files/rnv-GTFS_134.zip;https://opendata.rnv-online.de/dataset/gtfs-general-transit-feed-specification;opendata@rnv-online.de;Ja +RVS;dl-de/by-2.0 / Shapes ODbL;Datensatz der NVBW GmbH, Shapes (C) OpenStreetMap Mitwirkende;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_mit_Liniennetz/rvs.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/fahrplandaten-mit-liniennetz/;opendata@nvbw.de;Nein +SBAHN-S;cc-by-4.0;Deutsche Bahn AG;Ja;http://download-data.deutschebahn.com/static/datasets/sbahn_stuttgart_gtfs/017_001_S-Bahn_Stuttgart_J19.zip;https://data.deutschebahn.com/dataset/data-s-bahn-stuttgart-gtfs;dbopendata@deutschebahn.com;Ja +SBG;dl-de/by-2.0 / Shapes ODbL;Datensatz der NVBW GmbH, Shapes (C) OpenStreetMap Mitwirkende;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_mit_Liniennetz/sbg.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/fahrplandaten-mit-liniennetz/;opendata@nvbw.de;Nein +SWEG;dl-de/by-2.0 / Shapes ODbL;Datensatz der NVBW GmbH, Shapes (C) OpenStreetMap Mitwirkende;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_mit_Liniennetz/sweg.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/fahrplandaten-mit-liniennetz/;opendata@nvbw.de;Nein +SPNV-BW;dl-de/by-2.0 / Shapes ODbL;Datensatz der NVBW GmbH, Shapes (C) OpenStreetMap Mitwirkende;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_mit_Liniennetz/bwspnv.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/ +SWEG;dl-de/by-2.0 / Shapes ODbL;Datensatz der NVBW GmbH, Shapes (C) OpenStreetMap Mitwirkende;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_mit_Liniennetz/sweg.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/ +SWHN;dl-de/by-2.0 / Shapes ODbL;Datensatz der NVBW GmbH, Shapes (C) OpenStreetMap Mitwirkende;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_mit_Liniennetz/swhn.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/fahrplandaten-mit-liniennetz/;opendata@nvbw.de;Nein +SWU;ODbL-1.0;;Ja;https://gtfs.swu.de/daten/SWU.zip;https://www.swu.de/privatkunden/service/mobilitaet/gtfs-daten/;Christian.Burst@swu.de;Ja +TGO;dl-de/by-2.0 / Shapes ODbL;Datensatz der NVBW GmbH, Shapes (C) OpenStreetMap Mitwirkende;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_mit_Liniennetz/tgo.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/fahrplandaten-mit-liniennetz/;opendata@nvbw.de;Nein +TUTicket;dl-de/by-2.0 / Shapes ODbL;Datensatz der NVBW GmbH, Shapes (C) OpenStreetMap Mitwirkende;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_mit_Liniennetz/tuticket.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/fahrplandaten-mit-liniennetz/;opendata@nvbw.de;Nein +VAGFR;dl-de/by-2.0;VAG Freiburg;Ja;https://www.vag-freiburg.de/fileadmin/gtfs/VAGFR.zip; https://www.vag-freiburg.de/fahrplan/fahrplanauskunft;Klaus.Funke@vagfr.de;Nein +VBB;cc-by;Verkersverbund Berlin-Brandenburg GmbH;Nein;https://www.vbb.de/media/download/2029;https://www.vbb.de/unsere-themen/vbbdigital/api-entwicklerinfos/datensaetze;;Nein +VGC;dl-de/by-2.0 / Shapes ODbL;Datensatz der NVBW GmbH, Shapes (C) OpenStreetMap Mitwirkende;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_mit_Liniennetz/vgc.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/fahrplandaten-mit-liniennetz/;opendata@nvbw.de;Nein +VGF;dl-de/by-2.0 / Shapes ODbL;Datensatz der NVBW GmbH, Shapes (C) OpenStreetMap Mitwirkende;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_mit_Liniennetz/vgf.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/fahrplandaten-mit-liniennetz/;opendata@nvbw.de;Nein +VGN;cc-by-3.0-de;VGN – Ver­kehrs­ver­bund Groß­raum Nürn­berg GmbH;Ja;http://www.vgn.de/opendata/GTFS.zip;https://www.vgn.de/web-entwickler/open-data/;info@vgn.de;Nein +VMT;cc-by-nd;VMT GmbH;Ja;https://www.vmt-thueringen.de/fileadmin/user_upload/Open_Data/VMT_GTFS.zip;https://www.vmt-thueringen.de/auskunft/open-data/;;Nein +VRN;cc-by;Verkehrsverbund Rhein-Neckar GmbH;Nein;https://www.vrn.de/mam/service/downloads/vrn_gtfs.zip;https://www.vrn.de/service/entwickler/gtfs/index.html;dqs@vrn.de;Ja +VRR;cc-by;Verkehrsverbund Rhein-Ruhr AöR;Nein;https://openvrr.de/dataset/c415abd6-3b63-4a1f-8a17-9b77cf5f09ec/resource/a1e4fad2-fc11-4cd6-bb47-4bec24424f7c/download/2019_06_03_google_transit_verbundweit_mit_evu.zip;https://www.openvrr.de/dataset/gtfs;;Nein +VRS;cc-by;Verkehrsverbund Rhein-Sieg GmbH;Ja;http://download.vrsinfo.de/gtfs/google_transit.zip;https://www.vrsinfo.de/fahrplan/oepnv-daten-fuer-webentwickler.html;api@vrsinfo.de;Nein +VPE;dl-de/by-2.0 / Shapes ODbL;Datensatz der NVBW GmbH, Shapes (C) OpenStreetMap Mitwirkende;Ja;https://www.nvbw.de/fileadmin/nvbw/open-data/Fahrplandaten_mit_Liniennetz/vpe.zip;https://www.nvbw.de/aufgaben/digitale-mobilitaet/fahrplandaten-mit-liniennetz/;opendata@nvbw.de;Nein +VVS;cc-by;Verkehrs- und Tarifverbund Stuttgart Gmbh;Ja;https://www.openvvs.de/dataset/e66f03e4-79f2-41d0-90f1-166ca609e491/resource/bfbb59c7-767c-4bca-bbb2-d8d32a3e0378/download/vvs_gtfs.zip;https://www.openvvs.de/dataset/gtfs-daten;;Ja diff --git a/config/gtfs-rules/VGC.rule b/config/gtfs-rules/VGC.rule new file mode 100644 index 0000000..564f30b --- /dev/null +++ b/config/gtfs-rules/VGC.rule @@ -0,0 +1,33 @@ +## Fix missing lat/lon: + +# grep "000000" -r */stops.txt +# vgc.gtfs/stops.txt:de:08235:10449:0:4,"Nagold, Uferstr.",0.000000,0.000000 +# vgc.gtfs/stops.txt:de:08235:10451:0:3,"Nagold, Kreuzertal",0.000000,0.000000 +# vgc.gtfs/stops.txt:de:08235:3181:0:3,"Speßhardt, Friedhof",0.000000,0.000000 +# vgc.gtfs/stops.txt:de:08235:5660:2:3,"Neuenbürg, Eyachbrücke",0.000000,0.000000 +# vgc.gtfs/stops.txt:de:08235:5660:2:4,"Neuenbürg, Eyachbrücke",0.000000,0.000000 +# vgc.gtfs/stops.txt:de:08235:7330:0:3,"Wildberg, Bildungszentr.",0.000000,0.000000 +# vgc.gtfs/stops.txt:de:08235:7330:0:4,"Wildberg, Bildungszentr.",0.000000,0.000000 +# vgc.gtfs/stops.txt:de:08235:7330:0:5,"Wildberg, Bildungszentr.",0.000000,0.000000 +# vgc.gtfs/stops.txt:de:08235:7330:0:7,"Wildberg, Bildungszentr.",0.000000,0.000000 +# vgc.gtfs/stops.txt:gen:8235:10263:0:3,"Unterschwandorf, Löwen",0.000000,0.000000 +# vgc.gtfs/stops.txt:gen:8235:2730:0:3,"Enzkl., Adventure Golfpark",0.000000,0.000000 +# vgc.gtfs/stops.txt:gen:8235:2730:0:4,"Enzkl., Adventure Golfpark",0.000000,0.000000 + +# Guessing de:08235:10449:0:4,"Nagold, Uferstr. +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08235:10449:0:4"}, "update":{"stop_lat":"48.54923", "stop_lon":"8.71451"}} +# de:08235:10451:0:3,"Nagold, Kreuzertal", Location via info@vbn-bvn.de +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08235:10451:0:3"}, "update":{"stop_lat":"48.551247", "stop_lon":"8.737486"}} +# Speßhardt location via info@vbn-bvn.de +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08235:5660:2:3"}, "update":{"stop_lat":"48.714458", "stop_lon":"8.690857"}} +# Eyachbrücke location via info@vbn-bvn.de +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08235:5660:2:3"}, "update":{"stop_lat":"48.816358", "stop_lon":"8.577080", "stop_id":"de:08235:5660:2:1"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08235:5660:2:4"}, "update":{"stop_lat":"48.816326", "stop_lon":"8.577326", "stop_id":"de:08235:5660:2:2"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08235:7330:0:3"}, "update":{"stop_lat":"48.6241902615607", "stop_lon":"8.73791091051151"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08235:7330:0:4"}, "update":{"stop_lat":"48.6241812690453", "stop_lon":"8.73791095696822"}} +# Coordinates are estimates +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08235:7330:0:5"}, "update":{"stop_lat":"48.6241812690453", "stop_lon":"8.73791095696822"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08235:7330:0:7"}, "update":{"stop_lat":"48.6241812690453", "stop_lon":"8.73791095696822"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"gen:8235:10263:0:3"}, "update":{"stop_lat":"48.54465837060639", "stop_lon":"8.671057378464063", "stop_id":"de:08235:10263:0:3"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"gen:8235:2730:0:3"}, "update":{"stop_lat":"48.66741682564984", "stop_lon":"8.474423251273537", "stop_id":"de:08235:2730:0:3"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"gen:8235:2730:0:4"}, "update":{"stop_lat":"48.66741682564984", "stop_lon":"8.474423251273537", "stop_id":"de:08235:2730:0:4"}} diff --git a/config/gtfs-rules/VGF.rule b/config/gtfs-rules/VGF.rule new file mode 100644 index 0000000..a129a3a --- /dev/null +++ b/config/gtfs-rules/VGF.rule @@ -0,0 +1,33 @@ +## Fix missing lat/lon, wrong stop ids/names: + +# grep "000000" -r */stops.txt +# vgf.gtfs/stops.txt:de:08237:1065:0:1,FDS Berufsschule,0.000000,0.000000 +# vgf.gtfs/stops.txt:de:08237:1065:0:2,FDS Berufsschule,0.000000,0.000000 +# vgf.gtfs/stops.txt:de:08237:4004,Dornstetten Hochgericht,0.000000,0.000000 +# vgf.gtfs/stops.txt:de:08237:4352:0:4,Lützenhardt ZOB,0.000000,0.000000 +# vgf.gtfs/stops.txt:de:08237:4506,Pfalzgrafenweiler Sportanlagen,0.000000,0.000000 +# vgf.gtfs/stops.txt:de:08237:4645,Bösingen Ortsausgang,0.000000,0.000000 +# vgf.gtfs/stops.txt:gen:8237:3335,Betzweiler Betrieb,0.000000,0.000000 +# vgf.gtfs/stops.txt:gen:8237:4005:1:2,Dornstetten ZOB,0.000000,0.000000 +# vgf.gtfs/stops.txt:gen:8237:4005:1:4,Dornstetten ZOB,0.000000,0.000000 +# vgf.gtfs/stops.txt:gen:8237:4005:1:5,Dornstetten ZOB,0.000000,0.000000 +# vgf.gtfs/stops.txt:gen:8237:4005:1:6,Dornstetten ZOB,0.000000,0.000000 +# vgf.gtfs/stops.txt:gen:8325:9131,Dornhan Feuerwehr,0.000000,0.000000 + +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08237:1065:0:1"}, "update":{"stop_lat":"48.469387", "stop_lon":"8.417926", "stop_id": "de:08237:1101:0:1"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08237:1065:0:2"}, "update":{"stop_lat":"48.469387", "stop_lon":"8.417926", "stop_id": "de:08237:1101:0:2"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08237:4004"}, "update":{"stop_lat":"48.474133", "stop_lon":"8.488702"}} +# Stop has contradicting ifopt ids in nvbw haltestellen list (de:08237:4401 for stop vs de:08237:4352:0:4 for quai) +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08237:4352:0:4"}, "update":{"stop_lat":"48.4860261716506", "stop_lon":"8.565269769834273", "stop_id": "de:08237:4352:0:4"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08237:4506"}, "update":{"stop_lat":"48.530459", "stop_lon":"8.577677"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08237:4645"}, "update":{"stop_lat":"48.538909", "stop_lon":"8.596061"}} +# Not sure, if quais are correct +{"op":"update", "match":{"file":"stops.txt", "stop_id":"gen:8237:3335"}, "update":{"stop_lat":"48.36568038549475", "stop_lon":"8.48193022090397","stop_id": "de:08237:3331:0:1", "stop_name":"Betzweiler Cafe Walter"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"gen:8237:3331"}, "update":{"stop_id": "de:08237:3331:0:2", "stop_name":"Betzweiler Cafe Walter"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"gen:8237:4005:1:2"}, "update":{"stop_lat":"48.469958", "stop_lon":"8.501346", "stop_id": "de:08237:4005:1:2"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"gen:8237:4005:1:4"}, "update":{"stop_lat":"48.469958", "stop_lon":"8.501346", "stop_id": "de:08237:4005:1:4"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"gen:8237:4005:1:5"}, "update":{"stop_lat":"48.469958", "stop_lon":"8.501346", "stop_id": "de:08237:4005:1:5"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"gen:8237:4005:1:6"}, "update":{"stop_lat":"48.469958", "stop_lon":"8.501346", "stop_id": "de:08237:4005:1:6"}} +# gen:8325:9131 See https://www.vvr-info.de/fps/f19_hr_stand_20181209.pdf +# To avoid duplicates, add 0:1 as virtual quai... +{"op":"update", "match":{"file":"stops.txt", "stop_id":"gen:8325:9131"}, "update":{"stop_lat":"48.351671", "stop_lon":"8.502698","stop_id": "de:08325:27090:0:1", "stop_name":"Dornhan Freudenst. Straße"}} diff --git a/config/gtfs-rules/VRN.rule b/config/gtfs-rules/VRN.rule new file mode 100644 index 0000000..79a7f2e --- /dev/null +++ b/config/gtfs-rules/VRN.rule @@ -0,0 +1,20 @@ +## Fix missing lat/lon, wrong stop ids/names: +# Remove all stops without coordinates + +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08237:1065:0:1"}, "update":{"stop_lat":"48.469387", "stop_lon":"8.417926", "stop_id": "de:08237:1101:0:1"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08237:1065:0:2"}, "update":{"stop_lat":"48.469387", "stop_lon":"8.417926", "stop_id": "de:08237:1101:0:2"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08237:4004"}, "update":{"stop_lat":"48.474133", "stop_lon":"8.488702"}} +# Stop has contradicting ifopt ids in nvbw haltestellen list (de:08237:4401 for stop vs de:08237:4352:0:4 for quai) +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08237:4352:0:4"}, "update":{"stop_lat":"48.4860261716506", "stop_lon":"8.565269769834273", "stop_id": "de:08237:4352:0:4"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08237:4506"}, "update":{"stop_lat":"48.530459", "stop_lon":"8.577677"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08237:4645"}, "update":{"stop_lat":"48.538909", "stop_lon":"8.596061"}} +# Not sure, if quais are correct +{"op":"update", "match":{"file":"stops.txt", "stop_id":"gen:8237:3335"}, "update":{"stop_lat":"48.36568038549475", "stop_lon":"8.48193022090397","stop_id": "de:08237:3331:0:1", "stop_name":"Betzweiler Cafe Walter"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"gen:8237:3331"}, "update":{"stop_id": "de:08237:3331:0:2", "stop_name":"Betzweiler Cafe Walter"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"gen:8237:4005:1:2"}, "update":{"stop_lat":"48.469958", "stop_lon":"8.501346", "stop_id": "de:08237:4005:1:2"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"gen:8237:4005:1:4"}, "update":{"stop_lat":"48.469958", "stop_lon":"8.501346", "stop_id": "de:08237:4005:1:4"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"gen:8237:4005:1:5"}, "update":{"stop_lat":"48.469958", "stop_lon":"8.501346", "stop_id": "de:08237:4005:1:5"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"gen:8237:4005:1:6"}, "update":{"stop_lat":"48.469958", "stop_lon":"8.501346", "stop_id": "de:08237:4005:1:6"}} +# gen:8325:9131 See https://www.vvr-info.de/fps/f19_hr_stand_20181209.pdf +# To avoid duplicates, add 0:1 as virtual quai... +{"op":"update", "match":{"file":"stops.txt", "stop_id":"gen:8325:9131"}, "update":{"stop_lat":"48.351671", "stop_lon":"8.502698","stop_id": "de:08325:27090:0:1", "stop_name":"Dornhan Freudenst. Straße"}} diff --git a/config/gtfs-rules/VVS.rule b/config/gtfs-rules/VVS.rule new file mode 100644 index 0000000..07014fe --- /dev/null +++ b/config/gtfs-rules/VVS.rule @@ -0,0 +1,4 @@ +## Fix wrong coordinates: + +# de:08115:4800:0:3,,Herrenberg Waldfriedhof,,48.602371216,8.9029130936,,,0,,,0, +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08115:4800:0:3"}, "update":{"stop_lat":"48.6020352", "stop_lon":"8.9036348"}} \ No newline at end of file diff --git a/config/gtfs-rules/naldo.rule b/config/gtfs-rules/naldo.rule new file mode 100644 index 0000000..5964a7a --- /dev/null +++ b/config/gtfs-rules/naldo.rule @@ -0,0 +1,16 @@ +## Fix missing lat/lon: + +# de:08237:30965,Nordstetten Josef-Hezel-Straße,,,,onses/8003085_1.json +# de:08237:5122,Eutingen im Gäu Daimlerstraße,,,,onses/8001004_1.json +# de:08237:7715,Nordstetten Schule,,,,onses/8003085_1.json +# de:08237:8032,Eyach Sommerhalde,,,,ses/8001004_100.json + +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08237:30965"}, "update":{"stop_lat":"48.440657", "stop_lon":"8.701297"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08237:5122"}, "update":{"stop_lat":"48.47754", "stop_lon":"8.74097"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08237:7715"}, "update":{"stop_lat":"48.436585", "stop_lon":"8.698151"}} +{"op":"update", "match":{"file":"stops.txt", "stop_id":"de:08237:8032"}, "update":{"stop_lat":"48.449008", "stop_lon":"8.784465"}} + +# (temp) fix calendar +{"op":"update", "match": {"file":"calendar.txt", "service_id":"1"}, "update":{"start_date":"20180806", "end_date":"20190803"}} +{"op":"update", "match": {"file":"calendar.txt", "service_id":"6"}, "update":{"start_date":"20180806", "end_date":"20190803"}} +{"op":"update", "match": {"file":"calendar.txt", "service_id":"7"}, "update":{"start_date":"20180806", "end_date":"20190803"}} diff --git a/config/osm/bw_buffered.poly b/config/osm/bw_buffered.poly new file mode 100644 index 0000000..fb0005f --- /dev/null +++ b/config/osm/bw_buffered.poly @@ -0,0 +1,32 @@ +bw_buffered +1 + 8.2012939 49.6996149 + 7.9980469 49.0378679 + 7.6409912 48.6546858 + 7.4267578 48.1294344 + 7.3388672 47.7873255 + 7.4102783 47.5394554 + 7.7838135 47.4095029 + 8.1518555 47.4652362 + 8.4484863 47.3313772 + 8.7011719 47.3276540 + 8.8494873 47.3871931 + 9.0417480 47.5023590 + 9.2779541 47.4280873 + 9.5965576 47.3127587 + 9.9755859 47.4800885 + 10.3985596 47.5876417 + 10.4370117 48.0780789 + 10.3491211 48.3306913 + 10.6292725 48.5893258 + 10.6732178 48.9621874 + 10.4974365 49.1817034 + 10.3546143 49.5180764 + 10.0799561 49.6960618 + 9.7723389 49.8238091 + 9.6240234 49.8981729 + 9.1571045 49.8733977 + 8.9099121 49.7564289 + 8.2012939 49.6996149 +END +END diff --git a/config/osm/park_ride_transform.xml b/config/osm/park_ride_transform.xml new file mode 100644 index 0000000..ff0de5b --- /dev/null +++ b/config/osm/park_ride_transform.xml @@ -0,0 +1,711 @@ + + + + + Add park_ride tag to parkings close to stations + Add park_ride tag to parkings close to stations identified by id, if it does not exist already + + + + +4613241 +4706391 +5087637 +13076754 +19093743 +22701914 +23025336 +23026086 +23033123 +23056557 +23056560 +23067833 +24247285 +24314539 +24676367 +24676368 +24866280 +24930313 +24980910 +25048475 +25075410 +25356981 +25369900 +25381084 +25524772 +25694143 +25705737 +25705739 +25767594 +25813242 +25822965 +25822966 +26180268 +26833521 +27244046 +27383646 +27431057 +27518746 +27660987 +27819162 +28199218 +28203621 +28488228 +29028679 +29107467 +30483979 +30782506 +30782507 +31220748 +31452283 +31858649 +31858809 +31957310 +32121885 +32130730 +32685313 +32805452 +33462457 +33462458 +33700161 +33778012 +35148219 +35223061 +36054702 +36385452 +36730291 +36736235 +36911239 +36951087 +37056663 +37484345 +37488924 +37549558 +37633086 +37772238 +38608680 +39934090 +39950590 +40325704 +41501157 +41568451 +41587102 +41592801 +41805275 +41982734 +42263661 +42263666 +42263667 +42488122 +42710500 +44811575 +48215206 +48624926 +49615784 +49615789 +50516031 +51155558 +51155559 +51272432 +52189448 +53049051 +57802842 +59604117 +59848732 +61328677 +61731203 +62127695 +62127696 +67086592 +67371897 +68700577 +69001313 +76256081 +79134857 +81512765 +81512782 +81512783 +87257216 +87315135 +87786997 +89072677 +93047068 +93048275 +94544685 +96627426 +97718481 +98663115 +98663116 +103057881 +106057728 +108091791 +108130413 +109403196 +110244064 +111360147 +111444041 +111444098 +111444135 +111444193 +111444217 +112317258 +113218999 +115954097 +117646100 +118297742 +118698799 +118725065 +118727081 +120133432 +120515601 +121120659 +121804533 +121804539 +121804549 +121804557 +122553798 +124282374 +124333728 +126015120 +126353532 +128083092 +129897406 +130332535 +131578824 +134627655 +136316786 +136372327 +137384882 +137384883 +138538683 +138538685 +138538692 +138700336 +138700337 +142429382 +142429384 +142502406 +145259983 +147436102 +148348662 +149585336 +149597071 +150205018 +150728845 +150728848 +152390594 +152822215 +152822216 +152974605 +157878267 +158035026 +158434798 +158437585 +159154369 +159710547 +160279462 +161155999 +161431475 +162384510 +162982841 +163916403 +163917183 +163989782 +164876796 +166424879 +166720250 +166824913 +166830263 +167295114 +167295122 +168222465 +168222466 +168244602 +168746776 +168874743 +173153674 +173215322 +173928872 +173961638 +175805799 +175953920 +176663939 +176776909 +178870651 +179129718 +179243568 +180104229 +180478865 +180619876 +181068768 +181069249 +184786665 +186784768 +186784769 +186959833 +191000574 +191000576 +191000577 +191496966 +192888873 +193216447 +193298214 +195029229 +196696240 +196705672 +196746133 +197067659 +198005841 +198740187 +199212761 +199926299 +200495148 +200495151 +202662248 +202671147 +203039276 +203163589 +203360265 +205252936 +205385870 +208493839 +212464429 +212661342 +213123534 +215775574 +215775576 +215775577 +216999478 +217877738 +218280954 +218655755 +218655756 +220209810 +220783888 +222911122 +222911123 +222911129 +223742125 +224982325 +225541191 +225784449 +227354150 +227621108 +228016806 +228461318 +229617698 +229617708 +230077479 +231855864 +232204647 +232441068 +232441069 +233261111 +234406632 +234802645 +235170044 +235635326 +237016557 +237327752 +237327753 +241138259 +241373675 +241991054 +242189113 +244030591 +244084742 +244628956 +244628962 +245671593 +248918211 +249694461 +249694462 +250450216 +251226322 +252604643 +253929309 +256159994 +256294936 +256984119 +257221200 +257313633 +258250376 +260763186 +260765612 +261271210 +262833540 +266109299 +267242063 +268144931 +270896195 +271683980 +271683983 +272396593 +272420771 +274107210 +276984953 +276984956 +277304744 +277860035 +279061126 +279379056 +279499970 +281801888 +281803506 +281826798 +281826799 +282527558 +284838278 +284838282 +285958381 +287091779 +288709511 +288709512 +288709514 +288803793 +289014646 +292292184 +292292907 +292552579 +292552581 +292552582 +292552584 +293901823 +295115033 +295553566 +297413376 +298127087 +298759976 +299684208 +299953017 +300625931 +300625932 +301625951 +303508667 +303813837 +303813839 +303815267 +303815268 +304189562 +304189563 +304189564 +304189565 +305348764 +306241073 +307495030 +307495031 +308734137 +309858178 +315894631 +316106099 +316121169 +316273206 +317565360 +320452851 +320539925 +321165481 +321170652 +323723928 +324802592 +326991769 +327315451 +331455937 +332408623 +332409044 +333027655 +333091200 +333091202 +337200677 +339586995 +339586996 +339586998 +340398040 +340398045 +340590040 +344123596 +345572200 +345759031 +347057644 +347310514 +353643278 +354176037 +356531610 +356531611 +356604875 +357448218 +359371479 +359890122 +364222048 +364766289 +364766557 +365453097 +366570270 +366972371 +366972372 +372528512 +372528820 +372529262 +372619238 +372619248 +372633111 +372633114 +374015919 +374015920 +376880783 +378232231 +378307212 +380060003 +380529983 +381243621 +383412664 +384192463 +387890641 +388016387 +389632719 +389632721 +395653582 +395653584 +396996657 +396996658 +396996666 +398801872 +400463207 +401007207 +404111902 +408852280 +409051606 +410820748 +414048808 +417816633 +418043391 +418286365 +422297382 +424538435 +424545693 +427248499 +428966632 +428966645 +429365632 +429365635 +429365637 +429365639 +435016159 +435016160 +435182983 +436344786 +436901509 +436901515 +441392454 +444240277 +444240280 +444240291 +446922812 +447348295 +447348296 +447761451 +451699758 +456969455 +458671769 +459569191 +462952839 +464907229 +465501288 +475390345 +475528747 +475618983 +476876068 +480093618 +480215517 +481656567 +481818995 +482094970 +482094979 +482094980 +485211889 +489733944 +490899082 +490899083 +490899084 +500913457 +501235739 +501785936 +502972489 +506734483 +507516533 +510235556 +513025879 +513025880 +513971652 +513971653 +515493743 +524766400 +526579218 +529044226 +533424722 +536875492 +536875494 +538837208 +538837213 +542941762 +546401881 +550161615 +556746449 +556746452 +558534578 +558905456 +568193614 +574624027 +582392313 +582392314 +589069077 +591703107 +592826718 +593284451 +593284454 +593284456 +593286720 +600388102 +601833519 +602475945 +603023121 +603023124 +603024023 +608629049 +609328317 +609328318 +613347100 +620573922 +628219410 +633009051 +633254907 +633254912 +638257204 +642295640 +656355892 +658009268 + + + + + + 4 + 5 + + + + + + + + + + + + Add park_ride=hov tag to Parken and Mitfahren parkings + Add park_ride=hov tag to Parken and Mitfahren identified by id + + + + 47094273 +238162187 +26818901 +27665787 +4770490 +13997759 +35049302 +663635074 +495432091 +109088694 +109088692 +33310815 +40593348 +318993498 +318078740 +31543555 +203357579 +26582063 +28967177 +28114370 +206440150 +318421808 +360817828 +133551915 +81114195 +40553084 +27078593 +100847744 +318712806 +45205088 +78361392 +173149261 +117557987 +380677875 +282865821 +90210406 +515168292 +26335711 +198776829 +663270441 +7345877 +663639048 +474423981 +140184273 +199252733 +40870491 +663617769 +107632314 +360048225 +229212508 +197562673 +28025364 +32076135 +586643464 +329801481 +125194340 +98931661 +99512406 +31281518 +177592176 +177681157 +30628218 +446175726 +164372550 +161359734 +329564770 +40429724 +105789136 +168655577 +481789339 +79864367 +531763087 +562753306 +152342991 +99550405 +232938012 +135342490 +146186055 +32441871 +170408619 +382877933 +248419469 +662908475 +127550022 +299261916 +638590787 +204187413 + + + + +1452214 +5574641 + + + + + + + + + + diff --git a/makefile b/makefile new file mode 100644 index 0000000..932f08e --- /dev/null +++ b/makefile @@ -0,0 +1,29 @@ + +ALL = hbg bw +all : $(ALL:%=data/gtfs_validated/%.merged.gtfs.zip) +.PHONY : all + +# https://www.gnu.org/software/make/manual/html_node/Automatic-Variables.html + +# Shortcuts for the (dockerized) transform/merge tools. +MERGE = docker run -v $(HOST_DATA):/data --rm mfdz/otp-data-tools java -Xmx14g -jar one-busaway-gtfs-merge/onebusaway-gtfs-merge-cli.jar --file=stops.txt --duplicateDetection=identity +TRANSFORM = docker run -v $(HOST_DATA):/data --rm mfdz/otp-data-tools java -Xmx6g -jar one-busaway-gtfs-transformer/onebusaway-gtfs-transformer-cli.jar + +# For every merged dataset, it's composing feeds should be listed. +# At first, we define a variable with all feed names, which subsquently gets expanded +# to the complete paths +HBG = SPNV-BW naldo.filtered VGC.filtered VVS.filtered +HBG_FILES = $(HBG:%=data/gtfs_validated/%.gtfs.zip) + +# For BW we build upon the already merged Hbg feed and all missing agencies +BW = VAGFR bodo DING Filsland HNV HVG KVSH KVV OstalbMobil RAB Rexer RVS SBG SWEG SWHN TGO TUTicket VGF.filtered VPE VRN SPNV-BW naldo.filtered VGC.filtered VVS.filtered +BW_FILES = $(BW:%=data/gtfs_validated/%.gtfs.zip) + +data/gtfs/hbg.merged.gtfs.zip: $(HBG_FILES) + $(MERGE) $^ /$@ + +data/gtfs/bw.merged.gtfs.zip: $(BW_FILES) + $(MERGE) $^ /$@ + +data/gtfs_validated/%.filtered.gtfs.zip: data/gtfs_validated/%.gtfs.zip data/gtfs-rules/%.rule + $(TRANSFORM) --transform=/data/gtfs-rules/$*.rule /data/gtfs/$*.gtfs.zip /$@ diff --git a/update_all.sh b/update_all.sh new file mode 100644 index 0000000..f4c9fde --- /dev/null +++ b/update_all.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +./update_osm.sh +./update_gtfs.sh +cp makefile /var/ +mkdir /var/data/gtfs-rules/ +cp -p config/gtfs-rules/* /var/data/gtfs-rules/ +pushd /var/ +make +# Copy validated gtfs files to download dir, as well as merged feeds +cp -p /var/data/gtfs_validated/*.zip /var/data/www/ +cp -p /var/data/gtfs/*.merged.zip /var/data/www/ +popd \ No newline at end of file diff --git a/update_gtfs.sh b/update_gtfs.sh new file mode 100755 index 0000000..6a45a83 --- /dev/null +++ b/update_gtfs.sh @@ -0,0 +1,133 @@ +#!/bin/sh + +export DATA_DIR=/var/data +export GTFS_DIR=$DATA_DIR/gtfs +export GTFS_VALIDATED_DIR=$DATA_DIR/gtfs_validated +export GTFS_SOURCES_CSV=./config/gtfs-feeds.csv +export REPORT_PUBLISH_DIR=$DATA_DIR/www/ +export SUMMARY_FILE=$GTFS_DIR/index.html + + +function augment_shapes { + # extract gtfs + # TODO GTFS fixes should go into gtfs-rules + rm -rf "$GTFS_DIR/$1.gtfs" + unzip -o -d $GTFS_DIR/$1.gtfs $GTFS_DIR/$1.gtfs.zip + + if [ "$1" == "VVS" ]; then + # remove errornous transfers + echo "Fixing VVS..." + rm $GTFS_DIR/$1.gtfs/transfers.txt + fi + # call pfaedle + docker run --rm -v "$HOST_DATA":/data:rw mfdz/pfaedle --inplace -x /data/osm/bw-buffered.osm /data/gtfs/$1.gtfs + # zip and move gtfs-out + zip -j $GTFS_DIR/$1.with-shapes.gtfs.zip $GTFS_DIR/$1.gtfs/*.txt + mv $GTFS_DIR/$1.with-shapes.gtfs.zip $REPORT_PUBLISH_DIR/gtfs +} + +function download_and_check { + export GTFS_FILE=$GTFS_DIR/$1.gtfs.zip + echo Download $2 + downloadurl=$2 + if [ -f $GTFS_FILE ]; then + # Check even non-permanent Urls, as link in CSV may have been changed + if [ $1 != "VBB" -a $1 != "HVV"]; then + response=$(curl -R -L -w "%{http_code}" -o $GTFS_FILE -z $GTFS_FILE $downloadurl) + fi + else + response=$(curl -R -L -w "%{http_code}" -o $GTFS_FILE $downloadurl) + fi + + case "$response" in + 200) docker run -t -v $HOST_DATA/gtfs:/gtfs mfdz/transitfeed feedvalidator_googletransit.py -o /gtfs/feedvalidator_$1.html -l 1000 -d /gtfs/$1.gtfs.zip 2>&1 | tail -1 > /$GTFS_DIR/$1.log + if [ "$7" == "Ja" ]; then + echo "Augment shapes for $1" + augment_shapes $1 $OSM_FILE + fi + ;; + 301) printf "Received: HTTP $response (file moved permanently) ==> $url\n" ;; + 304) printf "Received: HTTP $response (file unchanged) ==> $url\n" ;; + 404) printf "Received: HTTP $response (file not found) ==> $url\n" ;; + *) printf "Received: HTTP $response ==> $url\n" ;; + esac + + local ERRORS="" + local WARNINGS="" + local ERROR_REGEX='^.* ([0-9]*) error.*$' + local WARNING_REGEX='^.* ([0-9]*) warning.*$' + if [[ `cat $GTFS_DIR/$1.log` =~ $ERROR_REGEX ]]; then + ERRORS=${BASH_REMATCH[1]} + else + cp $GTFS_FILE $GTFS_VALIDATED_DIR + fi + if [[ `cat $GTFS_DIR/$1.log` =~ $WARNING_REGEX ]]; then + WARNINGS=${BASH_REMATCH[1]} + fi + + echo " + $1 + `date -r $GTFS_DIR/$1.gtfs.zip +%Y-%m-%d` + $5 + $6 + Download + Report + $ERRORS + $WARNINGS + " >> $SUMMARY_FILE +} + +mkdir -p $GTFS_DIR +mkdir -p $GTFS_VALIDATED_DIR +mkdir -p $REPORT_PUBLISH_DIR +echo " + + + +GTFS-Publikationen +

GTFS-Publikationen

+

Nachfolgend sind für die uns derzeit bekannten GTFS-Veröffentlichungen deutscher Verkehrsunternehmen und- verbünde die +Ergebnisse der feedvalidator-Prüfung mit dem Google Transitfeed Feedvalidator aufgelistet.

+

HINWEIS: Einige Verkehrsverbünde veröffentlichen Datensätze derzeit unter einer versionsbezogenen URL. VBB und HVV rufen wir nicht automatisiert ab, +da Last-Modified/If-Modified-Since derzeit nicht unterstützt werden bzw. der Datensatz nicht unter eine permanten URL bereitgestellt wird. +Für diese können wir nicht automatisch die aktuellste Version prüfen und hier listen. Wir freuen uns über einen Hinweis, sollte es aktuellere Daten oder auch +weitere Datenquellen geben.

+

Feedback bitte an "hb at mfdz de"

+ + + + + + + + + +" > $SUMMARY_FILE + + + +while IFS=';' read -r name lizenz nammensnennung permanent downloadurl infourl email addshapes +do + if ! [ "$name" == "shortname" ]; then # ignore first line + download_and_check $name $downloadurl $permanent $infourl "$lizenz" "$nammensnennung" "$addshapes" + fi +done < $GTFS_SOURCES_CSV + +echo "
VerbundDatumLizenzNammensnennungDownloadValidierungFehlerWarnungen
+ +

Unter github/mfdz/GTFS-Issues sind weitere Probleme oder Erweiterungswünsche +dokumentiert.

+

Weitere Informationen:

+ + +" >> $SUMMARY_FILE + + +cp $GTFS_DIR/*.html $REPORT_PUBLISH_DIR/ diff --git a/update_osm.sh b/update_osm.sh new file mode 100755 index 0000000..ac50993 --- /dev/null +++ b/update_osm.sh @@ -0,0 +1,36 @@ +export OSM_DIR=/var/data/osm +export OUT_DIR=/var/data/www +# HOST_DATA should be set + +mkdir -p $OSM_DIR + +# Download or update OSM files +if [ ! -f "$OSM_DIR/alsace-latest.osm.pbf" ]; then + curl https://download.geofabrik.de/europe/france/alsace-latest.osm.pbf > $OSM_DIR/alsace-latest.osm.pbf +else + docker run -v $HOST_DATA/osm:/osm mfdz/pyosmium pyosmium-up-to-date /osm/alsace-latest.osm.pbf +fi + +if [ ! -f "$OSM_DIR/dach-latest.osm.pbf" ]; then + curl https://download.geofabrik.de/europe/dach-latest.osm.pbf > $OSM_DIR/dach-latest.osm.pbf +else + docker run -v $HOST_DATA/osm:/osm mfdz/pyosmium pyosmium-up-to-date /osm/dach-latest.osm.pbf +fi + + +cp config/osm/* $OSM_DIR +# Extract from PBF using a poly file +docker run -v $HOST_DATA/osm:/osm mfdz/pyosmium ls -l /osm +docker run -v $HOST_DATA/osm:/osm mfdz/pyosmium osmium extract -p /osm/bw_buffered.poly --no-progress -o /osm/alsace-extracted.osm.pbf -O /osm/alsace-latest.osm.pbf +docker run -v $HOST_DATA/osm:/osm mfdz/pyosmium osmium extract -p /osm/bw_buffered.poly --no-progress -o /osm/dach-extracted.osm.pbf -O /osm/dach-latest.osm.pbf + +# Set park_ride tag for well known parkings +docker run -v $HOST_DATA/osm:/osm mfdz/osmosis:0.47-1-gd370b8c4 --read-pbf /osm/dach-extracted.osm.pbf --tt file=/osm/park_ride_transform.xml stats=/osm/park_ride_stats.log --write-pbf /osm/dach-extracted-pr.osm.pbf + +# Merge files +docker run -v $HOST_DATA/osm:/osm mfdz/pyosmium osmium merge -o /osm/bw-buffered.osm.pbf -O /osm/alsace-extracted.osm.pbf /osm/dach-extracted-pr.osm.pbf + +# Extract osm format for GTFS shape enhancement +docker run -v $HOST_DATA/osm:/osm mfdz/pyosmium osmium cat /osm/bw-buffered.osm.pbf -o /osm/bw-buffered.osm -O + +mv $OSM_DIR/bw-buffered.osm.pbf $OUT_DIR/bw-buffered.osm.pbf \ No newline at end of file