From b984348799132dbcaf8551c42d1c5a9021a8eb4c Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Thu, 22 Feb 2024 11:18:03 -0300 Subject: [PATCH 1/3] Cleaning and reorganizing files, adding install scripts --- Makefile.am | 8 +- docs/Dev/bootstrapsh.md | 2 +- docs/Dev/install-with-docker.md | 6 +- docs/Dev/replicator.md | 2 +- docs/Dev/utility.md | 25 +-- docs/Replication/Run.md | 8 +- docs/get-started/Build.md | 59 +++--- docs/get-started/Install.md | 8 + docs/get-started/Run.md | 8 +- {utils => setup}/bootstrap.sh | 12 +- .../raw-underpass.sql => setup/db/indexes.sql | 0 utils/raw-underpass.lua => setup/db/raw.lua | 0 setup/{ => db}/setupdb.sh | 0 setup/{ => db}/underpass.sql | 0 setup/install-macos-arm64.sh | 58 ++++++ setup/install-macos.sh | 59 ++++++ setup/install.sh | 52 +++++ utils/raw.lua | 188 ----------------- utils/raw_with_ref.lua | 193 ------------------ utils/underpassmon.py | 110 ---------- 20 files changed, 225 insertions(+), 573 deletions(-) rename {utils => setup}/bootstrap.sh (92%) rename utils/raw-underpass.sql => setup/db/indexes.sql (100%) rename utils/raw-underpass.lua => setup/db/raw.lua (100%) rename setup/{ => db}/setupdb.sh (100%) rename setup/{ => db}/underpass.sql (100%) create mode 100644 setup/install-macos-arm64.sh create mode 100644 setup/install-macos.sh create mode 100644 setup/install.sh delete mode 100644 utils/raw.lua delete mode 100644 utils/raw_with_ref.lua delete mode 100644 utils/underpassmon.py diff --git a/Makefile.am b/Makefile.am index 407fe8d3..fbcf135f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -50,7 +50,7 @@ BOOST_LIBS = $(BOOST_DATE_TIME_LIB) \ $(BOOST_PYTHON_LIB) SQL_FILES = \ - setup/underpass.sql + setup/db/underpass.sql libunderpass_la_SOURCES = \ src/utils/log.cc src/utils/log.hh \ @@ -69,9 +69,9 @@ libunderpass_la_SOURCES = \ src/bootstrap/bootstrap.cc src/bootstrap/bootstrap.hh \ src/utils/geoutil.cc src/utils/geoutil.hh \ src/utils/geo.cc src/utils/geo.hh \ - src/tils/yaml.hh src/utils/yaml.cc \ + src/utils/yaml.hh src/utils/yaml.cc \ src/data/pq.hh src/data/pq.cc \ - setup/setupdb.sh + setup/db/setupdb.sh if JEMALLOC libunderpass_la_LDFLAGS = -avoid-version -ljemalloc @@ -107,7 +107,7 @@ AM_CPPFLAGS = \ # clutter to the output. This should only be enabled when doing performance tuning. -DTIMING_DEBUG -pkgdata_DATA = $(SQL_FILES) setup/setupdb.sh $(PYTHON_UTILS) +pkgdata_DATA = $(SQL_FILES) setup/db/setupdb.sh $(PYTHON_UTILS) EXTRA_DIST = \ $(SQL_FILES) \ diff --git a/docs/Dev/bootstrapsh.md b/docs/Dev/bootstrapsh.md index 74eff92e..a9ab79e3 100644 --- a/docs/Dev/bootstrapsh.md +++ b/docs/Dev/bootstrapsh.md @@ -24,7 +24,7 @@ This will: 2. Download raw data from GeoFabrik 3. Run osm2psql for import it 4. Convert the country .poly file to .geojson and install config files -5. Run ./underpass --bootstrap for bootstrapping the validation table +5. Run underpass --bootstrap for bootstrapping the validation table ### Script options diff --git a/docs/Dev/install-with-docker.md b/docs/Dev/install-with-docker.md index eba3719e..b7f19e87 100644 --- a/docs/Dev/install-with-docker.md +++ b/docs/Dev/install-with-docker.md @@ -58,19 +58,19 @@ displaying raw data and validation results on top of the OSM map. Run this command for start processing data from 2 days ago: ```sh -docker exec -w /code/build underpass ./underpass -t $(date +%Y-%m-%dT%H:%M:%S -d "2 days ago") +docker exec -w /code/build underpass underpass -t $(date +%Y-%m-%dT%H:%M:%S -d "2 days ago") ``` On MacOS, the date command is different: ```sh -docker exec -w /code/build underpass ./underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S) +docker exec -w /code/build underpass underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S) ``` For running underpass as a daemon, use the `-d` option: ```sh -docker exec -d -w /code/build underpass ./underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S) +docker exec -d -w /code/build underpass underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S) ``` ### Stop and start underpass diff --git a/docs/Dev/replicator.md b/docs/Dev/replicator.md index a59d5fd0..663d9412 100644 --- a/docs/Dev/replicator.md +++ b/docs/Dev/replicator.md @@ -19,7 +19,7 @@ sources, they are analyzed, and applied to the appropriate database. Most of the work is done by the Underpass library. *underpass* is the utility program that a uses the library. - ./underpass -h + underpass -h -h [ --help ] display help -s [ --server arg] database server (defaults to localhost) -m [ --monitor] Start monitoring planet diff --git a/docs/Dev/utility.md b/docs/Dev/utility.md index fe648381..98a65a02 100644 --- a/docs/Dev/utility.md +++ b/docs/Dev/utility.md @@ -1,6 +1,6 @@ # Utility Programs -## data/setupdb.sh +## data/db/setupdb.sh This is a simple shell script that creates all the databases Underpass uses. It can also be used to initialize the two internal databases @@ -10,7 +10,7 @@ countries used to determine which country a change was made in for statistics collection. These are used to bootstrap a new Underpass installation. -## utils/bootstrap.sh +## setup/bootstrap.sh * [Using the bootstrap.sh script](/underpass/Dev/bootstrapsh) @@ -64,27 +64,6 @@ Converts .poly to .geojson python poly2geojson file.poly ``` -## utils/underpassmon - -Replication process log monitor - -Use this script for monitoring Underpass logs and estimate replication speed. - -You'll need to pass the `-v -d` parameters to the `underpass` command -in order to output the log file. - -Monitoring OsmChanges processing (default): - -```sh -python underpassmon.py -f underpass.log -``` - -Monitoring ChangeSets processing: - -```sh -python underpassmon.py -f underpass.log -m changesets -``` - ## utils/clean-osmchanges.sql You may want to run this query if you're not processing raw data diff --git a/docs/Replication/Run.md b/docs/Replication/Run.md index b2dd91a2..3a1862c8 100644 --- a/docs/Replication/Run.md +++ b/docs/Replication/Run.md @@ -2,17 +2,17 @@ Run this command for start processing data from 2 days ago -`./underpass -t $(date +%Y-%m-%dT%H:%M:%S -d "2 days ago")` +`underpass -t $(date +%Y-%m-%dT%H:%M:%S -d "2 days ago")` On MacOS, the date command is different -`./underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)` +`underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)` Using Docker, add `docker exec -d -w /code/build underpass` -`docker exec -w /code/build underpass ./underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)` +`docker exec -w /code/build underpass underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)` For running underpass as a daemon, use the -d option: -`docker exec -d -w /code/build underpass ./underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)` +`docker exec -d -w /code/build underpass underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)` diff --git a/docs/get-started/Build.md b/docs/get-started/Build.md index 6255d093..14629c57 100644 --- a/docs/get-started/Build.md +++ b/docs/get-started/Build.md @@ -36,63 +36,50 @@ $ ./autogen.sh && \ ../configure && make -j$(nproc) && sudo make install ``` -## MacOS (Intel) +## MacOS ### Install dependencies ``` +sudo port install boost brew install \ - autoconf \ - automake \ libtool \ - swig \ gdal \ pkg-config \ - openjdk@11 \ - libosmium \ - gumbo-parser \ - boost \ - boost-python \ - open-mpi \ - range-v3 \ - wget \ - unzip \ + openssl \ + protobuf \ + boost-python3 \ libxml++3 \ libpqxx \ - ccache \ - openssl \ - protobuf + gumbo-parser ``` -### Build +### Build (Intel) ```sh sh ./autogen.sh mkdir build && cd build -../configure CXXFLAGS="-I/usr/local/include \ - -I/usr/local/Homebrew/Cellar/libpqxx/7.7.5/include/ \ - -I/opt/homebrew/opt/openssl@3/include \ - -L/opt/homebrew/Cellar/boost-python3/1.82.0/lib \ - -L/opt/homebrew/Cellar/jemalloc/5.3.0/lib \ - -g -O0" CXX="ccache g++" +../configure CXXFLAGS=" \ + -I/usr/local/include \ + -L/opt/homebrew/lib \ + -I/opt/homebrew/include \ + -I/opt/homebrew/Cellar \ + -L/usr/local/Cellar" CXX="g++" make -j$(nproc) && sudo make install ``` ## MacOS (Sillicon) -Make sure you have installed all dependencies built for arm64 or as universal binaries. - -You can do this with brew using: - -`arch -arm64 brew install ` - -Then the process is similar to Intel but adding `-arch arm64` to `CXXFLAGS`: +The process is similar to Intel but adding `-arch arm64` to `CXXFLAGS`: ```sh -../configure CXXFLAGS="-arch arm64 -I/usr/local/include \ - -I/usr/local/Homebrew/Cellar/libpqxx/7.7.5/include/ \ - -I/opt/homebrew/opt/openssl@3/include \ - -L/opt/homebrew/Cellar/boost-python3/1.82.0/lib \ - -L/opt/homebrew/Cellar/jemalloc/5.3.0/lib \ - -g -O0" CXX="ccache g++" +sh ./autogen.sh +mkdir build && cd build +../configure CXXFLAGS=" -arch arm64\ + -I/usr/local/include \ + -L/opt/homebrew/lib \ + -I/opt/homebrew/include \ + -I/opt/homebrew/Cellar \ + -L/usr/local/Cellar" CXX="g++" +make -j$(nproc) && sudo make install ``` diff --git a/docs/get-started/Install.md b/docs/get-started/Install.md index 08e1d134..f8808f94 100644 --- a/docs/get-started/Install.md +++ b/docs/get-started/Install.md @@ -8,9 +8,17 @@ Check the [build documentation](/underpass/get-started/Build) ## Linux +`setup/install.sh` `apt-get install underpass` (not available yet) ## MacOS `brew install underpass` (not available yet) +### Intel + +`setup/install-macos.sh` + +### Sillicon + +`setup/install-macos-arm64.sh` diff --git a/docs/get-started/Run.md b/docs/get-started/Run.md index e0f4c47e..18a593c0 100644 --- a/docs/get-started/Run.md +++ b/docs/get-started/Run.md @@ -4,22 +4,22 @@ Run this command for start processing data from 2 days ago: -`./underpass -t $(date +%Y-%m-%dT%H:%M:%S -d "2 days ago")` +`underpass -t $(date +%Y-%m-%dT%H:%M:%S -d "2 days ago")` ### MacOS On MacOS, the date command is slighty different: -`./underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)` +`underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)` ### Docker If you're running Underpass on a Docker container: -`docker exec -w /code/build underpass ./underpass -t $(date +%Y-%m-%dT%H:%M:%S -d "2 days ago")` +`docker exec -w /code/build underpass underpass -t $(date +%Y-%m-%dT%H:%M:%S -d "2 days ago")` For running as a daemon: -`docker exec -d -w /code/build underpass ./underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)` +`docker exec -d -w /code/build underpass underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)` diff --git a/utils/bootstrap.sh b/setup/bootstrap.sh similarity index 92% rename from utils/bootstrap.sh rename to setup/bootstrap.sh index 9427b95a..4c4a6b0e 100644 --- a/utils/bootstrap.sh +++ b/setup/bootstrap.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2023 Humanitarian OpenStreetMap Team +# Copyright (c) 2023, 2024 Humanitarian OpenStreetMap Team # # This file is part of Underpass. # @@ -91,7 +91,7 @@ then echo "Cleaning database ..." PGPASSWORD=$PASS psql --host $HOST --user $USER --port $PORT $DB -c 'DROP TABLE IF EXISTS ways_poly; DROP TABLE IF EXISTS ways_line; DROP TABLE IF EXISTS nodes; DROP TABLE IF EXISTS way_refs; DROP TABLE IF EXISTS validation; DROP TABLE IF EXISTS changesets;' - PGPASSWORD=$PASS psql --host $HOST --user $USER --port $PORT $DB --file '../setup/underpass.sql' + PGPASSWORD=$PASS psql --host $HOST --user $USER --port $PORT $DB --file 'db/underpass.sql' if "$localfiles"; then @@ -103,8 +103,8 @@ then fi echo "Importing data (this will take some time) ..." - PGPASSWORD=$PASS osm2pgsql -H $HOST -U $USER -P $PORT -d $DB --extra-attributes --output=flex --style ./raw-underpass.lua $COUNTRY-latest.osm.pbf - PGPASSWORD=$PASS psql --host $HOST --user $USER --port $PORT $DB < raw-underpass.sql + PGPASSWORD=$PASS osm2pgsql -H $HOST -U $USER -P $PORT -d $DB --extra-attributes --output=flex --style ./db/raw.lua $COUNTRY-latest.osm.pbf + PGPASSWORD=$PASS psql --host $HOST --user $USER --port $PORT $DB < db/indexes.sql echo "Configuring Underpass ..." python3 poly2geojson.py $COUNTRY.poly @@ -117,9 +117,9 @@ then echo "Bootstrapping database ..." if "$use_docker"; then - docker exec -w /code/build -t underpass ./underpass --bootstrap + docker exec -w /code/build -t underpass underpass --bootstrap else - cd ../build && ./underpass --bootstrap + underpass --bootstrap fi echo "Done." echo " " diff --git a/utils/raw-underpass.sql b/setup/db/indexes.sql similarity index 100% rename from utils/raw-underpass.sql rename to setup/db/indexes.sql diff --git a/utils/raw-underpass.lua b/setup/db/raw.lua similarity index 100% rename from utils/raw-underpass.lua rename to setup/db/raw.lua diff --git a/setup/setupdb.sh b/setup/db/setupdb.sh similarity index 100% rename from setup/setupdb.sh rename to setup/db/setupdb.sh diff --git a/setup/underpass.sql b/setup/db/underpass.sql similarity index 100% rename from setup/underpass.sql rename to setup/db/underpass.sql diff --git a/setup/install-macos-arm64.sh b/setup/install-macos-arm64.sh new file mode 100644 index 00000000..cef9a601 --- /dev/null +++ b/setup/install-macos-arm64.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# +# Copyright (c) 2024 Humanitarian OpenStreetMap Team +# +# This file is part of Underpass. +# +# Underpass is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Underpass is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Underpass. If not, see . + +echo "Installing dependencies ..." +sudo port install boost +brew install \ + libtool \ + gdal \ + pkg-config \ + openssl \ + protobuf \ + boost-python3 \ + libxml++3 \ + libpqxx \ + gumbo-parser +cd .. + +echo "Setting up build ..." +./autogen.sh +mkdir build && cd build +../configure CXXFLAGS="-arch arm64 \ + -I/usr/local/include \ + -L/opt/homebrew/lib \ + -I/opt/homebrew/include \ + -I/opt/homebrew/Cellar/ \ + -L/usr/local/Cellar/ \ + -g -O2" CXX="g++" + +echo "Building ..." +make -j$(nproc) && sudo make install +echo "Setting up libs ..." +ln -s /usr/local/lib/libboost_date_time.dylib .libs +ln -s /usr/local/lib/libboost_system.dylib .libs +ln -s /usr/local/lib/libboost_filesystem.dylib .libs +ln -s /usr/local/lib/libboost_log.dylib .libs +ln -s /usr/local/lib/libboost_program_options.dylib .libs +ln -s /usr/local/lib/libboost_iostreams.dylib .libs +ln -s /usr/local/lib/libboost_thread.dylib .libs +ln -s /usr/local/lib/libboost_serialization.dylib .libs +ln -s /usr/local/lib/libboost_regex.dylib .libs + +echo "Done! now you may want to initialize the database with the bootstrap script" diff --git a/setup/install-macos.sh b/setup/install-macos.sh new file mode 100644 index 00000000..d10b832f --- /dev/null +++ b/setup/install-macos.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# +# Copyright (c) 2024 Humanitarian OpenStreetMap Team +# +# This file is part of Underpass. +# +# Underpass is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Underpass is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Underpass. If not, see . + +echo "Installing dependencies ..." +sudo port install boost +brew install \ + libtool \ + gdal \ + pkg-config \ + openssl \ + protobuf \ + boost-python3 \ + libxml++3 \ + libpqxx \ + gumbo-parser +cd .. + +echo "Setting up build ..." +./autogen.sh +mkdir build && cd build +../configure CXXFLAGS="-arch arm64 \ + -I/usr/local/include \ + -L/opt/homebrew/lib \ + -I/opt/homebrew/include \ + -I/opt/homebrew/Cellar/ \ + -L/usr/local/Cellar/ \ + -g -O2" CXX="g++" + +echo "Building ..." +make -j$(nproc) && sudo make install + +echo "Setting up libs ..." +ln -s /usr/local/lib/libboost_date_time.dylib .libs +ln -s /usr/local/lib/libboost_system.dylib .libs +ln -s /usr/local/lib/libboost_filesystem.dylib .libs +ln -s /usr/local/lib/libboost_log.dylib .libs +ln -s /usr/local/lib/libboost_program_options.dylib .libs +ln -s /usr/local/lib/libboost_iostreams.dylib .libs +ln -s /usr/local/lib/libboost_thread.dylib .libs +ln -s /usr/local/lib/libboost_serialization.dylib .libs +ln -s /usr/local/lib/libboost_regex.dylib .libs + +echo "Done! now you may want to initialize the database with the bootstrap script" \ No newline at end of file diff --git a/setup/install.sh b/setup/install.sh new file mode 100644 index 00000000..f28bd44b --- /dev/null +++ b/setup/install.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# +# Copyright (c) 2024 Humanitarian OpenStreetMap Team +# +# This file is part of Underpass. +# +# Underpass is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Underpass is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Underpass. If not, see . + +echo "Installing dependencies ..." + +sudo apt-get update \ + && apt-get install -y software-properties-common \ + && apt-get update && apt-get install -y \ + libboost-dev \ + autotools-dev \ + swig \ + pkg-config \ + gcc \ + build-essential \ + ccache \ + libboost-all-dev \ + dejagnu \ + libjemalloc-dev \ + libxml++2.6-dev \ + doxygen \ + libgdal-dev \ + libosmium2-dev \ + libpqxx-dev \ + postgresql \ + libgumbo-dev \ + librange-v3-dev +``` + +echo "Setting up build ..." +cd .. +./autogen.sh && mkdir build && cd build && ../configure + +echo "Building ..." +make -j$(nproc) && sudo make install + +echo "Done! now you may want to initialize the database with the bootstrap script" diff --git a/utils/raw.lua b/utils/raw.lua deleted file mode 100644 index 8a07a490..00000000 --- a/utils/raw.lua +++ /dev/null @@ -1,188 +0,0 @@ --- Copyright (c) 2020, 2021, 2023 Humanitarian OpenStreetMap Team --- --- This file is part of Underpass. --- --- Underpass is free software: you can redistribute it and/or modify --- it under the terms of the GNU General Public License as published by --- the Free Software Foundation, either version 3 of the License, or --- (at your option) any later version. --- --- Underpass is distributed in the hope that it will be useful, --- but WITHOUT ANY WARRANTY; without even the implied warranty of --- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the --- GNU General Public License for more details. --- --- You should have received a copy of the GNU General Public License --- along with Underpass. If not, see . - --- This is lua script for osm2pgsql in order to create and process custom schema to store incoming osm data efficiently - --- osm2pgsql --create -H localhost -U admin -P 5432 -d postgres -W --extra-attributes --output=flex --style ./raw.lua nepal-latest-internal.osm.pbf - - --- Set projection to 4326 -local srid = 4326 - -local tables = {} - -tables.nodes = osm2pgsql.define_table{ - name="nodes", - -- This will generate a derived nodes table which stores all the nodes feature with their point geometry - ids = {type='node',id_column = 'osm_id' }, - columns = { - { column = 'uid', type = 'int' }, - { column = 'user', type = 'text' }, - { column = 'version', type = 'int' }, - { column = 'changeset', type = 'int' }, - { column = 'timestamp', sql_type = 'timestamp' }, - { column = 'tags', type = 'jsonb' }, - { column = 'geom', type = 'point', projection = srid }, - { column = 'country', type= 'int', create_only = true }, - - } - -} - -tables.ways_line = osm2pgsql.define_table{ - name="ways_line", - -- This will generate a derived ways line table which stores all the ways feature with linestring geometry - ids = {type='way',id_column = 'osm_id' }, - columns = { - { column = 'uid', type = 'int' }, - { column = 'user', type = 'text' }, - { column = 'version', type = 'int' }, - { column = 'changeset', type = 'int' }, - { column = 'timestamp', sql_type = 'timestamp' }, - { column = 'tags', type = 'jsonb' }, - { column = 'geom', type = 'linestring', projection = srid }, - { column = 'country', sql_type= 'int[]', create_only = true }, - - } - -} - -tables.ways_poly = osm2pgsql.define_table{ - name="ways_poly", - -- This will generate a derived ways poly table which stores all the ways feature with polygon geometry - ids = {type='way',id_column = 'osm_id' }, - columns = { - { column = 'uid', type = 'int' }, - { column = 'user', type = 'text' }, - { column = 'version', type = 'int' }, - { column = 'changeset', type = 'int' }, - { column = 'timestamp', sql_type = 'timestamp' }, - -- This will store tags as jsonb type - { column = 'tags', type = 'jsonb' }, - { column = 'geom', type = 'polygon', projection = srid }, - { column = 'grid', type = 'int', create_only = true }, - { column = 'country', type= 'int', create_only = true }, - } - -} - -tables.rels = osm2pgsql.define_table{ - name="relations", - -- This will generate a derived realtion table which stores all the relation feature to query without storing meta data parts and members - - ids = {type='relation', id_column = 'osm_id' }, - columns = { - - { column = 'uid', type = 'int' }, - { column = 'user', type = 'text' }, - { column = 'version', type = 'int' }, - { column = 'changeset', type = 'int' }, - { column = 'timestamp', sql_type = 'timestamp' }, - { column = 'tags', type = 'jsonb' }, - { column = 'geom', type = 'geometry', projection = srid }, - { column = 'country',sql_type= 'int[]', create_only = true }, - - - } -} - --- Returns true if there are no tags left. -function clean_tags(tags) - tags.odbl = nil - tags.created_by = nil - tags.source = nil - tags['source:ref'] = nil - - return next(tags) == nil -end - -function osm2pgsql.process_node(object) - - if clean_tags(object.tags) then - return - end - - tables.nodes:add_row({ - uid = object.uid, - user = object.user, - version = object.version, - changeset = object.changeset, - timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), - tags = object.tags, - geom = { create = 'point' } - }) -end - -function osm2pgsql.process_way(object) - if clean_tags(object.tags) then - return - end - - if object.is_closed and #object.nodes>3 then - tables.ways_poly:add_row({ - uid = object.uid, - user = object.user, - version = object.version, - changeset = object.changeset, - timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), - nodes=object.nodes, - tags = object.tags, - geom = { create = 'area' } - }) - else - tables.ways_line:add_row({ - uid = object.uid, - user = object.user, - version = object.version, - changeset = object.changeset, - timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), - nodes=object.nodes, - tags = object.tags, - geom = { create = 'line' } - }) - end -end - -function osm2pgsql.process_relation(object) - if clean_tags(object.tags) then - return - end - if object.tags.type == 'multipolygon' or object.tags.type == 'boundary' then - tables.rels:add_row({ - uid = object.uid, - user = object.user, - version = object.version, - changeset = object.changeset, - timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), - members=object.members, - tags = object.tags, - geom = { create = 'area' } - }) - else - tables.rels:add_row({ - uid = object.uid, - user = object.user, - version = object.version, - changeset = object.changeset, - timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), - members=object.members, - tags = object.tags, - geom= { create = 'line' } - }) - end -end - diff --git a/utils/raw_with_ref.lua b/utils/raw_with_ref.lua deleted file mode 100644 index 05527cb7..00000000 --- a/utils/raw_with_ref.lua +++ /dev/null @@ -1,193 +0,0 @@ --- Copyright (c) 2020, 2021, 2023 Humanitarian OpenStreetMap Team - --- This program is free software: you can redistribute it and/or modify --- it under the terms of the GNU Affero General Public License as --- published by the Free Software Foundation, either version 3 of the --- License, or (at your option) any later version. - --- This program is distributed in the hope that it will be useful, --- but WITHOUT ANY WARRANTY; without even the implied warranty of --- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the --- GNU Affero General Public License for more details. - --- You should have received a copy of the GNU Affero General Public License --- along with this program. If not, see . - --- Humanitarian OpenStreetmap Team --- 1100 13th Street NW Suite 800 Washington, D.C. 20005 --- --- This is lua script for osm2pgsql in order to create and process custom schema to store incoming osm data efficiently - --- osm2pgsql --create -H localhost -U admin -P 5432 -d postgres -W --extra-attributes --output=flex --style ./raw.lua nepal-latest-internal.osm.pbf - - --- Set projection to 4326 -local srid = 4326 - -local tables = {} - -tables.nodes = osm2pgsql.define_table{ - name="nodes", - -- This will generate a derived nodes table which stores all the nodes feature with their point geometry - ids = {type='node',id_column = 'osm_id' }, - columns = { - { column = 'uid', type = 'int' }, - { column = 'user', type = 'text' }, - { column = 'version', type = 'int' }, - { column = 'changeset', type = 'int' }, - { column = 'timestamp', sql_type = 'timestamp' }, - { column = 'tags', type = 'jsonb' }, - { column = 'geom', type = 'point', projection = srid }, - { column = 'country', sql_type= 'int[]', create_only = true }, - - } - -} - -tables.ways_line = osm2pgsql.define_table{ - name="ways_line", - -- This will generate a derived ways line table which stores all the ways feature with linestring geometry - ids = {type='way',id_column = 'osm_id' }, - columns = { - { column = 'uid', type = 'int' }, - { column = 'user', type = 'text' }, - { column = 'version', type = 'int' }, - { column = 'changeset', type = 'int' }, - { column = 'timestamp', sql_type = 'timestamp' }, - { column = 'tags', type = 'jsonb' }, - { column = 'refs', type= 'text', sql_type = 'bigint[]'}, - { column = 'geom', type = 'linestring', projection = srid }, - { column = 'country', sql_type= 'int[]', create_only = true }, - - } - -} - -tables.ways_poly = osm2pgsql.define_table{ - name="ways_poly", - -- This will generate a derived ways poly table which stores all the ways feature with polygon geometry - ids = {type='way',id_column = 'osm_id' }, - columns = { - { column = 'uid', type = 'int' }, - { column = 'user', type = 'text' }, - { column = 'version', type = 'int' }, - { column = 'changeset', type = 'int' }, - { column = 'timestamp', sql_type = 'timestamp' }, - -- This will store tags as jsonb type - { column = 'tags', type = 'jsonb' }, - { column = 'refs', type= 'text', sql_type = 'bigint[]'}, - { column = 'geom', type = 'polygon', projection = srid }, - { column = 'grid', type = 'int', create_only = true }, - { column = 'country', sql_type= 'int[]', create_only = true }, - } - -} - -tables.rels = osm2pgsql.define_table{ - name="relations", - -- This will generate a derived realtion table which stores all the relation feature to query without storing meta data parts and members - - ids = {type='relation', id_column = 'osm_id' }, - columns = { - - { column = 'uid', type = 'int' }, - { column = 'user', type = 'text' }, - { column = 'version', type = 'int' }, - { column = 'changeset', type = 'int' }, - { column = 'timestamp', sql_type = 'timestamp' }, - { column = 'tags', type = 'jsonb' }, - { column = 'refs', type = 'jsonb'}, - { column = 'geom', type = 'geometry', projection = srid }, - { column = 'country',sql_type= 'int[]', create_only = true }, - - } -} - --- Returns true if there are no tags left. -function clean_tags(tags) - tags.odbl = nil - -- tags.created_by = nil - tags['source:ref'] = nil - return next(tags) == nil -end - -function osm2pgsql.process_node(object) - - if clean_tags(object.tags) then - return - end - - tables.nodes:add_row({ - uid = object.uid, - user = object.user, - version = object.version, - changeset = object.changeset, - timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), - tags = object.tags, - geom = { create = 'point' } - }) -end - -function osm2pgsql.process_way(object) - if clean_tags(object.tags) then - return - end - - if object.is_closed and #object.nodes>3 then - tables.ways_poly:add_row({ - uid = object.uid, - user = object.user, - version = object.version, - changeset = object.changeset, - timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), - tags = object.tags, - refs = '{' .. table.concat(object.nodes, ',') .. '}', - geom = { create = 'area' }, - - }) - else - tables.ways_line:add_row({ - uid = object.uid, - user = object.user, - version = object.version, - changeset = object.changeset, - timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), - tags = object.tags, - refs = '{' .. table.concat(object.nodes, ',') .. '}', - geom = { create = 'line' }, - - }) - end -end - -function osm2pgsql.process_relation(object) - if clean_tags(object.tags) then - return - end - if object.tags.type == 'multipolygon' or object.tags.type == 'boundary' then - tables.rels:add_row({ - uid = object.uid, - user = object.user, - version = object.version, - changeset = object.changeset, - timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), - tags = object.tags, - geom = { create = 'area' }, - refs = object.members - - }) - else - tables.rels:add_row({ - uid = object.uid, - user = object.user, - version = object.version, - changeset = object.changeset, - timestamp = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), - tags = object.tags, - geom= { create = 'line' }, - refs = object.members - - }) - end -end - diff --git a/utils/underpassmon.py b/utils/underpassmon.py deleted file mode 100644 index a8033d26..00000000 --- a/utils/underpassmon.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/python3 -# -# Copyright (c) 2020, 2021, 2023 Humanitarian OpenStreetMap Team -# -# This file is part of Underpass. -# -# Underpass is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Underpass is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Underpass. If not, see . - -''' - Replication process log monitor - - Use this script for monitoring Underpass logs and - estimate replication speed. - - Monitoring OsmChanges processing (default): - python underpassmon.py -f underpass.log - - Monitoring ChangeSets processing: - python underpassmon.py -f underpass.log -m changesets -''' - -import re -from datetime import datetime -import time -import argparse - - -SEARCH_STRING = { - "osmchanges": "final_entry: ", - "changesets": "last_closed_at: " -} - -class UnderpassLogMonitor: - - def __init__(self, mode): - self.start = (datetime.now(), None) - self.lastDateTime = None - self.elapsedTime = None - self.processed = None - self.searchString = SEARCH_STRING[mode] - - def getDateFromStr(self, str): - return datetime.strptime(str, '%Y-%b-%d %H:%M:%S') - - def getDateFromLogLine(self, logLine): - result = re.search(self.searchString, logLine) - if result: - dateString = logLine[ - result.start() + len(self.searchString):result.start() + - len(self.searchString) + 20 - ] - return self.getDateFromStr(dateString) - return None - - def getInfo(self, line): - try: - current = (datetime.now(), self.getDateFromLogLine(line)) - if current[1]: - if self.start[1] == None: - self.start = current - if self.lastDateTime == None or current[1] > self.lastDateTime: - self.lastDateTime = current[1] - self.processed = self.lastDateTime - self.start[1] - self.elapsedTime = datetime.now() - self.start[0] - except: - pass - - def follow(self, filename): - with open(filename) as f: - f.seek(0,2) - while True: - pos = f.tell() - line = f.readline() - if not line: - f.seek(pos) - time.sleep(.2) - else: - self.getInfo(line) - self.dump() - - def dump(self): - if self.processed: - oneYear = (365*24*60*60 * self.elapsedTime.total_seconds() / self.processed.total_seconds()) / 60 / 60 - print("Started:", self.start[1].strftime('%Y-%b-%d %H:%M:%S')) - print("Current:", self.lastDateTime.strftime('%Y-%b-%d %H:%M:%S')) - print("Processed [", self.processed, "hs ] in [", self.elapsedTime, "hs ]") - print( "[EST] 1 year in ", round(oneYear,2), "hs") - print("") - -def main(): - args = argparse.ArgumentParser() - args.add_argument("--file", "-f", help="Underpass logfile", type=str, default="underpass.log") - args.add_argument("--mode", "-m", help="String for search before date", type=str, default="osmchanges") - args = args.parse_args() - ulog = UnderpassLogMonitor(args.mode) - ulog.follow(args.file) - -if __name__ == "__main__": - main() From d16fa5f897d317f82ee56aeb7a9cf08ea1147c5e Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Thu, 22 Feb 2024 15:54:57 -0300 Subject: [PATCH 2/3] Updating docs, dependencies, build scripts --- Makefile.am | 2 ++ README.md | 2 +- config/default.yaml | 2 +- configure.ac | 2 +- docker/underpass-config.yaml | 8 ++++++-- docker/underpass.dockerfile | 11 ++++++----- docs/get-started/Setup.md | 15 ++++++++------- python/dbapi/requirements.txt | 2 +- python/restapi/config.py | 1 + src/underpass.cc | 2 +- 10 files changed, 28 insertions(+), 19 deletions(-) diff --git a/Makefile.am b/Makefile.am index fbcf135f..3b2677fb 100644 --- a/Makefile.am +++ b/Makefile.am @@ -153,8 +153,10 @@ endif install-data-hook: $(MKDIR_P) $(DESTDIR)/$(pkglibdir) $(MKDIR_P) /etc/underpass + cp -rvp $(srcdir)/setup/service/underpass.service /etc/systemd/system/ cp -rvp $(srcdir)/config/priority.geojson /etc/underpass/ cp -rvp $(srcdir)/config/replicator /etc/underpass/ + cp -rvp $(srcdir)/config/validate /etc/underpass/ cp -rvp $(srcdir)/config/stats /etc/underpass/ cp -rvp $(srcdir)/config/default.yaml /etc/underpass/ $(MKDIR_P) $(DESTDIR)/$(docdir) diff --git a/README.md b/README.md index 67c50022..ea28aecc 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ It **updates a local copy of the OSM database** in near real-time, and provides We've deployed a rudimentary demo that keeps a database up-to-date for (some country), rendering buildings and highlighting the ones identified as "un-squared": -[See the development demo](https://underpass.live) +[https://underpass.live](https://underpass.live) Screenshot 2023-11-22 at 10 32 56 diff --git a/config/default.yaml b/config/default.yaml index b960e224..e0a79e37 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -3,7 +3,7 @@ config: - underpass_db_url: - underpass:underpass@localhost:5432/underpass - planet_servers: - - planet.openstreetmap.org + - planet.maps.mail.ru - destdir_base: - /usr/local/lib/underpass/data/ diff --git a/configure.ac b/configure.ac index 13a15067..547a99f3 100644 --- a/configure.ac +++ b/configure.ac @@ -239,7 +239,7 @@ if test x"${enable_python}" = x"yes"; then AX_BOOST_PYTHON AC_DEFINE(USE_PYTHON, [1], [Enable Python binding]) fi -AM_CONDITIONAL(ENABLE_PYTHON, [ test x"${enable_python}" != x ]) +AM_CONDITIONAL(ENABLE_PYTHON, [ test x"${enable_python}" != x"no" ]) AC_PATH_PROG(DOXYGEN, doxygen) AM_CONDITIONAL(ENABLE_DOXYGEN, [ test x"$DOXYGEN" != x ]) diff --git a/docker/underpass-config.yaml b/docker/underpass-config.yaml index cb4ad12e..cdc52f08 100644 --- a/docker/underpass-config.yaml +++ b/docker/underpass-config.yaml @@ -1,4 +1,8 @@ # Underpass config file config: - - underpass_db_url: - - underpass@postgis/underpass \ No newline at end of file + - underpass_db_url: + - underpass@postgis/underpass + - planet_servers: + - planet.maps.mail.ru + - destdir_base: + - /usr/local/lib/underpass/data/ diff --git a/docker/underpass.dockerfile b/docker/underpass.dockerfile index f80fc6f2..a49c53b2 100644 --- a/docker/underpass.dockerfile +++ b/docker/underpass.dockerfile @@ -74,11 +74,12 @@ CMD ["make", "check", "-j", "$(nproc)"] FROM deps as debug # Local debug with all dev deps ENV LD_LIBRARY_PATH=/code/build/.libs/ +COPY --from=build /usr/local/lib/ /usr/local/lib/ COPY --from=build /usr/local/bin /usr/local/bin COPY --from=build /code/build/.libs/ \ /code/build/.libs/ -COPY --from=build /usr/local/lib/underpass/config \ - /usr/local/lib/underpass/config +COPY --from=build /etc/underpass \ + /etc/underpass WORKDIR /code/build # Add non-root user RUN useradd -r -u 1001 -m -c "hotosm account" -d /home/appuser -s /bin/false appuser @@ -105,10 +106,10 @@ RUN set -ex \ "osm2pgsql" \ && rm -rf /var/lib/apt/lists/* COPY --from=build /usr/local/bin /usr/local/bin -COPY --from=build /code/build/.libs/ \ +COPY --from=build /usr/local/lib/ \ /usr/local/lib/ -COPY --from=build /usr/local/lib/underpass/config \ - /usr/local/lib/underpass/config +COPY --from=build /etc/underpass \ + /etc/underpass WORKDIR /code/build # Add non-root user RUN useradd -r -u 1001 -m -c "hotosm account" -d /home/appuser -s /bin/false appuser diff --git a/docs/get-started/Setup.md b/docs/get-started/Setup.md index d4843b0a..ef049002 100644 --- a/docs/get-started/Setup.md +++ b/docs/get-started/Setup.md @@ -1,4 +1,4 @@ -## Boostrap the database +## Boostraping the database You can prepare your Underpass installation with data for a specific country. @@ -34,17 +34,19 @@ apt install osm2pgsql ### Bootstrap -Now you can go to the `utils` directory and run the boostrap script: +Go to the `setup` directory and run the boostrap script: cd utils -chmod +x bootstrap.sh -./bootstrap.sh -r south-america -c uruguay -p 5432 -u underpass +./bootstrap.sh -r south-america -c uruguay ``` -Use `-p ` and `-u ` for the database. +Use `-u ` `-h ` `-d ` `-d ` for the database connection. -Regions (-r) are: +If you installed Underpass with Docker, you might use the `-p 5439 -k yes` options. + +./bootstrap.sh -r south-america -c uruguay -p 5439 -k yes +Regions (-r) are: africa asia australia-oceania @@ -58,4 +60,3 @@ Countries (-c) is the name of the country inside the region. Data is downloaded from GeoFabrik, if you are not sure of what name you need to use, please check there. For advanced users, check the [boostrap script documentation](/underpass/Dev/bootstrapsh). - diff --git a/python/dbapi/requirements.txt b/python/dbapi/requirements.txt index eaef2e86..4cf8abbb 100644 --- a/python/dbapi/requirements.txt +++ b/python/dbapi/requirements.txt @@ -1 +1 @@ -psycopg2==2.9.6 \ No newline at end of file +asyncpg==0.29.0 \ No newline at end of file diff --git a/python/restapi/config.py b/python/restapi/config.py index c3588594..8bdfe850 100644 --- a/python/restapi/config.py +++ b/python/restapi/config.py @@ -5,6 +5,7 @@ "http://localhost", "http://localhost:5000", "http://localhost:3000", + "http://localhost:8080", "http://127.0.0.1", "http://127.0.0.1:5000" ] diff --git a/src/underpass.cc b/src/underpass.cc index 9a5f4977..b16bec4c 100644 --- a/src/underpass.cc +++ b/src/underpass.cc @@ -107,7 +107,7 @@ main(int argc, char *argv[]) ("changeseturl", opts::value(), "Starting URL path for ChangeSet (ex. 000/075/000), takes precedence over 'timestamp' option") ("frequency,f", opts::value(), "Update frequency (hourly, daily), default minutely)") ("timestamp,t", opts::value>(), "Starting timestamp (can be used 2 times to set a range)") - ("import,i", opts::value(), "Initialize OSM database with datafile") + // ("import,i", opts::value(), "Initialize OSM database with datafile") ("boundary,b", opts::value(), "Boundary polygon file name") ("osmnoboundary", "Disable boundary polygon for OsmChanges") ("oscnoboundary", "Disable boundary polygon for Changesets") From 50431ad1b4743c7382c6ee306e9bdc2716637397 Mon Sep 17 00:00:00 2001 From: Emillio Mariscal Date: Thu, 22 Feb 2024 15:57:21 -0300 Subject: [PATCH 3/3] Clean code --- docs/get-started/Build.md | 2 +- setup/install-macos-arm64.sh | 6 +++--- setup/install-macos.sh | 2 +- setup/install.sh | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/get-started/Build.md b/docs/get-started/Build.md index 14629c57..a3c6ef59 100644 --- a/docs/get-started/Build.md +++ b/docs/get-started/Build.md @@ -41,7 +41,7 @@ $ ./autogen.sh && \ ### Install dependencies ``` -sudo port install boost +sudo port install boost brew install \ libtool \ gdal \ diff --git a/setup/install-macos-arm64.sh b/setup/install-macos-arm64.sh index cef9a601..d67d942e 100644 --- a/setup/install-macos-arm64.sh +++ b/setup/install-macos-arm64.sh @@ -18,7 +18,7 @@ # along with Underpass. If not, see . echo "Installing dependencies ..." -sudo port install boost +sudo port install boost brew install \ libtool \ gdal \ @@ -51,8 +51,8 @@ ln -s /usr/local/lib/libboost_filesystem.dylib .libs ln -s /usr/local/lib/libboost_log.dylib .libs ln -s /usr/local/lib/libboost_program_options.dylib .libs ln -s /usr/local/lib/libboost_iostreams.dylib .libs -ln -s /usr/local/lib/libboost_thread.dylib .libs -ln -s /usr/local/lib/libboost_serialization.dylib .libs +ln -s /usr/local/lib/libboost_thread.dylib .libs +ln -s /usr/local/lib/libboost_serialization.dylib .libs ln -s /usr/local/lib/libboost_regex.dylib .libs echo "Done! now you may want to initialize the database with the bootstrap script" diff --git a/setup/install-macos.sh b/setup/install-macos.sh index d10b832f..f378b522 100644 --- a/setup/install-macos.sh +++ b/setup/install-macos.sh @@ -18,7 +18,7 @@ # along with Underpass. If not, see . echo "Installing dependencies ..." -sudo port install boost +sudo port install boost brew install \ libtool \ gdal \ diff --git a/setup/install.sh b/setup/install.sh index f28bd44b..5e46c490 100644 --- a/setup/install.sh +++ b/setup/install.sh @@ -45,7 +45,7 @@ sudo apt-get update \ echo "Setting up build ..." cd .. ./autogen.sh && mkdir build && cd build && ../configure - + echo "Building ..." make -j$(nproc) && sudo make install