Merge pull request #471 from hotosm/feature/updateAndRefactor

Feature/update and refactor

Makefile.am

@@ -50,7 +50,7 @@ BOOST_LIBS = $(BOOST_DATE_TIME_LIB) \
 	$(BOOST_PYTHON_LIB)
 
 SQL_FILES = \
-	setup/underpass.sql
+	setup/db/underpass.sql
 
 libunderpass_la_SOURCES = \
 	src/utils/log.cc src/utils/log.hh \
@@ -69,9 +69,9 @@ libunderpass_la_SOURCES = \
 	src/bootstrap/bootstrap.cc src/bootstrap/bootstrap.hh \
 	src/utils/geoutil.cc src/utils/geoutil.hh \
 	src/utils/geo.cc src/utils/geo.hh \
-	src/tils/yaml.hh src/utils/yaml.cc \
+	src/utils/yaml.hh src/utils/yaml.cc \
 	src/data/pq.hh src/data/pq.cc \
-	setup/setupdb.sh
+	setup/db/setupdb.sh
 
 if JEMALLOC
 libunderpass_la_LDFLAGS = -avoid-version -ljemalloc
@@ -107,7 +107,7 @@ AM_CPPFLAGS = \
 # clutter to the output. This should only be enabled when doing performance tuning.
 	-DTIMING_DEBUG
 
-pkgdata_DATA = $(SQL_FILES) setup/setupdb.sh $(PYTHON_UTILS)
+pkgdata_DATA = $(SQL_FILES) setup/db/setupdb.sh $(PYTHON_UTILS)
 
 EXTRA_DIST = \
 	$(SQL_FILES) \
@@ -153,8 +153,10 @@ endif
 install-data-hook:
 	$(MKDIR_P) $(DESTDIR)/$(pkglibdir)
 	$(MKDIR_P) /etc/underpass
+	cp -rvp $(srcdir)/setup/service/underpass.service /etc/systemd/system/
 	cp -rvp $(srcdir)/config/priority.geojson /etc/underpass/
 	cp -rvp $(srcdir)/config/replicator /etc/underpass/
+	cp -rvp $(srcdir)/config/validate /etc/underpass/
 	cp -rvp $(srcdir)/config/stats /etc/underpass/
 	cp -rvp $(srcdir)/config/default.yaml /etc/underpass/
 	$(MKDIR_P) $(DESTDIR)/$(docdir)

README.md

@@ -12,7 +12,7 @@ It **updates a local copy of the OSM database** in near real-time, and provides
 We've deployed a rudimentary demo that keeps a database up-to-date for (some country),
 rendering buildings and highlighting the ones identified as "un-squared":
 
-[See the development demo](https://underpass.live)
+[https://underpass.live](https://underpass.live)
 
 <img width="993" alt="Screenshot 2023-11-22 at 10 32 56" src="https://github.com/hotosm/underpass/assets/1226194/73622131-b69f-4716-bf95-9e195efdbba9">
 

config/default.yaml

@@ -3,7 +3,7 @@ config:
     - underpass_db_url:
         - underpass:underpass@localhost:5432/underpass
     - planet_servers:
-        - planet.openstreetmap.org
+        - planet.maps.mail.ru
     - destdir_base:
         - /usr/local/lib/underpass/data/
 

configure.ac

@@ -239,7 +239,7 @@ if test x"${enable_python}" = x"yes"; then
   AX_BOOST_PYTHON
   AC_DEFINE(USE_PYTHON, [1], [Enable Python binding])
 fi
-AM_CONDITIONAL(ENABLE_PYTHON, [ test x"${enable_python}" != x ])
+AM_CONDITIONAL(ENABLE_PYTHON, [ test x"${enable_python}" != x"no" ])
 
 AC_PATH_PROG(DOXYGEN, doxygen)
 AM_CONDITIONAL(ENABLE_DOXYGEN, [ test x"$DOXYGEN" != x ])

docker/underpass-config.yaml

@@ -1,4 +1,8 @@
 # Underpass config file
 config:
-  - underpass_db_url:
-    - underpass@postgis/underpass
+    - underpass_db_url:
+        - underpass@postgis/underpass
+    - planet_servers:
+        - planet.maps.mail.ru
+    - destdir_base:
+        - /usr/local/lib/underpass/data/

docker/underpass.dockerfile

@@ -74,11 +74,12 @@ CMD ["make", "check", "-j", "$(nproc)"]
 FROM deps as debug
 # Local debug with all dev deps
 ENV LD_LIBRARY_PATH=/code/build/.libs/
+COPY --from=build /usr/local/lib/ /usr/local/lib/
 COPY --from=build /usr/local/bin /usr/local/bin
 COPY --from=build /code/build/.libs/ \
     /code/build/.libs/
-COPY --from=build /usr/local/lib/underpass/config \
-    /usr/local/lib/underpass/config
+COPY --from=build /etc/underpass \
+    /etc/underpass
 WORKDIR /code/build
 # Add non-root user
 RUN useradd -r -u 1001 -m -c "hotosm account" -d /home/appuser -s /bin/false appuser
@@ -105,10 +106,10 @@ RUN set -ex \
         "osm2pgsql" \
     && rm -rf /var/lib/apt/lists/*
 COPY --from=build /usr/local/bin /usr/local/bin
-COPY --from=build /code/build/.libs/ \
+COPY --from=build /usr/local/lib/ \
     /usr/local/lib/
-COPY --from=build /usr/local/lib/underpass/config \
-    /usr/local/lib/underpass/config
+COPY --from=build /etc/underpass \
+    /etc/underpass
 WORKDIR /code/build
 # Add non-root user
 RUN useradd -r -u 1001 -m -c "hotosm account" -d /home/appuser -s /bin/false appuser

docs/Dev/bootstrapsh.md

@@ -24,7 +24,7 @@ This will:
 2. Download raw data from GeoFabrik
 3. Run osm2psql for import it
 4. Convert the country .poly file to .geojson and install config files
-5. Run ./underpass --bootstrap for bootstrapping the validation table
+5. Run underpass --bootstrap for bootstrapping the validation table
 
 ### Script options
 

docs/Dev/install-with-docker.md

@@ -58,19 +58,19 @@ displaying raw data and validation results on top of the OSM map.
 Run this command for start processing data from 2 days ago:
 
 ```sh
-docker exec -w /code/build underpass ./underpass -t $(date +%Y-%m-%dT%H:%M:%S -d "2 days ago")
+docker exec -w /code/build underpass underpass -t $(date +%Y-%m-%dT%H:%M:%S -d "2 days ago")
 ```
 
 On MacOS, the date command is different:
 
 ```sh
-docker exec -w /code/build underpass ./underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)
+docker exec -w /code/build underpass underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)
 ```
 
 For running underpass as a daemon, use the `-d` option:
 
 ```sh
-docker exec -d -w /code/build underpass ./underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)
+docker exec -d -w /code/build underpass underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)
 ```
 
 ### Stop and start underpass

docs/Dev/replicator.md

@@ -19,7 +19,7 @@ sources, they are analyzed, and applied to the appropriate
 database. Most of the work is done by the Underpass
 library. *underpass* is the utility program that a uses the library. 
 
-	./underpass -h
+	underpass -h
 	-h [ --help ]         display help
 	-s [ --server arg]    database server (defaults to localhost)
 	-m [ --monitor]       Start monitoring planet

docs/Dev/utility.md

@@ -1,6 +1,6 @@
 # Utility Programs
 
-## data/setupdb.sh
+## data/db/setupdb.sh
 
 This is a simple shell script that creates all the databases Underpass
 uses. It can also be used to initialize the two internal databases
@@ -10,7 +10,7 @@ countries used to determine which country a change was made in for
 statistics collection. These are used to bootstrap a new Underpass
 installation.
 
-## utils/bootstrap.sh
+## setup/bootstrap.sh
 
 * [Using the bootstrap.sh script](/underpass/Dev/bootstrapsh)
 
@@ -64,27 +64,6 @@ Converts .poly to .geojson
 python poly2geojson file.poly
 ```
 
-## utils/underpassmon
-
-Replication process log monitor
-
-Use this script for monitoring Underpass logs and estimate replication speed.
-
-You'll need to pass the `-v -d` parameters to the `underpass` command
-in order to output the log file.
-
-Monitoring OsmChanges processing (default):
-
-```sh
-python underpassmon.py -f underpass.log
-```
-
-Monitoring ChangeSets processing:
-
-```sh
-python underpassmon.py -f underpass.log -m changesets
-```
-
 ## utils/clean-osmchanges.sql
 
 You may want to run this query if you're not processing raw data

docs/Replication/Run.md

@@ -2,17 +2,17 @@
 
 Run this command for start processing data from 2 days ago
 
-`./underpass -t $(date +%Y-%m-%dT%H:%M:%S -d "2 days ago")`
+`underpass -t $(date +%Y-%m-%dT%H:%M:%S -d "2 days ago")`
 
 On MacOS, the date command is different
 
-`./underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)`
+`underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)`
 
 Using Docker, add `docker exec -d -w /code/build underpass`
 
-`docker exec -w /code/build underpass ./underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)`
+`docker exec -w /code/build underpass underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)`
 
 For running underpass as a daemon, use the -d option:
 
-`docker exec -d -w /code/build underpass ./underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)`
+`docker exec -d -w /code/build underpass underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)`
 

docs/get-started/Build.md

@@ -36,63 +36,50 @@ $ ./autogen.sh && \
   ../configure && make -j$(nproc) && sudo make install
 ```
 
-## MacOS (Intel)
+## MacOS
 
 ### Install dependencies
 
 ```
+sudo port install boost
 brew install \
-    autoconf \
-    automake \
     libtool \
-    swig \
     gdal \
     pkg-config \
-    openjdk@11 \
-    libosmium \
-    gumbo-parser \
-    boost \
-    boost-python \
-    open-mpi \
-    range-v3 \
-    wget \
-    unzip \
+    openssl \
+    protobuf \
+    boost-python3 \
     libxml++3 \
     libpqxx \
-    ccache \
-    openssl \
-    protobuf
+    gumbo-parser
 ```
 
-### Build 
+### Build (Intel)
 
 ```sh
 sh ./autogen.sh
 mkdir build && cd build
-../configure CXXFLAGS="-I/usr/local/include \
-    -I/usr/local/Homebrew/Cellar/libpqxx/7.7.5/include/ \
-    -I/opt/homebrew/opt/openssl@3/include \
-    -L/opt/homebrew/Cellar/boost-python3/1.82.0/lib \
-    -L/opt/homebrew/Cellar/jemalloc/5.3.0/lib \
-    -g -O0" CXX="ccache g++"
+../configure CXXFLAGS=" \
+    -I/usr/local/include \
+    -L/opt/homebrew/lib \
+    -I/opt/homebrew/include \
+    -I/opt/homebrew/Cellar \
+    -L/usr/local/Cellar" CXX="g++"
 make -j$(nproc) && sudo make install
 ```
 
 ## MacOS (Sillicon)
 
-Make sure you have installed all dependencies built for arm64 or as universal binaries.
-
-You can do this with brew using:
-
-`arch -arm64 brew install <package>`
-
-Then the process is similar to Intel but adding `-arch arm64` to `CXXFLAGS`:
+The process is similar to Intel but adding `-arch arm64` to `CXXFLAGS`:
 
 ```sh
-../configure CXXFLAGS="-arch arm64 -I/usr/local/include \
-    -I/usr/local/Homebrew/Cellar/libpqxx/7.7.5/include/ \
-    -I/opt/homebrew/opt/openssl@3/include \
-    -L/opt/homebrew/Cellar/boost-python3/1.82.0/lib \
-    -L/opt/homebrew/Cellar/jemalloc/5.3.0/lib \
-    -g -O0" CXX="ccache g++"
+sh ./autogen.sh
+mkdir build && cd build
+../configure CXXFLAGS=" -arch arm64\
+    -I/usr/local/include \
+    -L/opt/homebrew/lib \
+    -I/opt/homebrew/include \
+    -I/opt/homebrew/Cellar \
+    -L/usr/local/Cellar" CXX="g++"
+make -j$(nproc) && sudo make install
 ```

docs/get-started/Install.md

@@ -8,9 +8,17 @@ Check the [build documentation](/underpass/get-started/Build)
 
 ## Linux
 
+`setup/install.sh`
 <s>`apt-get install underpass`</s> (not available yet)
 
 ## MacOS
 
 <s>`brew install underpass`</s> (not available yet)
 
+### Intel
+
+`setup/install-macos.sh`
+
+### Sillicon
+
+`setup/install-macos-arm64.sh`

docs/get-started/Run.md

@@ -4,22 +4,22 @@
 
 Run this command for start processing data from 2 days ago:
 
-`./underpass -t $(date +%Y-%m-%dT%H:%M:%S -d "2 days ago")`
+`underpass -t $(date +%Y-%m-%dT%H:%M:%S -d "2 days ago")`
 
 ### MacOS
 
 On MacOS, the date command is slighty different:
 
-`./underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)`
+`underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)`
 
 ### Docker
 
 If you're running Underpass on a Docker container:
 
-`docker exec -w /code/build underpass ./underpass -t $(date +%Y-%m-%dT%H:%M:%S -d "2 days ago")`
+`docker exec -w /code/build underpass underpass -t $(date +%Y-%m-%dT%H:%M:%S -d "2 days ago")`
 
 For running as a daemon:
 
-`docker exec -d -w /code/build underpass ./underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)`
+`docker exec -d -w /code/build underpass underpass -t $(date -v -2d +%Y-%m-%dT%H:%M:%S)`
 
 

docs/get-started/Setup.md

@@ -1,4 +1,4 @@
-## Boostrap the database
+## Boostraping the database
 
 You can prepare your Underpass installation with data for a specific country.
 
@@ -34,17 +34,19 @@ apt install osm2pgsql
 
 ### Bootstrap
 
-Now you can go to the `utils` directory and run the boostrap script:
+Go to the `setup` directory and run the boostrap script:
 
 cd utils
-chmod +x bootstrap.sh 
-./bootstrap.sh -r south-america -c uruguay -p 5432 -u underpass
+./bootstrap.sh -r south-america -c uruguay
 ```
 
-Use `-p <PORT>` and `-u <USERNAME>` for the database.
+Use `-u <USERNAME>` `-h <HOST>` `-d <DATABASE>` `-d <PORT>` for the database connection.
 
-Regions (-r) are:
+If you installed Underpass with Docker, you might use the `-p 5439 -k yes` options.
+
+./bootstrap.sh -r south-america -c uruguay -p 5439 -k yes
 
+Regions (-r) are:
     africa
     asia
     australia-oceania
@@ -58,4 +60,3 @@ Countries (-c) is the name of the country inside the region.
 Data is downloaded from GeoFabrik, if you are not sure of what name you need to use, please check there.
 
 For advanced users, check the [boostrap script documentation](/underpass/Dev/bootstrapsh).
-

python/dbapi/requirements.txt

@@ -1 +1 @@
-psycopg2==2.9.6
+asyncpg==0.29.0

python/restapi/config.py

@@ -5,6 +5,7 @@
     "http://localhost",
     "http://localhost:5000",
     "http://localhost:3000",
+    "http://localhost:8080",
     "http://127.0.0.1",
     "http://127.0.0.1:5000"
 ]