Use postgres container to load data + and webbben/pygeoapi-river-runn…

…er for pygeoapi image (#1) * Switch to postgres provider for merit * Move data loader to db container * Delete Dockerfile_loader * Delete entrypoint.sh * Build .gpkg for postgis * update pygeoapi config
ksonda · Sep 22, 2021 · 3c379a6 · 3c379a6
1 parent 2569466
commit 3c379a6
Show file tree

Hide file tree

Showing 7 changed files with 385 additions and 156 deletions.
diff --git a/pygeoapi/Dockerfile b/pygeoapi/Dockerfile
@@ -1,17 +1,5 @@
 FROM webbben/pygeoapi-river-runner:latest
 
-#Add data directory
-RUN mkdir /data
-
-#last updated 2021-09-14 00:09:28 UTC-5 
-ADD https://www.hydroshare.org/resource/4a22e88e689949afa1cf71ae009eaf1b/data/contents/merit_plus_simplify.gpkg /data/
-
-#last updated 2021-09-14 00:019:50 UTC-5 
-ADD https://www.hydroshare.org/resource/4a22e88e689949afa1cf71ae009eaf1b/data/contents/e_merit_cats.gpkg /data/
-
-#last updated 2021-09-14 00:019:50 UTC-5 
-ADD https://www.hydroshare.org/resource/4a22e88e689949afa1cf71ae009eaf1b/data/contents/w_merit_cats.gpkg /data/
-
 #add requirements and mods
 COPY ./pygeoapi.config.yml /pygeoapi/local.config.yml
 COPY ./schemas.opengis.net /opt/schemas.opengis.net

diff --git a/pygeoapi/Dockerfile_db b/pygeoapi/Dockerfile_db
@@ -0,0 +1,23 @@
+FROM ubuntu:20.04 AS ZIP
+
+#last updated 2021-09-22 00:09:28 UTC-5
+ADD https://prod-is-usgs-sb-prod-publish.s3.amazonaws.com/614a8864d34e0df5fb97572d/merit_plus_simplify.zip /
+
+#last updated 2021-09-14 00:019:50 UTC-5 
+# ADD https://www.hydroshare.org/resource/4a22e88e689949afa1cf71ae009eaf1b/data/contents/e_merit_cats.gpkg /data/
+
+#last updated 2021-09-14 00:019:50 UTC-5 
+# ADD https://www.hydroshare.org/resource/4a22e88e689949afa1cf71ae009eaf1b/data/contents/w_merit_cats.gpkg /data/
+
+RUN apt-get update \
+    && apt-get install unzip \
+    && unzip '/*.zip'
+
+FROM kartoza/postgis:latest
+
+# Add data directory
+RUN mkdir /data/
+
+COPY --from=ZIP /merit_plus_simplify.gpkg /data/merit_plus_simplify.gpkg
+
+COPY ./build.sh /docker-entrypoint-initdb.d/build.sh
diff --git a/pygeoapi/Dockerfile_gcp b/pygeoapi/Dockerfile_gcp
@@ -1,17 +1,5 @@
 FROM webbben/pygeoapi-river-runner:latest
 
-#Add data directory
-RUN mkdir /data
-
-#last updated 2021-09-14 00:09:28 UTC-5 
-ADD https://www.hydroshare.org/resource/4a22e88e689949afa1cf71ae009eaf1b/data/contents/merit_plus_simplify.gpkg /data/
-
-#last updated 2021-09-14 00:019:50 UTC-5 
-ADD https://www.hydroshare.org/resource/4a22e88e689949afa1cf71ae009eaf1b/data/contents/e_merit_cats.gpkg /data/
-
-#last updated 2021-09-14 00:019:50 UTC-5 
-ADD https://www.hydroshare.org/resource/4a22e88e689949afa1cf71ae009eaf1b/data/contents/w_merit_cats.gpkg /data/
-
 #add requirements and mods
 COPY ./pygeoapi.config.gcp.yml /pygeoapi/local.config.yml
 COPY ./schemas.opengis.net /opt/schemas.opengis.net

diff --git a/pygeoapi/build.sh b/pygeoapi/build.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+# =================================================================
+#
+# Authors: Just van den Broecke <[email protected]>
+#
+# Copyright (c) 2019 Just van den Broecke
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+# =================================================================
+echo "START /build.sh"
+
+set +e
+echo "Begining build"
+# python3 /scripts/ogr2ogr.py \
+#     -f PGDump -lco LAUNDER=NO -lco DROP_TABLE=OFF | gzip > /data/merit_.sql.gz \
+#     /data/merit_plus_simplify.gpkg
+while [ ! -f /data/merit_plus_simplify.gpkg ]; do echo 1; sleep 1; done
+ogr2ogr \
+	-f PostgreSQL \
+	PG:"host='localhost' \
+	    user='${POSTGRES_USER}' \
+		password='${POSTGRES_PASSWORD}' \
+		dbname='${POSTGRES_DB}'" \
+	/data/merit_plus_simplify.gpkg
+
+echo "Done"
+# ogr2ogr \
+# 	-f PostgreSQL \
+# 	PG:"host='localhost' \
+# 	    user='${POSTGRES_USER}' \
+# 		password='${POSTGRES_PASSWORD}' \
+# 		dbname='${POSTGRES_DB}'" \
+# 	/data/e_merit_cats.gpkg 
+
+# ogr2ogr \
+# 	-f PostgreSQL \
+# 	PG:"host='localhost' \
+# 	    user='${POSTGRES_USER}' \
+# 		password='${POSTGRES_PASSWORD}' \
+# 		dbname='${POSTGRES_DB}'" \
+# 	/data/w_merit_cats.gpkg 
+
+# rm -rf /data/
diff --git a/pygeoapi/docker-compose.yml b/pygeoapi/docker-compose.yml
@@ -32,12 +32,40 @@ version: "3"
 services:
 
   pygeoapi:
-    image: webbben/pygeoapi-river-runner:latest
-    ports:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    restart: always
+    ports: 
       - 5050:80
+    depends_on: 
+      - db
+    environment:
+      POSTGRES_HOST: db
+      POSTGRES_USER: root
+      POSTGRES_PASSWORD: password
+      POSTGRES_DB: merit
     volumes:
       - ./pygeoapi.config.yml:/pygeoapi/local.config.yml
-      - ./merit_plus_simplify.gpkg:/data/merit_plus_simplify.gpkg
-      - ./e_merit_cats.gpkg:/data/e_merit_cats.gpkg
-      - ./w_merit_cats.gpkg:/data/w_merit_cats.gpkg
       - ./schemas.opengis.net:/opt/schemas.opengis.net
+
+  db:
+    build:
+      context: .
+      dockerfile: Dockerfile_db
+    restart: always
+    ports: 
+      - 5432:5432
+    environment:
+      POSTGRES_USER: root
+      POSTGRES_PASSWORD: password
+      POSTGRES_DB: merit
+    volumes:
+      - ./merit_plus_simplify.gpkg:/data/merit_plus_simplify.gpkg
+      - ./build.sh:/docker-entrypoint-initdb.d/build.sh
+
+  adminer:
+    image: adminer
+    restart: always
+    ports: 
+      - 8080:8080
diff --git a/pygeoapi/pygeoapi.config.gcp.yml b/pygeoapi/pygeoapi.config.gcp.yml
@@ -100,12 +100,22 @@ resources:
                 - rivers
         context:
             - schema: https://schema.org/
-              nameID: schema:name
+              nameid: schema:name
         links:
             - type: application/html
               rel: canonical
-              title: data source
-              href: https://www.hydroshare.org/resource/4a22e88e689949afa1cf71ae009eaf1b/data/contents/merit_plus_simplify.gpkg
+              title: source
+              href: https://www.sciencebase.gov/catalog/item/614a8864d34e0df5fb97572d
+              hreflang: en-US
+            - type: application/html
+              rel: canonical
+              title: download
+              href: https://prod-is-usgs-sb-prod-publish.s3.amazonaws.com/614a8864d34e0df5fb97572d/merit_plus_simplify.zip
+              hreflang: en-US
+            - type: application/html
+              rel: canonical
+              title: information
+              href: https://www.usgs.gov/core-science-systems/ngp/national-hydrography/value-added-attributes-vaas
               hreflang: en-US
         extents:
             spatial:
@@ -116,76 +126,136 @@ resources:
                 end: null
         providers:
             - type: feature
-              name: SQLiteGPKG
-              data: /data/merit_plus_simplify.gpkg
+              name: PostgreSQL
+              data:
+                host: ${POSTGRES_HOST}
+                dbname: ${POSTGRES_DB}
+                user: ${POSTGRES_USER}
+                password: ${POSTGRES_PASSWORD}
+                search_path: [public]
+              geom_field: wkb_geometry
               id_field: comid
               table: merit_plus
 
-    eastcatchments:
-        type: collection
-        title: 
-            en: Eastern Hemisphere Merit Catchments
-        description: 
-            en: Simplified Eastern Hemisphere Merit Catchments
+    river-runner:
+        type: process
+        version: 0.1.0
+        id: river-runner
+        title:
+            en: River Runner
+        description:
+            en: A process that takes a set of coordinates in the world, and returns the largest flowpath from it to its terminal flowpoint.
         keywords:
             en:
-                - catchments
-        context:
-            - schema: https://schema.org/
-              COMID: schema:identifier
+                - rivers
+                - river-runner
         links:
             - type: application/html
               rel: canonical
-              title: data source
-              href: https://www.hydroshare.org/resource/4a22e88e689949afa1cf71ae009eaf1b/data/contents/e_merit_cats.gpkg
+              title: source
+              href: https://www.sciencebase.gov/catalog/item/614a8864d34e0df5fb97572d
               hreflang: en-US
-        extents:
-            spatial:
-                bbox: [-180,-90,180,90]
-                crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84
-            temporal:
-                begin: null
-                end: null
-        providers:
-            - type: feature
-              name: SQLiteGPKG
-              data: /data/e_merit_cats.gpkg
-              id_field: COMID
-              table: merit_cats
-
-    westcatchments:
-        type: collection
-        title: 
-            en: Western Hemisphere Merit Catchments
-        description: 
-            en: Simplified Western Hemisphere Merit Catchments
-        keywords:
-            en:
-                - catchments
-        context:
-            - schema: https://schema.org/
-              COMID: schema:identifier
-        links:
             - type: application/html
-              rel: canonical
-              title: data source
-              href: https://www.hydroshare.org/resource/4a22e88e689949afa1cf71ae009eaf1b/data/contents/w_merit_cats.gpkg
+              rel: cannonical
+              title: github
+              href: https://github.com/ksonda/global-river-runner
+              hreflang: en-US
+            - type: application/html
+              rel: cannonical
+              title: application
+              href: https://river-runner.samlearner.com/
               hreflang: en-US
-        extents:
-            spatial:
-                bbox: [-180,-90,180,90]
-                crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84
-            temporal:
-                begin: null
-                end: null
-        providers:
-            - type: feature
-              name: SQLiteGPKG
-              data: /data/w_merit_cats.gpkg
-              id_field: COMID
-              table: merit_cats      
-
-    river-runner:
-        type: process
         processor:
             name: RiverRunner
+        inputs:
+            bbox:
+                title:
+                    en: Bounding Box
+                description:
+                    en: Boundary box to begin a river runner query from
+                keywords:
+                    en: [box, coordinates]
+                schema:
+                    type: object
+                    default: []
+                minOccurs: 0
+                maxOccurs: 1
+                metadata: null
+            lat:
+                title: 
+                    en: Latitude
+                description: 
+                    en: Latitude of a point
+                keywords:
+                    en: [latitude, coordinate, eastwest]
+                schema:
+                    type: number
+                    default: null
+                minOccurs: 0
+                maxOccurs: 1
+                metadata: null
+            lng:
+                title: 
+                    en: Longitude
+                description: 
+                    en: Longitude of a point
+                keywords:
+                    en: [longitude, coordinate, northsouth]
+                schema:
+                    type: number
+                    default: null
+                minOccurs: 0
+                maxOccurs: 1
+                metadata: null
+            latlng:
+                title:
+                    en: Latitude and Longitude
+                description:
+                    en: Coordinates in order [Long, Lat]
+                keywords:
+                    en: [coordinates, world, point]
+                schema:
+                    type: object
+                    default: []
+                minOccurs: 0
+                maxOccurs: 1
+                metadata: null
+            sorted:
+                title: 
+                    en: Sorted
+                description: 
+                    en: 'Sort features by flow direction'
+                keywords:
+                    en: [downstream, upstream, unset]
+                schema:
+                    type: string
+                    default: downstream
+                minOccurs: 0
+                maxOccurs: 1
+                metadata: null
+            sortby:
+                title:
+                    en: Sort By
+                description:
+                    en: 'Property to sort featurs with'
+                keywords:
+                    en: [sort, hydroseq, nameid, comid]
+                schema:
+                    type: string
+                    default: hydroseq
+                minOccurs: 0
+                maxOccurs: 1
+                metadata: null
+        outputs:
+            echo:
+                title:
+                    en: Feature Collection
+                description:
+                    en: A geoJSON Feature Collection of the River Runner process
+                schema:
+                    type: object
+                    contentMediaType: application/json
+        example:
+            inputs:
+                bbox: [-86.2, 39.7, -86.15, 39.75]
+                sorted: downstream