From b0554b4b626fd677a4ddb06a67173047c942fa68 Mon Sep 17 00:00:00 2001 From: Tariq Soliman Date: Wed, 29 Jan 2025 15:45:38 -0800 Subject: [PATCH] #622 Fix docker-compose depends_on (#623) * #622 Fix docker-compose depends_on * #622 Add v1 docker-compose and fix CMD-SHELL * #622 Small tweaks * #622 Update docker-compose.sample * #622 Fix docker-compose cmd-shell --- API/Backend/Utils/routes/utils.js | 144 +++++++++++++++++++++ docker-compose.sample.yml | 49 +++++++- docker-compose.v1-sample.yml | 171 +++++++++++++++++++++++++ python-environment.yml | 1 + sample.env | 2 +- scripts/init-db.js | 28 +++-- scripts/server.js | 199 +----------------------------- 7 files changed, 380 insertions(+), 214 deletions(-) create mode 100644 docker-compose.v1-sample.yml diff --git a/API/Backend/Utils/routes/utils.js b/API/Backend/Utils/routes/utils.js index 03b02785..2ef43302 100644 --- a/API/Backend/Utils/routes/utils.js +++ b/API/Backend/Utils/routes/utils.js @@ -6,6 +6,9 @@ const express = require("express"); const router = express.Router(); const fs = require("fs"); const path = require("path"); +const exec = require("child_process").exec; +const execFile = require("child_process").execFile; + const Sequelize = require("sequelize"); const { sequelizeSTAC } = require("../../../connection"); @@ -227,4 +230,145 @@ router.get("/queryTilesetTimes", function (req, res) { else queryTilesetTimesDir(req, res); }); +// API +// TODO: move to API/Backend +//TEST +router.get("/healthcheck", function (req, res) { + res.send("Alive and Well!"); +}); + +// TODO: Remove or move to Setup structure. Some are definitely still used. + +//utils getprofile +router.post("/getprofile", function (req, res) { + const path = encodeURIComponent(req.body.path); + const lat1 = encodeURIComponent(req.body.lat1); + const lon1 = encodeURIComponent(req.body.lon1); + const lat2 = encodeURIComponent(req.body.lat2); + const lon2 = encodeURIComponent(req.body.lon2); + const steps = encodeURIComponent(req.body.steps); + const axes = encodeURIComponent(req.body.axes); + + execFile( + "python", + [ + "private/api/2ptsToProfile.py", + path, + lat1, + lon1, + lat2, + lon2, + steps, + axes, + 1, + ], + function (error, stdout, stderr) { + if (error) { + logger("warn", error); + res.status(400).send(); + } else { + res.send(stdout.replace(/None/g, null)); + } + } + ); +}); + +//utils getbands +router.post("/getbands", function (req, res) { + const path = encodeURIComponent(req.body.path); + const x = encodeURIComponent(req.body.x); + const y = encodeURIComponent(req.body.y); + const xyorll = encodeURIComponent(req.body.xyorll); + const bands = encodeURIComponent(req.body.bands); + + execFile( + "python", + ["private/api/BandsToProfile.py", path, x, y, xyorll, bands], + function (error, stdout, stderr) { + if (error) { + logger("warn", error); + res.status(400).send(); + } else { + res.send(stdout); + } + } + ); +}); + +//utils ll2aerll +router.post("/ll2aerll", function (req, res) { + const lng = encodeURIComponent(req.body.lng); + const lat = encodeURIComponent(req.body.lat); + const height = encodeURIComponent(req.body.height); + const target = encodeURIComponent(req.body.target); + const time = encodeURIComponent(req.body.time) + .replace(/%20/g, " ") + .replace(/%3A/g, ":"); + const obsRefFrame = encodeURIComponent(req.body.obsRefFrame) || "IAU_MARS"; + const obsBody = encodeURIComponent(req.body.obsBody) || "MARS"; + const includeSunEarth = + encodeURIComponent(req.body.includeSunEarth) || "False"; + + const isCustom = encodeURIComponent(req.body.isCustom) || "False"; + const customAz = encodeURIComponent(req.body.customAz); + const customEl = encodeURIComponent(req.body.customEl); + const customRange = encodeURIComponent(req.body.customRange); + + execFile( + "python", + [ + "private/api/ll2aerll.py", + lng, + lat, + height, + target, + time, + obsRefFrame, + obsBody, + includeSunEarth, + isCustom, + customAz, + customEl, + customRange, + ], + function (error, stdout, stderr) { + if (error) logger("error", "ll2aerll failure:", "server", null, error); + res.send(stdout); + } + ); +}); + +//utils chronos (spice time converter) +router.post("/chronice", function (req, res) { + const body = encodeURIComponent(req.body.body); + const target = encodeURIComponent(req.body.target); + const fromFormat = encodeURIComponent(req.body.from); + const time = encodeURIComponent(req.body.time) + .replace(/%20/g, " ") + .replace(/%3A/g, ":"); + + execFile( + "python", + ["private/api/chronice.py", body, target, fromFormat, time], + function (error, stdout, stderr) { + if (error) logger("error", "chronice failure:", "server", null, error); + res.send(stdout); + } + ); +}); + +//utils chronos (spice time converter) +router.get("/proj42wkt", function (req, res) { + const proj4 = encodeURIComponent(req.query.proj4); + + execFile( + "python", + ["private/api/proj42wkt.py", proj4], + function (error, stdout, stderr) { + if (error) logger("error", "proj42wkt failure:", "server", null, error); + res.send(stdout); + } + ); +}); + module.exports = router; diff --git a/docker-compose.sample.yml b/docker-compose.sample.yml index 0881d9f0..cad3b75a 100644 --- a/docker-compose.sample.yml +++ b/docker-compose.sample.yml @@ -1,11 +1,28 @@ services: mmgis: - build: . + # build: . + image: ghcr.io/nasa-ammos/mmgis:development depends_on: - - db + db: + condition: service_healthy + restart: true env_file: .env ports: - 8888:8888 + healthcheck: + # Use https if using HTTPS= env + test: + [ + "CMD", + "curl", + "-XGET", + "-k", + "http://mmgis:8888/API/utils/healthcheck", + ] + interval: 20s + retries: 5 + start_period: 15s + timeout: 8s restart: on-failure volumes: - ./Missions:/usr/src/app/Missions @@ -26,7 +43,12 @@ services: - DB_MIN_CONN_SIZE=1 - DB_MAX_CONN_SIZE=1 depends_on: - - db + mmgis: + condition: service_healthy + restart: true + db: + condition: service_healthy + restart: true command: bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h db -p 5432 && uvicorn stac_fastapi.pgstac.app:app --host 0.0.0.0 --port 8881" volumes: - ./adjacent-servers/docker-scripts:/tmp/scripts @@ -56,7 +78,12 @@ services: - DB_MAX_CONN_SIZE=10 command: bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h db -p 5432 && /start.sh" depends_on: - - db + mmgis: + condition: service_healthy + restart: true + db: + condition: service_healthy + restart: true volumes: - ./adjacent-servers/docker-scripts:/tmp/scripts - ./Missions:/Missions @@ -151,7 +178,12 @@ services: - TITILER_PGSTAC_SEARCH_EXITWHENFULL=FALSE - TITILER_PGSTAC_SEARCH_SKIPCOVERED=FALSE depends_on: - - db + mmgis: + condition: service_healthy + restart: true + db: + condition: service_healthy + restart: true command: bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h db -p 5432 && /start.sh" volumes: - ./Missions:/Missions @@ -161,9 +193,16 @@ services: image: postgis/postgis:16-3.4-alpine env_file: .env environment: + - POSTGRES_USER=${DB_USER} - POSTGRES_PASSWORD_OFF=Rename to 'POSTGRES_PASSWORD' and replace this with an initial root password for the DB ports: - 5432 + healthcheck: + test: ["CMD-SHELL", "pg_isready -d postgres -U ${DB_USER}"] + interval: 10s + retries: 10 + start_period: 10s + timeout: 8s restart: on-failure volumes: - mmgis-db:/var/lib/postgresql/data diff --git a/docker-compose.v1-sample.yml b/docker-compose.v1-sample.yml new file mode 100644 index 00000000..0881d9f0 --- /dev/null +++ b/docker-compose.v1-sample.yml @@ -0,0 +1,171 @@ +services: + mmgis: + build: . + depends_on: + - db + env_file: .env + ports: + - 8888:8888 + restart: on-failure + volumes: + - ./Missions:/usr/src/app/Missions + - ./ssl:/usr/src/app/ssl + + stac-fastapi: + image: ghcr.io/stac-utils/stac-fastapi-pgstac:3.0.0 + ports: + - 8881 + environment: + # Postgres connection + - POSTGRES_USER=username + - POSTGRES_PASS=password + - POSTGRES_DBNAME=mmgis-stac + - POSTGRES_HOST_READER=db + - POSTGRES_HOST_WRITER=db + - POSTGRES_PORT=5432 + - DB_MIN_CONN_SIZE=1 + - DB_MAX_CONN_SIZE=1 + depends_on: + - db + command: bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h db -p 5432 && uvicorn stac_fastapi.pgstac.app:app --host 0.0.0.0 --port 8881" + volumes: + - ./adjacent-servers/docker-scripts:/tmp/scripts + - ./Missions:/Missions + + tipg: + image: ghcr.io/developmentseed/tipg:0.7.2 + ports: + - 8882 + environment: + # Application + - HOST=0.0.0.0 + - PORT=8882 + # https://github.com/tiangolo/uvicorn-gunicorn-docker#web_concurrency + - WEB_CONCURRENCY=10 + # https://github.com/tiangolo/uvicorn-gunicorn-docker#workers_per_core + # - WORKERS_PER_CORE=1 + # https://github.com/tiangolo/uvicorn-gunicorn-docker#max_workers + # - MAX_WORKERS=10 + # Postgres connection + - POSTGRES_USER=username + - POSTGRES_PASS=password + - POSTGRES_DBNAME=mmgis-stac + - POSTGRES_HOST=db + - POSTGRES_PORT=5432 + - DB_MIN_CONN_SIZE=1 + - DB_MAX_CONN_SIZE=10 + command: bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h db -p 5432 && /start.sh" + depends_on: + - db + volumes: + - ./adjacent-servers/docker-scripts:/tmp/scripts + - ./Missions:/Missions + + titiler: + # TODO: remove once https://github.com/rasterio/rasterio-wheels/issues/69 is resolved + # See https://github.com/developmentseed/titiler/discussions/387 + platform: linux/amd64 + image: ghcr.io/developmentseed/titiler-uvicorn:0.18.6 + ports: + - 8883 + environment: + # Application + - HOST=0.0.0.0 + - PORT=8883 + # Uvicorn + # http://www.uvicorn.org/settings/#production + - WEB_CONCURRENCY=1 + # GDAL config + - CPL_TMPDIR=/tmp + - GDAL_CACHEMAX=75% + - GDAL_INGESTED_BYTES_AT_OPEN=32768 + - GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR + - GDAL_HTTP_MERGE_CONSECUTIVE_RANGES=YES + - GDAL_HTTP_MULTIPLEX=YES + - GDAL_HTTP_VERSION=2 + - PYTHONWARNINGS=ignore + - VSI_CACHE=TRUE + - VSI_CACHE_SIZE=536870912 + # GDAL VSI Config + # https://gdal.org/user/virtual_file_systems.html#vsis3-aws-s3-files + # https://gdal.org/user/virtual_file_systems.html#vsigs-google-cloud-storage-files + # https://gdal.org/user/virtual_file_systems.html#vsiaz-microsoft-azure-blob-files + # - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + # - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + # TiTiler config + # - TITILER_API_DISABLE_STAC=TRUE/FALSE + # - TITILER_API_DISABLE_MOSAIC=TRUE/FALSE + # - TITILER_API_DISABLE_COG=TRUE/FALSE + # - TITILER_API_CORS_ORIGIN=url.io,url.xyz + # - TITILER_API_CACHECONTROL=public, max-age=3600 + # - TITILER_API_DEBUG=TRUE/FALSE + # - MOSAIC_CONCURRENCY= # will default to `RIO_TILER_MAX_THREADS` + # rio-tiler config + # - RIO_TILER_MAX_THREADS= + volumes: + - ./Missions:/Missions + + titiler-pgstac: + # At the time of writing, rasterio and psycopg wheels are not available for arm64 arch + # so we force the image to be built with linux/amd64 + platform: linux/amd64 + image: ghcr.io/stac-utils/titiler-pgstac:1.4.0 + ports: + - 8884 + environment: + # Application + - HOST=0.0.0.0 + - PORT=8884 + # https://github.com/tiangolo/uvicorn-gunicorn-docker#web_concurrency + - WEB_CONCURRENCY=1 + # https://github.com/tiangolo/uvicorn-gunicorn-docker#workers_per_core + - WORKERS_PER_CORE=1 + # https://github.com/tiangolo/uvicorn-gunicorn-docker#max_workers + - MAX_WORKERS=10 + # Postgres connection + - POSTGRES_USER=username + - POSTGRES_PASS=password + - POSTGRES_DBNAME=mmgis-stac + - POSTGRES_HOST=db + - POSTGRES_PORT=5432 + - DB_MIN_CONN_SIZE=1 + - DB_MAX_CONN_SIZE=10 + # - DB_MAX_QUERIES=10 + # - DB_MAX_IDLE=10 + # GDAL Config + - CPL_TMPDIR=/tmp + - GDAL_CACHEMAX=75% + - GDAL_INGESTED_BYTES_AT_OPEN=32768 + - GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR + - GDAL_HTTP_MERGE_CONSECUTIVE_RANGES=YES + - GDAL_HTTP_MULTIPLEX=YES + - GDAL_HTTP_VERSION=2 + - VSI_CACHE=TRUE + - VSI_CACHE_SIZE=536870912 + # TiTiler Config + - MOSAIC_CONCURRENCY=1 + # AWS S3 endpoint config + - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + # TiTiler pgSTAC + - TITILER_PGSTAC_SEARCH_EXITWHENFULL=FALSE + - TITILER_PGSTAC_SEARCH_SKIPCOVERED=FALSE + depends_on: + - db + command: bash -c "bash /tmp/scripts/wait-for-it.sh -t 120 -h db -p 5432 && /start.sh" + volumes: + - ./Missions:/Missions + - ./adjacent-servers/docker-scripts:/tmp/scripts + + db: + image: postgis/postgis:16-3.4-alpine + env_file: .env + environment: + - POSTGRES_PASSWORD_OFF=Rename to 'POSTGRES_PASSWORD' and replace this with an initial root password for the DB + ports: + - 5432 + restart: on-failure + volumes: + - mmgis-db:/var/lib/postgresql/data +volumes: + mmgis-db: diff --git a/python-environment.yml b/python-environment.yml index c94e5904..9960658d 100644 --- a/python-environment.yml +++ b/python-environment.yml @@ -8,6 +8,7 @@ dependencies: - pip: - numpy==2.1.0 - requests==2.32.3 + - rio_stac==0.8.1 - spiceypy==5.1.2 - pymap3d==3.0.1 - pypgstac==0.8.6 diff --git a/sample.env b/sample.env index 7f13da86..7237b9d9 100644 --- a/sample.env +++ b/sample.env @@ -80,7 +80,7 @@ LINK_PREVIEW_DESCRIPTION= DISABLE_LINK_SHORTENER=false #DB -# If using docker, DB_HOST is the database container name +# If using docker, DB_HOST is the database service name (db) DB_HOST=localhost # Postgres' default port is 5432 DB_PORT=5432 diff --git a/scripts/init-db.js b/scripts/init-db.js index 9ce2441e..66bb9946 100644 --- a/scripts/init-db.js +++ b/scripts/init-db.js @@ -1,8 +1,11 @@ const Sequelize = require("sequelize"); const logger = require("../API/logger"); +const utils = require("../API/utils"); const execSync = require("child_process").execSync; require("dotenv").config({ path: __dirname + "/../.env" }); +const isDocker = utils.isDocker(); + initializeDatabase() .then(() => { logger("info", "Finished successfully.", "connection"); @@ -59,16 +62,21 @@ async function initializeDatabase() { function keepGoingSTAC() { try { - const output = execSync(`pypgstac migrate`, { - env: { - PYTHONUTF8: 1, - PGHOST: process.env.DB_HOST, - PGPORT: process.env.DB_PORT, - PGUSER: process.env.DB_USER, - PGDATABASE: "mmgis-stac", - PGPASSWORD: process.env.DB_PASS, - }, - }); + const output = execSync( + `${ + isDocker ? `source ~/.bashrc && micromamba run -n mmgis ` : `` + }pypgstac migrate`, + { + env: { + PYTHONUTF8: 1, + PGHOST: process.env.DB_HOST, + PGPORT: process.env.DB_PORT, + PGUSER: process.env.DB_USER, + PGDATABASE: "mmgis-stac", + PGPASSWORD: process.env.DB_PASS, + }, + } + ); logger( "info", `Conformed the mmgis-stac database to pgstac.`, diff --git a/scripts/server.js b/scripts/server.js index bbaa3e6e..bf8ead9e 100644 --- a/scripts/server.js +++ b/scripts/server.js @@ -11,8 +11,7 @@ const cookieParser = require("cookie-parser"); const express = require("express"); var swaggerUi = require("swagger-ui-express"); var swaggerDocumentMain = require("../documentation/pages/swaggers/swaggerMain.json"); -var exec = require("child_process").exec; -var execFile = require("child_process").execFile; + const createError = require("http-errors"); const cors = require("cors"); const logger = require("../API/logger"); @@ -654,202 +653,6 @@ setups.getBackendSetups(function (setups) { } ); - // API - // TODO: move to API/Backend - //TEST - app.post(`${ROOT_PATH}/api/test`, function (req, res) { - res.send("Hello World!"); - }); - - // TODO: Remove or move to Setup structure. Some are definitely still used. - - //utils getprofile - app.post( - `${ROOT_PATH}/api/utils/getprofile`, - ensureUser(), - ensureGroup(permissions.users), - function (req, res) { - const path = encodeURIComponent(req.body.path); - const lat1 = encodeURIComponent(req.body.lat1); - const lon1 = encodeURIComponent(req.body.lon1); - const lat2 = encodeURIComponent(req.body.lat2); - const lon2 = encodeURIComponent(req.body.lon2); - const steps = encodeURIComponent(req.body.steps); - const axes = encodeURIComponent(req.body.axes); - - execFile( - "python", - [ - "private/api/2ptsToProfile.py", - path, - lat1, - lon1, - lat2, - lon2, - steps, - axes, - 1, - ], - function (error, stdout, stderr) { - if (error) { - logger("warn", error); - res.status(400).send(); - } else { - res.send(stdout.replace(/None/g, null)); - } - } - ); - } - ); - - //utils getbands - app.post( - `${ROOT_PATH}/api/utils/getbands`, - ensureUser(), - ensureGroup(permissions.users), - function (req, res) { - const path = encodeURIComponent(req.body.path); - const x = encodeURIComponent(req.body.x); - const y = encodeURIComponent(req.body.y); - const xyorll = encodeURIComponent(req.body.xyorll); - const bands = encodeURIComponent(req.body.bands); - - execFile( - "python", - ["private/api/BandsToProfile.py", path, x, y, xyorll, bands], - function (error, stdout, stderr) { - if (error) { - logger("warn", error); - res.status(400).send(); - } else { - res.send(stdout); - } - } - ); - } - ); - - //utils ll2aerll - app.post( - `${ROOT_PATH}/api/utils/ll2aerll`, - ensureUser(), - ensureGroup(permissions.users), - function (req, res) { - const lng = encodeURIComponent(req.body.lng); - const lat = encodeURIComponent(req.body.lat); - const height = encodeURIComponent(req.body.height); - const target = encodeURIComponent(req.body.target); - const time = encodeURIComponent(req.body.time) - .replace(/%20/g, " ") - .replace(/%3A/g, ":"); - const obsRefFrame = - encodeURIComponent(req.body.obsRefFrame) || "IAU_MARS"; - const obsBody = encodeURIComponent(req.body.obsBody) || "MARS"; - const includeSunEarth = - encodeURIComponent(req.body.includeSunEarth) || "False"; - - const isCustom = encodeURIComponent(req.body.isCustom) || "False"; - const customAz = encodeURIComponent(req.body.customAz); - const customEl = encodeURIComponent(req.body.customEl); - const customRange = encodeURIComponent(req.body.customRange); - - execFile( - "python", - [ - "private/api/ll2aerll.py", - lng, - lat, - height, - target, - time, - obsRefFrame, - obsBody, - includeSunEarth, - isCustom, - customAz, - customEl, - customRange, - ], - function (error, stdout, stderr) { - if (error) - logger("error", "ll2aerll failure:", "server", null, error); - res.send(stdout); - } - ); - } - ); - - //utils chronos (spice time converter) - app.post( - `${ROOT_PATH}/api/utils/chronice`, - ensureUser(), - ensureGroup(permissions.users), - function (req, res) { - const body = encodeURIComponent(req.body.body); - const target = encodeURIComponent(req.body.target); - const fromFormat = encodeURIComponent(req.body.from); - const time = encodeURIComponent(req.body.time) - .replace(/%20/g, " ") - .replace(/%3A/g, ":"); - - execFile( - "python", - ["private/api/chronice.py", body, target, fromFormat, time], - function (error, stdout, stderr) { - if (error) - logger("error", "chronice failure:", "server", null, error); - res.send(stdout); - } - ); - } - ); - - //utils chronos (spice time converter) - app.get( - `${ROOT_PATH}/api/utils/proj42wkt`, - ensureUser(), - ensureGroup(permissions.users), - function (req, res) { - const proj4 = encodeURIComponent(req.query.proj4); - - execFile( - "python", - ["private/api/proj42wkt.py", proj4], - function (error, stdout, stderr) { - if (error) - logger("error", "proj42wkt failure:", "server", null, error); - res.send(stdout); - } - ); - } - ); - - /* - //http://localhost:8888/test/timeLayer?start=2022-05-12T16:10:11.648750Z&end=2022-05-12T16:25:25.084933Z - app.get("/test/timeLayer", (req, res) => { - res.send({ - type: "FeatureCollection", - properties: { crs_code: "IAU2000:30100" }, - features: [ - { - type: "Feature", - geometry: { - type: "LineString", - coordinates: [ - [137.3717, parseFloat("-4.666" + parseInt(Math.random() * 100))], - [137.3797, parseFloat("-4.667" + parseInt(Math.random() * 100))], - ], - }, - properties: { - event_utc: [req.query.start, req.query.end], - crs_code: "IAU2000:30100", - }, - }, - ], - }); - }); - */ - // Validate envs if (process.env.NODE_ENV === "development") { console.log(chalk.cyan("Validating Environment Variables...\n"));