From 193364161f0d2a66eda82c95034f1763dbeda15b Mon Sep 17 00:00:00 2001 From: alexeh Date: Tue, 31 Jan 2023 17:17:39 +0300 Subject: [PATCH] Add tiler microservice --- .editorconfig | 3 ++ .github/workflows/testing-tiler.yml | 34 ++++++++++++++++++ api/config/custom-environment-variables.json | 5 +++ api/config/default.json | 6 ++++ api/config/development.json | 5 +++ api/config/test.json | 5 +++ .../authentication.controller.ts | 9 +++++ .../scenarios.access-control.service.ts | 2 +- api/src/modules/scenarios/scenario.entity.ts | 7 ++-- docker-compose.yml | 18 ++++++++++ infrastructure/kubernetes/.terraform.lock.hcl | 9 +++++ tiler/.env.default | 4 +++ tiler/Dockerfile | 35 +++++++++++++++++++ tiler/__init__.py | 0 tiler/app/__init__.py | 0 tiler/app/config/__init__.py | 0 tiler/app/config/config.py | 15 ++++++++ tiler/app/main.py | 26 ++++++++++++++ tiler/app/middlewares/__init__.py | 0 tiler/app/middlewares/auth_middleware.py | 32 +++++++++++++++++ tiler/app/middlewares/url_injector.py | 8 +++++ tiler/app/test/__init__.py | 0 tiler/app/test/test_health.py | 11 ++++++ tiler/requirements.txt | 4 +++ 24 files changed, 235 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/testing-tiler.yml create mode 100644 tiler/.env.default create mode 100644 tiler/Dockerfile create mode 100644 tiler/__init__.py create mode 100644 tiler/app/__init__.py create mode 100644 tiler/app/config/__init__.py create mode 100644 tiler/app/config/config.py create mode 100644 tiler/app/main.py create mode 100644 tiler/app/middlewares/__init__.py create mode 100644 tiler/app/middlewares/auth_middleware.py create mode 100644 tiler/app/middlewares/url_injector.py create mode 100644 tiler/app/test/__init__.py create mode 100644 tiler/app/test/test_health.py create mode 100644 tiler/requirements.txt diff --git a/.editorconfig b/.editorconfig index 0c6685d1c..d209c1661 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,6 +10,9 @@ charset = utf-8 trim_trailing_whitespace = true insert_final_newline = true +[*.py] +indent_size = 4 + [*.md] indent_style = tab indent_size = 4 diff --git a/.github/workflows/testing-tiler.yml b/.github/workflows/testing-tiler.yml new file mode 100644 index 000000000..4c9446206 --- /dev/null +++ b/.github/workflows/testing-tiler.yml @@ -0,0 +1,34 @@ +name: Tiler Tests + +on: + + push: + paths: + - 'tiler/**' + + workflow_dispatch: + +jobs: + test: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.10] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Tiler tests + env: + # add environment variables for tests + run: | + pytest diff --git a/api/config/custom-environment-variables.json b/api/config/custom-environment-variables.json index f515f132e..c50dbd18b 100644 --- a/api/config/custom-environment-variables.json +++ b/api/config/custom-environment-variables.json @@ -47,6 +47,11 @@ "cachePort": "DB_CACHE_PORT", "cacheDatabase": "DB_CACHE_DATABASE" }, + + "tiler": { + "host": "TILER_HOST", + "port": "TILER_PORT" + }, "fileUploads": { "sizeLimit": "FILE_SIZE_LIMIT", "storagePath": "FILE_STORAGE_PATH" diff --git a/api/config/default.json b/api/config/default.json index d0cdcebe9..2b2b300a0 100644 --- a/api/config/default.json +++ b/api/config/default.json @@ -55,6 +55,12 @@ "cacheHost": "localhost", "cacheDatabase": 3 }, + + "tiler": { + "host": "localhost", + "port": 4000 + }, + "fileUploads": { "sizeLimit": 8388608, "storagePath": "/tmp/csv-uploads" diff --git a/api/config/development.json b/api/config/development.json index c5e48fad2..68e6608d1 100644 --- a/api/config/development.json +++ b/api/config/development.json @@ -14,6 +14,11 @@ "log" ] }, + + "tiler": { + "host": "localhost", + "port": 4000 + }, "auth": { "requireUserAccountActivation": false, "jwt": { diff --git a/api/config/test.json b/api/config/test.json index 7a967726a..59f4a2c1a 100644 --- a/api/config/test.json +++ b/api/config/test.json @@ -9,6 +9,11 @@ "logging": false, "cacheEnabled": false }, + + "tiler": { + "host": "localhost", + "port": 4000 + }, "server": { "loggerLevel": [ "error", diff --git a/api/src/modules/authentication/authentication.controller.ts b/api/src/modules/authentication/authentication.controller.ts index d678cb380..61d6e8e57 100644 --- a/api/src/modules/authentication/authentication.controller.ts +++ b/api/src/modules/authentication/authentication.controller.ts @@ -80,6 +80,15 @@ export class AuthenticationController { await this.authenticationService.validateActivationToken(activationToken); } + /** + * @description: This endpoint is exclusively to validate requests sent to the tiler + * service + */ + @Get('validate-token') + async validateToken(): Promise { + return { message: 'valid token' }; + } + /** * @debt Make sure (and add e2e tests to check for regressions) that we * gracefully handle situations where a user's username has changed between diff --git a/api/src/modules/authorization/modules/scenarios.access-control.service.ts b/api/src/modules/authorization/modules/scenarios.access-control.service.ts index 262ae468a..b8c7cef96 100644 --- a/api/src/modules/authorization/modules/scenarios.access-control.service.ts +++ b/api/src/modules/authorization/modules/scenarios.access-control.service.ts @@ -4,7 +4,7 @@ import { Repository } from 'typeorm'; import { Scenario } from 'modules/scenarios/scenario.entity'; /** - * @desciption: Service extending base AccessControl which receives a fresh request object from it each time. + * @description: Service extending base AccessControl which receives a fresh request object from it each time. * Use this class ta perform user's specific authorisation over scenarios, based on roles/permissions */ diff --git a/api/src/modules/scenarios/scenario.entity.ts b/api/src/modules/scenarios/scenario.entity.ts index 13c7e9a39..c5f1bcc08 100644 --- a/api/src/modules/scenarios/scenario.entity.ts +++ b/api/src/modules/scenarios/scenario.entity.ts @@ -42,11 +42,14 @@ export class Scenario extends TimestampedBaseEntity { @Column({ nullable: true }) description?: string; - @ApiPropertyOptional() + @ApiPropertyOptional({ + type: 'boolean', + description: 'Make a Scenario public to all users', + }) @Column({ type: 'boolean', default: false, nullable: false }) isPublic!: boolean; - @ApiProperty() + @ApiProperty({ enum: SCENARIO_STATUS }) @Column({ type: 'enum', enum: SCENARIO_STATUS, diff --git a/docker-compose.yml b/docker-compose.yml index 931e7bfd8..8bc41ff20 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,6 +40,24 @@ services: - postgresql - redis + tiler: + container_name: landgriffon-tiler + build: + context: ./tiler + args: + - TILER_TEST_COG_FILENAME=${TILER_TEST_COG_FILENAME} + + env_file: ${ENVFILE} + environment: + - API_URL=${API_URL} + - API_PORT=${API_PORT} + - S3_BUCKET_URL=${S3_BUCKET_URL} + ports: + - "${TILER_SERVICE_PORT}:4000" + volumes: + - ./tiler:/opt/landgriffon-tiler/data/local-dev + restart: unless-stopped + redis: build: context: ./redis diff --git a/infrastructure/kubernetes/.terraform.lock.hcl b/infrastructure/kubernetes/.terraform.lock.hcl index fbc4591cc..3e73a50d8 100644 --- a/infrastructure/kubernetes/.terraform.lock.hcl +++ b/infrastructure/kubernetes/.terraform.lock.hcl @@ -6,6 +6,7 @@ provider "registry.terraform.io/gavinbunney/kubectl" { constraints = "~> 1.14.0" hashes = [ "h1:gLFn+RvP37sVzp9qnFCwngRjjFV649r6apjxvJ1E/SE=", + "h1:mX2AOFIMIxJmW5kM8DT51gloIOKCr9iT6W8yodnUyfs=", "zh:0350f3122ff711984bbc36f6093c1fe19043173fad5a904bce27f86afe3cc858", "zh:07ca36c7aa7533e8325b38232c77c04d6ef1081cb0bac9d56e8ccd51f12f2030", "zh:0c351afd91d9e994a71fe64bbd1662d0024006b3493bb61d46c23ea3e42a7cf5", @@ -22,6 +23,7 @@ provider "registry.terraform.io/hashicorp/aws" { version = "4.34.0" constraints = "~> 4.34.0" hashes = [ + "h1:JRqeU/5qR61U+z86mC68C5hp0XHZXxmRK9dupTIAhGg=", "h1:TMVXbfjowAI4MjMDCU7AJwCUzfufoSC/v6/v85sAOlg=", "zh:2bdc9b908008c1e874d8ba7e2cfabd856cafb63c52fef51a1fdeef2f5584bffd", "zh:43c5364e3161be3856e56494cbb8b21d513fc05875f1b40e66e583602154dd0a", @@ -42,6 +44,7 @@ provider "registry.terraform.io/hashicorp/github" { version = "5.3.0" hashes = [ "h1:DVgl7G+BOUt0tBLW3LwCuJh7FmliwG4Y+KiELX4gN9U=", + "h1:pFsKVGjnvAUu9Scqkk3W0EdjEXtgkTz2qxKYMMA/Bww=", "zh:1ad22c2d5b02f16ff6281e471be93d9e33f102020e7d88b2a86fd97b7f2c3728", "zh:1d3968417f7cd87678d505afd12d8d753e692dd90c6ba0f52e7b150c69649eaf", "zh:1fd5c610488671e7685ebd9b4afaf6fb86f5540f4a9df03a6ef4d449aec761c2", @@ -64,6 +67,7 @@ provider "registry.terraform.io/hashicorp/helm" { constraints = "~> 2.7.0" hashes = [ "h1:YXQgYy5YoqnMgKwlgRmkkUhlSKAX2RMOMujb86ua3jU=", + "h1:quavRe9VlwM06DoCgMckuj+5T48g+lfG75pip+iIbFQ=", "zh:01f7428823c169e20c051e363e580093b874d32e64fe8feab665cc9d1d599691", "zh:089511b2b363d9bd4d47cb0975e4612c0ae02bdac6185e8872ded7e229d27192", "zh:0b4ab015e114a3f73b320b716d9aa081b378736389e85aa13c6aba430c219029", @@ -83,6 +87,7 @@ provider "registry.terraform.io/hashicorp/kubernetes" { version = "2.14.0" constraints = "~> 2.14.0" hashes = [ + "h1:4zSUEWLVFn2Sji7mWT64XQGWwBQVDqTGXGfW4ZBB16U=", "h1:FFeFf2j2ipbMlrbhmIv8M7bzX3Zq8SQHeFkkQGALh1k=", "zh:1363fcd6eb3c63113eaa6947a4e7a9f78a6974ea344e89b662d97a78e2ccb70c", "zh:166352455666b7d584705ceeb00f24fb9b884ab84e3a1a6019dc45d6539c9174", @@ -103,6 +108,7 @@ provider "registry.terraform.io/hashicorp/null" { version = "3.1.1" hashes = [ "h1:71sNUDvmiJcijsvfXpiLCz0lXIBSsEJjMxljt7hxMhw=", + "h1:Pctug/s/2Hg5FJqjYcTM0kPyx3AoYK1MpRWO0T9V2ns=", "zh:063466f41f1d9fd0dd93722840c1314f046d8760b1812fa67c34de0afcba5597", "zh:08c058e367de6debdad35fc24d97131c7cf75103baec8279aba3506a08b53faf", "zh:73ce6dff935150d6ddc6ac4a10071e02647d10175c173cfe5dca81f3d13d8afe", @@ -122,6 +128,7 @@ provider "registry.terraform.io/hashicorp/random" { version = "3.4.3" constraints = "~> 3.4.3" hashes = [ + "h1:tL3katm68lX+4lAncjQA9AXL4GR/VM+RPwqYf4D2X8Q=", "h1:xZGZf18JjMS06pFa4NErzANI98qi59SEcBsOcS2P2yQ=", "zh:41c53ba47085d8261590990f8633c8906696fa0a3c4b384ff6a7ecbf84339752", "zh:59d98081c4475f2ad77d881c4412c5129c56214892f490adf11c7e7a5a47de9b", @@ -141,6 +148,7 @@ provider "registry.terraform.io/hashicorp/random" { provider "registry.terraform.io/hashicorp/template" { version = "2.2.0" hashes = [ + "h1:0wlehNaxBX7GJQnPfQwTNvvAf38Jm0Nv7ssKGMaG6Og=", "h1:94qn780bi1qjrbC3uQtjJh3Wkfwd5+tTtJHOb7KTg9w=", "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", @@ -160,6 +168,7 @@ provider "registry.terraform.io/integrations/github" { constraints = "~> 5.3.0" hashes = [ "h1:DVgl7G+BOUt0tBLW3LwCuJh7FmliwG4Y+KiELX4gN9U=", + "h1:pFsKVGjnvAUu9Scqkk3W0EdjEXtgkTz2qxKYMMA/Bww=", "zh:1ad22c2d5b02f16ff6281e471be93d9e33f102020e7d88b2a86fd97b7f2c3728", "zh:1d3968417f7cd87678d505afd12d8d753e692dd90c6ba0f52e7b150c69649eaf", "zh:1fd5c610488671e7685ebd9b4afaf6fb86f5540f4a9df03a6ef4d449aec761c2", diff --git a/tiler/.env.default b/tiler/.env.default new file mode 100644 index 000000000..4fa88bad8 --- /dev/null +++ b/tiler/.env.default @@ -0,0 +1,4 @@ +# TILER +REQUIRE_AUTH= +API_HOST= +S3_BUCKET_URL= diff --git a/tiler/Dockerfile b/tiler/Dockerfile new file mode 100644 index 000000000..6f110ea86 --- /dev/null +++ b/tiler/Dockerfile @@ -0,0 +1,35 @@ +FROM python:3.11 +LABEL maintainer="hello@vizzuality.com" + +ENV NAME landgriffon-tiler +ENV USER $NAME +ENV APP_HOME /opt/$NAME + +ARG TILE_LANDGRIFFON_COG_FILENAME +# @todo: The data should be retrieved from a S3 bucket in LG World +#ARG DATA_CORE_COG_SOURCE_URL + +#ARG DATA_CORE_COG_CHECKSUM + +RUN addgroup $USER && adduser --shell /bin/bash --disabled-password --ingroup $USER $USER + +WORKDIR $APP_HOME +RUN chown $USER:$USER $APP_HOME + +COPY requirements.txt requirements.txt + +RUN pip install --no-cache-dir --upgrade -r ./requirements.txt + + +COPY --chown=$USER:$USER app ./app + +#ADD --chown=$USER:$USER --checksum=${DATA_CORE_COG_CHECKSUM} ${DATA_CORE_COG_SOURCE_URL} ./data/${TILE_LANDGRIFFON_COG_FILENAME} +#RUN find ./data -type f -exec chmod ugo-w '{}' \; + +EXPOSE 4000 +USER $USER + +# @todo Consider whether to use a shell script as entrypoint to allow for +# different run modes (e.g. debug, dev, prod, etc.) via Docker commands + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "4000"] diff --git a/tiler/__init__.py b/tiler/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tiler/app/__init__.py b/tiler/app/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tiler/app/config/__init__.py b/tiler/app/config/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tiler/app/config/config.py b/tiler/app/config/config.py new file mode 100644 index 000000000..85b336e1d --- /dev/null +++ b/tiler/app/config/config.py @@ -0,0 +1,15 @@ +from pydantic import BaseSettings +from os import getenv +from functools import lru_cache + + +class Settings(BaseSettings): + api_url: str = getenv('API_HOST') + api_port: str = getenv('API_PORT') + s3_bucket_url: str = getenv('S3_BUCKET_URL') + require_auth: str = getenv('REQUIRE_AUTH') + + +@lru_cache() +def get_settings(): + return Settings() diff --git a/tiler/app/main.py b/tiler/app/main.py new file mode 100644 index 000000000..90ab98248 --- /dev/null +++ b/tiler/app/main.py @@ -0,0 +1,26 @@ +from fastapi import FastAPI, Query +from fastapi.middleware.cors import CORSMiddleware +from titiler.core import TilerFactory +from titiler.core.errors import DEFAULT_STATUS_CODES, add_exception_handlers +from titiler.core.middleware import TotalTimeMiddleware, LoggerMiddleware +from .middlewares.auth_middleware import AuthMiddleware +from .middlewares.url_injector import inject_s3_url + +app = FastAPI(title="LandGriffon Tiler! Because why keep life simple?") +app.add_middleware(TotalTimeMiddleware) +app.add_middleware(LoggerMiddleware) +app.add_middleware(AuthMiddleware) +app.add_middleware(CORSMiddleware, allow_origins=["*"], + allow_methods=["GET"], + allow_headers=["*"], ) + +# single COG tiler. One file can have multiple bands +cog = TilerFactory(router_prefix="/cog", path_dependency=inject_s3_url) +app.include_router(cog.router, tags=["Cloud Optimized GeoTIFF"], prefix="/cog", ) +add_exception_handlers(app, DEFAULT_STATUS_CODES) + + +@app.get("/health", description="Health Check", tags=["Health Check"]) +def ping(): + """Health check.""" + return {"ping": "pong"} diff --git a/tiler/app/middlewares/__init__.py b/tiler/app/middlewares/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tiler/app/middlewares/auth_middleware.py b/tiler/app/middlewares/auth_middleware.py new file mode 100644 index 000000000..2898b662a --- /dev/null +++ b/tiler/app/middlewares/auth_middleware.py @@ -0,0 +1,32 @@ +import requests +from fastapi import Request, Response, HTTPException +from starlette.middleware.base import BaseHTTPMiddleware +import os +from app.config.config import get_settings + +api_url = get_settings().api_url +api_port = get_settings().api_port + + +class AuthMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + env = os.environ.get('PYTHON_ENV') + if env == 'dev': + return await call_next(request) + else: + try: + token = request.headers.get("Authorization") + if not token: + return Response(status_code=400, content="No token found") + headers = { + "Authorization": token + } + response = requests.get(f"{api_url}:{api_port}/auth/validate-token", headers=headers) + if response.status_code != 200: + return Response(status_code=401, content="Unauthorized") + return await call_next(request) + + + except Exception as e: + print(e) + raise HTTPException(status_code=500, detail=str(e)) diff --git a/tiler/app/middlewares/url_injector.py b/tiler/app/middlewares/url_injector.py new file mode 100644 index 000000000..f4fdbf174 --- /dev/null +++ b/tiler/app/middlewares/url_injector.py @@ -0,0 +1,8 @@ +from fastapi.params import Query + +from ..config.config import get_settings + + +def inject_s3_url(url: str | None = Query(default=None, description="Optional dataset URL")) -> str: + s3_url = get_settings().s3_bucket_url + return s3_url + url diff --git a/tiler/app/test/__init__.py b/tiler/app/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tiler/app/test/test_health.py b/tiler/app/test/test_health.py new file mode 100644 index 000000000..6ea7f23d6 --- /dev/null +++ b/tiler/app/test/test_health.py @@ -0,0 +1,11 @@ +from fastapi.testclient import TestClient +from ..main import app + +client = TestClient(app) + + +def test_auth_middleware(): + """Should throw a Unauthorized Exception if no token has been provided""" + response = client.get("/cog/info") + assert response.status_code == 400 + assert response.text == 'No token found' diff --git a/tiler/requirements.txt b/tiler/requirements.txt new file mode 100644 index 000000000..15d218774 --- /dev/null +++ b/tiler/requirements.txt @@ -0,0 +1,4 @@ +titiler.application==0.11.0 +uvicorn +requests~=2.28.2 +pytest~=7.2.1