From 6c1ee3aab481093b3b0b8d9bc2e1d9fbef6dcedc Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Sat, 13 Apr 2024 17:22:13 +0530 Subject: [PATCH 01/22] scaffolding fastapi app for whatsapp analytics --- .env.example | 5 +++ .github/workflows/pre-commit.yml | 16 +++++++ .gitignore | 8 ++++ .pre-commit-config.yaml | 14 +++++++ README.md | 42 ++++++++++++++++++- app/__init__.py | 1 + app/helpers/__init__.py | 1 + app/helpers/db_helper.py | 9 ++++ app/mixins.py | 10 +++++ app/models/__init__.py | 1 + app/models/webhook_transaction_log.py | 25 +++++++++++ app/services/__init__.py | 1 + .../webhook_transaction_log_service.py | 33 +++++++++++++++ config.py | 14 +++++++ main.py | 27 ++++++++++++ requirements-dev.txt | 4 ++ requirements.txt | 7 ++++ 17 files changed, 216 insertions(+), 2 deletions(-) create mode 100644 .env.example create mode 100644 .github/workflows/pre-commit.yml create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 app/__init__.py create mode 100644 app/helpers/__init__.py create mode 100644 app/helpers/db_helper.py create mode 100644 app/mixins.py create mode 100644 app/models/__init__.py create mode 100644 app/models/webhook_transaction_log.py create mode 100644 app/services/__init__.py create mode 100644 app/services/webhook_transaction_log_service.py create mode 100644 config.py create mode 100644 main.py create mode 100644 requirements-dev.txt create mode 100644 requirements.txt diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..b6368bb --- /dev/null +++ b/.env.example @@ -0,0 +1,5 @@ +DB_USER= +DB_PASSWORD= +DB_NAME= +DB_HOST= +DB_PORT= diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..cac6e42 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,16 @@ +name: pre-commit + +on: + pull_request: + push: + branches: [main] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: '3.9' + - uses: pre-commit/action@v2.0.0 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..61fcdca --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +*.pyc +__pycache__/ + +# Environments +.env +.venv +env/ +venv/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..8b33f4f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,14 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-json + - id: check-yaml + - id: check-merge-conflict + - id: check-added-large-files +- repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black diff --git a/README.md b/README.md index 0cb4ac5..bc5b2a8 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,41 @@ -### WhatsApp Webhook Analytics +### WhatsApp Webhook Analytics -Handling and processing Incoming webhook request configured at Glific. \ No newline at end of file +Handling and processing Incoming webhook request configured at Glific. + +## Installation + +### Prerequisite +1. pyenv +2. python 3.12 + +### Steps +1. Clone the repository + ```sh + git clone https://github.com/DostEducation/whatsapp-webhook-analytics.git + ``` +2. Switch to project folder and setup the vertual environment + ```sh + cd whatsapp-webhook-analytics + python -m venv venv + ``` +3. Activate the virtual environment + ```sh + source ./venv/bin/activate + ``` +4. Install the dependencies: + ```sh + pip install -r requirements-dev.txt + ``` +5. Set up your .env file by copying .env.example + ```sh + cp .env.example .env + ``` +6. Add/update variables in your `.env` file for your environment. +7. Run the following command to get started with pre-commit + ```sh + pre-commit install + ``` +8. Start the server by following command + ```sh + functions_framework --target=handle_payload --debug + ``` diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..1eb109e --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +from app.mixins import * diff --git a/app/helpers/__init__.py b/app/helpers/__init__.py new file mode 100644 index 0000000..7c280a4 --- /dev/null +++ b/app/helpers/__init__.py @@ -0,0 +1 @@ +from app.helpers.db_helper import * diff --git a/app/helpers/db_helper.py b/app/helpers/db_helper.py new file mode 100644 index 0000000..ee6a4f8 --- /dev/null +++ b/app/helpers/db_helper.py @@ -0,0 +1,9 @@ +from sqlalchemy import create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker +from config import SQLALCHEMY_DATABASE_URL + +engine = create_engine(SQLALCHEMY_DATABASE_URL) +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) +Base = declarative_base() +db = SessionLocal() diff --git a/app/mixins.py b/app/mixins.py new file mode 100644 index 0000000..d3ce93e --- /dev/null +++ b/app/mixins.py @@ -0,0 +1,10 @@ +from pydantic import BaseModel, Field +from datetime import datetime + + +class TimestampMixin(BaseModel): + created_on: datetime = Field(default_factory=datetime.now) + updated_on: datetime = Field(default_factory=datetime.now) + + class Config: + orm_mode = True diff --git a/app/models/__init__.py b/app/models/__init__.py new file mode 100644 index 0000000..8198557 --- /dev/null +++ b/app/models/__init__.py @@ -0,0 +1 @@ +from app.models.webhook_transaction_log import * diff --git a/app/models/webhook_transaction_log.py b/app/models/webhook_transaction_log.py new file mode 100644 index 0000000..d2aa0c3 --- /dev/null +++ b/app/models/webhook_transaction_log.py @@ -0,0 +1,25 @@ +from app import TimestampMixin +from pydantic import BaseModel +from sqlalchemy import Column, Integer, Text, Boolean +from sqlalchemy.orm import declarative_base + +Base = declarative_base() + + +class WebhookTransactionLog(Base): + __tablename__ = "webhook_transaction_log" + + id = Column(Integer, primary_key=True) + payload = Column(Text) + processed = Column(Boolean, nullable=False) + attempts = Column(Integer, nullable=False, default=0) + + +class WebhookTransactionLogSchema(BaseModel): + payload: str + processed: bool + attempts: int + timestamps: TimestampMixin + + class Config: + orm_mode = True diff --git a/app/services/__init__.py b/app/services/__init__.py new file mode 100644 index 0000000..e13367e --- /dev/null +++ b/app/services/__init__.py @@ -0,0 +1 @@ +from app.services.webhook_transaction_log_service import * diff --git a/app/services/webhook_transaction_log_service.py b/app/services/webhook_transaction_log_service.py new file mode 100644 index 0000000..bd25326 --- /dev/null +++ b/app/services/webhook_transaction_log_service.py @@ -0,0 +1,33 @@ +import json +from app.helpers import db +from app import models + + +class WebhookTransactionLogService: + def create_new_webhook_log(self, jsonData): + try: + data = json.dumps(jsonData) + new_webhook_log = models.WebhookTransactionLog( + payload=data, + processed=False, + attempts=0, + ) + db.add(new_webhook_log) + db.commit() + return new_webhook_log + except Exception as e: + print( + f"Error while creating new webhook log. Webhook: {jsonData}. Error message: {e}" + ) + finally: + db.close() + + def mark_webhook_log_as_processed(self, webhook_log): + try: + webhook_log.processed = True + db.add(webhook_log) + db.commit() + except Exception as e: + print(f"Error while marking webhook log as processed. Error message: {e}") + finally: + db.close() diff --git a/config.py b/config.py new file mode 100644 index 0000000..f2a47b3 --- /dev/null +++ b/config.py @@ -0,0 +1,14 @@ +import os + +# Database configuration +POSTGRES = { + "user": os.environ.get("DB_USER"), + "password": os.environ.get("DB_PASSWORD"), + "database": os.environ.get("DB_NAME"), + "host": os.environ.get("DB_HOST"), + "port": os.environ.get("DB_PORT"), +} + +SQLALCHEMY_DATABASE_URL = ( + "postgresql://%(user)s:%(password)s@%(host)s:%(port)s/%(database)s" % POSTGRES +) diff --git a/main.py b/main.py new file mode 100644 index 0000000..eb9aeb2 --- /dev/null +++ b/main.py @@ -0,0 +1,27 @@ +from fastapi import FastAPI +from agraffe import Agraffe +from app.services import WebhookTransactionLogService + +app = FastAPI() + + +def handle_payload(request): + if request.method == "POST": + try: + jsonData = request.get_json() + if jsonData: + handle_webhook(jsonData) + except Exception as e: + print(f"Error: {e}") + return jsonData + else: + return "System does not accepts GET request" + + +def handle_webhook(jsonData): + transaction_log_service = WebhookTransactionLogService() + webhook_log = transaction_log_service.create_new_webhook_log(jsonData) + transaction_log_service.mark_webhook_log_as_processed(webhook_log) + + +handler = Agraffe.entry_point(app) diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..94dd033 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,4 @@ +# Includes dev dependencies on top of requirements.txt +-r requirements.txt +functions-framework==3.5.0 +pre-commit==3.5.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f6a9e48 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +agraffe==0.7.0 +fastapi==0.70.0 +psycopg2-binary==2.9.1 +pydantic==1.9.0 +sqlalchemy==1.4.26 +uvicorn==0.15.0 +setuptools==69.2.0 From dd9f22e1dea5f6a997019b807b7c569aab6680c6 Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Mon, 15 Apr 2024 11:06:06 +0530 Subject: [PATCH 02/22] merged requirements files --- README.md | 2 +- requirements-dev.txt | 4 ---- requirements.txt | 5 ++++- 3 files changed, 5 insertions(+), 6 deletions(-) delete mode 100644 requirements-dev.txt diff --git a/README.md b/README.md index bc5b2a8..5136014 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Handling and processing Incoming webhook request configured at Glific. ``` 4. Install the dependencies: ```sh - pip install -r requirements-dev.txt + pip install -r requirements.txt ``` 5. Set up your .env file by copying .env.example ```sh diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 94dd033..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Includes dev dependencies on top of requirements.txt --r requirements.txt -functions-framework==3.5.0 -pre-commit==3.5.0 diff --git a/requirements.txt b/requirements.txt index f6a9e48..ac9ace3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,10 @@ agraffe==0.7.0 fastapi==0.70.0 +functions-framework==3.5.0 +pre-commit==3.5.0 +psycopg2==2.9.9 psycopg2-binary==2.9.1 -pydantic==1.9.0 +pydantic==1.10.0 sqlalchemy==1.4.26 uvicorn==0.15.0 setuptools==69.2.0 From a42c908170d327661d20b68e3a461a0ede87b7a5 Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Mon, 15 Apr 2024 12:20:27 +0530 Subject: [PATCH 03/22] created database conectivity to the app --- config.py | 13 ++++++++----- requirements.txt | 1 + 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/config.py b/config.py index f2a47b3..a70674e 100644 --- a/config.py +++ b/config.py @@ -1,12 +1,15 @@ import os +from dotenv import load_dotenv + +load_dotenv() # Database configuration POSTGRES = { - "user": os.environ.get("DB_USER"), - "password": os.environ.get("DB_PASSWORD"), - "database": os.environ.get("DB_NAME"), - "host": os.environ.get("DB_HOST"), - "port": os.environ.get("DB_PORT"), + "user": os.getenv("DB_USER"), + "password": os.getenv("DB_PASSWORD"), + "database": os.getenv("DB_NAME"), + "host": os.getenv("DB_HOST"), + "port": os.getenv("DB_PORT"), } SQLALCHEMY_DATABASE_URL = ( diff --git a/requirements.txt b/requirements.txt index ac9ace3..3795fa3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ pydantic==1.10.0 sqlalchemy==1.4.26 uvicorn==0.15.0 setuptools==69.2.0 +python-dotenv==1.0.1 From 94f9f2f7b01560aa68b3a454e0bb1b7e13a30b8b Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Mon, 15 Apr 2024 13:30:12 +0530 Subject: [PATCH 04/22] added logger in the application --- .env.example | 1 + app/services/webhook_transaction_log_service.py | 7 +++++-- config.py | 2 ++ main.py | 5 ++++- requirements.txt | 1 + 5 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index b6368bb..500c8e1 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,4 @@ +ENVIRONMENT= DB_USER= DB_PASSWORD= DB_NAME= diff --git a/app/services/webhook_transaction_log_service.py b/app/services/webhook_transaction_log_service.py index bd25326..9f66574 100644 --- a/app/services/webhook_transaction_log_service.py +++ b/app/services/webhook_transaction_log_service.py @@ -1,6 +1,7 @@ import json from app.helpers import db from app import models +from app.utils.loggingutils import logger class WebhookTransactionLogService: @@ -16,7 +17,7 @@ def create_new_webhook_log(self, jsonData): db.commit() return new_webhook_log except Exception as e: - print( + logger.error( f"Error while creating new webhook log. Webhook: {jsonData}. Error message: {e}" ) finally: @@ -28,6 +29,8 @@ def mark_webhook_log_as_processed(self, webhook_log): db.add(webhook_log) db.commit() except Exception as e: - print(f"Error while marking webhook log as processed. Error message: {e}") + logger.error( + f"Error while marking webhook log as processed. Error message: {e}" + ) finally: db.close() diff --git a/config.py b/config.py index a70674e..71ab255 100644 --- a/config.py +++ b/config.py @@ -3,6 +3,8 @@ load_dotenv() +ENVIRONMENT = os.environ.get("ENVIRONMENT", "development") + # Database configuration POSTGRES = { "user": os.getenv("DB_USER"), diff --git a/main.py b/main.py index eb9aeb2..92e4af4 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,7 @@ from fastapi import FastAPI from agraffe import Agraffe from app.services import WebhookTransactionLogService +from app.utils.loggingutils import logger app = FastAPI() @@ -12,7 +13,9 @@ def handle_payload(request): if jsonData: handle_webhook(jsonData) except Exception as e: - print(f"Error: {e}") + logger.error( + f"Exception occured while handling the webhook payload: {jsonData}. Error: {e}" + ) return jsonData else: return "System does not accepts GET request" diff --git a/requirements.txt b/requirements.txt index 3795fa3..0436d28 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ sqlalchemy==1.4.26 uvicorn==0.15.0 setuptools==69.2.0 python-dotenv==1.0.1 +google-cloud-logging==3.10.0 From 0e9aa9f8c4a220249c0a11d8360a827cb5cbcfa7 Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Mon, 15 Apr 2024 13:30:52 +0530 Subject: [PATCH 05/22] added logger in the application --- app/utils/__init__.py | 1 + app/utils/loggingutils.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 app/utils/__init__.py create mode 100644 app/utils/loggingutils.py diff --git a/app/utils/__init__.py b/app/utils/__init__.py new file mode 100644 index 0000000..0ea5005 --- /dev/null +++ b/app/utils/__init__.py @@ -0,0 +1 @@ +from app.utils.loggingutils import * diff --git a/app/utils/loggingutils.py b/app/utils/loggingutils.py new file mode 100644 index 0000000..e840cdd --- /dev/null +++ b/app/utils/loggingutils.py @@ -0,0 +1,16 @@ +import logging +import config + +from google.cloud import logging as gcloud_logging + +logger = logging.getLogger() +logging.basicConfig(level=logging.DEBUG) + +if config.ENVIRONMENT == "development": + log_handler = logger.handlers[0] + logger.addHandler(log_handler) +else: + log_client = gcloud_logging.Client() + log_client.setup_logging() + log_handler = log_client.get_default_handler() + logger.addHandler(log_handler) From 701d27f80cb1f7cb741509ff2e6f32d1c961c294 Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Mon, 15 Apr 2024 16:00:20 +0530 Subject: [PATCH 06/22] Updated CI checks for the better coding standards --- .pre-commit-config.yaml | 14 ++++++++++++-- app/helpers/db_helper.py | 1 + app/mixins.py | 3 ++- app/models/__init__.py | 1 + app/models/webhook_transaction_log.py | 7 +++---- app/services/webhook_transaction_log_service.py | 3 ++- app/utils/loggingutils.py | 5 +++-- config.py | 1 + main.py | 3 ++- requirements.txt | 6 +++--- 10 files changed, 30 insertions(+), 14 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8b33f4f..bb390b7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v4.6.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -8,7 +8,17 @@ repos: - id: check-yaml - id: check-merge-conflict - id: check-added-large-files + - id: debug-statements + - id: requirements-txt-fixer +- repo: https://github.com/pre-commit/mirrors-isort + rev: 'v5.10.1' + hooks: + - id: isort +- repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v1.9.0' + hooks: + - id: mypy - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 24.4.0 hooks: - id: black diff --git a/app/helpers/db_helper.py b/app/helpers/db_helper.py index ee6a4f8..db2b068 100644 --- a/app/helpers/db_helper.py +++ b/app/helpers/db_helper.py @@ -1,6 +1,7 @@ from sqlalchemy import create_engine from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker + from config import SQLALCHEMY_DATABASE_URL engine = create_engine(SQLALCHEMY_DATABASE_URL) diff --git a/app/mixins.py b/app/mixins.py index d3ce93e..670f098 100644 --- a/app/mixins.py +++ b/app/mixins.py @@ -1,6 +1,7 @@ -from pydantic import BaseModel, Field from datetime import datetime +from pydantic import BaseModel, Field + class TimestampMixin(BaseModel): created_on: datetime = Field(default_factory=datetime.now) diff --git a/app/models/__init__.py b/app/models/__init__.py index 8198557..b4786ad 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -1 +1,2 @@ +from app.models.base import * from app.models.webhook_transaction_log import * diff --git a/app/models/webhook_transaction_log.py b/app/models/webhook_transaction_log.py index d2aa0c3..21c952e 100644 --- a/app/models/webhook_transaction_log.py +++ b/app/models/webhook_transaction_log.py @@ -1,9 +1,8 @@ -from app import TimestampMixin from pydantic import BaseModel -from sqlalchemy import Column, Integer, Text, Boolean -from sqlalchemy.orm import declarative_base +from sqlalchemy import Boolean, Column, Integer, Text -Base = declarative_base() +from app import TimestampMixin +from app.models.base import Base class WebhookTransactionLog(Base): diff --git a/app/services/webhook_transaction_log_service.py b/app/services/webhook_transaction_log_service.py index 9f66574..0ef5d4e 100644 --- a/app/services/webhook_transaction_log_service.py +++ b/app/services/webhook_transaction_log_service.py @@ -1,6 +1,7 @@ import json -from app.helpers import db + from app import models +from app.helpers import db from app.utils.loggingutils import logger diff --git a/app/utils/loggingutils.py b/app/utils/loggingutils.py index e840cdd..be5c134 100644 --- a/app/utils/loggingutils.py +++ b/app/utils/loggingutils.py @@ -1,12 +1,13 @@ import logging -import config from google.cloud import logging as gcloud_logging +from config import ENVIRONMENT + logger = logging.getLogger() logging.basicConfig(level=logging.DEBUG) -if config.ENVIRONMENT == "development": +if ENVIRONMENT == "development": log_handler = logger.handlers[0] logger.addHandler(log_handler) else: diff --git a/config.py b/config.py index 71ab255..d8543d4 100644 --- a/config.py +++ b/config.py @@ -1,4 +1,5 @@ import os + from dotenv import load_dotenv load_dotenv() diff --git a/main.py b/main.py index 92e4af4..de3377d 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,6 @@ -from fastapi import FastAPI from agraffe import Agraffe +from fastapi import FastAPI + from app.services import WebhookTransactionLogService from app.utils.loggingutils import logger diff --git a/requirements.txt b/requirements.txt index 0436d28..75960b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ agraffe==0.7.0 fastapi==0.70.0 functions-framework==3.5.0 +google-cloud-logging==3.10.0 pre-commit==3.5.0 psycopg2==2.9.9 psycopg2-binary==2.9.1 pydantic==1.10.0 +python-dotenv==1.0.1 +setuptools==69.2.0 sqlalchemy==1.4.26 uvicorn==0.15.0 -setuptools==69.2.0 -python-dotenv==1.0.1 -google-cloud-logging==3.10.0 From bb309cd185ad2ec2dfa660bdba1a3385b27abe35 Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Mon, 15 Apr 2024 16:07:11 +0530 Subject: [PATCH 07/22] Fixed error: Invalid base class 'Base'. --- app/models/base.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 app/models/base.py diff --git a/app/models/base.py b/app/models/base.py new file mode 100644 index 0000000..860e542 --- /dev/null +++ b/app/models/base.py @@ -0,0 +1,3 @@ +from sqlalchemy.ext.declarative import declarative_base + +Base = declarative_base() From a21de5fb8efe6278243d1682cc44aa5caab05c7c Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Mon, 15 Apr 2024 16:22:11 +0530 Subject: [PATCH 08/22] Updated gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 61fcdca..e72b932 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.pyc __pycache__/ +.mypy_cache # Environments .env From 8bbf2718fac771d8508c032d4c0d03e5fbacefc0 Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Mon, 15 Apr 2024 18:21:17 +0530 Subject: [PATCH 09/22] Added migrations for maintaining the database schemas through automation --- .gitignore | 1 + .pre-commit-config.yaml | 1 + alembic/README | 1 + alembic/env.py | 75 +++++++++++++++++++ alembic/script.py.mako | 26 +++++++ .../27b68e693a86_initial_migrations.py | 40 ++++++++++ app/mixins.py | 14 ++-- app/models/webhook_transaction_log.py | 13 +--- requirements.txt | 2 + 9 files changed, 155 insertions(+), 18 deletions(-) create mode 100644 alembic/README create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/27b68e693a86_initial_migrations.py diff --git a/.gitignore b/.gitignore index e72b932..cedccb5 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ __pycache__/ .venv env/ venv/ +alembic.ini diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bb390b7..4b85b3c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,6 +18,7 @@ repos: rev: 'v1.9.0' hooks: - id: mypy + exclude: alembic - repo: https://github.com/psf/black rev: 24.4.0 hooks: diff --git a/alembic/README b/alembic/README new file mode 100644 index 0000000..2500aa1 --- /dev/null +++ b/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..406f014 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,75 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config, pool + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +from app import models + +target_metadata = models.Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..fbc4b07 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/27b68e693a86_initial_migrations.py b/alembic/versions/27b68e693a86_initial_migrations.py new file mode 100644 index 0000000..a37fa6c --- /dev/null +++ b/alembic/versions/27b68e693a86_initial_migrations.py @@ -0,0 +1,40 @@ +"""initial migrations + +Revision ID: 27b68e693a86 +Revises: +Create Date: 2024-04-15 18:11:38.625368 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "27b68e693a86" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "webhook_transaction_log", + sa.Column("created_on", sa.DateTime(), nullable=True), + sa.Column("updated_on", sa.DateTime(), nullable=True), + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("payload", sa.Text(), nullable=True), + sa.Column("processed", sa.Boolean(), nullable=False), + sa.Column("attempts", sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint("id"), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("webhook_transaction_log") + # ### end Alembic commands ### diff --git a/app/mixins.py b/app/mixins.py index 670f098..32a8d56 100644 --- a/app/mixins.py +++ b/app/mixins.py @@ -1,11 +1,13 @@ from datetime import datetime -from pydantic import BaseModel, Field +from sqlalchemy import Column, DateTime +from sqlalchemy.ext.declarative import declared_attr -class TimestampMixin(BaseModel): - created_on: datetime = Field(default_factory=datetime.now) - updated_on: datetime = Field(default_factory=datetime.now) +class TimestampMixin: + created_on = Column(DateTime, default=datetime.utcnow) + updated_on = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) - class Config: - orm_mode = True + @declared_attr + def __tablename__(cls): + return cls.__name__.lower() diff --git a/app/models/webhook_transaction_log.py b/app/models/webhook_transaction_log.py index 21c952e..b76dd6b 100644 --- a/app/models/webhook_transaction_log.py +++ b/app/models/webhook_transaction_log.py @@ -1,24 +1,13 @@ -from pydantic import BaseModel from sqlalchemy import Boolean, Column, Integer, Text from app import TimestampMixin from app.models.base import Base -class WebhookTransactionLog(Base): +class WebhookTransactionLog(TimestampMixin, Base): __tablename__ = "webhook_transaction_log" id = Column(Integer, primary_key=True) payload = Column(Text) processed = Column(Boolean, nullable=False) attempts = Column(Integer, nullable=False, default=0) - - -class WebhookTransactionLogSchema(BaseModel): - payload: str - processed: bool - attempts: int - timestamps: TimestampMixin - - class Config: - orm_mode = True diff --git a/requirements.txt b/requirements.txt index 75960b9..557c9a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,9 @@ agraffe==0.7.0 +alembic==1.13.1 fastapi==0.70.0 functions-framework==3.5.0 google-cloud-logging==3.10.0 +Mako==1.3.3 pre-commit==3.5.0 psycopg2==2.9.9 psycopg2-binary==2.9.1 From d45eac2b1452450493b6643b78a7f78ae40c5507 Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Mon, 15 Apr 2024 18:24:37 +0530 Subject: [PATCH 10/22] Added migrations for maintaining the database schemas through automation --- ...ial_migrations.py => 19b23a59f196_initial_migrations.py} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename alembic/versions/{27b68e693a86_initial_migrations.py => 19b23a59f196_initial_migrations.py} (91%) diff --git a/alembic/versions/27b68e693a86_initial_migrations.py b/alembic/versions/19b23a59f196_initial_migrations.py similarity index 91% rename from alembic/versions/27b68e693a86_initial_migrations.py rename to alembic/versions/19b23a59f196_initial_migrations.py index a37fa6c..07dd4c9 100644 --- a/alembic/versions/27b68e693a86_initial_migrations.py +++ b/alembic/versions/19b23a59f196_initial_migrations.py @@ -1,8 +1,8 @@ """initial migrations -Revision ID: 27b68e693a86 +Revision ID: 19b23a59f196 Revises: -Create Date: 2024-04-15 18:11:38.625368 +Create Date: 2024-04-15 18:22:25.267651 """ @@ -13,7 +13,7 @@ from alembic import op # revision identifiers, used by Alembic. -revision: str = "27b68e693a86" +revision: str = "19b23a59f196" down_revision: Union[str, None] = None branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None From fb8ed036ce08d8bd053349039bc96e769eda1300 Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Mon, 15 Apr 2024 18:58:51 +0530 Subject: [PATCH 11/22] Upgraded the packages to the latest --- requirements.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 557c9a2..22dd660 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,14 @@ agraffe==0.7.0 alembic==1.13.1 -fastapi==0.70.0 +fastapi==0.110.1 functions-framework==3.5.0 google-cloud-logging==3.10.0 Mako==1.3.3 -pre-commit==3.5.0 +pre-commit==3.7.0 psycopg2==2.9.9 -psycopg2-binary==2.9.1 -pydantic==1.10.0 +psycopg2-binary==2.9.9 +pydantic==2.7.0 python-dotenv==1.0.1 setuptools==69.2.0 -sqlalchemy==1.4.26 -uvicorn==0.15.0 +sqlalchemy==2.0.29 +uvicorn==0.29.0 From d6652a598dc7050586f4d3140d238e7d99778595 Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Tue, 16 Apr 2024 11:01:44 +0530 Subject: [PATCH 12/22] Updated return messages --- main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index de3377d..6fe9523 100644 --- a/main.py +++ b/main.py @@ -17,9 +17,9 @@ def handle_payload(request): logger.error( f"Exception occured while handling the webhook payload: {jsonData}. Error: {e}" ) - return jsonData + return "Success" else: - return "System does not accepts GET request" + return "Currently, the system does not accept a GET request" def handle_webhook(jsonData): From 8848671732f6a7198de0d44c06343c9dc3ae5308 Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Wed, 17 Apr 2024 18:35:21 +0530 Subject: [PATCH 13/22] Updating FastAPI to Flask --- app/__init__.py | 9 +++++++++ main.py | 10 +++------- requirements.txt | 10 +++------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 1eb109e..83b4f5a 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1 +1,10 @@ +from flask import Flask +from flask_sqlalchemy import SQLAlchemy + from app.mixins import * + +app = Flask(__name__) +db = SQLAlchemy(app) +app.config.from_object("config") + +db.init_app(app) diff --git a/main.py b/main.py index 6fe9523..7bfe30f 100644 --- a/main.py +++ b/main.py @@ -1,12 +1,11 @@ -from agraffe import Agraffe -from fastapi import FastAPI +import functions_framework from app.services import WebhookTransactionLogService from app.utils.loggingutils import logger -app = FastAPI() - +### Endpoint for Cloud function +@functions_framework.http def handle_payload(request): if request.method == "POST": try: @@ -26,6 +25,3 @@ def handle_webhook(jsonData): transaction_log_service = WebhookTransactionLogService() webhook_log = transaction_log_service.create_new_webhook_log(jsonData) transaction_log_service.mark_webhook_log_as_processed(webhook_log) - - -handler = Agraffe.entry_point(app) diff --git a/requirements.txt b/requirements.txt index 22dd660..d3cfdee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,10 @@ -agraffe==0.7.0 alembic==1.13.1 -fastapi==0.110.1 +Flask==3.0.3 +Flask-SQLAlchemy==3.1.1 functions-framework==3.5.0 google-cloud-logging==3.10.0 -Mako==1.3.3 pre-commit==3.7.0 psycopg2==2.9.9 -psycopg2-binary==2.9.9 -pydantic==2.7.0 python-dotenv==1.0.1 -setuptools==69.2.0 sqlalchemy==2.0.29 -uvicorn==0.29.0 +Werkzeug==3.0.2 From 21773af4155d15ca0b11057a677e3b9254d81ad8 Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Thu, 18 Apr 2024 13:12:02 +0530 Subject: [PATCH 14/22] upgrded fastapi app to flask app --- .gitignore | 1 - alembic/README | 1 - alembic/env.py | 75 ------------------- alembic/script.py.mako | 26 ------- .../19b23a59f196_initial_migrations.py | 40 ---------- app/__init__.py | 10 --- app/helpers/__init__.py | 1 - app/helpers/db_helper.py | 10 --- app/mixins.py | 13 ---- app/models/__init__.py | 2 - app/models/base.py | 3 - app/models/webhook_transaction_log.py | 13 ---- app/services/__init__.py | 1 - .../webhook_transaction_log_service.py | 37 --------- app/utils/__init__.py | 1 - app/utils/loggingutils.py | 17 ----- config.py | 35 ++++++--- main.py | 24 +++--- requirements.txt | 2 +- 19 files changed, 40 insertions(+), 272 deletions(-) delete mode 100644 alembic/README delete mode 100644 alembic/env.py delete mode 100644 alembic/script.py.mako delete mode 100644 alembic/versions/19b23a59f196_initial_migrations.py delete mode 100644 app/__init__.py delete mode 100644 app/helpers/__init__.py delete mode 100644 app/helpers/db_helper.py delete mode 100644 app/mixins.py delete mode 100644 app/models/__init__.py delete mode 100644 app/models/base.py delete mode 100644 app/models/webhook_transaction_log.py delete mode 100644 app/services/__init__.py delete mode 100644 app/services/webhook_transaction_log_service.py delete mode 100644 app/utils/__init__.py delete mode 100644 app/utils/loggingutils.py diff --git a/.gitignore b/.gitignore index cedccb5..e72b932 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,3 @@ __pycache__/ .venv env/ venv/ -alembic.ini diff --git a/alembic/README b/alembic/README deleted file mode 100644 index 2500aa1..0000000 --- a/alembic/README +++ /dev/null @@ -1 +0,0 @@ -Generic single-database configuration. diff --git a/alembic/env.py b/alembic/env.py deleted file mode 100644 index 406f014..0000000 --- a/alembic/env.py +++ /dev/null @@ -1,75 +0,0 @@ -from logging.config import fileConfig - -from sqlalchemy import engine_from_config, pool - -from alembic import context - -# this is the Alembic Config object, which provides -# access to the values within the .ini file in use. -config = context.config - -# Interpret the config file for Python logging. -# This line sets up loggers basically. -if config.config_file_name is not None: - fileConfig(config.config_file_name) - -# add your model's MetaData object here -# for 'autogenerate' support -from app import models - -target_metadata = models.Base.metadata - -# other values from the config, defined by the needs of env.py, -# can be acquired: -# my_important_option = config.get_main_option("my_important_option") -# ... etc. - - -def run_migrations_offline() -> None: - """Run migrations in 'offline' mode. - - This configures the context with just a URL - and not an Engine, though an Engine is acceptable - here as well. By skipping the Engine creation - we don't even need a DBAPI to be available. - - Calls to context.execute() here emit the given string to the - script output. - - """ - url = config.get_main_option("sqlalchemy.url") - context.configure( - url=url, - target_metadata=target_metadata, - literal_binds=True, - dialect_opts={"paramstyle": "named"}, - ) - - with context.begin_transaction(): - context.run_migrations() - - -def run_migrations_online() -> None: - """Run migrations in 'online' mode. - - In this scenario we need to create an Engine - and associate a connection with the context. - - """ - connectable = engine_from_config( - config.get_section(config.config_ini_section, {}), - prefix="sqlalchemy.", - poolclass=pool.NullPool, - ) - - with connectable.connect() as connection: - context.configure(connection=connection, target_metadata=target_metadata) - - with context.begin_transaction(): - context.run_migrations() - - -if context.is_offline_mode(): - run_migrations_offline() -else: - run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako deleted file mode 100644 index fbc4b07..0000000 --- a/alembic/script.py.mako +++ /dev/null @@ -1,26 +0,0 @@ -"""${message} - -Revision ID: ${up_revision} -Revises: ${down_revision | comma,n} -Create Date: ${create_date} - -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -${imports if imports else ""} - -# revision identifiers, used by Alembic. -revision: str = ${repr(up_revision)} -down_revision: Union[str, None] = ${repr(down_revision)} -branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} -depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} - - -def upgrade() -> None: - ${upgrades if upgrades else "pass"} - - -def downgrade() -> None: - ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/19b23a59f196_initial_migrations.py b/alembic/versions/19b23a59f196_initial_migrations.py deleted file mode 100644 index 07dd4c9..0000000 --- a/alembic/versions/19b23a59f196_initial_migrations.py +++ /dev/null @@ -1,40 +0,0 @@ -"""initial migrations - -Revision ID: 19b23a59f196 -Revises: -Create Date: 2024-04-15 18:22:25.267651 - -""" - -from typing import Sequence, Union - -import sqlalchemy as sa - -from alembic import op - -# revision identifiers, used by Alembic. -revision: str = "19b23a59f196" -down_revision: Union[str, None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "webhook_transaction_log", - sa.Column("created_on", sa.DateTime(), nullable=True), - sa.Column("updated_on", sa.DateTime(), nullable=True), - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("payload", sa.Text(), nullable=True), - sa.Column("processed", sa.Boolean(), nullable=False), - sa.Column("attempts", sa.Integer(), nullable=False), - sa.PrimaryKeyConstraint("id"), - ) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table("webhook_transaction_log") - # ### end Alembic commands ### diff --git a/app/__init__.py b/app/__init__.py deleted file mode 100644 index 83b4f5a..0000000 --- a/app/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from flask import Flask -from flask_sqlalchemy import SQLAlchemy - -from app.mixins import * - -app = Flask(__name__) -db = SQLAlchemy(app) -app.config.from_object("config") - -db.init_app(app) diff --git a/app/helpers/__init__.py b/app/helpers/__init__.py deleted file mode 100644 index 7c280a4..0000000 --- a/app/helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from app.helpers.db_helper import * diff --git a/app/helpers/db_helper.py b/app/helpers/db_helper.py deleted file mode 100644 index db2b068..0000000 --- a/app/helpers/db_helper.py +++ /dev/null @@ -1,10 +0,0 @@ -from sqlalchemy import create_engine -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker - -from config import SQLALCHEMY_DATABASE_URL - -engine = create_engine(SQLALCHEMY_DATABASE_URL) -SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) -Base = declarative_base() -db = SessionLocal() diff --git a/app/mixins.py b/app/mixins.py deleted file mode 100644 index 32a8d56..0000000 --- a/app/mixins.py +++ /dev/null @@ -1,13 +0,0 @@ -from datetime import datetime - -from sqlalchemy import Column, DateTime -from sqlalchemy.ext.declarative import declared_attr - - -class TimestampMixin: - created_on = Column(DateTime, default=datetime.utcnow) - updated_on = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) - - @declared_attr - def __tablename__(cls): - return cls.__name__.lower() diff --git a/app/models/__init__.py b/app/models/__init__.py deleted file mode 100644 index b4786ad..0000000 --- a/app/models/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from app.models.base import * -from app.models.webhook_transaction_log import * diff --git a/app/models/base.py b/app/models/base.py deleted file mode 100644 index 860e542..0000000 --- a/app/models/base.py +++ /dev/null @@ -1,3 +0,0 @@ -from sqlalchemy.ext.declarative import declarative_base - -Base = declarative_base() diff --git a/app/models/webhook_transaction_log.py b/app/models/webhook_transaction_log.py deleted file mode 100644 index b76dd6b..0000000 --- a/app/models/webhook_transaction_log.py +++ /dev/null @@ -1,13 +0,0 @@ -from sqlalchemy import Boolean, Column, Integer, Text - -from app import TimestampMixin -from app.models.base import Base - - -class WebhookTransactionLog(TimestampMixin, Base): - __tablename__ = "webhook_transaction_log" - - id = Column(Integer, primary_key=True) - payload = Column(Text) - processed = Column(Boolean, nullable=False) - attempts = Column(Integer, nullable=False, default=0) diff --git a/app/services/__init__.py b/app/services/__init__.py deleted file mode 100644 index e13367e..0000000 --- a/app/services/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from app.services.webhook_transaction_log_service import * diff --git a/app/services/webhook_transaction_log_service.py b/app/services/webhook_transaction_log_service.py deleted file mode 100644 index 0ef5d4e..0000000 --- a/app/services/webhook_transaction_log_service.py +++ /dev/null @@ -1,37 +0,0 @@ -import json - -from app import models -from app.helpers import db -from app.utils.loggingutils import logger - - -class WebhookTransactionLogService: - def create_new_webhook_log(self, jsonData): - try: - data = json.dumps(jsonData) - new_webhook_log = models.WebhookTransactionLog( - payload=data, - processed=False, - attempts=0, - ) - db.add(new_webhook_log) - db.commit() - return new_webhook_log - except Exception as e: - logger.error( - f"Error while creating new webhook log. Webhook: {jsonData}. Error message: {e}" - ) - finally: - db.close() - - def mark_webhook_log_as_processed(self, webhook_log): - try: - webhook_log.processed = True - db.add(webhook_log) - db.commit() - except Exception as e: - logger.error( - f"Error while marking webhook log as processed. Error message: {e}" - ) - finally: - db.close() diff --git a/app/utils/__init__.py b/app/utils/__init__.py deleted file mode 100644 index 0ea5005..0000000 --- a/app/utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from app.utils.loggingutils import * diff --git a/app/utils/loggingutils.py b/app/utils/loggingutils.py deleted file mode 100644 index be5c134..0000000 --- a/app/utils/loggingutils.py +++ /dev/null @@ -1,17 +0,0 @@ -import logging - -from google.cloud import logging as gcloud_logging - -from config import ENVIRONMENT - -logger = logging.getLogger() -logging.basicConfig(level=logging.DEBUG) - -if ENVIRONMENT == "development": - log_handler = logger.handlers[0] - logger.addHandler(log_handler) -else: - log_client = gcloud_logging.Client() - log_client.setup_logging() - log_handler = log_client.get_default_handler() - logger.addHandler(log_handler) diff --git a/config.py b/config.py index d8543d4..83b6780 100644 --- a/config.py +++ b/config.py @@ -1,20 +1,37 @@ +"""Flask configuration.""" + import os +from urllib.parse import quote + +FLASK_APP = os.environ.get("FLASK_APP", "development") -from dotenv import load_dotenv +if FLASK_APP == "development": + from os import path -load_dotenv() + from dotenv import load_dotenv + + basedir = path.abspath(path.dirname(__file__)) + load_dotenv(path.join(basedir, ".env")) -ENVIRONMENT = os.environ.get("ENVIRONMENT", "development") # Database configuration POSTGRES = { - "user": os.getenv("DB_USER"), - "password": os.getenv("DB_PASSWORD"), - "database": os.getenv("DB_NAME"), - "host": os.getenv("DB_HOST"), - "port": os.getenv("DB_PORT"), + "user": os.environ.get("DB_USER"), + "password": os.environ.get("DB_PASSWORD"), + "database": os.environ.get("DB_NAME"), + "host": os.environ.get("DB_HOST"), + "port": os.environ.get("DB_PORT"), + "connection_name": os.environ.get("CONNECTION_NAME"), } -SQLALCHEMY_DATABASE_URL = ( +SQLALCHEMY_DATABASE_URI = ( "postgresql://%(user)s:%(password)s@%(host)s:%(port)s/%(database)s" % POSTGRES ) +print(SQLALCHEMY_DATABASE_URI) + +# For socket based connection +if FLASK_APP in ("production", "staging"): + SQLALCHEMY_DATABASE_URI = ( + "postgresql://%(user)s:%(password)s@/%(database)s?host=%(connection_name)s/" + % POSTGRES + ) diff --git a/main.py b/main.py index 7bfe30f..6dc8e34 100644 --- a/main.py +++ b/main.py @@ -1,22 +1,24 @@ import functions_framework -from app.services import WebhookTransactionLogService -from app.utils.loggingutils import logger +from api import app +from api.services import WebhookTransactionLogService +from api.utils.loggingutils import logger ### Endpoint for Cloud function @functions_framework.http def handle_payload(request): if request.method == "POST": - try: - jsonData = request.get_json() - if jsonData: - handle_webhook(jsonData) - except Exception as e: - logger.error( - f"Exception occured while handling the webhook payload: {jsonData}. Error: {e}" - ) - return "Success" + with app.app_context(): + try: + jsonData = request.get_json() + if jsonData: + handle_webhook(jsonData) + except Exception as e: + logger.error( + f"Exception occured while handling the webhook payload: {jsonData}. Error: {e}" + ) + return "Success" else: return "Currently, the system does not accept a GET request" diff --git a/requirements.txt b/requirements.txt index d3cfdee..a968a88 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -alembic==1.13.1 Flask==3.0.3 +Flask-Migrate==4.0.7 Flask-SQLAlchemy==3.1.1 functions-framework==3.5.0 google-cloud-logging==3.10.0 From a51fef1ab0279fbadc8c42f0b86da50a7e243f9a Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Thu, 18 Apr 2024 13:12:19 +0530 Subject: [PATCH 15/22] upgrded fastapi app to flask app --- config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/config.py b/config.py index 83b6780..ca151fb 100644 --- a/config.py +++ b/config.py @@ -1,7 +1,6 @@ """Flask configuration.""" import os -from urllib.parse import quote FLASK_APP = os.environ.get("FLASK_APP", "development") From 88188c67c9f6c7660b5febf7c2e14669877918ee Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Thu, 18 Apr 2024 13:13:00 +0530 Subject: [PATCH 16/22] upgrded fastapi app to flask app --- api/__init__.py | 6 ++++ api/db_models/__init__.py | 1 + api/db_models/webhook_transaction_log.py | 12 +++++++ api/helpers/__init__.py | 1 + api/helpers/db_helper.py | 35 +++++++++++++++++++ api/mixins.py | 10 ++++++ api/services/__init__.py | 1 + .../webhook_transaction_log_service.py | 33 +++++++++++++++++ api/utils/__init__.py | 1 + api/utils/loggingutils.py | 18 ++++++++++ manage.py | 12 +++++++ 11 files changed, 130 insertions(+) create mode 100644 api/__init__.py create mode 100644 api/db_models/__init__.py create mode 100644 api/db_models/webhook_transaction_log.py create mode 100644 api/helpers/__init__.py create mode 100644 api/helpers/db_helper.py create mode 100644 api/mixins.py create mode 100644 api/services/__init__.py create mode 100644 api/services/webhook_transaction_log_service.py create mode 100644 api/utils/__init__.py create mode 100644 api/utils/loggingutils.py create mode 100644 manage.py diff --git a/api/__init__.py b/api/__init__.py new file mode 100644 index 0000000..dc3a534 --- /dev/null +++ b/api/__init__.py @@ -0,0 +1,6 @@ +from flask import Flask +from flask_sqlalchemy import SQLAlchemy + +app = Flask(__name__) +app.config.from_object("config") +db = SQLAlchemy(app) diff --git a/api/db_models/__init__.py b/api/db_models/__init__.py new file mode 100644 index 0000000..4a63da3 --- /dev/null +++ b/api/db_models/__init__.py @@ -0,0 +1 @@ +from api.db_models.webhook_transaction_log import * diff --git a/api/db_models/webhook_transaction_log.py b/api/db_models/webhook_transaction_log.py new file mode 100644 index 0000000..1f8c4e3 --- /dev/null +++ b/api/db_models/webhook_transaction_log.py @@ -0,0 +1,12 @@ +from api import db +from api.mixins import TimestampMixin + + +class WebhookTransactionLog(TimestampMixin, db.Model): + + __tablename__ = "webhook_transaction_log" + id = db.Column(db.Integer, primary_key=True) + payload = db.Column(db.Text) + processed = db.Column(db.Boolean, nullable=False) + gg = db.Column(db.Boolean, nullable=False) + attempts = db.Column(db.Integer, nullable=False, default="0") diff --git a/api/helpers/__init__.py b/api/helpers/__init__.py new file mode 100644 index 0000000..101f855 --- /dev/null +++ b/api/helpers/__init__.py @@ -0,0 +1 @@ +from api.helpers.db_helper import * diff --git a/api/helpers/db_helper.py b/api/helpers/db_helper.py new file mode 100644 index 0000000..fd026af --- /dev/null +++ b/api/helpers/db_helper.py @@ -0,0 +1,35 @@ +import traceback + +import psycopg2 + +from api import app, db +from api.utils.loggingutils import logger + + +def get_db_connection(connection_data={}): + db_name = connection_data.get("db_name", app.config["DB_NAME"]) + db_user = connection_data.get("db_user", app.config["DB_USER"]) + db_host = connection_data.get("db_host", app.config["DB_HOST"]) + db_pwd = connection_data.get("db_pwd", app.config["DB_PWD"]) + + db_conn_string = f"dbname={db_name} user={db_user} host={db_host} password={db_pwd}" + + try: + return psycopg2.connect(db_conn_string) + except Exception as e: + logger.error( + f"DB Helper: Unable to connect to the database: {db_conn_string}. Error message:{e}" + ) + return None + + +def save(data): + try: + db.session.add(data) + db.session.commit() + except Exception as e: + logger.error( + f"Error occurred while committing the data in the database. Error message: {e}" + ) + logger.debug(traceback.format_exc()) + db.session.rollback() diff --git a/api/mixins.py b/api/mixins.py new file mode 100644 index 0000000..0e964af --- /dev/null +++ b/api/mixins.py @@ -0,0 +1,10 @@ +from __future__ import absolute_import + +from datetime import datetime + +from api import db + + +class TimestampMixin: + created_on = db.Column(db.DateTime, default=datetime.now) + updated_on = db.Column(db.DateTime, onupdate=datetime.now, default=datetime.now) diff --git a/api/services/__init__.py b/api/services/__init__.py new file mode 100644 index 0000000..3873888 --- /dev/null +++ b/api/services/__init__.py @@ -0,0 +1 @@ +from api.services.webhook_transaction_log_service import * diff --git a/api/services/webhook_transaction_log_service.py b/api/services/webhook_transaction_log_service.py new file mode 100644 index 0000000..8ea3174 --- /dev/null +++ b/api/services/webhook_transaction_log_service.py @@ -0,0 +1,33 @@ +import json + +from api import db, db_models +from api.helpers import db_helper +from api.utils.loggingutils import logger + + +class WebhookTransactionLogService: + def create_new_webhook_log(self, jsonData): + try: + data = json.dumps(jsonData) + new_webhook_log = db_models.WebhookTransactionLog( + payload=data, + processed=False, + attempts=0, + ) + db_helper.save(new_webhook_log) + return new_webhook_log + except Exception as e: + logger.error( + f"Error while creating new webhook log. Webhook: {jsonData}. Error message: {e}" + ) + return None + + def mark_webhook_log_as_processed(self, webhook_log): + try: + webhook_log.processed = True + db_helper.save(webhook_log) + except Exception as e: + logger.error( + f"Error while marking webhook log as processed. Error message: {e}" + ) + return None diff --git a/api/utils/__init__.py b/api/utils/__init__.py new file mode 100644 index 0000000..03eb9ab --- /dev/null +++ b/api/utils/__init__.py @@ -0,0 +1 @@ +from api.utils.loggingutils import * diff --git a/api/utils/loggingutils.py b/api/utils/loggingutils.py new file mode 100644 index 0000000..e872aaa --- /dev/null +++ b/api/utils/loggingutils.py @@ -0,0 +1,18 @@ +import logging +import os + +from google.cloud import logging as gcloud_logging + +from api import app + +logger = logging.getLogger() +logging.basicConfig(level=logging.DEBUG) + +if os.environ.get("FLASK_ENV", "development"): + log_handler = logger.handlers[0] + logger.addHandler(log_handler) +else: + log_client = gcloud_logging.Client() + log_client.setup_logging() + log_handler = log_client.get_default_handler() + app.logger.addHandler(log_handler) diff --git a/manage.py b/manage.py new file mode 100644 index 0000000..cdad6ed --- /dev/null +++ b/manage.py @@ -0,0 +1,12 @@ +from __future__ import absolute_import + +from flask.cli import FlaskGroup +from flask_migrate import Migrate + +from api import app, db + +migrate = Migrate(app, db) +cli = FlaskGroup(app) + +if __name__ == "__main__": + cli() From 23bd5403398e97387b4d7f000c454639f9aeab1c Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Thu, 18 Apr 2024 14:18:02 +0530 Subject: [PATCH 17/22] added flask migration --- .env.example | 2 +- api/db_models/__init__.py | 1 - api/models/__init__.py | 5 + .../webhook_transaction_log.py | 1 - api/services/__init__.py | 2 +- .../webhook_transaction_log_service.py | 4 +- api/utils/__init__.py | 2 +- config.py | 1 - migrations/README | 1 + migrations/alembic.ini | 50 ++++++++ migrations/env.py | 108 ++++++++++++++++++ migrations/script.py.mako | 24 ++++ .../c4ae3a3aac32_initial_migration.py | 37 ++++++ 13 files changed, 230 insertions(+), 8 deletions(-) delete mode 100644 api/db_models/__init__.py create mode 100644 api/models/__init__.py rename api/{db_models => models}/webhook_transaction_log.py (88%) create mode 100644 migrations/README create mode 100644 migrations/alembic.ini create mode 100644 migrations/env.py create mode 100644 migrations/script.py.mako create mode 100644 migrations/versions/c4ae3a3aac32_initial_migration.py diff --git a/.env.example b/.env.example index 500c8e1..62cd471 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,4 @@ -ENVIRONMENT= +FLASK_APP= DB_USER= DB_PASSWORD= DB_NAME= diff --git a/api/db_models/__init__.py b/api/db_models/__init__.py deleted file mode 100644 index 4a63da3..0000000 --- a/api/db_models/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from api.db_models.webhook_transaction_log import * diff --git a/api/models/__init__.py b/api/models/__init__.py new file mode 100644 index 0000000..a478b05 --- /dev/null +++ b/api/models/__init__.py @@ -0,0 +1,5 @@ +from sqlalchemy.ext.declarative import declarative_base + +Base = declarative_base() + +from .webhook_transaction_log import * diff --git a/api/db_models/webhook_transaction_log.py b/api/models/webhook_transaction_log.py similarity index 88% rename from api/db_models/webhook_transaction_log.py rename to api/models/webhook_transaction_log.py index 1f8c4e3..c7a8720 100644 --- a/api/db_models/webhook_transaction_log.py +++ b/api/models/webhook_transaction_log.py @@ -8,5 +8,4 @@ class WebhookTransactionLog(TimestampMixin, db.Model): id = db.Column(db.Integer, primary_key=True) payload = db.Column(db.Text) processed = db.Column(db.Boolean, nullable=False) - gg = db.Column(db.Boolean, nullable=False) attempts = db.Column(db.Integer, nullable=False, default="0") diff --git a/api/services/__init__.py b/api/services/__init__.py index 3873888..097f248 100644 --- a/api/services/__init__.py +++ b/api/services/__init__.py @@ -1 +1 @@ -from api.services.webhook_transaction_log_service import * +from .webhook_transaction_log_service import * diff --git a/api/services/webhook_transaction_log_service.py b/api/services/webhook_transaction_log_service.py index 8ea3174..4d5b639 100644 --- a/api/services/webhook_transaction_log_service.py +++ b/api/services/webhook_transaction_log_service.py @@ -1,6 +1,6 @@ import json -from api import db, db_models +from api import models from api.helpers import db_helper from api.utils.loggingutils import logger @@ -9,7 +9,7 @@ class WebhookTransactionLogService: def create_new_webhook_log(self, jsonData): try: data = json.dumps(jsonData) - new_webhook_log = db_models.WebhookTransactionLog( + new_webhook_log = models.WebhookTransactionLog( payload=data, processed=False, attempts=0, diff --git a/api/utils/__init__.py b/api/utils/__init__.py index 03eb9ab..dedfb26 100644 --- a/api/utils/__init__.py +++ b/api/utils/__init__.py @@ -1 +1 @@ -from api.utils.loggingutils import * +from .loggingutils import * diff --git a/config.py b/config.py index ca151fb..e6b8487 100644 --- a/config.py +++ b/config.py @@ -26,7 +26,6 @@ SQLALCHEMY_DATABASE_URI = ( "postgresql://%(user)s:%(password)s@%(host)s:%(port)s/%(database)s" % POSTGRES ) -print(SQLALCHEMY_DATABASE_URI) # For socket based connection if FLASK_APP in ("production", "staging"): diff --git a/migrations/README b/migrations/README new file mode 100644 index 0000000..0e04844 --- /dev/null +++ b/migrations/README @@ -0,0 +1 @@ +Single-database configuration for Flask. diff --git a/migrations/alembic.ini b/migrations/alembic.ini new file mode 100644 index 0000000..ec9d45c --- /dev/null +++ b/migrations/alembic.ini @@ -0,0 +1,50 @@ +# A generic, single database configuration. + +[alembic] +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic,flask_migrate + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[logger_flask_migrate] +level = INFO +handlers = +qualname = flask_migrate + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 0000000..93471b1 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,108 @@ +import logging +from logging.config import fileConfig + +from alembic import context +from flask import current_app + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) +logger = logging.getLogger("alembic.env") + + +def get_engine(): + try: + # this works with Flask-SQLAlchemy<3 and Alchemical + return current_app.extensions["migrate"].db.get_engine() + except (TypeError, AttributeError): + # this works with Flask-SQLAlchemy>=3 + return current_app.extensions["migrate"].db.engine + + +def get_engine_url(): + try: + return get_engine().url.render_as_string(hide_password=False).replace("%", "%%") + except AttributeError: + return str(get_engine().url).replace("%", "%%") + + +# add your model's MetaData object here +# for 'autogenerate' support +from api import models + +target_metadata = models.Base.metadata +config.set_main_option("sqlalchemy.url", get_engine_url()) +target_db = current_app.extensions["migrate"].db + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def get_metadata(): + if hasattr(target_db, "metadatas"): + return target_db.metadatas[None] + return target_db.metadata + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure(url=url, target_metadata=get_metadata(), literal_binds=True) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html + def process_revision_directives(context, revision, directives): + if getattr(config.cmd_opts, "autogenerate", False): + script = directives[0] + if script.upgrade_ops.is_empty(): + directives[:] = [] + logger.info("No changes in schema detected.") + + conf_args = current_app.extensions["migrate"].configure_args + if conf_args.get("process_revision_directives") is None: + conf_args["process_revision_directives"] = process_revision_directives + + connectable = get_engine() + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=get_metadata(), **conf_args + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..2c01563 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/c4ae3a3aac32_initial_migration.py b/migrations/versions/c4ae3a3aac32_initial_migration.py new file mode 100644 index 0000000..aedbb23 --- /dev/null +++ b/migrations/versions/c4ae3a3aac32_initial_migration.py @@ -0,0 +1,37 @@ +"""Initial migration. + +Revision ID: c4ae3a3aac32 +Revises: +Create Date: 2024-04-18 14:12:03.526440 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "c4ae3a3aac32" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "webhook_transaction_log", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("payload", sa.Text(), nullable=True), + sa.Column("processed", sa.Boolean(), nullable=False), + sa.Column("attempts", sa.Integer(), nullable=False), + sa.Column("created_on", sa.DateTime(), nullable=True), + sa.Column("updated_on", sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("webhook_transaction_log") + # ### end Alembic commands ### From 63abb51e08d1b541230d656ab42904da5e8ee098 Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Thu, 18 Apr 2024 16:34:19 +0530 Subject: [PATCH 18/22] added more pre-commit hooks --- .github/workflows/pre-commit.yml | 4 +++- .pre-commit-config.yaml | 10 ++++++++++ api/helpers/db_helper.py | 10 +++++++--- api/mixins.py | 4 +++- .../webhook_transaction_log_service.py | 6 ++++-- config.py | 7 ++++--- main.py | 5 +++-- migrations/env.py | 19 ++++++++++++------- .../c4ae3a3aac32_initial_migration.py | 1 - 9 files changed, 46 insertions(+), 20 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index cac6e42..a133ba9 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -3,7 +3,9 @@ name: pre-commit on: pull_request: push: - branches: [main] + branches: + - develop + - main jobs: pre-commit: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4b85b3c..f8457b9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,3 +23,13 @@ repos: rev: 24.4.0 hooks: - id: black + args: [--line-length=79] +- repo: https://github.com/PyCQA/flake8 + rev: 7.0.0 + hooks: + - id: flake8 + exclude: __init__.py +- repo: https://github.com/PyCQA/docformatter + rev: v1.5.0 + hooks: + - id: docformatter diff --git a/api/helpers/db_helper.py b/api/helpers/db_helper.py index fd026af..b532bd4 100644 --- a/api/helpers/db_helper.py +++ b/api/helpers/db_helper.py @@ -12,13 +12,16 @@ def get_db_connection(connection_data={}): db_host = connection_data.get("db_host", app.config["DB_HOST"]) db_pwd = connection_data.get("db_pwd", app.config["DB_PWD"]) - db_conn_string = f"dbname={db_name} user={db_user} host={db_host} password={db_pwd}" + db_conn_string = ( + f"dbname={db_name} user={db_user} host={db_host} password={db_pwd}" + ) try: return psycopg2.connect(db_conn_string) except Exception as e: logger.error( - f"DB Helper: Unable to connect to the database: {db_conn_string}. Error message:{e}" + f"DB Helper: Unable to connect to the database: {db_conn_string}." + f"Error message:{e}" ) return None @@ -29,7 +32,8 @@ def save(data): db.session.commit() except Exception as e: logger.error( - f"Error occurred while committing the data in the database. Error message: {e}" + "Error occurred while committing the data in the database." + f"Error message: {e}" ) logger.debug(traceback.format_exc()) db.session.rollback() diff --git a/api/mixins.py b/api/mixins.py index 0e964af..886c46c 100644 --- a/api/mixins.py +++ b/api/mixins.py @@ -7,4 +7,6 @@ class TimestampMixin: created_on = db.Column(db.DateTime, default=datetime.now) - updated_on = db.Column(db.DateTime, onupdate=datetime.now, default=datetime.now) + updated_on = db.Column( + db.DateTime, onupdate=datetime.now, default=datetime.now + ) diff --git a/api/services/webhook_transaction_log_service.py b/api/services/webhook_transaction_log_service.py index 4d5b639..f9a2409 100644 --- a/api/services/webhook_transaction_log_service.py +++ b/api/services/webhook_transaction_log_service.py @@ -18,7 +18,8 @@ def create_new_webhook_log(self, jsonData): return new_webhook_log except Exception as e: logger.error( - f"Error while creating new webhook log. Webhook: {jsonData}. Error message: {e}" + f"Error while creating new webhook log. Webhook: {jsonData}." + f"Error message: {e}" ) return None @@ -28,6 +29,7 @@ def mark_webhook_log_as_processed(self, webhook_log): db_helper.save(webhook_log) except Exception as e: logger.error( - f"Error while marking webhook log as processed. Error message: {e}" + f"Error while marking webhook log as processed." + f"Error message: {e}" ) return None diff --git a/config.py b/config.py index e6b8487..e5c549c 100644 --- a/config.py +++ b/config.py @@ -20,16 +20,17 @@ "database": os.environ.get("DB_NAME"), "host": os.environ.get("DB_HOST"), "port": os.environ.get("DB_PORT"), - "connection_name": os.environ.get("CONNECTION_NAME"), + "conn_str": os.environ.get("CONNECTION_NAME"), } SQLALCHEMY_DATABASE_URI = ( - "postgresql://%(user)s:%(password)s@%(host)s:%(port)s/%(database)s" % POSTGRES + "postgresql://%(user)s:%(password)s@%(host)s:%(port)s/%(database)s" + % POSTGRES ) # For socket based connection if FLASK_APP in ("production", "staging"): SQLALCHEMY_DATABASE_URI = ( - "postgresql://%(user)s:%(password)s@/%(database)s?host=%(connection_name)s/" + "postgresql://%(user)s:%(password)s@/%(database)s?host=%(conn_str)s/" % POSTGRES ) diff --git a/main.py b/main.py index 6dc8e34..897bbdf 100644 --- a/main.py +++ b/main.py @@ -5,7 +5,7 @@ from api.utils.loggingutils import logger -### Endpoint for Cloud function +# Endpoint for Cloud function @functions_framework.http def handle_payload(request): if request.method == "POST": @@ -16,7 +16,8 @@ def handle_payload(request): handle_webhook(jsonData) except Exception as e: logger.error( - f"Exception occured while handling the webhook payload: {jsonData}. Error: {e}" + f"Exception while handling the webhook payload: {jsonData}" + f"Error: {e}" ) return "Success" else: diff --git a/migrations/env.py b/migrations/env.py index 93471b1..18e0bb4 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -4,6 +4,8 @@ from alembic import context from flask import current_app +from api import models + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config @@ -25,14 +27,17 @@ def get_engine(): def get_engine_url(): try: - return get_engine().url.render_as_string(hide_password=False).replace("%", "%%") + return ( + get_engine() + .url.render_as_string(hide_password=False) + .replace("%", "%%") + ) except AttributeError: return str(get_engine().url).replace("%", "%%") # add your model's MetaData object here # for 'autogenerate' support -from api import models target_metadata = models.Base.metadata config.set_main_option("sqlalchemy.url", get_engine_url()) @@ -60,10 +65,11 @@ def run_migrations_offline(): Calls to context.execute() here emit the given string to the script output. - """ url = config.get_main_option("sqlalchemy.url") - context.configure(url=url, target_metadata=get_metadata(), literal_binds=True) + context.configure( + url=url, target_metadata=get_metadata(), literal_binds=True + ) with context.begin_transaction(): context.run_migrations() @@ -72,9 +78,8 @@ def run_migrations_offline(): def run_migrations_online(): """Run migrations in 'online' mode. - In this scenario we need to create an Engine - and associate a connection with the context. - + In this scenario we need to create an Engine and associate a + connection with the context. """ # this callback is used to prevent an auto-migration from being generated diff --git a/migrations/versions/c4ae3a3aac32_initial_migration.py b/migrations/versions/c4ae3a3aac32_initial_migration.py index aedbb23..6e2f0c9 100644 --- a/migrations/versions/c4ae3a3aac32_initial_migration.py +++ b/migrations/versions/c4ae3a3aac32_initial_migration.py @@ -3,7 +3,6 @@ Revision ID: c4ae3a3aac32 Revises: Create Date: 2024-04-18 14:12:03.526440 - """ import sqlalchemy as sa From aef62ff086b3eedf4e33c33705c3b7d2d2164efe Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Thu, 18 Apr 2024 16:37:38 +0530 Subject: [PATCH 19/22] updated readme.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 5136014..4cef832 100644 --- a/README.md +++ b/README.md @@ -19,9 +19,15 @@ Handling and processing Incoming webhook request configured at Glific. python -m venv venv ``` 3. Activate the virtual environment + + For mac-os: ```sh source ./venv/bin/activate ``` + For win os: + ```sh + venv\Scripts\Activate.ps1 + ``` 4. Install the dependencies: ```sh pip install -r requirements.txt From db8d89e45e90139435acad5848415c4ec2dcefda Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Thu, 18 Apr 2024 16:49:25 +0530 Subject: [PATCH 20/22] updated readme.md --- README.md | 24 ++++++++++++++++++++---- api/helpers/__init__.py | 2 +- api/helpers/db_helper.py | 24 +----------------------- 3 files changed, 22 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 4cef832..ebc1495 100644 --- a/README.md +++ b/README.md @@ -20,11 +20,11 @@ Handling and processing Incoming webhook request configured at Glific. ``` 3. Activate the virtual environment - For mac-os: + **For mac-os:** ```sh source ./venv/bin/activate ``` - For win os: + **For win os:** ```sh venv\Scripts\Activate.ps1 ``` @@ -37,11 +37,27 @@ Handling and processing Incoming webhook request configured at Glific. cp .env.example .env ``` 6. Add/update variables in your `.env` file for your environment. -7. Run the following command to get started with pre-commit +7. Run these commands to add environment variables in the system. + + **For Windows** + ```sh + $env:FLASK_APP="manage.py" + $env:PYTHONPATH="" + ``` + **For Mac** + ```sh + export FLASK_APP=manage.py + export PYTHONPATH=path-of-the-project + ``` +8. Upgrade DB to the latest version using this command. + ```sh + flask db upgrade + ``` +9. Run the following command to get started with pre-commit ```sh pre-commit install ``` -8. Start the server by following command +10. Start the server by following command ```sh functions_framework --target=handle_payload --debug ``` diff --git a/api/helpers/__init__.py b/api/helpers/__init__.py index 101f855..e91aa3c 100644 --- a/api/helpers/__init__.py +++ b/api/helpers/__init__.py @@ -1 +1 @@ -from api.helpers.db_helper import * +from .db_helper import * diff --git a/api/helpers/db_helper.py b/api/helpers/db_helper.py index b532bd4..45648d2 100644 --- a/api/helpers/db_helper.py +++ b/api/helpers/db_helper.py @@ -1,31 +1,9 @@ import traceback -import psycopg2 - -from api import app, db +from api import db from api.utils.loggingutils import logger -def get_db_connection(connection_data={}): - db_name = connection_data.get("db_name", app.config["DB_NAME"]) - db_user = connection_data.get("db_user", app.config["DB_USER"]) - db_host = connection_data.get("db_host", app.config["DB_HOST"]) - db_pwd = connection_data.get("db_pwd", app.config["DB_PWD"]) - - db_conn_string = ( - f"dbname={db_name} user={db_user} host={db_host} password={db_pwd}" - ) - - try: - return psycopg2.connect(db_conn_string) - except Exception as e: - logger.error( - f"DB Helper: Unable to connect to the database: {db_conn_string}." - f"Error message:{e}" - ) - return None - - def save(data): try: db.session.add(data) From c48fa2b9961d31dcf5fde16cf2f232a2db74d9b4 Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Thu, 18 Apr 2024 17:30:08 +0530 Subject: [PATCH 21/22] updated suggested chages --- .env.example | 1 + .github/workflows/pre-commit.yml | 2 +- README.md | 8 ++--- api/helpers/__init__.py | 1 - api/helpers/db_helper.py | 17 --------- .../webhook_transaction_log_service.py | 6 ++-- api/utils/__init__.py | 1 + api/utils/loggingutils.py | 3 +- config.py | 7 ++-- .../c4ae3a3aac32_initial_migration.py | 36 ------------------- 10 files changed, 15 insertions(+), 67 deletions(-) delete mode 100644 api/helpers/__init__.py delete mode 100644 api/helpers/db_helper.py delete mode 100644 migrations/versions/c4ae3a3aac32_initial_migration.py diff --git a/.env.example b/.env.example index 62cd471..166f935 100644 --- a/.env.example +++ b/.env.example @@ -4,3 +4,4 @@ DB_PASSWORD= DB_NAME= DB_HOST= DB_PORT= +LOGGING_LEVEL= diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index a133ba9..ad1a6ee 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -14,5 +14,5 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: - python-version: '3.9' + python-version: '3.12.1' - uses: pre-commit/action@v2.0.0 diff --git a/README.md b/README.md index ebc1495..fd429c5 100644 --- a/README.md +++ b/README.md @@ -20,13 +20,13 @@ Handling and processing Incoming webhook request configured at Glific. ``` 3. Activate the virtual environment - **For mac-os:** + **For Windows** ```sh - source ./venv/bin/activate + venv\Scripts\Activate.ps1 ``` - **For win os:** + **For Mac** ```sh - venv\Scripts\Activate.ps1 + source ./venv/bin/activate ``` 4. Install the dependencies: ```sh diff --git a/api/helpers/__init__.py b/api/helpers/__init__.py deleted file mode 100644 index e91aa3c..0000000 --- a/api/helpers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .db_helper import * diff --git a/api/helpers/db_helper.py b/api/helpers/db_helper.py deleted file mode 100644 index 45648d2..0000000 --- a/api/helpers/db_helper.py +++ /dev/null @@ -1,17 +0,0 @@ -import traceback - -from api import db -from api.utils.loggingutils import logger - - -def save(data): - try: - db.session.add(data) - db.session.commit() - except Exception as e: - logger.error( - "Error occurred while committing the data in the database." - f"Error message: {e}" - ) - logger.debug(traceback.format_exc()) - db.session.rollback() diff --git a/api/services/webhook_transaction_log_service.py b/api/services/webhook_transaction_log_service.py index f9a2409..b3cb880 100644 --- a/api/services/webhook_transaction_log_service.py +++ b/api/services/webhook_transaction_log_service.py @@ -1,7 +1,7 @@ import json from api import models -from api.helpers import db_helper +from api.utils import db_utils from api.utils.loggingutils import logger @@ -14,7 +14,7 @@ def create_new_webhook_log(self, jsonData): processed=False, attempts=0, ) - db_helper.save(new_webhook_log) + db_utils.save(new_webhook_log) return new_webhook_log except Exception as e: logger.error( @@ -26,7 +26,7 @@ def create_new_webhook_log(self, jsonData): def mark_webhook_log_as_processed(self, webhook_log): try: webhook_log.processed = True - db_helper.save(webhook_log) + db_utils.save(webhook_log) except Exception as e: logger.error( f"Error while marking webhook log as processed." diff --git a/api/utils/__init__.py b/api/utils/__init__.py index dedfb26..e9792ab 100644 --- a/api/utils/__init__.py +++ b/api/utils/__init__.py @@ -1 +1,2 @@ +from .db_utils import * from .loggingutils import * diff --git a/api/utils/loggingutils.py b/api/utils/loggingutils.py index e872aaa..87ba28a 100644 --- a/api/utils/loggingutils.py +++ b/api/utils/loggingutils.py @@ -4,9 +4,10 @@ from google.cloud import logging as gcloud_logging from api import app +from config import LOGGING_LEVEL logger = logging.getLogger() -logging.basicConfig(level=logging.DEBUG) +logging.basicConfig(level=LOGGING_LEVEL) if os.environ.get("FLASK_ENV", "development"): log_handler = logger.handlers[0] diff --git a/config.py b/config.py index e5c549c..ffa94d8 100644 --- a/config.py +++ b/config.py @@ -5,12 +5,9 @@ FLASK_APP = os.environ.get("FLASK_APP", "development") if FLASK_APP == "development": - from os import path - from dotenv import load_dotenv - basedir = path.abspath(path.dirname(__file__)) - load_dotenv(path.join(basedir, ".env")) + load_dotenv() # Database configuration @@ -34,3 +31,5 @@ "postgresql://%(user)s:%(password)s@/%(database)s?host=%(conn_str)s/" % POSTGRES ) + +LOGGING_LEVEL = os.environ.get("LOGGING_LEVEL", "DEBUG") diff --git a/migrations/versions/c4ae3a3aac32_initial_migration.py b/migrations/versions/c4ae3a3aac32_initial_migration.py deleted file mode 100644 index 6e2f0c9..0000000 --- a/migrations/versions/c4ae3a3aac32_initial_migration.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Initial migration. - -Revision ID: c4ae3a3aac32 -Revises: -Create Date: 2024-04-18 14:12:03.526440 -""" - -import sqlalchemy as sa -from alembic import op - -# revision identifiers, used by Alembic. -revision = "c4ae3a3aac32" -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "webhook_transaction_log", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("payload", sa.Text(), nullable=True), - sa.Column("processed", sa.Boolean(), nullable=False), - sa.Column("attempts", sa.Integer(), nullable=False), - sa.Column("created_on", sa.DateTime(), nullable=True), - sa.Column("updated_on", sa.DateTime(), nullable=True), - sa.PrimaryKeyConstraint("id"), - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table("webhook_transaction_log") - # ### end Alembic commands ### From 5c47b4be2ed45ea1bed2d812ec082c1e67cbbc38 Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Thu, 18 Apr 2024 17:30:39 +0530 Subject: [PATCH 22/22] updated suggested chages --- api/utils/db_utils.py | 17 +++++++++ .../70bea50174bf_initial_migration.py | 36 +++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 api/utils/db_utils.py create mode 100644 migrations/versions/70bea50174bf_initial_migration.py diff --git a/api/utils/db_utils.py b/api/utils/db_utils.py new file mode 100644 index 0000000..45648d2 --- /dev/null +++ b/api/utils/db_utils.py @@ -0,0 +1,17 @@ +import traceback + +from api import db +from api.utils.loggingutils import logger + + +def save(data): + try: + db.session.add(data) + db.session.commit() + except Exception as e: + logger.error( + "Error occurred while committing the data in the database." + f"Error message: {e}" + ) + logger.debug(traceback.format_exc()) + db.session.rollback() diff --git a/migrations/versions/70bea50174bf_initial_migration.py b/migrations/versions/70bea50174bf_initial_migration.py new file mode 100644 index 0000000..992373d --- /dev/null +++ b/migrations/versions/70bea50174bf_initial_migration.py @@ -0,0 +1,36 @@ +"""Initial migration. + +Revision ID: 70bea50174bf +Revises: +Create Date: 2024-04-18 17:18:19.882178 +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "70bea50174bf" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "webhook_transaction_log", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("payload", sa.Text(), nullable=True), + sa.Column("processed", sa.Boolean(), nullable=False), + sa.Column("attempts", sa.Integer(), nullable=False), + sa.Column("created_on", sa.DateTime(), nullable=True), + sa.Column("updated_on", sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("webhook_transaction_log") + # ### end Alembic commands ###