Scaffolding of a FLASK app for WhatsApp analytics

.env.example

@@ -0,0 +1,7 @@
+FLASK_APP=
+DB_USER=
+DB_PASSWORD=
+DB_NAME=
+DB_HOST=
+DB_PORT=
+LOGGING_LEVEL=

.github/workflows/pre-commit.yml

@@ -0,0 +1,18 @@
+name: pre-commit
+
+on:
+  pull_request:
+  push:
+    branches:
+      - develop
+      - main
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.12.1'
+    - uses: pre-commit/[email protected]

.gitignore

@@ -0,0 +1,9 @@
+*.pyc
+__pycache__/
+.mypy_cache
+
+# Environments
+.env
+.venv
+env/
+venv/

.pre-commit-config.yaml

@@ -0,0 +1,35 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-json
+    -   id: check-yaml
+    -   id: check-merge-conflict
+    -   id: check-added-large-files
+    -   id: debug-statements
+    -   id: requirements-txt-fixer
+- repo: https://github.com/pre-commit/mirrors-isort
+  rev: 'v5.10.1'
+  hooks:
+    - id: isort
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: 'v1.9.0'
+  hooks:
+    - id: mypy
+      exclude: alembic
+-   repo: https://github.com/psf/black
+    rev: 24.4.0
+    hooks:
+    -   id: black
+        args: [--line-length=79]
+-   repo: https://github.com/PyCQA/flake8
+    rev: 7.0.0
+    hooks:
+    -   id: flake8
+        exclude: __init__.py
+- repo: https://github.com/PyCQA/docformatter
+  rev: v1.5.0
+  hooks:
+    - id: docformatter

README.md

@@ -1,3 +1,63 @@
-### WhatsApp Webhook Analytics 
+### WhatsApp Webhook Analytics
 
-Handling and processing Incoming webhook request configured at Glific.
+Handling and processing Incoming webhook request configured at Glific.
+
+## Installation
+
+### Prerequisite
+1. pyenv
+2. python 3.12
+
+### Steps
+1. Clone the repository
+    ```sh
+    git clone https://github.com/DostEducation/whatsapp-webhook-analytics.git
+    ```
+2. Switch to project folder and setup the vertual environment
+    ```sh
+    cd whatsapp-webhook-analytics
+    python -m venv venv
+    ```
+3. Activate the virtual environment
+
+    **For Windows**
+    ```sh
+    venv\Scripts\Activate.ps1
+    ```
+    **For Mac**
+    ```sh
+    source ./venv/bin/activate
+    ```
+4. Install the dependencies:
+    ```sh
+    pip install -r requirements.txt
+    ```
+5. Set up your .env file by copying .env.example
+    ```sh
+    cp .env.example .env
+    ```
+6. Add/update variables in your `.env` file for your environment.
+7. Run these commands to add environment variables in the system.
+
+   **For Windows**
+    ```sh
+    $env:FLASK_APP="manage.py"
+    $env:PYTHONPATH="<Path of your project, eg: C:\Users\whatsapp-webhook-analytics>"
+    ```
+    **For Mac**
+    ```sh
+    export FLASK_APP=manage.py
+    export PYTHONPATH=path-of-the-project
+    ```
+8. Upgrade DB to the latest version using this command.
+    ```sh
+     flask db upgrade
+    ```
+9. Run the following command to get started with pre-commit
+    ```sh
+    pre-commit install
+    ```
+10. Start the server by following command
+    ```sh
+    functions_framework --target=handle_payload --debug
+    ```

api/__init__.py

@@ -0,0 +1,6 @@
+from flask import Flask
+from flask_sqlalchemy import SQLAlchemy
+
+app = Flask(__name__)
+app.config.from_object("config")
+db = SQLAlchemy(app)

api/mixins.py

@@ -0,0 +1,12 @@
+from __future__ import absolute_import
+
+from datetime import datetime
+
+from api import db
+
+
+class TimestampMixin:
+    created_on = db.Column(db.DateTime, default=datetime.now)
+    updated_on = db.Column(
+        db.DateTime, onupdate=datetime.now, default=datetime.now
+    )

api/models/__init__.py

@@ -0,0 +1,5 @@
+from sqlalchemy.ext.declarative import declarative_base
+
+Base = declarative_base()
+
+from .webhook_transaction_log import *

api/models/webhook_transaction_log.py

@@ -0,0 +1,11 @@
+from api import db
+from api.mixins import TimestampMixin
+
+
+class WebhookTransactionLog(TimestampMixin, db.Model):
+
+    __tablename__ = "webhook_transaction_log"
+    id = db.Column(db.Integer, primary_key=True)
+    payload = db.Column(db.Text)
+    processed = db.Column(db.Boolean, nullable=False)
+    attempts = db.Column(db.Integer, nullable=False, default="0")

api/services/__init__.py

@@ -0,0 +1 @@
+from .webhook_transaction_log_service import *

api/services/webhook_transaction_log_service.py

@@ -0,0 +1,35 @@
+import json
+
+from api import models
+from api.utils import db_utils
+from api.utils.loggingutils import logger
+
+
+class WebhookTransactionLogService:
+    def create_new_webhook_log(self, jsonData):
+        try:
+            data = json.dumps(jsonData)
+            new_webhook_log = models.WebhookTransactionLog(
+                payload=data,
+                processed=False,
+                attempts=0,
+            )
+            db_utils.save(new_webhook_log)
+            return new_webhook_log
+        except Exception as e:
+            logger.error(
+                f"Error while creating new webhook log. Webhook: {jsonData}."
+                f"Error message: {e}"
+            )
+            return None
+
+    def mark_webhook_log_as_processed(self, webhook_log):
+        try:
+            webhook_log.processed = True
+            db_utils.save(webhook_log)
+        except Exception as e:
+            logger.error(
+                f"Error while marking webhook log as processed."
+                f"Error message: {e}"
+            )
+            return None

api/utils/__init__.py

@@ -0,0 +1,2 @@
+from .db_utils import *
+from .loggingutils import *

api/utils/db_utils.py

@@ -0,0 +1,17 @@
+import traceback
+
+from api import db
+from api.utils.loggingutils import logger
+
+
+def save(data):
+    try:
+        db.session.add(data)
+        db.session.commit()
+    except Exception as e:
+        logger.error(
+            "Error occurred while committing the data in the database."
+            f"Error message: {e}"
+        )
+        logger.debug(traceback.format_exc())
+        db.session.rollback()

api/utils/loggingutils.py

@@ -0,0 +1,19 @@
+import logging
+import os
+
+from google.cloud import logging as gcloud_logging
+
+from api import app
+from config import LOGGING_LEVEL
+
+logger = logging.getLogger()
+logging.basicConfig(level=LOGGING_LEVEL)
+
+if os.environ.get("FLASK_ENV", "development"):
+    log_handler = logger.handlers[0]
+    logger.addHandler(log_handler)
+else:
+    log_client = gcloud_logging.Client()
+    log_client.setup_logging()
+    log_handler = log_client.get_default_handler()
+    app.logger.addHandler(log_handler)

config.py

@@ -0,0 +1,35 @@
+"""Flask configuration."""
+
+import os
+
+FLASK_APP = os.environ.get("FLASK_APP", "development")
+
+if FLASK_APP == "development":
+    from dotenv import load_dotenv
+
+    load_dotenv()
+
+
+# Database configuration
+POSTGRES = {
+    "user": os.environ.get("DB_USER"),
+    "password": os.environ.get("DB_PASSWORD"),
+    "database": os.environ.get("DB_NAME"),
+    "host": os.environ.get("DB_HOST"),
+    "port": os.environ.get("DB_PORT"),
+    "conn_str": os.environ.get("CONNECTION_NAME"),
+}
+
+SQLALCHEMY_DATABASE_URI = (
+    "postgresql://%(user)s:%(password)s@%(host)s:%(port)s/%(database)s"
+    % POSTGRES
+)
+
+# For socket based connection
+if FLASK_APP in ("production", "staging"):
+    SQLALCHEMY_DATABASE_URI = (
+        "postgresql://%(user)s:%(password)s@/%(database)s?host=%(conn_str)s/"
+        % POSTGRES
+    )
+
+LOGGING_LEVEL = os.environ.get("LOGGING_LEVEL", "DEBUG")

main.py

@@ -0,0 +1,30 @@
+import functions_framework
+
+from api import app
+from api.services import WebhookTransactionLogService
+from api.utils.loggingutils import logger
+
+
+# Endpoint for Cloud function
+@functions_framework.http
+def handle_payload(request):
+    if request.method == "POST":
+        with app.app_context():
+            try:
+                jsonData = request.get_json()
+                if jsonData:
+                    handle_webhook(jsonData)
+            except Exception as e:
+                logger.error(
+                    f"Exception while handling the webhook payload: {jsonData}"
+                    f"Error: {e}"
+                )
+            return "Success"
+    else:
+        return "Currently, the system does not accept a GET request"
+
+
+def handle_webhook(jsonData):
+    transaction_log_service = WebhookTransactionLogService()
+    webhook_log = transaction_log_service.create_new_webhook_log(jsonData)
+    transaction_log_service.mark_webhook_log_as_processed(webhook_log)

manage.py

@@ -0,0 +1,12 @@
+from __future__ import absolute_import
+
+from flask.cli import FlaskGroup
+from flask_migrate import Migrate
+
+from api import app, db
+
+migrate = Migrate(app, db)
+cli = FlaskGroup(app)
+
+if __name__ == "__main__":
+    cli()

migrations/README

@@ -0,0 +1 @@
+Single-database configuration for Flask.

migrations/alembic.ini

@@ -0,0 +1,50 @@
+# A generic, single database configuration.
+
+[alembic]
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic,flask_migrate
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[logger_flask_migrate]
+level = INFO
+handlers =
+qualname = flask_migrate
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S