Scaffolding of a FLASK app for WhatsApp analytics

.env.example

@@ -0,0 +1,6 @@
+ENVIRONMENT=
+DB_USER=
+DB_PASSWORD=
+DB_NAME=
+DB_HOST=
+DB_PORT=

.github/workflows/pre-commit.yml

@@ -0,0 +1,16 @@
+name: pre-commit
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+    - uses: pre-commit/[email protected]

.gitignore

@@ -0,0 +1,10 @@
+*.pyc
+__pycache__/
+.mypy_cache
+
+# Environments
+.env
+.venv
+env/
+venv/
+alembic.ini

.pre-commit-config.yaml

@@ -0,0 +1,25 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-json
+    -   id: check-yaml
+    -   id: check-merge-conflict
+    -   id: check-added-large-files
+    -   id: debug-statements
+    -   id: requirements-txt-fixer
+- repo: https://github.com/pre-commit/mirrors-isort
+  rev: 'v5.10.1'
+  hooks:
+    - id: isort
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: 'v1.9.0'
+  hooks:
+    - id: mypy
+      exclude: alembic
+-   repo: https://github.com/psf/black
+    rev: 24.4.0
+    hooks:
+    -   id: black

README.md

@@ -1,3 +1,41 @@
-### WhatsApp Webhook Analytics 
+### WhatsApp Webhook Analytics
 
-Handling and processing Incoming webhook request configured at Glific.
+Handling and processing Incoming webhook request configured at Glific.
+
+## Installation
+
+### Prerequisite
+1. pyenv
+2. python 3.12
+
+### Steps
+1. Clone the repository
+    ```sh
+    git clone https://github.com/DostEducation/whatsapp-webhook-analytics.git
+    ```
+2. Switch to project folder and setup the vertual environment
+    ```sh
+    cd whatsapp-webhook-analytics
+    python -m venv venv
+    ```
+3. Activate the virtual environment
+    ```sh
+    source ./venv/bin/activate
+    ```
+4. Install the dependencies:
+    ```sh
+    pip install -r requirements.txt
+    ```
+5. Set up your .env file by copying .env.example
+    ```sh
+    cp .env.example .env
+    ```
+6. Add/update variables in your `.env` file for your environment.
+7. Run the following command to get started with pre-commit
+    ```sh
+    pre-commit install
+    ```
+8. Start the server by following command
+    ```sh
+    functions_framework --target=handle_payload --debug
+    ```

alembic/README

@@ -0,0 +1 @@
+Generic single-database configuration.

alembic/env.py

@@ -0,0 +1,75 @@
+from logging.config import fileConfig
+
+from sqlalchemy import engine_from_config, pool
+
+from alembic import context
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+from app import models
+
+target_metadata = models.Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(connection=connection, target_metadata=target_metadata)
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

alembic/script.py.mako

@@ -0,0 +1,26 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

alembic/versions/19b23a59f196_initial_migrations.py

@@ -0,0 +1,40 @@
+"""initial migrations
+
+Revision ID: 19b23a59f196
+Revises:
+Create Date: 2024-04-15 18:22:25.267651
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "19b23a59f196"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "webhook_transaction_log",
+        sa.Column("created_on", sa.DateTime(), nullable=True),
+        sa.Column("updated_on", sa.DateTime(), nullable=True),
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("payload", sa.Text(), nullable=True),
+        sa.Column("processed", sa.Boolean(), nullable=False),
+        sa.Column("attempts", sa.Integer(), nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("webhook_transaction_log")
+    # ### end Alembic commands ###

app/__init__.py

@@ -0,0 +1 @@
+from app.mixins import *

app/helpers/__init__.py

@@ -0,0 +1 @@
+from app.helpers.db_helper import *

app/helpers/db_helper.py

@@ -0,0 +1,10 @@
+from sqlalchemy import create_engine
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import sessionmaker
+
+from config import SQLALCHEMY_DATABASE_URL
+
+engine = create_engine(SQLALCHEMY_DATABASE_URL)
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+Base = declarative_base()
+db = SessionLocal()

app/mixins.py

@@ -0,0 +1,13 @@
+from datetime import datetime
+
+from sqlalchemy import Column, DateTime
+from sqlalchemy.ext.declarative import declared_attr
+
+
+class TimestampMixin:
+    created_on = Column(DateTime, default=datetime.utcnow)
+    updated_on = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+
+    @declared_attr
+    def __tablename__(cls):
+        return cls.__name__.lower()

app/models/__init__.py

@@ -0,0 +1,2 @@
+from app.models.base import *
+from app.models.webhook_transaction_log import *

app/models/base.py

@@ -0,0 +1,3 @@
+from sqlalchemy.ext.declarative import declarative_base
+
+Base = declarative_base()

app/models/webhook_transaction_log.py

@@ -0,0 +1,13 @@
+from sqlalchemy import Boolean, Column, Integer, Text
+
+from app import TimestampMixin
+from app.models.base import Base
+
+
+class WebhookTransactionLog(TimestampMixin, Base):
+    __tablename__ = "webhook_transaction_log"
+
+    id = Column(Integer, primary_key=True)
+    payload = Column(Text)
+    processed = Column(Boolean, nullable=False)
+    attempts = Column(Integer, nullable=False, default=0)

app/services/__init__.py

@@ -0,0 +1 @@
+from app.services.webhook_transaction_log_service import *

app/services/webhook_transaction_log_service.py

@@ -0,0 +1,37 @@
+import json
+
+from app import models
+from app.helpers import db
+from app.utils.loggingutils import logger
+
+
+class WebhookTransactionLogService:
+    def create_new_webhook_log(self, jsonData):
+        try:
+            data = json.dumps(jsonData)
+            new_webhook_log = models.WebhookTransactionLog(
+                payload=data,
+                processed=False,
+                attempts=0,
+            )
+            db.add(new_webhook_log)
+            db.commit()
+            return new_webhook_log
+        except Exception as e:
+            logger.error(
+                f"Error while creating new webhook log. Webhook: {jsonData}. Error message: {e}"
+            )
+        finally:
+            db.close()
+
+    def mark_webhook_log_as_processed(self, webhook_log):
+        try:
+            webhook_log.processed = True
+            db.add(webhook_log)
+            db.commit()
+        except Exception as e:
+            logger.error(
+                f"Error while marking webhook log as processed. Error message: {e}"
+            )
+        finally:
+            db.close()

app/utils/__init__.py

@@ -0,0 +1 @@
+from app.utils.loggingutils import *

app/utils/loggingutils.py

@@ -0,0 +1,17 @@
+import logging
+
+from google.cloud import logging as gcloud_logging
+
+from config import ENVIRONMENT
+
+logger = logging.getLogger()
+logging.basicConfig(level=logging.DEBUG)
+
+if ENVIRONMENT == "development":
+    log_handler = logger.handlers[0]
+    logger.addHandler(log_handler)
+else:
+    log_client = gcloud_logging.Client()
+    log_client.setup_logging()
+    log_handler = log_client.get_default_handler()
+    logger.addHandler(log_handler)

config.py

@@ -0,0 +1,20 @@
+import os
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+ENVIRONMENT = os.environ.get("ENVIRONMENT", "development")
+
+# Database configuration
+POSTGRES = {
+    "user": os.getenv("DB_USER"),
+    "password": os.getenv("DB_PASSWORD"),
+    "database": os.getenv("DB_NAME"),
+    "host": os.getenv("DB_HOST"),
+    "port": os.getenv("DB_PORT"),
+}
+
+SQLALCHEMY_DATABASE_URL = (
+    "postgresql://%(user)s:%(password)s@%(host)s:%(port)s/%(database)s" % POSTGRES
+)