From 9ecdfce99fb6fa0dab485d17cc0e16a4466e12fb Mon Sep 17 00:00:00 2001 From: Sachin Bisht Date: Tue, 5 Mar 2024 11:37:56 +0530 Subject: [PATCH] Added some more linting hooks in the pre-commit --- .env.example | 12 ++--- .github/workflows/lint.yml | 30 ----------- .github/workflows/main.yml | 11 ++-- .pre-commit-config.yaml | 52 ++++++++++++++----- README.md | 51 +++++++++--------- SLACKAPP.md | 27 ++++------ .../bigquery_helper/bigquery_helper.py | 1 - app/helpers/notification_helper.py | 6 +++ app/services/__init__.py | 2 +- app/services/budget_service.py | 18 +++---- app/services/slack_service.py | 6 ++- config.py | 1 + main.py | 24 ++++++--- requirements-dev.txt | 6 +-- requirements.txt | 3 +- utils/loggingutils.py | 4 +- 16 files changed, 132 insertions(+), 122 deletions(-) delete mode 100644 .github/workflows/lint.yml diff --git a/.env.example b/.env.example index 11b46a8..096154d 100644 --- a/.env.example +++ b/.env.example @@ -1,10 +1,10 @@ # Add env variables here -TESTING= +DATASET_ID= +DATASET_LOCATION= DEBUG= -SLACK_TOKEN= -SLACK_CHANNEL= ENVIRONMENT= -DATASET_ID= -TABLE_ID= GOOGLE_APPLICATION_CREDENTIALS= -DATASET_LOCATION= +SLACK_CHANNEL= +SLACK_TOKEN= +TABLE_ID= +TESTING= diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 94ef245..0000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,30 +0,0 @@ - name: Lint - - on: # yamllint disable-line rule:truthy - push: null - pull_request: null - - jobs: - build: - name: Lint - runs-on: ubuntu-latest - - permissions: - contents: read - packages: read - # To report GitHub Actions status checks - statuses: write - - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - # super-linter needs the full git history to get the - # list of files that changed across commits - fetch-depth: 0 - - - name: Super-linter - uses: super-linter/super-linter@v6.3.0 # x-release-please-version - env: - # To report GitHub Actions status checks - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3c4384e..e3234bc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,8 +6,6 @@ on: - opened - synchronize -permissions: write-all - jobs: review: runs-on: ubuntu-latest @@ -24,17 +22,16 @@ jobs: OPENAI_API_MODEL: "gpt-4" exclude: "**/*.json, **/*.md" - - name: Check for Failures - id: check-failures - run: echo ::set-output name=failures::$(echo "${{ steps.ai-review.outputs.review_result }}" | jq -r '.failures') + - name: Set Failures Output + run: echo "failures=${{ steps.ai-review.outputs.review_result }}" | jq -r '.failures' >> $GITHUB_ENV - name: Add Comment on Failure - if: steps.check-failures.outputs.failures != null + if: ${{ steps.ai-review.outputs.review_result != '{}' }} uses: actions/github-script@v4 with: github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const failures = JSON.parse('${{ steps.check-failures.outputs.failures }}'); + const failures = JSON.parse(process.env.failures); const issueBody = "### AI Code Review Failures\n\n"; failures.forEach(failure => { issueBody += `- ${failure}\n`; diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8b33f4f..72097ce 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,14 +1,40 @@ +exclude: "^\ + (third-party/.*)\ + " + repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 - hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-json - - id: check-yaml - - id: check-merge-conflict - - id: check-added-large-files -- repo: https://github.com/psf/black - rev: 22.3.0 - hooks: - - id: black + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-added-large-files # prevents giant files from being committed. + - id: check-case-conflict # checks for files that would conflict in case-insensitive filesystems. + - id: check-merge-conflict # checks for files that contain merge conflict strings. + - id: check-yaml # checks yaml files for parseable syntax. + - id: detect-private-key # detects the presence of private keys. + - id: end-of-file-fixer # ensures that a file is either empty, or ends with one newline. + - id: fix-byte-order-marker # removes utf-8 byte order marker. + - id: mixed-line-ending # replaces or checks mixed line ending. + - id: requirements-txt-fixer # sorts entries in requirements.txt. + - id: trailing-whitespace # trims trailing whitespace. + + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v4.0.0-alpha.8 + hooks: + - id: prettier + files: \.(js|ts|jsx|tsx|css|less|html|json|markdown|md|yaml|yml)$ + + - repo: https://github.com/psf/black + rev: 24.2.0 + hooks: + - id: black + + - repo: https://github.com/PyCQA/isort + rev: 5.13.2 + hooks: + - id: isort + args: [--profile=black] + + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v17.0.6 + hooks: + - id: clang-format diff --git a/README.md b/README.md index fc316d6..618e3d6 100644 --- a/README.md +++ b/README.md @@ -1,47 +1,50 @@ # gcp-budget-alerts-service -The micro-service is for tracking and monitoring the spending amount at the GCP service. This micro-service elevates an alert whenever the overall budget surpasses or meets predefined thresholds. + +The microservice is for tracking and monitoring the spending amount at the GCP service. This microservice elevates an alert whenever the overall budget surpasses or meets predefined thresholds. ## Architecture ![_ Expected architecture for the Alert service](https://github.com/Sachinbisht27/gcp-budget-alerts-service/assets/96137915/86e87e12-824a-46a9-ba95-07edca694285) -
Architecture for the Alert Service
+
Architecture for the Alert Service
## Installation Guideline ### Prerequisite + 1. pyenv 2. python 3.8 3. A Slack App for delivering messages on the channel. [Setup Guideline](./SLACKAPP.md) ### Steps + 1. Clone the repository - ```sh - git clone https://github.com/Sachinbisht27/gcp-budget-alerts-service.git - ``` + ```sh + git clone https://github.com/Sachinbisht27/gcp-budget-alerts-service.git + ``` 2. Switch to project folder and setup the vertual environment - ```sh - cd gcp-alerts - python -m venv venv - ``` + ```sh + cd gcp-alerts + python -m venv venv + ``` 3. Activate the virtual environment - ```sh - source ./venv/bin/activate - ``` + ```sh + source ./venv/bin/activate + ``` 4. Install the dependencies: - ```sh - pip install -r requirements-dev.txt - ``` + ```sh + pip install -r requirements-dev.txt + ``` 5. Set up your .env file by copying .env.example - ```sh - cp .env.example .env - ``` + ```sh + cp .env.example .env + ``` 6. Add/update variables in your `.env` file for your environment. 7. Run the following command to get started with pre-commit - ```sh - pre-commit install - ``` + ```sh + pre-commit install + ``` 8. Start the server by following command - ```sh - functions_framework --target=handle --debug - ``` + ```sh + functions_framework --target=handle --debug + ``` diff --git a/SLACKAPP.md b/SLACKAPP.md index c36faf8..ef26a98 100644 --- a/SLACKAPP.md +++ b/SLACKAPP.md @@ -1,58 +1,49 @@ # Create a Slack App -# Steps to create an app on Slack +## Steps to create an app on Slack 1. Go to the channel on which you want to install the app 2. Go to the view all members icon on the top right of the channel - + ![image](https://github.com/Sachinbisht27/gcp-budget-alerts-service/assets/96137915/13cc0d92-076e-4fd5-b985-fb5102efac79) - 3. Click on **add app** option 4. Click on **View app directory** 5. Click on the **build** option at the top right of the menu 6. Create an app from scratch 7. Provide a name to the app and assign it to a workspace 8. Go to OAuth and Permission menu - + ![image (9)](https://github.com/Sachinbisht27/gcp-budget-alerts-service/assets/96137915/252b96f9-ce02-4c74-bcdc-0527b01c3742) - 9. Under the scope section, give the app permission to write messages - + ![image (10)](https://github.com/Sachinbisht27/gcp-budget-alerts-service/assets/96137915/6183d4aa-4577-4fe8-af35-5c8baf28d93c) - 10. Install the app to the workspace - + ![image (11)](https://github.com/Sachinbisht27/gcp-budget-alerts-service/assets/96137915/4b75f0a7-0c34-4abe-b31d-120ed99d8679) - 11. Give permission to the app - + ![image (12)](https://github.com/Sachinbisht27/gcp-budget-alerts-service/assets/96137915/1ef99851-9258-4c87-88f3-22dd7971d665) - 12. Once the app is installed copy the OAuth Token as it will be used for authentication. - + ![image (14)](https://github.com/Sachinbisht27/gcp-budget-alerts-service/assets/96137915/355d7d13-9cef-4d6b-9a6d-894c96685e95) - 13. Go to the channel where you want to add app. 14. Click on the `channel name` . 15. Click on the integrations tab here. - + ![Untitled](https://github.com/Sachinbisht27/gcp-budget-alerts-service/assets/96137915/cc301091-2403-4541-ad55-305224bc7f7c) - 16. Then click on the `Add apps`. - + ![Untitled (2)](https://github.com/Sachinbisht27/gcp-budget-alerts-service/assets/96137915/1cf34c76-a38a-4f85-86c7-dd4ef0241246) - 17. Then recognize the newly created app and add to the channel. ![Untitled (3)](https://github.com/Sachinbisht27/gcp-budget-alerts-service/assets/96137915/17cf0b6f-d8bb-4033-81f4-8bc64e543a8e) - You are done installing the app. Enjoy! diff --git a/app/helpers/bigquery_helper/bigquery_helper.py b/app/helpers/bigquery_helper/bigquery_helper.py index c606a8d..d8e5a4b 100644 --- a/app/helpers/bigquery_helper/bigquery_helper.py +++ b/app/helpers/bigquery_helper/bigquery_helper.py @@ -1,6 +1,5 @@ from google.cloud import bigquery - client = bigquery.Client() diff --git a/app/helpers/notification_helper.py b/app/helpers/notification_helper.py index c0e1dfd..10fbb7b 100644 --- a/app/helpers/notification_helper.py +++ b/app/helpers/notification_helper.py @@ -1,3 +1,9 @@ +""" +Notification Helper Module + +This module provides functions for sending notifications. +""" + from app import services diff --git a/app/services/__init__.py b/app/services/__init__.py index 01ffcf5..4edc291 100644 --- a/app/services/__init__.py +++ b/app/services/__init__.py @@ -1,2 +1,2 @@ -from .slack_service import * from .budget_service import * +from .slack_service import * diff --git a/app/services/budget_service.py b/app/services/budget_service.py index b22ace3..94816d7 100644 --- a/app/services/budget_service.py +++ b/app/services/budget_service.py @@ -1,23 +1,24 @@ -import config import datetime -from google.cloud import bigquery +from google.cloud import bigquery +import config +from app.helpers import bigquery_helper, notification_helper from app.services import templates -from app.helpers import bigquery_helper -from app.helpers import notification_helper from app.services.queries import budget_table - client = bigquery.Client() class BudgetService: + """Handles budget-related operations.""" + def __init__(self): self.dataset_id = config.DATASET_ID self.table_id = config.ALERT_THRESHOLD_TABLE_ID def handle(self, alert_attrs, alert_data): + """Handles budget alerts.""" threshold = float(alert_data.get("alertThresholdExceeded")) * 100 if not self.is_new_threshold_greater(threshold): return @@ -30,10 +31,7 @@ def handle(self, alert_attrs, alert_data): current_year_month = datetime.datetime.utcnow().strftime("%Y-%m") year_month_of_budget_interval = interval.strftime("%Y-%m") - # Check if the alert is for the previous month if year_month_of_budget_interval != current_year_month: - ## The pub/sub is sending us alerts for the previous month on the first day of the new month. - ## To handle this condition, we compare the Month-Year of the budget alert interval with the current Month-Year. return billing_id = alert_attrs.get("billingAccountId") @@ -47,10 +45,11 @@ def handle(self, alert_attrs, alert_data): notify = notification_helper.notify(slack_block) - if notify == True: + if notify is True: self.insert_new_threshold(cost, budget, budget_name, threshold) def is_new_threshold_greater(self, threshold): + """Checks if the new threshold is greater.""" query_to_get_existing_threshold = budget_table.get_existing_threshold_query( client, self.dataset_id, self.table_id ) @@ -60,6 +59,7 @@ def is_new_threshold_greater(self, threshold): return last_existing_threshold is None or threshold > last_existing_threshold def insert_new_threshold(self, cost, budget, budget_name, threshold): + """Inserts a new threshold into the database.""" query_to_insert_threshold = budget_table.get_insert_threshold_query( client, self.dataset_id, self.table_id, cost, budget, budget_name, threshold ) diff --git a/app/services/slack_service.py b/app/services/slack_service.py index 64fdde5..535128e 100644 --- a/app/services/slack_service.py +++ b/app/services/slack_service.py @@ -1,7 +1,9 @@ -import config +from ssl import SSLContext + import slack + +import config from utils import logger -from ssl import SSLContext class SlackService: diff --git a/config.py b/config.py index b0bacab..6c85d6c 100644 --- a/config.py +++ b/config.py @@ -4,6 +4,7 @@ if ENVIRONMENT == "development": from os import path + from dotenv import load_dotenv basedir = path.abspath(path.dirname(__file__)) diff --git a/main.py b/main.py index 369000a..519ca94 100644 --- a/main.py +++ b/main.py @@ -1,12 +1,26 @@ -import json +""" +Module: main +Description: Contains the main function for handling billing alerts. +""" + import base64 +import json -from utils import logger from app import services +from utils import logger -# Endpoint of the cloud function. def handle(payload, context): + """ + Function to handle billing alerts. + + Args: + payload (dict): Payload containing alert attributes and data. + context (dict): Context information. + + Returns: + dict: Response message and status code. + """ try: alert_attrs = payload.get("attributes") alert_data = json.loads(base64.b64decode(payload.get("data")).decode("utf-8")) @@ -16,12 +30,10 @@ def handle(payload, context): alert_attrs, alert_data, ) - budget_service = services.BudgetService() budget_service.handle(alert_attrs, alert_data) except Exception as e: - logger.error(f"Error in main function: {e}") - logger.error(f"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx Testing codacy CI check xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx") + logger.error("Error in main function: %s", e) return {"message": "Something went wrong!"}, 400 return {"message": "Success"}, 200 diff --git a/requirements-dev.txt b/requirements-dev.txt index 0fa6829..a7ba2bd 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,6 @@ -pre-commit==2.20.0 -python-dotenv==0.20.0 -slackclient==2.9.4 functions-framework==3.2.0 google-cloud-logging==3.5.0 google_cloud_bigquery==3.11.4 +pre-commit==2.20.0 +python-dotenv==0.20.0 +slackclient==2.9.4 diff --git a/requirements.txt b/requirements.txt index edc0ae0..bd5887c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ -slackclient==2.9.4 functions-framework==3.2.0 google-cloud-logging==3.5.0 google_cloud_bigquery==3.11.4 +pre-commit==3.6.2 +slackclient==2.9.4 Werkzeug==2.3.7 diff --git a/utils/loggingutils.py b/utils/loggingutils.py index fc1ec54..af3eb04 100644 --- a/utils/loggingutils.py +++ b/utils/loggingutils.py @@ -1,7 +1,9 @@ import logging -import config + from google.cloud import logging as gcloud_logging +import config + logger = logging.getLogger() logging.basicConfig(level=logging.DEBUG)