diff --git a/1_otc_services_dict.py b/1_otc_services_dict.py index 01e49d6..45c8e68 100644 --- a/1_otc_services_dict.py +++ b/1_otc_services_dict.py @@ -5,13 +5,14 @@ import base64 import logging -import os import time import psycopg2 import requests import yaml +from classes import Database, EnvVariables + logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') start_time = time.time() @@ -20,33 +21,8 @@ BASE_URL = "https://gitea.eco.tsi-dev.otc-service.com/api/v1" -gitea_token = os.getenv("GITEA_TOKEN") -headers = { - "Authorization": f"token {gitea_token}" -} - -db_host = os.getenv("DB_HOST") -db_port = os.getenv("DB_PORT") -db_csv = os.getenv("DB_CSV") -db_orph = os.getenv("DB_ORPH") -db_zuul = os.getenv("DB_ZUUL") -db_user = os.getenv("DB_USER") -db_password = os.getenv("DB_PASSWORD") - - -def connect_to_db(db): - logging.info("Connecting to Postgres (%s)...", db) - try: - return psycopg2.connect( - host=db_host, - port=db_port, - dbname=db, - user=db_user, - password=db_password - ) - except psycopg2.Error as e: - logging.error("Connecting to Postgres: an error occurred while trying to connect to the database: %s", e) - return None +env_vars = EnvVariables() +database = Database(env_vars) def create_rtc_table(conn_csv, cur_csv, table_name): @@ -86,7 +62,7 @@ def create_doc_table(conn_csv, cur_csv, table_name): def get_pretty_category_names(base_dir, category_dir): - response = requests.get(f"{BASE_URL}{category_dir}", timeout=10, headers=headers) + response = requests.get(f"{BASE_URL}{category_dir}", timeout=10) response.raise_for_status() all_files = [item['path'] for item in response.json() if item['type'] == 'file'] @@ -94,7 +70,7 @@ def get_pretty_category_names(base_dir, category_dir): for file_path in all_files: if file_path.endswith('.yaml'): - response = requests.get(f"{BASE_URL}{base_dir}{file_path}", timeout=10, headers=headers) + response = requests.get(f"{BASE_URL}{base_dir}{file_path}", timeout=10) response.raise_for_status() file_content_base64 = response.json()['content'] @@ -109,7 +85,7 @@ def get_pretty_category_names(base_dir, category_dir): def get_service_categories(base_dir, category_dir, services_dir): pretty_names = get_pretty_category_names(base_dir, category_dir) - response = requests.get(f"{BASE_URL}{services_dir}", timeout=10, headers=headers) + response = requests.get(f"{BASE_URL}{services_dir}", timeout=10) response.raise_for_status() all_files = [item['path'] for item in response.json() if item['type'] == 'file'] @@ -117,7 +93,7 @@ def get_service_categories(base_dir, category_dir, services_dir): for file_path in all_files: if file_path.endswith('.yaml'): - response = requests.get(f"{BASE_URL}{base_dir}{file_path}", timeout=10, headers=headers) + response = requests.get(f"{BASE_URL}{base_dir}{file_path}", timeout=10) response.raise_for_status() file_content_base64 = response.json()['content'] @@ -139,7 +115,7 @@ def get_service_categories(base_dir, category_dir, services_dir): def get_docs_info(base_dir, doc_dir): - response = requests.get(f"{BASE_URL}{doc_dir}", timeout=10, headers=headers) + response = requests.get(f"{BASE_URL}{doc_dir}", timeout=10) response.raise_for_status() all_files = [item['path'] for item in response.json() if item['type'] == 'file'] @@ -147,7 +123,7 @@ def get_docs_info(base_dir, doc_dir): for file_path in all_files: if file_path.endswith('.yaml'): - response = requests.get(f"{BASE_URL}{base_dir}{file_path}", timeout=10, headers=headers) + response = requests.get(f"{BASE_URL}{base_dir}{file_path}", timeout=10) response.raise_for_status() file_content_base64 = response.json()['content'] @@ -179,7 +155,7 @@ def insert_services_data(item, conn_csv, cur_csv, table_name): def get_squad_description(styring_url): - response = requests.get(styring_url, timeout=10, headers=headers) + response = requests.get(styring_url, timeout=10) response.raise_for_status() file_content_base64 = response.json()['content'] @@ -266,15 +242,15 @@ def main(base_dir, rtctable, doctable, styring_path): services_dir = f"{base_dir}otc_metadata/data/services" category_dir = f"{base_dir}otc_metadata/data/service_categories" doc_dir = f"{base_dir}otc_metadata/data/documents" - styring_url = f"{BASE_URL}{styring_path}{gitea_token}" + styring_url = f"{BASE_URL}{styring_path}{env_vars.gitea_token}" - conn_orph = connect_to_db(db_orph) + conn_orph = database.connect_to_db(env_vars.db_orph) cur_orph = conn_orph.cursor() - conn_zuul = connect_to_db(db_zuul) + conn_zuul = database.connect_to_db(env_vars.db_zuul) cur_zuul = conn_zuul.cursor() - conn_csv = connect_to_db(db_csv) + conn_csv = database.connect_to_db(env_vars.db_csv) cur_csv = conn_csv.cursor() conns = [conn_orph, conn_zuul] @@ -315,7 +291,7 @@ def main(base_dir, rtctable, doctable, styring_path): main(BASE_DIR_REGULAR, BASE_RTC_TABLE, BASE_DOC_TABLE, STYRING_URL_REGULAR) main(BASE_DIR_SWISS, f"{BASE_RTC_TABLE}_swiss", f"{BASE_DOC_TABLE}_swiss", STYRING_URL_SWISS) - conn_csv = connect_to_db(db_csv) + conn_csv = database.connect_to_db(env_vars.db_csv) cur_csv = conn_csv.cursor() add_obsolete_services(conn_csv, cur_csv) conn_csv.commit() diff --git a/3_github_info.py b/3_github_info.py index 6b3171e..7c1d0a5 100644 --- a/3_github_info.py +++ b/3_github_info.py @@ -3,53 +3,22 @@ """ import logging -import os import re import time -import psycopg2 import requests from github import Github +from classes import Database, EnvVariables + logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') start_time = time.time() logging.info("-------------------------GITHUB INFO SCRIPT IS RUNNING-------------------------") -github_token = os.getenv("GITHUB_TOKEN") -github_fallback_token = os.getenv("GITHUB_FALLBACK_TOKEN") - -db_host = os.getenv("DB_HOST") -db_port = os.getenv("DB_PORT") -db_name = os.getenv("DB_ORPH") # Here we're using dedicated postgres db for orphan PRs only -db_user = os.getenv("DB_USER") -db_password = os.getenv("DB_PASSWORD") - - -def check_env_variables(): - required_env_vars = [ - "GITHUB_TOKEN", "DB_HOST", "DB_PORT", - "DB_NAME", "DB_USER", "DB_PASSWORD", "GITEA_TOKEN" - ] - for var in required_env_vars: - if os.getenv(var) is None: - raise Exception(f"Missing environment variable: {var}") - - -def connect_to_db(db_name): - logging.info(f"Connecting to Postgres ({db_name})...") - try: - return psycopg2.connect( - host=db_host, - port=db_port, - dbname=db_name, - user=db_user, - password=db_password - ) - except psycopg2.Error as e: - logging.info("Connecting to Postgres: an error occurred while trying to connect to the database: %s", e) - return None +env_vars = EnvVariables() +database = Database(env_vars) def extract_pull_links(cur, table_name): @@ -129,30 +98,29 @@ def update_orphaned_prs(org_str, cur, conn, rows, auto_prs, table_name): def main(org, gorg, table_name, token): - check_env_variables() g = Github(token) ghorg = g.get_organization(gorg) repo_names = [repo.name for repo in ghorg.get_repos()] - conn = connect_to_db(db_name) - cur = conn.cursor() + conn_orph = database.connect_to_db(env_vars.db_orph) + cur_orph = conn_orph.cursor() - pull_links = extract_pull_links(cur, table_name) + pull_links = extract_pull_links(cur_orph, table_name) auto_prs = [] logging.info("Gathering PRs info...") for repo_name in repo_names: - auto_prs += get_auto_prs(gorg, repo_name, github_token, pull_links) + auto_prs += get_auto_prs(gorg, repo_name, env_vars.github_token, pull_links) - add_github_columns(cur, conn, table_name) + add_github_columns(cur_orph, conn_orph, table_name) - cur.execute(f'SELECT id, "Auto PR URL" FROM {table_name};') - rows = cur.fetchall() + cur_orph.execute(f'SELECT id, "Auto PR URL" FROM {table_name};') + rows = cur_orph.fetchall() - update_orphaned_prs(org, cur, conn, rows, auto_prs, table_name) + update_orphaned_prs(org, cur_orph, conn_orph, rows, auto_prs, table_name) - cur.close() - conn.close() + cur_orph.close() + conn_orph.close() if __name__ == "__main__": @@ -162,13 +130,13 @@ def main(org, gorg, table_name, token): DONE = False try: - main(ORG_STRING, GH_ORG_STR, ORPH_TABLE, github_token) - main(f"{ORG_STRING}-swiss", f"{GH_ORG_STR}-swiss", f"{ORPH_TABLE}_swiss", github_token) + main(ORG_STRING, GH_ORG_STR, ORPH_TABLE, env_vars.github_token) + main(f"{ORG_STRING}-swiss", f"{GH_ORG_STR}-swiss", f"{ORPH_TABLE}_swiss", env_vars.github_token) DONE = True except Exception as e: logging.info(f"Error has been occurred: {e}") - main(ORG_STRING, GH_ORG_STR, ORPH_TABLE, github_fallback_token) - main(f"{ORG_STRING}-swiss", f"{GH_ORG_STR}-swiss", f"{ORPH_TABLE}_swiss", github_fallback_token) + main(ORG_STRING, GH_ORG_STR, ORPH_TABLE, env_vars.github_fallback_token) + main(f"{ORG_STRING}-swiss", f"{GH_ORG_STR}-swiss", f"{ORPH_TABLE}_swiss", env_vars.github_fallback_token) DONE = True if DONE: logging.info("Github operations successfully done!") diff --git a/5_open_issues.py b/5_open_issues.py index 6025fbf..d482417 100644 --- a/5_open_issues.py +++ b/5_open_issues.py @@ -4,7 +4,6 @@ import json import logging -import os import re import time from datetime import datetime @@ -13,6 +12,8 @@ import requests from github import Github +from classes import Database, EnvVariables + logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') start_time = time.time() @@ -22,40 +23,8 @@ GITEA_API_ENDPOINT = "https://gitea.eco.tsi-dev.otc-service.com/api/v1" session = requests.Session() -gitea_token = os.getenv("GITEA_TOKEN") -github_token = os.getenv("GITHUB_TOKEN") -github_fallback_token = os.getenv("GITHUB_FALLBACK_TOKEN") - -db_host = os.getenv("DB_HOST") -db_port = os.getenv("DB_PORT") -db_name = os.getenv("DB_CSV") # here we're using main postgres table since we don't need orphan PRs -db_user = os.getenv("DB_USER") -db_password = os.getenv("DB_PASSWORD") - - -def check_env_variables(): - required_env_vars = [ - "GITHUB_TOKEN", "DB_HOST", "DB_PORT", - "DB_NAME", "DB_USER", "DB_PASSWORD", "GITEA_TOKEN" - ] - for var in required_env_vars: - if os.getenv(var) is None: - raise Exception(f"Missing environment variable: {var}") - - -def connect_to_db(db_name): - logging.info("Connecting to Postgres %s...", db_name) - try: - return psycopg2.connect( - host=db_host, - port=db_port, - dbname=db_name, - user=db_user, - password=db_password - ) - except psycopg2.Error as e: - logging.error("Connecting to Postgres: an error occurred while trying to connect to the database %s:", e) - return None +env_vars = EnvVariables() +database = Database(env_vars) def create_open_issues_table(conn, cur, table_name): @@ -79,7 +48,7 @@ def create_open_issues_table(conn, cur, table_name): logging.info("Table %s has been created successfully", table_name) except psycopg2.Error as e: logging.error("Tables creating: an error occurred while trying to create a table %s in the " - "database %s: %s", table_name, db_name, e) + "database %s: %s", table_name, env_vars.db_csv, e) def get_gitea_issues(gitea_token, gitea_org): @@ -145,7 +114,7 @@ def get_github_issues(github_token, repo_names, gh_org): def get_issues_table(gh_org, gitea_issues, github_issues, cur, conn, table_name): - logging.info("Posting data to Postgres (%s)...", db_name) + logging.info("Posting data to Postgres (%s)...", env_vars.db_csv) try: for tea in gitea_issues: environment = "Gitea" @@ -245,26 +214,25 @@ def update_squad_and_title(conn, cur, table_name, rtc): def main(org, gh_org, table_name, rtc, token): - check_env_variables() g = Github(token) github_org = g.get_organization(gh_org) repo_names = [repo.name for repo in github_org.get_repos()] logging.info("%s repos have been processed", len(repo_names)) - gitea_issues = get_gitea_issues(gitea_token, org) - github_issues = get_github_issues(github_token, repo_names, gh_org) - conn = connect_to_db(db_name) - cur = conn.cursor() + gitea_issues = get_gitea_issues(env_vars.gitea_token, org) + github_issues = get_github_issues(env_vars.github_token, repo_names, gh_org) + conn_csv = database.connect_to_db(env_vars.db_csv) + cur_csv = conn_csv.cursor() - cur.execute( + cur_csv.execute( f'''DROP TABLE IF EXISTS {table_name}''' ) - conn.commit() + conn_csv.commit() - create_open_issues_table(conn, cur, table_name) - get_issues_table(org, gitea_issues, github_issues, cur, conn, table_name) - update_squad_and_title(conn, cur, table_name, rtc) - conn.close() + create_open_issues_table(conn_csv, cur_csv, table_name) + get_issues_table(org, gitea_issues, github_issues, cur_csv, conn_csv, table_name) + update_squad_and_title(conn_csv, cur_csv, table_name, rtc) + conn_csv.close() if __name__ == '__main__': @@ -275,14 +243,15 @@ def main(org, gh_org, table_name, rtc, token): DONE = False try: - main(ORG_STRING, GH_ORG_STRING, OPEN_TABLE, RTC_TABLE, github_token) - main(f"{ORG_STRING}-swiss", f"{GH_ORG_STRING}-swiss", f"{OPEN_TABLE}_swiss", f"{RTC_TABLE}_swiss", github_token) + main(ORG_STRING, GH_ORG_STRING, OPEN_TABLE, RTC_TABLE, env_vars.github_token) + main(f"{ORG_STRING}-swiss", f"{GH_ORG_STRING}-swiss", f"{OPEN_TABLE}_swiss", f"{RTC_TABLE}_swiss", + env_vars.github_token) DONE = True except Exception as e: logging.error("An error occurred: %s", e) - main(ORG_STRING, GH_ORG_STRING, OPEN_TABLE, RTC_TABLE, github_fallback_token) + main(ORG_STRING, GH_ORG_STRING, OPEN_TABLE, RTC_TABLE, env_vars.github_fallback_token) main(f"{ORG_STRING}-swiss", f"{GH_ORG_STRING}-swiss", f"{OPEN_TABLE}_swiss", f"{RTC_TABLE}_swiss", - github_fallback_token) + env_vars.github_fallback_token) DONE = True if DONE: logging.info("Github operations successfully done!") diff --git a/6_last_commit_info.py b/6_last_commit_info.py index d1a04d5..239a018 100644 --- a/6_last_commit_info.py +++ b/6_last_commit_info.py @@ -3,7 +3,6 @@ """ import logging -import os import shutil import tempfile import time @@ -12,45 +11,16 @@ import psycopg2 from github import Github +from classes import Database, EnvVariables + logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') start_time = time.time() logging.info("-------------------------LAST COMMIT INFO SCRIPT IS RUNNING-------------------------") -github_token = os.getenv("GITHUB_TOKEN") -github_fallback_token = os.getenv("GITHUB_FALLBACK_TOKEN") - -db_host = os.getenv("DB_HOST") -db_port = os.getenv("DB_PORT") -db_name = os.getenv("DB_CSV") # Here we're using main postgres db since we don't need orphan PRs -db_user = os.getenv("DB_USER") -db_password = os.getenv("DB_PASSWORD") - - -def check_env_variables(): - required_env_vars = [ - "GITHUB_TOKEN", "DB_HOST", "DB_PORT", - "DB_NAME", "DB_USER", "DB_PASSWORD", "GITEA_TOKEN" - ] - for var in required_env_vars: - if os.getenv(var) is None: - raise Exception("Missing environment variable: %s" % var) - - -def connect_to_db(db_name): - logging.info("Connecting to Postgres (%s)...", db_name) - try: - return psycopg2.connect( - host=db_host, - port=db_port, - dbname=db_name, - user=db_user, - password=db_password - ) - except psycopg2.Error as e: - logging.error("Connecting to Postgres: an error occurred while trying to connect to the database: %s", e) - return None +env_vars = EnvVariables() +database = Database(env_vars) def create_commits_table(conn, cur, table_name): @@ -156,19 +126,18 @@ def update_squad_and_title(conn, cur, table_name, rtc): def main(gorg, table_name, rtc, gh_str, token): - check_env_variables() g = Github(token) org = g.get_organization(gorg) - conn = connect_to_db(db_name) - cur = conn.cursor() - cur.execute(f"DROP TABLE IF EXISTS {table_name}") - create_commits_table(conn, cur, table_name) + conn_csv = database.connect_to_db(env_vars.db_csv) + cur_csv = conn_csv.cursor() + cur_csv.execute(f"DROP TABLE IF EXISTS {table_name}") + create_commits_table(conn_csv, cur_csv, table_name) logging.info("Searching for a most recent commit in umn/source...") - get_last_commit(org, conn, cur, "umn/source", gh_str, table_name) + get_last_commit(org, conn_csv, cur_csv, "umn/source", gh_str, table_name) logging.info("Searching for a most recent commit in api-ref/source...") - get_last_commit(org, conn, cur, "api-ref/source", gh_str, table_name) - update_squad_and_title(conn, cur, table_name, rtc) - conn.commit() + get_last_commit(org, conn_csv, cur_csv, "api-ref/source", gh_str, table_name) + update_squad_and_title(conn_csv, cur_csv, table_name, rtc) + conn_csv.commit() if __name__ == "__main__": @@ -178,14 +147,15 @@ def main(gorg, table_name, rtc, gh_str, token): DONE = False try: - main(GH_ORG_STR, COMMIT_TABLE, RTC_TABLE, GH_ORG_STR, github_token) - main(f"{GH_ORG_STR}-swiss", f"{COMMIT_TABLE}_swiss", f"{RTC_TABLE}_swiss", f"{GH_ORG_STR}-swiss", github_token) + main(GH_ORG_STR, COMMIT_TABLE, RTC_TABLE, GH_ORG_STR, env_vars.github_token) + main(f"{GH_ORG_STR}-swiss", f"{COMMIT_TABLE}_swiss", f"{RTC_TABLE}_swiss", f"{GH_ORG_STR}-swiss", + env_vars.github_token) DONE = True except Exception as e: logging.info("Error has been occurred: %s", e) - main(GH_ORG_STR, COMMIT_TABLE, RTC_TABLE, GH_ORG_STR, github_fallback_token) + main(GH_ORG_STR, COMMIT_TABLE, RTC_TABLE, GH_ORG_STR, env_vars.github_fallback_token) main(f"{GH_ORG_STR}-swiss", f"{COMMIT_TABLE}_swiss", f"{RTC_TABLE}_swiss", f"{GH_ORG_STR}-swiss", - github_fallback_token) + env_vars.github_fallback_token) DONE = True if DONE: logging.info("Github operations successfully done!") diff --git a/7_request_changes.py b/7_request_changes.py index 67f3b7b..a5d4693 100644 --- a/7_request_changes.py +++ b/7_request_changes.py @@ -1,6 +1,5 @@ import json import logging -import os import re import time from datetime import datetime @@ -8,6 +7,8 @@ import psycopg2 import requests +from classes import Database, EnvVariables + logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') start_time = time.time() @@ -16,38 +17,9 @@ gitea_api_endpoint = "https://gitea.eco.tsi-dev.otc-service.com/api/v1" session = requests.Session() -gitea_token = os.getenv("GITEA_TOKEN") - -db_host = os.getenv("DB_HOST") -db_port = os.getenv("DB_PORT") -db_csv = os.getenv("DB_CSV") # main postgres db, where open PRs tables for both public and hybrid clouds are stored -db_user = os.getenv("DB_USER") -db_password = os.getenv("DB_PASSWORD") - - -def check_env_variables(): - required_env_vars = [ - "DB_HOST", "DB_PORT", - "DB_CSV", "DB_USER", "DB_PASSWORD", "GITEA_TOKEN" - ] - for var in required_env_vars: - if os.getenv(var) is None: - raise Exception("Missing environment variable: %s", var) - -def connect_to_db(db_name): - logging.info("Connecting to Postgres (%s)...", db_name) - try: - return psycopg2.connect( - host=db_host, - port=db_port, - dbname=db_name, - user=db_user, - password=db_password - ) - except psycopg2.Error as e: - logging.error("Connecting to Postgres: an error occurred while trying to connect to the database: %s", e) - return None +env_vars = EnvVariables() +database = Database(env_vars) def create_prs_table(conn, cur, table_name): @@ -77,7 +49,6 @@ def get_repos(cur, rtc): try: cur.execute(f"SELECT DISTINCT \"Repository\" FROM {rtc} WHERE \"Env\" = 'public';") repos = [row[0] for row in cur.fetchall()] - # repos.append('doc-exports') if not repos: logging.info("No repositories found.") except Exception as e: @@ -92,7 +63,7 @@ def get_pr_number(org, repo): while True: try: repo_resp = session.get(f"{gitea_api_endpoint}/repos/{org}/{repo}/pulls?state=open&page={page}" - f"&limit=1000&token={gitea_token}") + f"&limit=1000&token={env_vars.gitea_token}") repo_resp.raise_for_status() pull_requests = json.loads(repo_resp.content.decode()) except requests.exceptions.HTTPError as e: @@ -130,7 +101,7 @@ def process_pr_reviews(org, repo, pr_number, changes_tab): reviews = [] try: reviews_resp = session.get(f"{gitea_api_endpoint}/repos/{org}/{repo}/pulls/{pr_number}/reviews?token=" - f"{gitea_token}") + f"{env_vars.gitea_token}") reviews_resp.raise_for_status() reviews = json.loads(reviews_resp.content.decode()) except requests.exceptions.HTTPError as e: @@ -157,7 +128,7 @@ def get_last_commit(org, repo, pr_number, reviewer_login, last_review_date, chan try: commits_resp = session.get(f"{gitea_api_endpoint}/repos/{org}/{repo}/pulls/{pr_number}/commits?token=" - f"{gitea_token}") + f"{env_vars.gitea_token}") commits_resp.raise_for_status() commits = json.loads(commits_resp.content.decode()) except requests.exceptions.RequestException as e: @@ -184,7 +155,7 @@ def get_last_commit(org, repo, pr_number, reviewer_login, last_review_date, chan def insert_data_postgres(org, repo, pr_number, conn, cur, activity_date, changes_tab): try: filtered_reviews_resp = session.get(f"{gitea_api_endpoint}/repos/{org}/{repo}/pulls/{pr_number}/" - f"reviews?token={gitea_token}") + f"reviews?token={env_vars.gitea_token}") filtered_reviews_resp.raise_for_status() filtered_reviews = json.loads(filtered_reviews_resp.content.decode()) except requests.exceptions.RequestException as e: @@ -223,7 +194,8 @@ def parent_pr_changes_check(cur, conn, org, changes_tab): for repo, pr_number in repo_pr_dict.items(): try: - pr_resp = session.get(f"{gitea_api_endpoint}/repos/{org}/{repo}/pulls/{pr_number}?token={gitea_token}") + pr_resp = session.get(f"{gitea_api_endpoint}/repos/{org}/{repo}/pulls/{pr_number}?token=" + f"{env_vars.gitea_token}") pr_resp.raise_for_status() except requests.exceptions.HTTPError as e: if e.response.status_code == 404: @@ -250,7 +222,7 @@ def parent_pr_changes_check(cur, conn, org, changes_tab): parent_reviews = [] try: parent_reviews_resp = session.get(f"{gitea_api_endpoint}/repos/{org}/{repo_name}/pulls/" - f"{parent_pr_number}/reviews?token={gitea_token}") + f"{parent_pr_number}/reviews?token={env_vars.gitea_token}") parent_reviews_resp.raise_for_status() parent_reviews = json.loads(parent_reviews_resp.content.decode()) except requests.exceptions.HTTPError as e: @@ -320,8 +292,6 @@ def update_squad_and_title(cur, conn, rtc, changes_tab): def main(org, rtc, changes_tab): - check_env_variables() - cur_csv.execute(f"DROP TABLE IF EXISTS {changes_tab}") conn_csv.commit() @@ -347,7 +317,7 @@ def main(org, rtc, changes_tab): rtc_table = "repo_title_category" changes_table = "requested_changes" - conn_csv = connect_to_db(db_csv) + conn_csv = database.connect_to_db(env_vars.db_csv) cur_csv = conn_csv.cursor() done = False diff --git a/9_scheduler.py b/9_scheduler.py index 1963072..baebe5c 100644 --- a/9_scheduler.py +++ b/9_scheduler.py @@ -3,15 +3,15 @@ """ import logging -import os import time from datetime import datetime from urllib.parse import quote -import psycopg2 import zulip from psycopg2.extras import DictCursor +from classes import Database, EnvVariables + logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') # timer for counting script time execution @@ -19,17 +19,15 @@ logging.info("-------------------------SCHEDULER IS RUNNING-------------------------") -# Database and Zulip configuration, environment vars are used -db_host = os.getenv("DB_HOST") -db_port = os.getenv("DB_PORT") -db_csv = os.getenv("DB_CSV") -db_orph = os.getenv("DB_ORPH") -db_user = os.getenv("DB_USER") -db_password = os.getenv("DB_PASSWORD") -api_key = os.getenv("OTC_BOT_API") +env_vars = EnvVariables() +database = Database(env_vars) # Zulip stream and topic mapping for each squad squad_streams = { + # "Database Squad": {"stream": "4grafana", "topic": "testing"}, + # "Big Data and AI Squad": {"stream": "4grafana", "topic": "testing"}, + # "Compute Squad": {"stream": "4grafana", "topic": "testing"}, + # "Network Squad": {"stream": "4grafana", "topic": "testing"} "Database Squad": {"stream": "Database Squad", "topic": "Doc alerts"}, "Big Data and AI Squad": {"stream": "bigdata & ai", "topic": "helpcenter_alerts"}, "Compute Squad": {"stream": "compute", "topic": "hc_alerts topic"}, @@ -37,35 +35,9 @@ } -def check_env_variables(): - required_env_vars = [ - "DB_HOST", "DB_PORT", - "DB_CSV", "DB_ORPH", "DB_USER", "DB_PASSWORD", "OTC_BOT_API" - ] - for var in required_env_vars: - if os.getenv(var) is None: - raise Exception(f"Missing environment variable: {var}") - - -def connect_to_db(db_name): - logging.info("Connecting to Postgres (%s)...", db_name) - try: - return psycopg2.connect( - host=db_host, - port=db_port, - dbname=db_name, - user=db_user, - password=db_password, - cursor_factory=DictCursor - ) - except psycopg2.Error as e: - logging.error("Connecting to Postgres: an error occurred while trying to connect to the database: %s", e) - return None - - def check_orphans(conn_orph, squad_name, stream_name, topic_name): results = [] - cur_orph = conn_orph.cursor() + cur_orph = conn_orph.cursor(cursor_factory=DictCursor) tables = ["open_prs", "open_prs_swiss"] for table in tables: # here each query marked with zone marker (Public or Hybrid) for bring it into message @@ -81,15 +53,14 @@ def check_orphans(conn_orph, squad_name, stream_name, topic_name): results = cur_orph.fetchall() if results: for row in results: - send_zulip_notification(row, api_key, stream_name, topic_name) + send_zulip_notification(row, env_vars.api_key, stream_name, topic_name) def check_open_issues(conn, squad_name, stream_name, topic_name): results = [] - cur = conn.cursor() + cur = conn.cursor(cursor_factory=DictCursor) tables = ["open_issues", "open_issues_swiss"] for table in tables: - # here each query marked with zone marker (Public or Hybrid) for bring it into message if table == "open_issues": logging.info("Checking %s for %s", table, squad_name) query = f"""SELECT *, 'Public' as zone, 'issue' as type FROM {table} WHERE "Squad" = '{squad_name}' AND @@ -104,12 +75,12 @@ def check_open_issues(conn, squad_name, stream_name, topic_name): results = cur.fetchall() if results: for row in results: - send_zulip_notification(row, api_key, stream_name, topic_name) + send_zulip_notification(row, env_vars.api_key, stream_name, topic_name) def check_outdated_docs(conn, squad_name, stream_name, topic_name): results = [] - cur = conn.cursor() + cur = conn.cursor(cursor_factory=DictCursor) tables = ["last_update_commit", "last_update_commit_swiss"] for table in tables: # here each query marked with zone marker (Public or Hybrid) for bring it into message @@ -125,7 +96,7 @@ def check_outdated_docs(conn, squad_name, stream_name, topic_name): results = cur.fetchall() if results: for row in results: - send_zulip_notification(row, api_key, stream_name, topic_name) + send_zulip_notification(row, env_vars.api_key, stream_name, topic_name) def send_zulip_notification(row, api_key, stream_name, topic_name): @@ -203,9 +174,8 @@ def send_zulip_notification(row, api_key, stream_name, topic_name): def main(): - check_env_variables() - conn = connect_to_db(db_csv) - conn_orph = connect_to_db(db_orph) + conn = database.connect_to_db(env_vars.db_csv) + conn_orph = database.connect_to_db(env_vars.db_orph) for squad_name, channel in squad_streams.items(): stream_name = channel["stream"] diff --git a/classes.py b/classes.py index e521192..fa9c441 100644 --- a/classes.py +++ b/classes.py @@ -25,6 +25,7 @@ def __init__(self): self.gitea_token = os.getenv("GITEA_TOKEN") self.github_token = os.getenv("GITHUB_TOKEN") self.github_fallback_token = os.getenv("GITHUB_FALLBACK_TOKEN") + self.api_key = os.getenv("OTC_BOT_API") self.check_env_variables() def check_env_variables(self):