diff --git a/EBSampleApp-Python.iml b/EBSampleApp-Python.iml deleted file mode 100644 index db04778c..00000000 --- a/EBSampleApp-Python.iml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - - - \ No newline at end of file diff --git a/Procfile b/Procfile deleted file mode 100644 index 7667e26a..00000000 --- a/Procfile +++ /dev/null @@ -1 +0,0 @@ -web: gunicorn -w 4 -k uvicorn.workers.UvicornWorker gene.main:app diff --git a/codebuild/deploy_eb_env.py b/codebuild/deploy_eb_env.py deleted file mode 100644 index b5c81d1c..00000000 --- a/codebuild/deploy_eb_env.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Module for deploying staging EB environment.""" -import boto3 -import time - -elasticbeanstalk = boto3.client('elasticbeanstalk') -servicecatalog = boto3.client('servicecatalog') -terminate_time = 12 -eb_app_name = "GeneNormalization" -eb_env_name = "GeneNormalization-staging-env" -sc_product_id = "prod-mmw6ymv2ntzl2" -print(f'Launching new Service Catalog Product for staging environment:' - f' {eb_app_name}') -sc_product_artifacts = \ - servicecatalog.list_provisioning_artifacts(ProductId=sc_product_id) -for artifact in sc_product_artifacts['ProvisioningArtifactDetails']: - if artifact['Active']: - provisioning_artifact_id = artifact['Id'] -try: - eb_provisioned_product = servicecatalog.provision_product( - ProductId=sc_product_id, - ProvisioningArtifactId=provisioning_artifact_id, - ProvisionedProductName=eb_env_name, - ProvisioningParameters=[ - { - 'Key': 'Env', - 'Value': eb_app_name - }, - { - 'Key': 'EnvType', - 'Value': 'staging' - }, - { - 'Key': 'TerminateTime', - 'Value': str(terminate_time) - } - ] - ) - eb_provisioned_product_Id = \ - eb_provisioned_product['RecordDetail']['ProvisionedProductId'] - product_status = servicecatalog.describe_provisioned_product( - Id=eb_provisioned_product_Id) - eb_provisioned_product_status = \ - product_status['ProvisionedProductDetail']['Status'] - while eb_provisioned_product_status == "UNDER_CHANGE": - time.sleep(10) - product_status = servicecatalog.describe_provisioned_product( - Id=eb_provisioned_product_Id) - eb_provisioned_product_status = \ - product_status['ProvisionedProductDetail']['Status'] - print(eb_provisioned_product_status) -except: # noqa: E722 - print("The EB environment is already running...") diff --git a/codebuild/deploy_eb_env_dev.py b/codebuild/deploy_eb_env_dev.py deleted file mode 100644 index a2ca8deb..00000000 --- a/codebuild/deploy_eb_env_dev.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Module for deploying staging EB environment.""" -import boto3 -import time - -elasticbeanstalk = boto3.client('elasticbeanstalk') -servicecatalog = boto3.client('servicecatalog') -terminate_time = 12 -eb_app_name = "GeneNormalization" -eb_env_name = "GeneNormalization-dev-env" -sc_product_id = "prod-mmw6ymv2ntzl2" -print(f'Launching new Service Catalog Product for staging environment:' - f' {eb_app_name}') -sc_product_artifacts = \ - servicecatalog.list_provisioning_artifacts(ProductId=sc_product_id) -for artifact in sc_product_artifacts['ProvisioningArtifactDetails']: - if artifact['Active']: - provisioning_artifact_id = artifact['Id'] -try: - eb_provisioned_product = servicecatalog.provision_product( - ProductId=sc_product_id, - ProvisioningArtifactId=provisioning_artifact_id, - ProvisionedProductName=eb_env_name, - ProvisioningParameters=[ - { - 'Key': 'Env', - 'Value': eb_app_name - }, - { - 'Key': 'EnvType', - 'Value': 'dev' - }, - { - 'Key': 'TerminateTime', - 'Value': str(terminate_time) - } - ] - ) - eb_provisioned_product_Id = \ - eb_provisioned_product['RecordDetail']['ProvisionedProductId'] - product_status = servicecatalog.describe_provisioned_product( - Id=eb_provisioned_product_Id) - eb_provisioned_product_status = \ - product_status['ProvisionedProductDetail']['Status'] - while eb_provisioned_product_status == "UNDER_CHANGE": - time.sleep(10) - product_status = servicecatalog.describe_provisioned_product( - Id=eb_provisioned_product_Id) - eb_provisioned_product_status = \ - product_status['ProvisionedProductDetail']['Status'] - print(eb_provisioned_product_status) -except: # noqa: E722 - print("The EB environment is already running...") diff --git a/codebuild/terminate_eb_env.py b/codebuild/terminate_eb_env.py deleted file mode 100644 index f65d5dd3..00000000 --- a/codebuild/terminate_eb_env.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Module for terminating staging EB environment.""" -import boto3 -import json -import time - -client = boto3.client('lambda') -servicecatalog = boto3.client('servicecatalog') -eb_env_name = "GeneNormalization-staging-env" -data = {"sc_provisioned_name": eb_env_name} -client.invoke(FunctionName='igm-inf-terminate-provisioned-product', - Payload=json.dumps(data)) -time.sleep(10) -provisioned_product = \ - servicecatalog.describe_provisioned_product(Name=eb_env_name) -eb_provisioned_product_Id =\ - provisioned_product['ProvisionedProductDetail']['Id'] -product_status = servicecatalog.describe_provisioned_product( - Id=eb_provisioned_product_Id) -eb_provisioned_product_status =\ - product_status['ProvisionedProductDetail']['Status'] -while eb_provisioned_product_status == "UNDER_CHANGE": - time.sleep(10) - try: - product_status = servicecatalog.describe_provisioned_product( - Id=eb_provisioned_product_Id) - eb_provisioned_product_status = \ - product_status['ProvisionedProductDetail']['Status'] - except: # noqa: E722 - eb_provisioned_product_status = "PRODUCT NOT FOUND" - print(eb_provisioned_product_status) diff --git a/codebuild/terminate_eb_env_dev.py b/codebuild/terminate_eb_env_dev.py deleted file mode 100644 index e9a99324..00000000 --- a/codebuild/terminate_eb_env_dev.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Module for terminating staging EB environment.""" -import boto3 -import json -import time - -client = boto3.client('lambda') -servicecatalog = boto3.client('servicecatalog') -eb_env_name = "GeneNormalization-dev-env" -data = {"sc_provisioned_name": eb_env_name} -client.invoke(FunctionName='igm-inf-terminate-provisioned-product', - Payload=json.dumps(data)) -time.sleep(10) -provisioned_product = \ - servicecatalog.describe_provisioned_product(Name=eb_env_name) -eb_provisioned_product_Id =\ - provisioned_product['ProvisionedProductDetail']['Id'] -product_status = servicecatalog.describe_provisioned_product( - Id=eb_provisioned_product_Id) -eb_provisioned_product_status =\ - product_status['ProvisionedProductDetail']['Status'] -while eb_provisioned_product_status == "UNDER_CHANGE": - time.sleep(10) - try: - product_status = servicecatalog.describe_provisioned_product( - Id=eb_provisioned_product_Id) - eb_provisioned_product_status = \ - product_status['ProvisionedProductDetail']['Status'] - except: # noqa: E722 - eb_provisioned_product_status = "PRODUCT NOT FOUND" - print(eb_provisioned_product_status) diff --git a/cron.yaml b/cron.yaml deleted file mode 100644 index 675190d5..00000000 --- a/cron.yaml +++ /dev/null @@ -1,5 +0,0 @@ -version: 1 -cron: - - name: "task1" - url: "/scheduled" - schedule: "* * * * *" diff --git a/gene/__init__.py b/gene/__init__.py index 80e22f99..825b02fb 100644 --- a/gene/__init__.py +++ b/gene/__init__.py @@ -1,18 +1,14 @@ """The VICC library for normalizing genes.""" -from .version import __version__ # noqa: F401 from pathlib import Path import logging from os import environ -APP_ROOT = Path(__file__).resolve().parents[0] +from .version import __version__ # noqa: F401 -if "GENE_NORM_EB_PROD" in environ: - LOG_FN = "/tmp/gene.log" -else: - LOG_FN = "gene.log" +APP_ROOT = Path(__file__).resolve().parents[0] logging.basicConfig( - filename=LOG_FN, + filename="gene.log", format="[%(asctime)s] - %(name)s - %(levelname)s : %(message)s") logger = logging.getLogger("gene") logger.setLevel(logging.DEBUG) @@ -25,11 +21,6 @@ logging.getLogger("biocommons.seqrepo.seqaliasdb.seqaliasdb").setLevel(logging.INFO) # noqa: E501 logging.getLogger("biocommons.seqrepo.fastadir.fastadir").setLevel(logging.INFO) # noqa: E501 -if "GENE_NORM_EB_PROD" in environ: - ch = logging.StreamHandler() - ch.setLevel(logging.INFO) - logger.addHandler(ch) - SEQREPO_DATA_PATH = environ.get("SEQREPO_DATA_PATH", "/usr/local/share/seqrepo/latest") diff --git a/gene/cli.py b/gene/cli.py index 16b48a7b..ff077981 100644 --- a/gene/cli.py +++ b/gene/cli.py @@ -8,7 +8,8 @@ from boto3.dynamodb.conditions import Key from gene import SOURCES -from gene.database import Database, confirm_aws_db_use +from gene.database import Database, confirm_aws_db_use, SKIP_AWS_DB_ENV_NAME, \ + VALID_AWS_ENV_NAMES, AWS_ENV_VAR_NAME from gene.etl import NCBI, HGNC, Ensembl # noqa: F401 from gene.etl.merge import Merge from gene.schemas import SourceName @@ -28,9 +29,9 @@ class CLI: help="The normalizer(s) you wish to update separated by spaces." ) @click.option( - '--prod', + '--aws_instance', is_flag=True, - help="Working in production environment." + help="Using AWS DynamodDB instance." ) @click.option( '--db_url', @@ -46,15 +47,21 @@ class CLI: is_flag=True, help='Update concepts for normalize endpoint from accepted sources.' ) - def update_normalizer_db(normalizer, prod, db_url, update_all, + def update_normalizer_db(normalizer, aws_instance, db_url, update_all, update_merged): """Update selected normalizer source(s) in the gene database.""" - # Sometimes GENE_NORM_EB_PROD is accidentally set. We should verify that - if "GENE_NORM_EB_PROD" in environ: - confirm_aws_db_use("PROD") - - if prod: - environ['GENE_NORM_PROD'] = "TRUE" + # If SKIP_AWS_CONFIRMATION is accidentally set, we should verify that the + # aws instance should actually be used + invalid_aws_msg = f"{AWS_ENV_VAR_NAME} must be set to one of {VALID_AWS_ENV_NAMES}" # noqa: E501 + aws_env_var_set = False + if AWS_ENV_VAR_NAME in environ: + aws_env_var_set = True + assert environ[AWS_ENV_VAR_NAME] in VALID_AWS_ENV_NAMES, invalid_aws_msg + confirm_aws_db_use(environ[AWS_ENV_VAR_NAME].upper()) + + if aws_env_var_set or aws_instance: + assert AWS_ENV_VAR_NAME in environ, invalid_aws_msg + environ[SKIP_AWS_DB_ENV_NAME] = "true" # this is already checked above db: Database = Database() else: if db_url: diff --git a/gene/database.py b/gene/database.py index c1efdac6..40fb4e11 100644 --- a/gene/database.py +++ b/gene/database.py @@ -1,18 +1,50 @@ """This module creates the database.""" -from gene import PREFIX_LOOKUP -from boto3.dynamodb.conditions import Key -from botocore.exceptions import ClientError +from enum import Enum +import sys +import logging from os import environ from typing import List, Optional, Dict, Any, Set + import boto3 import click -import sys -import logging +from boto3.dynamodb.conditions import Key +from botocore.exceptions import ClientError + +from gene import PREFIX_LOOKUP logger = logging.getLogger() logger.setLevel(logging.DEBUG) +# can be set to either `Dev`, `Staging`, or `Prod` +# ONLY set when wanting to access aws instance +AWS_ENV_VAR_NAME = "GENE_NORM_ENV" + +# Set to "true" if want to skip db confirmation check. Should ONLY be used for +# deployment needs +SKIP_AWS_DB_ENV_NAME = "SKIP_AWS_CONFIRMATION" + + +class AwsEnvName(str, Enum): + """AWS environment name that is being used""" + + DEVELOPMENT = "Dev" + STAGING = "Staging" + PRODUCTION = "Prod" + + +VALID_AWS_ENV_NAMES = {v.value for v in AwsEnvName.__members__.values()} + + +def confirm_aws_db_use(env_name: str) -> None: + """Check to ensure that AWS instance should actually be used.""" + if click.confirm(f"Are you sure you want to use the AWS {env_name} database?", + default=False): + click.echo(f"***GENE AWS {env_name.upper()} DATABASE IN USE***") + else: + click.echo("Exiting.") + sys.exit() + class Database: """The database class.""" @@ -23,19 +55,23 @@ def __init__(self, db_url: str = '', region_name: str = 'us-east-2'): :param str db_url: URL endpoint for DynamoDB source :param str region_name: default AWS region """ - if 'GENE_NORM_PROD' in environ or 'GENE_NORM_EB_PROD' in environ: + gene_concepts_table = "gene_concepts" # default + gene_metadata_table = "gene_metadata" # default + if AWS_ENV_VAR_NAME in environ: + aws_env = environ[AWS_ENV_VAR_NAME] + assert aws_env in VALID_AWS_ENV_NAMES, f"{AWS_ENV_VAR_NAME} must be one of {VALID_AWS_ENV_NAMES}" # noqa: E501 + + skip_confirmation = environ.get(SKIP_AWS_DB_ENV_NAME) + if (not skip_confirmation) or (skip_confirmation and skip_confirmation != "true"): # noqa: E501 + confirm_aws_db_use(environ[AWS_ENV_VAR_NAME]) + boto_params = { - 'region_name': region_name + "region_name": region_name } - if 'GENE_NORM_EB_PROD' not in environ: - # EB Instance should not have to confirm. - # This is used only for updating production via CLI - if click.confirm("Are you sure you want to use the " - "production database?", default=False): - click.echo("***GENE PRODUCTION DATABASE IN USE***") - else: - click.echo("Exiting.") - sys.exit() + + if aws_env == AwsEnvName.DEVELOPMENT: + gene_concepts_table = "gene_concepts_nonprod" + gene_metadata_table = "gene_metadata_nonprod" else: if db_url: endpoint_url = db_url @@ -52,13 +88,13 @@ def __init__(self, db_url: str = '', region_name: str = 'us-east-2'): self.dynamodb = boto3.resource('dynamodb', **boto_params) self.dynamodb_client = boto3.client('dynamodb', **boto_params) - # Create tables if nonexistent if not connecting to production database - if 'GENE_NORM_PROD' not in environ and\ - 'GENE_NORM_EB_PROD' not in environ and 'TEST' not in environ: + # Only create tables for local instance + envs_do_not_create_tables = {AWS_ENV_VAR_NAME, "TEST"} + if not set(envs_do_not_create_tables) & set(environ): self.create_db_tables() - self.genes = self.dynamodb.Table('gene_concepts') - self.metadata = self.dynamodb.Table('gene_metadata') + self.genes = self.dynamodb.Table(gene_concepts_table) + self.metadata = self.dynamodb.Table(gene_metadata_table) self.batch = self.genes.batch_writer() self.cached_sources = {}