diff --git a/.github/workflows/aws-preview.yml b/.github/workflows/aws-preview.yml index 0634c444..be39d85f 100644 --- a/.github/workflows/aws-preview.yml +++ b/.github/workflows/aws-preview.yml @@ -41,7 +41,10 @@ jobs: role-to-assume: arn:aws:iam::654654285942:role/Github-OIDC audience: sts.amazonaws.com aws-region: ${{ env.AWS_REGION }} - + # Sync .env from remote + - run: | + pip install -r scripts/requirements.txt + python scripts/sync_envs.py pull -t .aws/petercat-preview.toml # Build inside Docker containers - run: sam build --use-container --config-file .aws/petercat-preview.toml @@ -50,22 +53,4 @@ jobs: sam deploy \ --no-confirm-changeset \ --no-fail-on-empty-changeset \ - --config-file .aws/petercat-preview.toml \ - --parameter-overrides APIUrl="https://api-pre.petercat.chat" \ - WebUrl="https://www.petercat.chat" \ - GitHubAppID=${{ secrets.X_GITHUB_APP_ID }} \ - GithubAppsClientId=${{ secrets.X_GITHUB_APPS_CLIENT_ID }} \ - GithubAppsClientSecret=${{ secrets.X_GITHUB_APPS_CLIENT_SECRET }} \ - OpenAIAPIKey=${{ secrets.OPENAI_API_KEY }} \ - GeminiAPIKey=${{ secrets.GEMINI_API_KEY }} \ - SupabaseServiceKey=${{ secrets.SUPABASE_SERVICE_KEY }} \ - SupabaseUrl=${{ secrets.SUPABASE_URL }} \ - TavilyAPIKey=${{ secrets.TAVILY_API_KEY }} \ - APIIdentifier=${{ secrets.API_IDENTIFIER }} \ - FastAPISecretKey=${{ secrets.FASTAPI_SECRET_KEY }} \ - SQSQueueName=${{ secrets.SQS_QUEUE_NAME }} \ - SQSQueueUrl=${{ secrets.SQS_QUEUE_URL }} \ - GitHubToken=${{ secrets.X_GITHUB_TOKEN }} \ - Auth0Domain=${{ secrets.AUTH0_DOMAIN }} \ - Auth0ClientId=${{ secrets.AUTH0_CLIENT_ID }} \ - Auth0ClientSecret=${{ secrets.AUTH0_CLIENT_SECRET }} + --config-file .aws/petercat-preview.toml diff --git a/.github/workflows/aws-prod.yml b/.github/workflows/aws-prod.yml index 9a8ac29c..8e85c2d0 100644 --- a/.github/workflows/aws-prod.yml +++ b/.github/workflows/aws-prod.yml @@ -36,6 +36,10 @@ jobs: audience: sts.amazonaws.com aws-region: ${{ env.AWS_REGION }} + # Sync .env from remote + - run: | + pip install -r scripts/requirements.txt + python scripts/sync_envs.py pull -t .aws/petercat-preview.toml # Build inside Docker containers - run: sam build --use-container --config-file .aws/petercat-prod.toml @@ -44,22 +48,4 @@ jobs: sam deploy \ --no-confirm-changeset \ --no-fail-on-empty-changeset \ - --config-file .aws/petercat-prod.toml \ - --parameter-overrides APIUrl="https://api.petercat.chat" \ - WebUrl="https://www.petercat.chat" \ - GitHubAppID=${{ secrets.X_GITHUB_APP_ID }} \ - GithubAppsClientId=${{ secrets.X_GITHUB_APPS_CLIENT_ID }} \ - GithubAppsClientSecret=${{ secrets.X_GITHUB_APPS_CLIENT_SECRET }} \ - OpenAIAPIKey=${{ secrets.OPENAI_API_KEY }} \ - GeminiAPIKey=${{ secrets.GEMINI_API_KEY }} \ - SupabaseServiceKey=${{ secrets.SUPABASE_SERVICE_KEY }} \ - SupabaseUrl=${{ secrets.SUPABASE_URL }} \ - TavilyAPIKey=${{ secrets.TAVILY_API_KEY }} \ - APIIdentifier=${{ secrets.API_IDENTIFIER }} \ - FastAPISecretKey=${{ secrets.FASTAPI_SECRET_KEY }} \ - SQSQueueName=${{ secrets.SQS_QUEUE_NAME }} \ - SQSQueueUrl=${{ secrets.SQS_QUEUE_URL }} \ - GitHubToken=${{ secrets.X_GITHUB_TOKEN }} \ - Auth0Domain=${{ secrets.AUTH0_DOMAIN }} \ - Auth0ClientId=${{ secrets.AUTH0_CLIENT_ID }} \ - Auth0ClientSecret=${{ secrets.AUTH0_CLIENT_SECRET }} \ No newline at end of file + --config-file .aws/petercat-prod.toml \ No newline at end of file diff --git a/scripts/requirements.txt b/scripts/requirements.txt new file mode 100644 index 00000000..1c561043 --- /dev/null +++ b/scripts/requirements.txt @@ -0,0 +1,3 @@ +boto3>=1.34.84 +toml +pyyaml diff --git a/scripts/sync_envs.py b/scripts/sync_envs.py index 1a134897..a07f4467 100644 --- a/scripts/sync_envs.py +++ b/scripts/sync_envs.py @@ -1,16 +1,150 @@ import boto3 import io +import toml +import argparse +import yaml S3_BUCKET = "petercat-env-variables" ENV_FILE = ".env" +LOCAL_ENV_FILE = "./server/.env" s3 = boto3.resource('s3') - obj = s3.Object(S3_BUCKET, ENV_FILE) -data = io.BytesIO() -obj.download_fileobj(data) +def pull_envs(): + data = io.BytesIO() + + obj.download_fileobj(data) + + with open(LOCAL_ENV_FILE, 'wb') as f: + f.write(data.getvalue()) + +def snake_to_camel(snake_str): + """Convert snake_case string to camelCase.""" + components = snake_str.lower().split('_') + # Capitalize the first letter of each component except the first one + return ''.join(x.title() for x in components) + +def load_env_file(env_file): + """Load the .env file and return it as a dictionary with camelCase keys.""" + env_vars = {} + with open(env_file, 'r') as file: + for line in file: + line = line.strip() + if line and not line.startswith('#'): # Skip empty lines and comments + key, value = line.split('=', 1) + camel_case_key = snake_to_camel(key.strip()) + env_vars[camel_case_key] = value.strip() + return env_vars + +def generate_cloudformation_parameters(env_vars): + """Generate CloudFormation Parameters from dot-separated keys in env_vars.""" + parameters = {} + for param_name in env_vars: + parameters[param_name] = { + 'Type': 'String', + 'Description': f"Parameter for {param_name}" + } + return parameters + +class Ref: + """Custom representation for CloudFormation !Ref.""" + def __init__(self, ref): + self.ref = ref + +def ref_representer(dumper, data): + """Custom YAML representer for CloudFormation !Ref.""" + return dumper.represent_scalar('!Ref', data.ref, style='') + +def update_cloudformation_environment(env_vars = {}, cloudformation_template = "template.yml"): + """Update Environment Variables in CloudFormation template to use Parameters.""" + def cloudformation_tag_constructor(loader, tag_suffix, node): + """Handle CloudFormation intrinsic functions like !Ref, !GetAtt, etc.""" + return loader.construct_scalar(node) + + # Register constructors for CloudFormation intrinsic functions + yaml.SafeLoader.add_multi_constructor('!', cloudformation_tag_constructor) + yaml.SafeDumper.add_representer(Ref, ref_representer) + + with open(cloudformation_template, 'r') as file: + template = yaml.safe_load(file) + + parameters = generate_cloudformation_parameters(env_vars) + print(f"---> generate_cloudformation_parameters: {parameters}") + # Add parameters to the CloudFormation template + if 'Parameters' not in template: + template['Parameters'] = {} + template['Parameters'].update(parameters) + + # Update environment variables in the resources + for resource in template.get('Resources', {}).values(): + if 'Properties' in resource and 'Environment' in resource['Properties']: + env_vars_section = resource['Properties']['Environment'].get('Variables', {}) + for key in env_vars_section: + camel_key = snake_to_camel(key) + print(f"Environment Variables {camel_key}") + + if camel_key in env_vars: + env_vars_section[key] = Ref(camel_key) + + # Save the updated CloudFormation template + with open(cloudformation_template, 'w') as file: + yaml.safe_dump(template, file, default_style=None, default_flow_style=False) + +def load_config_toml(toml_file): + """Load the config.toml file and return its content as a dictionary.""" + with open(toml_file, 'r') as file: + config = toml.load(file) + return config + +def update_parameter_overrides(config, env_vars): + """Update the parameter_overrides in the config dictionary with values from env_vars.""" + parameter_overrides = [f"{key}={value}" for key, value in env_vars.items()] + config['default']['deploy']['parameters']['parameter_overrides'] = parameter_overrides + return config + +def save_config_toml(config, toml_file): + """Save the updated config back to the toml file.""" + with open(toml_file, 'w') as file: + toml.dump(config, file) + +def update_config_with_env(env_file: str = LOCAL_ENV_FILE, toml_file = ".aws/petercat-preview.toml"): + """Load env vars from a .env file and update them into a config.toml file.""" + pull_envs() + + env_vars = load_env_file(env_file) + config = load_config_toml(toml_file) + updated_config = update_parameter_overrides(config, env_vars) + save_config_toml(updated_config, toml_file) + + update_cloudformation_environment(env_vars) + +def main(): + parser = argparse.ArgumentParser(description="Update config.toml parameter_overrides with values from a .env file.") + + subparsers = parser.add_subparsers(dest='command', required=True, help='Sub-command help') + pull_parser = subparsers.add_parser('pull', help='Pull environment variables from a .env file and update samconfig.toml') + + pull_parser.add_argument( + '-e', '--env', + type=str, + default=LOCAL_ENV_FILE, + help='Path to the .env file (default: .env)' + ) + + pull_parser.add_argument( + '-t', '--template', + type=str, + required=True, + default=".aws/petercat-preview.toml", + help='Path to the CloudFormation template file' + ) + + args = parser.parse_args() + + if args.command == 'pull': + update_config_with_env(args.env, args.template) -with open("./server/.env", 'wb') as f: - f.write(data.getvalue()) \ No newline at end of file +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/template.yml b/template.yml index ccd9f471..8f9d50fb 100644 --- a/template.yml +++ b/template.yml @@ -1,172 +1,163 @@ AWSTemplateFormatVersion: '2010-09-09' -Transform: AWS::Serverless-2016-10-31 -Description: > - Streaming Bedrock Response with FastAPI on AWS Lambda +Description: 'Streaming Bedrock Response with FastAPI on AWS Lambda -# More info about Globals: https://github.com/awslabs/serverless-application-model/blob/master/docs/globals.rst + ' Globals: Function: Timeout: 300 - +Outputs: + FastAPIFunction: + Description: FastAPI Lambda Function ARN + Value: FastAPIFunction.Arn + FastAPIFunctionUrl: + Description: Function URL for FastAPI function + Value: FastAPIFunctionUrl.FunctionUrl + SQSSubscriptionFunction: + Description: SQS Subscription Function Lambda Function ARN + Value: SQSSubscriptionFunction.Arn + SQSSubscriptionFunctionUrl: + Description: Function URL for SQS Subscriptio function + Value: SQSSubscriptionFunctionUrl.FunctionUrl Parameters: - GitHubAppID: - Type: Number - Description: Github App Id - Default: 1 - GithubAppsClientId: - Type: String - Description: Github App Id - Default: 1 - GitHubToken: - Type: String - Description: Github Token - Default: 1 - WebUrl: + ApiIdentifier: + Description: Parameter for ApiIdentifier + Type: String + ApiUrl: + Description: Parameter for ApiUrl + Type: String + Auth0ClientId: + Description: Parameter for Auth0ClientId + Type: String + Auth0ClientSecret: + Description: Parameter for Auth0ClientSecret + Type: String + Auth0Domain: + Description: Parameter for Auth0Domain + Type: String + AwsRegionName: + Description: Parameter for AwsRegionName + Type: String + AwsSecretName: + Description: Parameter for AwsSecretName + Type: String + CorsOriginWhitelist: + Description: Parameter for CorsOriginWhitelist Type: String - Description: Web URL - Default: 1 - APIUrl: + FastapiSecretKey: + Description: Parameter for FastapiSecretKey Type: String - Description: API URL - Default: 1 - APIIdentifier: + GeminiApiKey: + Description: Parameter for GeminiApiKey Type: String - Description: APIIdentifier - Default: 1 - FastAPISecretKey: + OpenaiApiKey: + Description: Parameter for OpenaiApiKey Type: String - Description: FastAPISecretKey - Default: 1 - GithubAppsClientSecret: + RateLimitDuration: + Description: Parameter for RateLimitDuration Type: String - Description: Github App Id - Default: 1 - OpenAIAPIKey: + RateLimitEnabled: + Description: Parameter for RateLimitEnabled Type: String - Description: Github App Id - Default: 1 - GeminiAPIKey: + RateLimitRequests: + Description: Parameter for RateLimitRequests + Type: String + S3BucketName: + Description: Parameter for S3BucketName + Type: String + SqsQueueUrl: + Description: Parameter for SqsQueueUrl Type: String - Description: Gemini API Key - Default: 1 SupabaseServiceKey: + Description: Parameter for SupabaseServiceKey Type: String - Description: Github App Id - Default: 1 SupabaseUrl: + Description: Parameter for SupabaseUrl Type: String - Description: Github App Id - Default: 1 - TavilyAPIKey: + TavilyApiKey: + Description: Parameter for TavilyApiKey Type: String - Description: Github App Id - Default: 1 - SQSQueueName: + WebUrl: + Description: Parameter for WebUrl Type: String - Description: SQS Queue Name - Default: 1 - SQSQueueUrl: + XGithubAppId: + Description: Parameter for XGithubAppId Type: String - Description: SQS Queue Url - Default: 1 - Auth0Domain: + XGithubAppsClientId: + Description: Parameter for XGithubAppsClientId Type: String - Description: Auth0 Domain - Default: 1 - Auth0ClientId: + XGithubAppsClientSecret: + Description: Parameter for XGithubAppsClientSecret Type: String - Description: Auth0 ClientId - Default: 1 - Auth0ClientSecret: - Type: String - Description: Auth0 Clientt Secret - Default: 1 Resources: FastAPIFunction: - Type: AWS::Serverless::Function + Metadata: + DockerContext: server + DockerTag: v1 + Dockerfile: ../docker/Dockerfile.aws.lambda Properties: - PackageType: Image - MemorySize: 512 Environment: Variables: + API_IDENTIFIER: !Ref 'ApiIdentifier' + API_URL: !Ref 'ApiUrl' + AUTH0_CLIENT_ID: !Ref 'Auth0ClientId' + AUTH0_CLIENT_SECRET: !Ref 'Auth0ClientSecret' + AUTH0_DOMAIN: !Ref 'Auth0Domain' AWS_LWA_INVOKE_MODE: RESPONSE_STREAM - API_URL: !Ref APIUrl - WEB_URL: !Ref WebUrl - X_GITHUB_APP_ID: !Ref GitHubAppID - X_GITHUB_APPS_CLIENT_ID: !Ref GithubAppsClientId - X_GITHUB_APPS_CLIENT_SECRET: !Ref GithubAppsClientSecret - API_IDENTIFIER: !Ref APIIdentifier - FASTAPI_SECRET_KEY: !Ref FastAPISecretKey - OPENAI_API_KEY: !Ref OpenAIAPIKey - GEMINI_API_KEY: !Ref GeminiAPIKey - SUPABASE_SERVICE_KEY: !Ref SupabaseServiceKey - SUPABASE_URL: !Ref SupabaseUrl - GITHUB_TOKEN: !Ref GitHubToken - TAVILY_API_KEY: !Ref TavilyAPIKey - SQS_QUEUE_URL: !Ref SQSQueueUrl - AUTH0_DOMAIN: !Ref Auth0Domain - AUTH0_CLIENT_ID: !Ref Auth0ClientId - AUTH0_CLIENT_SECRET: !Ref Auth0ClientSecret + FASTAPI_SECRET_KEY: !Ref 'FastapiSecretKey' + GEMINI_API_KEY: !Ref 'GeminiApiKey' + GITHUB_TOKEN: GitHubToken + OPENAI_API_KEY: !Ref 'OpenaiApiKey' + SQS_QUEUE_URL: !Ref 'SqsQueueUrl' + SUPABASE_SERVICE_KEY: !Ref 'SupabaseServiceKey' + SUPABASE_URL: !Ref 'SupabaseUrl' + TAVILY_API_KEY: !Ref 'TavilyApiKey' + WEB_URL: !Ref 'WebUrl' + X_GITHUB_APPS_CLIENT_ID: !Ref 'XGithubAppsClientId' + X_GITHUB_APPS_CLIENT_SECRET: !Ref 'XGithubAppsClientSecret' + X_GITHUB_APP_ID: !Ref 'XGithubAppId' FunctionUrlConfig: AuthType: NONE InvokeMode: RESPONSE_STREAM + MemorySize: 512 + PackageType: Image Policies: - Statement: - - Sid: BedrockInvokePolicy - Effect: Allow - Action: + - Action: - bedrock:InvokeModelWithResponseStream + Effect: Allow Resource: '*' + Sid: BedrockInvokePolicy Tracing: Active + Type: AWS::Serverless::Function + SQSSubscriptionFunction: Metadata: - DockerContext: server - Dockerfile: ../docker/Dockerfile.aws.lambda + DockerContext: subscriber DockerTag: v1 - - SQSSubscriptionFunction: - Type: AWS::Serverless::Function + Dockerfile: ../docker/Dockerfile.subscriber Properties: - PackageType: Image - MemorySize: 512 - FunctionUrlConfig: - AuthType: NONE Environment: Variables: - SUPABASE_SERVICE_KEY: !Ref SupabaseServiceKey - SUPABASE_URL: !Ref SupabaseUrl - GITHUB_TOKEN: !Ref GitHubToken + GITHUB_TOKEN: GitHubToken + SUPABASE_SERVICE_KEY: !Ref 'SupabaseServiceKey' + SUPABASE_URL: !Ref 'SupabaseUrl' + FunctionUrlConfig: + AuthType: NONE + MemorySize: 512 + PackageType: Image Policies: - Statement: - - Sid: BedrockInvokePolicy - Effect: Allow - Action: + - Action: - bedrock:InvokeModelWithResponseStream - Resource: '*' - - Sid: SQSInvokePolicy Effect: Allow - Action: + Resource: '*' + Sid: BedrockInvokePolicy + - Action: - sqs:* + Effect: Allow Resource: '*' + Sid: SQSInvokePolicy - SQSPollerPolicy: - QueueName: - !Ref SQSQueueName + QueueName: SQSQueueName Tracing: Active - Metadata: - Dockerfile: ../docker/Dockerfile.subscriber - DockerContext: subscriber - DockerTag: v1 - -Outputs: - FastAPIFunctionUrl: - Description: "Function URL for FastAPI function" - Value: !GetAtt FastAPIFunctionUrl.FunctionUrl - FastAPIFunction: - Description: "FastAPI Lambda Function ARN" - Value: !GetAtt FastAPIFunction.Arn - - SQSSubscriptionFunctionUrl: - Description: "Function URL for SQS Subscriptio function" - Value: !GetAtt SQSSubscriptionFunctionUrl.FunctionUrl - SQSSubscriptionFunction: - Description: "SQS Subscription Function Lambda Function ARN" - Value: !GetAtt SQSSubscriptionFunction.Arn \ No newline at end of file + Type: AWS::Serverless::Function +Transform: AWS::Serverless-2016-10-31