Merge pull request #205 from FullStackWithLawrence/next

v0.7.0 - Pydantic Settings class + unit test automations
FullStackWithLawrence · Dec 18, 2023 · 358ad4d · 358ad4d
2 parents f4cd8f2 + 9a2b00d
commit 358ad4d
Show file tree

Hide file tree

Showing 179 changed files with 4,214 additions and 1,992 deletions.
diff --git a/.github/actions/tests/python/action.yml b/.github/actions/tests/python/action.yml
@@ -28,9 +28,6 @@ pinecone-environment:
   required: true
   type: string
 
-env:
-  REQUIREMENTS_PATH: "api/terraform/python/layer_genai/requirements.txt"
-
 runs:
   using: "composite"
   steps:
@@ -96,7 +93,6 @@ runs:
     - name: Install dependencies
       shell: bash
       run: |
-        cp -R ./api/terraform/python/layer_genai/openai_utils ${{ env.SITE_PACKAGES_PATH }}
         pip install -r ./requirements.txt
       env:
         SITE_PACKAGES_PATH: ${{ env.SITE_PACKAGES_PATH }}
@@ -109,20 +105,14 @@ runs:
         echo "OPENAI_API_KEY=${{ env.OPENAI_API_KEY }}" >> ./.env
         echo "PINECONE_API_KEY=${{ env.PINECONE_API_KEY }}" >> ./.env
         echo "PINECONE_ENVIRONMENT=${{ env.PINECONE_ENVIRONMENT }}" >> ./.env
+        echo "DEBUG_MODE=False" >> ./.env
       env:
         OPENAI_API_ORGANIZATION: ${{ inputs.openai-api-organization }}
         OPENAI_API_KEY: ${{ inputs.openai-api-key }}
         PINECONE_API_KEY: ${{ inputs.pinecone-api-key }}
         PINECONE_ENVIRONMENT: ${{ inputs.pinecone-environment }}
 
-    - name: Test lambda_openai_v2
-      shell: bash
-      run: |
-        cd ./api/terraform/python/lambda_openai_v2
-        pytest -v -s tests/
-
-    - name: Test lambda_langchain
+    - name: Run Python unit tests
       shell: bash
       run: |
-        cd ./api/terraform/python/lambda_langchain
-        pytest -v -s tests/
+        make api-test
diff --git a/.github/workflows/auto-assign.yml b/.github/workflows/auto-assign.yml
@@ -0,0 +1,19 @@
+name: Auto Assign
+on:
+  issues:
+    types: [opened]
+  pull_request:
+    types: [opened]
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - name: "Auto-assign issue"
+        uses: pozil/auto-assign-issue@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          assignees: lpm0073
+          numOfAssignee: 1
diff --git a/.github/workflows/runTests.yml b/.github/workflows/runTests.yml
@@ -17,6 +17,13 @@ jobs:
         id: checkout
         uses: actions/checkout@v4
 
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
       - name: Run Python tests
         uses: ./.github/actions/tests/python
         with:

diff --git a/.github/workflows/semanticVersionBump.yml b/.github/workflows/semanticVersionBump.yml
@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       VERSION_FILE: __version__.py
-      PACKAGE_PATH: ${{ github.workspace }}/api/terraform/python/layer_genai/openai_utils/
+      PACKAGE_PATH: ${{ github.workspace }}/api/terraform/python/openai_api/
 
     steps:
       - uses: actions/checkout@v4
@@ -110,8 +110,9 @@ jobs:
         id: update_version
         run: |
           echo "# -*- coding: utf-8 -*-" > ${{ env.VERSION_FILE }}
-          echo "__version__ = '${{ env.NEXT_VERSION }}'" >>  ${{ env.VERSION_FILE }}
-          echo "" >>  ${{ env.VERSION_FILE }}
+          echo "# DO NOT EDIT." >> ${{ env.VERSION_FILE }}
+          echo "# Managed via automated CI/CD in .github/workflows/semanticVersionBump.yml." >> ${{ env.VERSION_FILE }}
+          echo "__version__ = \"${{ env.NEXT_VERSION }}\"" >>  ${{ env.VERSION_FILE }}
           git config --local user.email "[email protected]"
           git config --local user.name "GitHub Action"
           git add ${{ env.VERSION_FILE }}

diff --git a/.github/workflows/testsPython.yml b/.github/workflows/testsPython.yml
@@ -0,0 +1,37 @@
+name: Python Unit Tests
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - "**.py"
+  push:
+    paths:
+      - "**.py"
+
+env:
+  python-version: "3.11"
+
+jobs:
+  python-unit-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        id: checkout
+        uses: actions/checkout@v4
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: Run Python tests
+        uses: ./.github/actions/tests/python
+        with:
+          python-version: "${{ env.python-version}}"
+          openai-api-organization: "${{ secrets.OPENAI_API_ORGANIZATION }}"
+          openai-api-key: "${{ secrets.OPENAI_API_KEY }}"
+          pinecone-api-key: "${{ secrets.PINECONE_API_KEY }}"
+          pinecone-environment: "${{ secrets.PINECONE_ENVIRONMENT }}"
diff --git a/.gitignore b/.gitignore
@@ -1,8 +1,14 @@
 # Jupyter Notebook
 .ipynb_checkpoints
+build
+dist
 
 .DS_Store
+
 #  Local .terraform directories
+.terraform.lock.hcl
+*.tfstate
+*.tfstate.*
 .terraform/
 .env
 lambda_dist_pkg

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -17,19 +17,19 @@ repos:
         types: [javascript]
         files: ^client/
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v3.1.0
+    rev: v4.0.0-alpha.4
     hooks:
       - id: prettier
   - repo: https://github.com/psf/black
-    rev: 23.11.0
+    rev: 23.12.0
     hooks:
       - id: black
   - repo: https://github.com/PyCQA/flake8
     rev: 6.1.0
     hooks:
       - id: flake8
   - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
       - id: isort
         args: ["--settings-path=pyproject.toml"]
@@ -41,7 +41,7 @@ repos:
         language: script
         types: [python]
   - repo: https://github.com/PyCQA/bandit
-    rev: 1.7.5
+    rev: 1.7.6
     hooks:
       - id: bandit
         args: ["-ll"]

diff --git a/.pylintrc b/.pylintrc
@@ -1,5 +1,4 @@
 [MASTER]
-init-hook='import sys; print(sys.executable); sys.path.append("/absolute/path/to/api/terraform/python/layer_genai/openai_utils")'
 ignore-paths =
   api/terraform/python/lambda_openai_v2/lambda_dist_pkg,
   api/terraform/python/lambda_openai/lambda_dist_pkg,

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,16 +1,30 @@
-## [0.6.5](https://github.com/FullStackWithLawrence/aws-openai/compare/v0.6.4...v0.6.5) (2023-12-05)
+# Change Log
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/).
+
+## [0.7.0]
 
+- replace Terraform template_file() with templatefile()
+- restructure Python modules. Place original lambda source code under openai_api and refactor common module so that it no longer needs to be pip installed. remove legacy lambda_openai.
+- refactor layer_genai to remove pip-installed common code.
+- refactor makefile and Github Actions so that these no longer have any special handling of Python source code.
+- switch from pytest to built-in python unittest
+- add Pydantic-based Settings class
+- add unit testing of both lambdas + common code
+
+## [0.6.5](https://github.com/FullStackWithLawrence/aws-openai/compare/v0.6.4...v0.6.5) (2023-12-05)
 
 ### Bug Fixes
 
-* force a new release ([b5f8ee6](https://github.com/FullStackWithLawrence/aws-openai/commit/b5f8ee6541befb08c7e7f6b3a13bd841bc44680f))
+- force a new release ([b5f8ee6](https://github.com/FullStackWithLawrence/aws-openai/commit/b5f8ee6541befb08c7e7f6b3a13bd841bc44680f))
 
 ## [0.6.4](https://github.com/FullStackWithLawrence/aws-openai/compare/v0.6.3...v0.6.4) (2023-11-24)
 
-
 ### Bug Fixes
 
-* simulated bug to trigger new release ([f373039](https://github.com/FullStackWithLawrence/aws-openai/commit/f37303924693b4ffeae424476d9fdf3595efb311))
+- simulated bug to trigger new release ([f373039](https://github.com/FullStackWithLawrence/aws-openai/commit/f37303924693b4ffeae424476d9fdf3595efb311))
 
 ## [0.6.3](https://github.com/FullStackWithLawrence/aws-openai/compare/v0.6.2...v0.6.3) (2023-11-21)
 

diff --git a/Makefile b/Makefile
@@ -8,7 +8,7 @@ else
     $(shell echo -e "OPENAI_API_ORGANIZATION=PLEASE-ADD-ME\nOPENAI_API_KEY=PLEASE-ADD-ME\nPINECONE_API_KEY=PLEASE-ADD-ME\nPINECONE_ENVIRONMENT=gcp-starter\nDEBUG_MODE=True\n" >> .env)
 endif
 
-.PHONY: analyze api-init api-activate api-lint api-clean api-test client-init client-lint client-update client-run client-build client-release
+.PHONY: analyze pre-commit api-init api-activate api-lint api-clean api-test client-init client-lint client-update client-run client-build client-release
 
 # Default target executed when no arguments are given to make.
 all: help
@@ -19,6 +19,14 @@ analyze:
 release:
 	git commit -m "fix: force a new release" --allow-empty && git push
 
+# -------------------------------------------------------------------------
+# Install and run pre-commit hooks
+# -------------------------------------------------------------------------
+pre-commit:
+	pre-commit install
+	pre-commit autoupdate
+	pre-commit run --all-files
+
 ######################
 # AWS API Gateway + Lambda + OpenAI
 ######################
@@ -32,14 +40,12 @@ api-init:
 	source venv/bin/activate && \
 	pip install --upgrade pip && \
 	pip install -r requirements.txt && \
-	cp -R ./api/terraform/python/layer_genai/openai_utils ./venv/lib/python3.11/site-packages/ && \
 	deactivate && \
 	cd ./api/terraform/python/layer_genai/ && \
 	python3.11 -m venv venv && \
 	source venv/bin/activate && \
 	pip install --upgrade pip && \
 	pip install -r requirements.txt && \
-	cp -R ./openai_utils ./venv/lib/python3.11/site-packages/ && \
 	deactivate && \
 	pre-commit install
 
@@ -48,20 +54,11 @@ api-activate:
 	pip install -r requirements.txt
 
 api-test:
-	cd ./api/terraform/python/lambda_langchain/ && pytest -v -s tests/
-	cd ../../../..
-	cd ./api/terraform/python/lambda_openai_v2/ && pytest -v -s tests/
-	cd ../../../..
-	cd ./api/terraform/python/lambda_openai/ && pytest -v -s tests/
-	cd ../../../..
+	python -m unittest discover -s api/terraform/python/openai_api/
 
 api-lint:
-	venv/bin/python3 -m pylint api/terraform/python/lambda_langchain/lambda_handler.py && \
-	venv/bin/python3 -m pylint api/terraform/python/lambda_openai/lambda_handler.py && \
-	venv/bin/python3 -m pylint api/terraform/python/lambda_openai_v2/lambda_handler.py && \
-	venv/bin/python3 -m pylint api/terraform/python/layer_genai/openai_utils && \
-	terraform fmt -recursive && \
-	pre-commit run --all-files && \
+	terraform fmt -recursive
+	pre-commit run --all-files
 	black ./api/terraform/python/
 
 api-clean:
@@ -123,7 +120,7 @@ client-release:
 help:
 	@echo '===================================================================='
 	@echo 'analyze             - generate code analysis report'
-	@echo 'relase              - force a new release'
+	@echo 'release             - force a new release'
 	@echo '-- AWS API Gateway + Lambda --'
 	@echo 'api-init            - create a Python virtual environment and install dependencies'
 	@echo 'api-activate        - activate the Python virtual environment'