From ba43da18ccdac7a644cc02ad21877be11211fda2 Mon Sep 17 00:00:00 2001 From: Eric Buckley Date: Mon, 16 Sep 2024 15:12:23 -0700 Subject: [PATCH] Adding workflows for unit tests and linting (#11) ## Description Adding 4 different GitHub workflows to help with managing code checks. 1. unit tests 2. linting 3. code vulnerabilities 4. dependency vulnerabilities ## Related Issues closes #7 ## Additional Notes Also removing the `scripts/vulnerability_checks.sh` script as we're going to try using GH tools (ie CodeQL and Dependabot) for managing those checks and won't need an independent script for it anymore. --- .github/dependabot.yml | 9 ++++ .../workflows/check_code_vulnerabilities.yml | 39 +++++++++++++++ .github/workflows/check_lint.yml | 31 ++++++++++++ .github/workflows/check_unit_tests.yml | 50 +++++++++++++++++++ README.md | 8 --- pyproject.toml | 3 +- scripts/vulnerability_check.sh | 11 ---- src/recordlinker/config.py | 2 +- src/recordlinker/linkage/config.py | 1 + src/recordlinker/linkage/models.py | 5 +- 10 files changed, 136 insertions(+), 23 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/check_code_vulnerabilities.yml create mode 100644 .github/workflows/check_lint.yml create mode 100644 .github/workflows/check_unit_tests.yml delete mode 100644 scripts/vulnerability_check.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..4198f1aa --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,9 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + commit-message: + prefix: "[deps] " + open-pull-requests-limit: 3 diff --git a/.github/workflows/check_code_vulnerabilities.yml b/.github/workflows/check_code_vulnerabilities.yml new file mode 100644 index 00000000..e78327aa --- /dev/null +++ b/.github/workflows/check_code_vulnerabilities.yml @@ -0,0 +1,39 @@ +name: code vulnerabilities check + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + codeql: + runs-on: ubuntu-latest + + permissions: + packages: read + actions: read + contents: read + security-events: write + + strategy: + matrix: + # Using a matrix in case we need to test Javascript code in the future + language: ['python'] + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" + config: | + # only scan the code in the src directory + paths: ["src"] diff --git a/.github/workflows/check_lint.yml b/.github/workflows/check_lint.yml new file mode 100644 index 00000000..2fa45236 --- /dev/null +++ b/.github/workflows/check_lint.yml @@ -0,0 +1,31 @@ +name: lint check + +# When the workflow will be triggered +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install '.[dev]' + + - name: Run lint checks + run: | + ruff check src/ diff --git a/.github/workflows/check_unit_tests.yml b/.github/workflows/check_unit_tests.yml new file mode 100644 index 00000000..e8fd8a76 --- /dev/null +++ b/.github/workflows/check_unit_tests.yml @@ -0,0 +1,50 @@ +name: unit tests check + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:13 + ports: + - 5432:5432 + env: + POSTGRES_PASSWORD: pw + POSTGRES_DB: testdb + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install '.[dev]' + + - name: Wait for PostgreSQL to be ready + run: | + until pg_isready -h localhost -U postgres; do sleep 1; done + + - name: Run unit tests + env: + MPI_DB_TYPE: postgres + MPI_DBNAME: testdb + MPI_HOST: localhost + MPI_PORT: 5432 + MPI_USER: postgres + MPI_PASSWORD: pw + run: | + pytest --cov=recordlinker --cov-report=xml tests/unit diff --git a/README.md b/README.md index fa2ff4d3..d3ebb7d1 100644 --- a/README.md +++ b/README.md @@ -45,14 +45,6 @@ To run a single unit test, use the following command: ./scripts/test_unit.sh tests/unit/test_linkage.py::test_link_record_against_mpi ``` -### Building the Docker Image - -To build the Docker image for the record linkage service from source code instead of downloading it from the DIBBs repository follow these steps. -1. Ensure that both [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) and [Docker](https://docs.docker.com/get-docker/) are installed. -2. Clone the DIBBs repository with `git clone https://github.com/CDCgov/phdi`. -3. Navigate to `/phdi/containers/record-linkage/`. -4. Run `docker build -t record-linkage .`. - ## Standard Notices ### Public Domain Standard Notice diff --git a/pyproject.toml b/pyproject.toml index 9d2c0d35..152434d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,9 +31,8 @@ dependencies = [ [project.optional-dependencies] dev = [ "pytest>=8.3", + "pytest-cov", "ruff", - "pip-audit", - "bandit", "mypy", "pyarrow", "httpx", diff --git a/scripts/vulnerability_check.sh b/scripts/vulnerability_check.sh deleted file mode 100644 index be88417d..00000000 --- a/scripts/vulnerability_check.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -# -# Run vulnerability checks on the project and its dependencies. -# -# Usage: scripts/vulnerability_check.sh - -set -e - -cd "$(dirname "$0")/.." - -pip-audit . && bandit -r src/ diff --git a/src/recordlinker/config.py b/src/recordlinker/config.py index ce39b6e2..0afaf509 100644 --- a/src/recordlinker/config.py +++ b/src/recordlinker/config.py @@ -1,8 +1,8 @@ from functools import lru_cache from typing import Optional -from pydantic_settings import BaseSettings from pydantic import Field +from pydantic_settings import BaseSettings class Settings(BaseSettings): diff --git a/src/recordlinker/linkage/config.py b/src/recordlinker/linkage/config.py index 2908a5cd..d4455740 100644 --- a/src/recordlinker/linkage/config.py +++ b/src/recordlinker/linkage/config.py @@ -2,6 +2,7 @@ from pydantic_settings import BaseSettings + class DBSettings(BaseSettings): mpi_db_type: str mpi_dbname: str diff --git a/src/recordlinker/linkage/models.py b/src/recordlinker/linkage/models.py index fa11e793..1d4c30a5 100644 --- a/src/recordlinker/linkage/models.py +++ b/src/recordlinker/linkage/models.py @@ -1,6 +1,9 @@ import uuid -from sqlalchemy import orm, ForeignKey, String, JSON +from sqlalchemy import ForeignKey +from sqlalchemy import JSON +from sqlalchemy import orm +from sqlalchemy import String class Base(orm.DeclarativeBase):