Migrate to poetry, update dependencies, add pre-commit

Migrate to poetry for dependency management
Setup pre-commit hooks
Fix django settings in upgrading to 4.x
Add __init__ files as necessary to keep module structure
Remove old test cases, reformat edited files

.github/workflows/cypress.yml

@@ -2,9 +2,9 @@ name: Cypress E2E
 
 on:
   push:
-    branches: [ master ]
+    branches: [master]
   pull_request:
-    branches: [ master ]
+    branches: [master]
 
 env:
   PYTHON_VERSION: 3.9
@@ -15,15 +15,15 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        containers: [ 1, 2, 3, 4 ]
+        containers: [1, 2, 3, 4]
     services:
       postgres:
         image: postgres:14
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: csm_web_dev
-        ports: [ "5432:5432" ]
+        ports: ["5432:5432"]
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
     steps:
       - name: Checkout
@@ -71,15 +71,15 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        containers: [ 1, 2, 3, 4 ]
+        containers: [1, 2, 3, 4]
     services:
       postgres:
         image: postgres:14
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: csm_web_dev
-        ports: [ "5432:5432" ]
+        ports: ["5432:5432"]
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
     steps:
       - name: Checkout

.github/workflows/django.yml

@@ -2,9 +2,9 @@ name: Django CI
 
 on:
   push:
-    branches: [ master ]
+    branches: [master]
   pull_request:
-    branches: [ master ]
+    branches: [master]
 
 env:
   PYTHON_VERSION: 3.9
@@ -13,26 +13,35 @@ jobs:
   pytest:
     name: "pytest"
     runs-on: ubuntu-latest
+
     services:
       postgres:
         image: postgres:14
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: csm_web_dev
-        ports: [ "5432:5432" ]
+        ports: ["5432:5432"]
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
 
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python ${{ env.PYTHON_VERSION }}
-        uses: actions/setup-python@v2
+      - name: Parse Python version from runtime.txt
+        run: |
+          sed 's/[^0-9.]//g' runtime.txt | head > .python-version
+      - name: Set up Python
+        uses: actions/setup-python@v4
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
-      - name: Install Dependencies
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+      - name: Install dependencies with Poetry
         run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          poetry install --no-root --with=dev
+          # add virtual environment to path for future steps
+          echo .venv/bin >> $GITHUB_PATH
       - name: Run Tests
         env:
           SECRET_KEY: ${{ secrets.SECRET_KEY }}

.github/workflows/nodejs.yml

@@ -5,9 +5,9 @@ name: Node.js CI
 
 on:
   push:
-    branches: [ master ]
+    branches: [master]
   pull_request:
-    branches: [ master ]
+    branches: [master]
 
 env:
   NODE_VERSION: 16.x

.github/workflows/pre-commit.yml

@@ -0,0 +1,38 @@
+name: pre-commit
+
+on:
+  pull_request:
+    branches: [master]
+  push:
+    branches: [master]
+
+jobs:
+  pre-commit:
+    name: "pre-commit"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/[email protected]
+      - name: Parse Python version from runtime.txt
+        run: |
+          sed 's/[^0-9.]//g' runtime.txt | head > .python-version
+      - name: Set up Python
+        uses: actions/setup-python@v4
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+      - name: Install dependencies with Poetry
+        run: |
+          poetry install --no-root --with=dev --no-interaction --no-ansi
+          # add virtual environment to path for future steps
+          echo $(poetry env info --path)/bin >> $GITHUB_PATH
+      - name: Run pre-commit hooks
+        uses: pre-commit/[email protected]
+        with:
+          # only check changed files
+          extra_args: "--files ${{ steps.changed-files.outputs.all_changed_files }}"

.gitignore

@@ -83,6 +83,9 @@ ipython_config.py
 # pyenv
 .python-version
 
+# asdf
+.tool-versions
+
 # celery beat schedule file
 celerybeat-schedule
 

.pre-commit-config.yaml

@@ -0,0 +1,47 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-merge-conflict
+      - id: detect-private-key
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: trailing-whitespace
+  # python
+  - repo: https://github.com/psf/black
+    rev: 22.12.0
+    hooks:
+      - id: black
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.11.4
+    hooks:
+      - id: isort
+  - repo: local
+    hooks:
+      - id: pylint
+        name: pylint
+        entry: pylint
+        language: system
+        types: [python]
+        args: [
+            "-rn" # only display messages
+          ]
+        verbose: true # always display output
+  # dependencies
+  - repo: https://github.com/python-poetry/poetry
+    rev: 1.5.0
+    hooks:
+      - id: poetry-check
+      - id: poetry-export
+        args: ["-f", "requirements.txt", "-o", "requirements.txt", "--without-hashes", "--with", "prod"]
+  # js
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.0.0-alpha.4
+    hooks:
+      - id: prettier
+  - repo: https://github.com/pre-commit/mirrors-eslint
+    rev: v8.30.0
+    hooks:
+      - id: eslint
+        exclude: 'webpack\.config\.js'
+        verbose: true # always display output

README.md

@@ -1,78 +1,92 @@
 # CSM Web Applications
+
 This repository contains webapps that help support the infrastructure of CSM. Currently, it only holds Scheduler (our section signup and attendance system), but more are on the way.
 
 If you're unfamiliar with CSM and/or its web applications, check out [this repository's releases](https://github.com/csmberkeley/csm_web/releases) for a peek at what our web applications look like and what we've been working on lately.
 
 ## Installing
+
 ### Requirements
+
 We don't know what specific minimum version you would need for any of the following software, but the most recent version of any of the below should work.
 
-* Python 3.8
-  * [virtualenv](https://packaging.python.org/guides/installing-using-pip-and-virtualenv/) (`pip install virtualenv`)
-  * [autopep8](https://pypi.org/project/autopep8/) (`pip install autopep8`)
-* [npm](https://www.npmjs.com/get-npm)
-  * [prettier](https://prettier.io/) (`npm install -g prettier`)
-  * [eslint](https://eslint.org/) (`npm install -g eslint`; `npm install -g eslint-plugin-react`)
-* Postgresql (`brew install postgresql && brew services start postgresql`)
-* [Heroku CLI](https://devcenter.heroku.com/articles/heroku-cli#download-and-install) (`brew tap heroku/brew && brew install heroku`)
-  * Create an account on [Heroku](https://id.heroku.com/login) and [login](https://devcenter.heroku.com/articles/heroku-cli#getting-started)
-* [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html) (`brew install awscli`)
-  * We use an S3 bucket to store course resources. See [here](https://aws.amazon.com/s3/) to get started.
-* Log in to AWS CLI (`aws configure `) This will prompt an interactive session to enter login credentials.
-  * AWS Access Key ID: (ask tech chair)
-  * AWS Secret Access Key: (ask tech chair)
-  * Default region name: us-east-1
-  * Default output format: json
+- Python 3.9.13
+  - It is recommended that you use [`pyenv`](https://github.com/pyenv/pyenv) to manage python versions, so that you can use a consistent python version for `csm_web`, and another python version for your other projects.
+- [`poetry`](https://python-poetry.org/docs/#installation)
+- `npm`
+  - It is recommended that you use [`nvm`](https://github.com/nvm-sh/nvm) to manage node/npm versions, so that you can use a consistent node/npm version for `csm_web`, and another verison for your other projects.
+- [PostgreSQL](https://www.postgresql.org/download/)
+- [Heroku CLI](https://devcenter.heroku.com/articles/heroku-cli#download-and-install)
+  - Create an account on [Heroku](https://id.heroku.com/login) and [login](https://devcenter.heroku.com/articles/heroku-cli#getting-started)
+- [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html)
+  - We use an S3 bucket to store course resources. See [here](https://aws.amazon.com/s3/) to get started.
+- Log in to AWS CLI (`aws configure`) This will prompt an interactive session to enter login credentials.
+  - AWS Access Key ID: (ask tech chair)
+  - AWS Secret Access Key: (ask tech chair)
+  - Default region name: `us-east-1`
+  - Default output format: `json`
 
 Other miscellaneous requirements will be installed by the commands below.
 
 ### Setup
-To ensure package version consistency and avoid polluting your global package installations, we highly recommend running the server with [virtualenv](https://virtualenv.pypa.io/en/stable/) (`pip install virtualenv`).
 
-From a terminal in the top level of the project directory, run `python3 -m virtualenv venv`.
+To ensure package version consistency and avoid polluting your global package installations, we highly recommend running the server with a virtual environment. Python's standard library includes [venv](https://docs.python.org/3/library/venv.html), so you do not need to install anything new here.
+
+Firstly, make sure you have the right python version (see `runtime.txt` for the expected python version to install). If you're using `pyenv` to manage python versions (this is recommended), you can install the specified python version with `pyenv install <version>`.
+
+From a terminal in the top level of the project directory, run `python3 -m venv venv`; if your system python version is different from the version required here, and you're using `pyenv`, run `PYENV_VERSION=<version> python3 -m venv venv` instead (for example, `PYENV_VERSION=3.9.13 python3 -m venv venv`). This will initialize a new virtual environment in the `venv/` folder, with the correct base python version.
 
 To activate the environment, run `source venv/bin/activate`. You will need to run this command every time you open a new terminal.
 
-Finally, run `./setup.sh`. This will install additional requirements needed by the server, and set up some necessary environment variables.
+Finally, run `./setup.sh`. This will install additional requirements needed by the server, and set up some necessary environment variables. You should _not_ be running this script after it has succeeded and set up the environment for the first time.
 
 ## Running
-To start the Django server, run `python csm_web/manage.py runserver` and visit `localhost:8000` in your browser.
 
-Run `python csm_web/manage.py createtestdata` to generate some test data. If you ran `./setup.sh`,
+To start the Django server, run `python3 csm_web/manage.py runserver` and visit `localhost:8000` in your browser.
+
+Run `python3 csm_web/manage.py createtestdata` to generate some test data. If you ran `./setup.sh`,
 this was done for you.
 
-*If you are working on the frontend*:
+_If you are working on the frontend_:
 
 Run `npm run watch`, which will automatically rebuild the JS bundle if any changes to the frontend JS are detected.
 Alternatively you can run `npm run dev` manually each time you make changes to the frontend.
 
 ## Troubleshooting
+
 ### `setup.sh` Errors
-* The following errors are likely caused by some quirks in our build system - if you set up a new virtual environment through normal commands, you may run into them. The solution for all of these should be to run `setup.sh` (you should be able to do this even after attempting to run pip/npm commands already).
-  * `django.core.exceptions.ImproperlyConfigured: The SECRET_KEY setting must not be empty.`
-  * When installing `psycopg2`, console output displays `ld: library not found for -lpq` or similar
-* During `./setup.sh` or `pip3 install` I'm getting a `psycopg2` install error that looks like this:
+
+- The following errors are likely caused by some quirks in our build system - if you set up a new virtual environment through normal commands, you may run into them. The solution for all of these should be to run `setup.sh` (you should be able to do this even after attempting to run pip/npm commands already).
+  - `django.core.exceptions.ImproperlyConfigured: The SECRET_KEY setting must not be empty.`
+  - When installing `psycopg2`, console output displays `ld: library not found for -lpq` or similar
+- During `./setup.sh` or `pip3 install` I'm getting a `psycopg2` install error that looks like this:
+
 ```
 Error: pg_config executable not found.
-    
+
     pg_config is required to build psycopg2 from source.  Please add the directory
     containing pg_config to the $PATH or specify the full executable path with the
     option:
 ```
-  * You need to install Postgres locally (see [this](https://stackoverflow.com/a/12037133) SO post)
-* During `./setup.sh`, this happens after `pip3 install` runs
+
+- You need to install Postgres locally (see [this](https://stackoverflow.com/a/12037133) SO post)
+- During `./setup.sh`, this happens after `pip3 install` runs
+
 ```
 ./setup.sh:41: no such file or directory: /Users/jhshi/Documents/csm/csm_web/venv/.project_dir
 ./setup.sh:44: no such file or directory: /Users/jhshi/Documents/csm/csm_web/venv/bin/activate
 ./setup.sh:47: no such file or directory: /Users/jhshi/Documents/csm/csm_web/venv/bin/activate
 ```
-  * This likely happens because you moved your folder after creating your virtualenv. Removing
-    and recreating your virtualenv should fix it.
+
+- This likely happens because you moved your folder after creating your virtualenv. Removing
+  and recreating your virtualenv should fix it.
 
 ### How do I access the `/admin` page?
-  * The `./setup.sh` script will create a user with username `demo_user` and password `pass`. You can access it by signing in through the admin page.
-  * If you wish to assign admin permissions to an account that uses OAuth (such as your Berkeley email), run the following commands in the Django shell (accessible by running `python3 csm_web/manage.py shell`):
-```
+
+- The `./setup.sh` script will create a user with username `demo_user` and password `pass`. You can access it by signing in through the admin page.
+- If you wish to assign admin permissions to an account that uses OAuth (such as your Berkeley email), run the following commands in the Django shell (accessible by running `python3 csm_web/manage.py shell`):
+
+```py
 from scheduler.models import *
 # replace "my_username" with the prefix of your Berkeley email, as in "[email protected]"
 user = User.objects.get(username="my_username")
@@ -82,25 +96,31 @@ user.save()
 ```
 
 ### Miscellaneous
+
 #### OSX: error on running `pip`
+
 Try replacing `pip` with `pip3` instead.
 
 #### OAuth errors when trying to sign in locally
+
 ```
 Error 401: invalid_client
 The OAuth client was not found.
 ```
+
 OAuth secrets are sourced from the heroku repository: you'll need to log in to heroku, add the
 appropriate remote, rerun `setup.sh`, and then reactivate your virtualenv.
 
 ### On `runserver`, long stack trace ending with this:
+
 ```
-django.db.utils.OperationalError: could not connect to server: Connection refused                           
-        Is the server running on host "localhost" (::1) and accepting                                       
-        TCP/IP connections on port 5432?                                                                    
-could not connect to server: Connection refused                                                             
-        Is the server running on host "localhost" (127.0.0.1) and accepting                                 
-        TCP/IP connections on port 5432?                                                                    
+django.db.utils.OperationalError: could not connect to server: Connection refused
+        Is the server running on host "localhost" (::1) and accepting
+        TCP/IP connections on port 5432?
+could not connect to server: Connection refused
+        Is the server running on host "localhost" (127.0.0.1) and accepting
+        TCP/IP connections on port 5432?
 ```
+
 Your postgres server is likely not running. On a mac (which is the only platform we've done local
 testing on), run `brew services start postgres` before invoking `runserver` again.