build: add python 3.11 and 3.12 ci checks

.github/workflows/ci.yml

@@ -11,59 +11,71 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        include:
-          - django-env: django32
-            testname: quality-and-jobs
-            targets: PYTHON_ENV=py38 requirements.js check_translations_up_to_date validate_translations clean_static static quality validate_js check_keywords
-          - django-env: django32
-            testname: test-python
-            targets: PYTHON_ENV=py38 requirements.js clean_static static validate_python
-          - django-env: django32
-            testname: acceptance-python
-            targets: PYTHON_ENV=py38 requirements.js clean_static static acceptance
-
+        python-version: ['py38', 'py311', 'py312']
+        django-env: ['django32']
+        test: ['acceptance-python', 'test-python', 'quality-and-jobs']
     steps:
       - uses: actions/checkout@v2
+      - name: Setup and Format Python Version
+        id: format_python_version
+        shell: bash
+        run: |
+          # Remove 'py' and insert a dot to format the version
+          FORMATTED_VERSION=${{ matrix.python-version }}   # e.g., py38
+          FORMATTED_VERSION=${FORMATTED_VERSION/py3/3.}    # becomes 3.8
+          # Set environment variables
+          echo "PYTHON_VERSION=$FORMATTED_VERSION" >> $GITHUB_ENV
       - name: Start container
         run: |
-          docker-compose -f ./.ci/docker-compose-ci.yml up -d
+          docker compose -f ./.ci/docker-compose-ci.yml up -d
+          docker exec ecommerce_testing bash -c "
+          sudo apt-get update -y &&
+          sudo apt-get install python$PYTHON_VERSION \
+          python$PYTHON_VERSION-dev \
+          python$PYTHON_VERSION-distutils \
+          default-libmysqlclient-dev build-essential pkg-config -y &&
+          curl -sS https://bootstrap.pypa.io/get-pip.py | python$PYTHON_VERSION;"
+        # Need to install pip from source here^ otherwise some packages don't get installed
       - name: Install dependencies
         run: |
           docker exec -t ecommerce_testing bash -c "
             cd /edx/app/ecommerce/ecommerce/ &&
-            python3 -m pip install tox
+            python$PYTHON_VERSION -m pip install tox
           "
       - name: Run tests
         run: |
           docker exec -t -e CI=1 ecommerce_testing bash -c "
             cd /edx/app/ecommerce/ecommerce/ &&
             PATH=\$PATH:/edx/app/ecommerce/nodeenvs/ecommerce/bin:/snap/bin
-            DJANGO_ENV=${{ matrix.django-env }} make ${{ matrix.targets }}
+            DJANGO_ENV=${{ matrix.django-env }} PYTHON_ENV=${{ matrix.python-version }} PYTHON_VERSION=$PYTHON_VERSION make ${{ matrix.test }}
           "
       - name: Run coverage
-        if: matrix.testname == 'test-python'
+        if: matrix.test == 'test-python'
         run: |
           docker exec ecommerce_testing /edx/app/ecommerce/ecommerce/.ci/run_coverage.sh
       - name: Setup Python
-        if: matrix.testname == 'test-python'
+        if: matrix.test == 'test-python' && matrix.python-version == 'py38'
         uses: actions/setup-python@v2
         with:
           python-version: "3.8"
           architecture: x64
       - name: Report coverage
-        if: matrix.testname == 'test-python'
+        if: matrix.test == 'test-python' && matrix.python-version == 'py38'
         uses: codecov/codecov-action@v3
         with:
           flags: unittests
           fail_ci_if_error: false
 
   docs:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.8', '3.11', '3.12']
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: "3.8"
+          python-version: ${{matrix.python-version}}
           architecture: x64
       - name: Install Dependencies
         run: pip install -r requirements/docs.txt -r requirements/tox.txt

Makefile

@@ -1,7 +1,8 @@
 NODE_BIN=./node_modules/.bin
 DIFF_COVER_BASE_BRANCH=master
-PYTHON_ENV=py38
+PYTHON_ENV_VAR=$(if $(PYTHON_ENV),$(PYTHON_ENV),py312)
 DJANGO_ENV_VAR=$(if $(DJANGO_ENV),$(DJANGO_ENV),django32)
+PYTHON_VERSION_VAR=$(if $(PYTHON_VERSION),$(PYTHON_VERSION),3.12)
 
 help:
 	@echo ''
@@ -45,17 +46,17 @@ requirements: requirements.js
 	pip3 install -r requirements/dev.txt --exists-action w
 
 requirements.tox:
-	pip3 install -U pip==20.0.2
+	pip3 install -U pip
 	pip3 install -r requirements/tox.txt --exists-action w
 
 production-requirements: requirements.js
 	pip3 install -r requirements.txt --exists-action w
 
 migrate: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-migrate
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-migrate
 
 serve: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-serve
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-serve
 
 clean:
 	find . -name '*.pyc' -delete
@@ -65,18 +66,18 @@ clean_static:
 	rm -rf assets/* ecommerce/static/build/*
 
 run_check_isort: requirements.tox
-	tox -e $(PYTHON_ENV)-check_isort
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-check_isort
 
 run_isort: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-run_isort
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-run_isort
 
 run_pycodestyle: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-pycodestyle
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-pycodestyle
 
 run_pep8: run_pycodestyle
 
 run_pylint: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-pylint
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-pylint
 
 quality: run_check_isort run_pycodestyle run_pylint
 
@@ -86,42 +87,42 @@ validate_js:
 	$(NODE_BIN)/gulp lint
 
 validate_python: clean requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-tests
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-tests
 
 acceptance: clean requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-acceptance
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-acceptance
 
 fast_validate_python: clean requirements.tox
-	DISABLE_ACCEPTANCE_TESTS=True tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-tests
+	DISABLE_ACCEPTANCE_TESTS=True python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-tests
 
 validate: validate_python validate_js quality
 
 theme_static: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-theme_static
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-theme_static
 
 static: requirements.js theme_static requirements.tox
 	$(NODE_BIN)/r.js -o build.js
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-static
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-static
 
 html_coverage: requirements.tox
-	tox -e $(PYTHON_ENV)-coverage_html
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-coverage_html
 
 diff_coverage: validate fast_diff_coverage
 
 fast_diff_coverage: requirements.tox
-	tox -e $(PYTHON_ENV)-fast_diff_coverage
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-fast_diff_coverage
 
 e2e: requirements.tox
-	tox -e $(PYTHON_ENV)-e2e
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-e2e
 
 extract_translations: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-extract_translations
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-extract_translations
 
 dummy_translations: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-dummy_translations
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-dummy_translations
 
 compile_translations: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-compile_translations
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-compile_translations
 
 fake_translations: extract_translations dummy_translations compile_translations
 
@@ -134,18 +135,18 @@ update_translations: pull_translations fake_translations
 
 # extract_translations should be called before this command can detect changes
 detect_changed_source_translations: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-detect_changed_translations
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-detect_changed_translations
 
 # @FIXME: skip detect_changed_source_translations until git diff works again (REV-2737)
 check_translations_up_to_date: fake_translations # detect_changed_source_translations
 
 # Validate translations
 validate_translations: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-validate_translations
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-validate_translations
 
 # Scan the Django models in all installed apps in this project for restricted field names
 check_keywords: requirements.tox
-	tox -e $(PYTHON_ENV)-${DJANGO_ENV_VAR}-check_keywords
+	python$(PYTHON_VERSION_VAR) -m tox -e $(PYTHON_ENV_VAR)-${DJANGO_ENV_VAR}-check_keywords
 
 COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt
 .PHONY: $(COMMON_CONSTRAINTS_TXT)
@@ -173,6 +174,12 @@ upgrade: $(COMMON_CONSTRAINTS_TXT)
 docs:
 	tox -e docs
 
+quality-and-jobs: requirements.js check_translations_up_to_date validate_translations clean_static static quality validate_js check_keywords
+
+test-python: requirements.js clean_static static validate_python
+
+acceptance-python: requirements.js clean_static static acceptance
+
 # Targets in a Makefile which do not produce an output file with the same name as the target name
 .PHONY: help requirements migrate serve clean validate_python quality validate_js validate html_coverage e2e \
 	extract_translations dummy_translations compile_translations fake_translations pull_translations \

ecommerce/core/tests/test_models.py

@@ -177,7 +177,7 @@ def test_user_details_uses_jwt(self):
 
         # Verify the headers passed to the API were correct.
         expected = {'Authorization': 'JWT {}'.format(token), }
-        self.assertDictContainsSubset(expected, last_request.headers)
+        self.assertLessEqual(expected.items(), last_request.headers.items())
 
     def test_no_user_details(self):
         """ Verify False is returned when there is a connection error. """

ecommerce/credit/tests/test_views.py

@@ -2,7 +2,6 @@
 Tests for the checkout page.
 """
 
-
 from datetime import timedelta
 
 import ddt
@@ -130,7 +129,19 @@ def _assert_success_checkout_page(self, sku=None):
 
         response = self.client.get(self.path)
         self.assertEqual(response.status_code, 200)
-        self.assertDictContainsSubset({'course': self.course}, response.context)
+
+        # assertDictContainsSubset is deprecated in Python version > 3.9
+        # response.context returns a ContextList object; the below statements will convert it to a dict
+        # assertLessEqual method is used instead of the deprecated assertDictContainsSubset method
+        context = {}
+        for i, ctx in enumerate(response.context):
+            if isinstance(ctx, dict):
+                context.update(ctx)
+            elif hasattr(ctx, '__iter__') and not isinstance(ctx, str):
+                for item in ctx:
+                    if isinstance(item, dict):
+                        context.update(item)
+        self.assertLessEqual({'course': self.course}.items(), context.items())
 
         self.assertContains(
             response,

ecommerce/enterprise/tests/test_utils.py

@@ -122,7 +122,7 @@ def test_post_enterprise_customer_user(self, mock_helpers, expected_return):
             self.learner.username
         )
 
-        self.assertDictContainsSubset(expected_return, response)
+        self.assertLessEqual(expected_return.items(), response.items())
 
     @responses.activate
     def test_ecu_needs_consent(self):

ecommerce/extensions/checkout/tests/test_views.py

@@ -338,7 +338,7 @@ def test_get_receipt_for_existing_order(self, mock_learner_data):
         }
 
         self.assertEqual(response.status_code, 200)
-        self.assertDictContainsSubset(context_data, response.context_data)
+        self.assertLessEqual(context_data.items(), response.context_data.items())
 
     @patch('ecommerce.extensions.checkout.views.fetch_enterprise_learner_data')
     @responses.activate
@@ -371,7 +371,7 @@ def test_get_receipt_for_existing_entitlement_order(self, mock_learner_data):
         }
 
         self.assertEqual(response.status_code, 200)
-        self.assertDictContainsSubset(context_data, response.context_data)
+        self.assertLessEqual(context_data.items(), response.context_data.items())
 
     @patch('ecommerce.extensions.checkout.views.fetch_enterprise_learner_data')
     @responses.activate
@@ -387,7 +387,7 @@ def test_get_receipt_for_existing_order_as_staff_user(self, mock_learner_data):
         }
 
         self.assertEqual(response.status_code, 200)
-        self.assertDictContainsSubset(context_data, response.context_data)
+        self.assertLessEqual(context_data.items(), response.context_data.items())
 
     @patch('ecommerce.extensions.checkout.views.fetch_enterprise_learner_data')
     @responses.activate
@@ -400,7 +400,7 @@ def test_get_receipt_for_existing_order_user_not_owner(self, mock_learner_data):
         context_data = {'order_history_url': self.site.siteconfiguration.build_lms_url('account/settings')}
 
         self.assertEqual(response.status_code, 404)
-        self.assertDictContainsSubset(context_data, response.context_data)
+        self.assertLessEqual(context_data.items(), response.context_data.items())
 
     @patch('ecommerce.extensions.checkout.views.fetch_enterprise_learner_data')
     @responses.activate
@@ -456,7 +456,7 @@ def test_dashboard_link_for_course_purchase(self, mock_learner_data):
         }
 
         self.assertEqual(response.status_code, 200)
-        self.assertDictContainsSubset(context_data, response.context_data)
+        self.assertLessEqual(context_data.items(), response.context_data.items())
 
     @patch('ecommerce.extensions.checkout.views.fetch_enterprise_learner_data')
     @responses.activate
@@ -482,7 +482,7 @@ def test_dashboard_link_for_bundle_purchase(self, mock_learner_data):
         }
 
         self.assertEqual(response.status_code, 200)
-        self.assertDictContainsSubset(context_data, response.context_data)
+        self.assertLessEqual(context_data.items(), response.context_data.items())
 
     @patch('ecommerce.extensions.checkout.views.fetch_enterprise_learner_data')
     @responses.activate

ecommerce/extensions/fulfillment/tests/test_modules.py

@@ -212,7 +212,7 @@ def test_enrollment_module_fulfill(self):
             'X-Forwarded-For': self.user.tracking_context['lms_ip'],
         }
 
-        self.assertDictContainsSubset(expected_headers, actual_headers)
+        self.assertLessEqual(expected_headers.items(), actual_headers.items())
         self.assertEqual(expected_body, actual_body)
 
     @responses.activate
@@ -377,7 +377,7 @@ def test_revoke_product(self):
             'X-Forwarded-For': self.user.tracking_context['lms_ip'],
         }
 
-        self.assertDictContainsSubset(expected_headers, actual_headers)
+        assert expected_headers.items() <= actual_headers.items()
         self.assertEqual(expected_body, actual_body)
 
     @responses.activate

ecommerce/extensions/payment/tests/processors/test_cybersource.py

@@ -49,7 +49,7 @@ def assert_processor_response_recorded(self, processor_name, transaction_id, res
         expected = {
             'requestID': transaction_id,
         }
-        self.assertDictContainsSubset(expected, ppr.response)
+        self.assertLessEqual(expected.items(), ppr.response.items())
         self.assertEqual(ppr.basket, basket)
 
         return ppr.id

ecommerce/extensions/voucher/tests/test_utils.py

@@ -523,7 +523,7 @@ def test_generate_coupon_report_with_deleted_product(self):
 
         __, rows = generate_coupon_report([query_coupon.attr.coupon_vouchers])
         self.assert_report_first_row(rows[0], query_coupon, first_voucher)
-        self.assertDictContainsSubset({'Redeemed For Course ID': 'Unknown'}, rows[2])
+        self.assertLessEqual({'Redeemed For Course ID': 'Unknown'}.items(), rows[2].items())
 
     def test_report_for_inactive_coupons(self):
         """ Verify the coupon report show correct status for inactive coupons. """