build: DEPRECATE requirements.txt in favor of poetry builds

- change: upgrade dockerfile to Python 3.12.5
- change: add (local) edu_sharing_client package to dockerfile before trying to install dependencies
  - this change was necessary to guarantee that fresh development environments can use the edu-sharing API client without "ModuleNotFoundError"s
- chore: update dependencies
- chore: update pyproject.toml to reflect Python 3.12 requirement

GitHub Workflows:
- build: skip .venv/ and edu_sharing_client directories during flake8 checks
- build: use poetry to run flake8 and pytest
- build: install dependencies with poetry in GitHub workflow

.github/workflows/python.yaml

@@ -33,17 +33,22 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-
           ${{ runner.os }}-
-    - name: Install dependencies
+    - name: Install Poetry via pip
       run: |
         python -m pip install --upgrade pip
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-        if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
-    - name: Lint with flake8
+        python -m pip install poetry
+    - name: Configure Poetry to use in-project .venv
+      run: |
+        python -m poetry config virtualenvs.in-project true
+    - name: Install Dependencies with Poetry
+      run: |
+        python -m poetry install
+    - name: Lint with flake8 (via Poetry)
       run: |
         # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        poetry run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude=.venv/,edu_sharing_openapi/
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+        poetry run flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude=.venv/,edu_sharing_openapi/
     - name: Test with pytest
       run: |
-        pytest
+        poetry run pytest

Dockerfile

@@ -1,17 +1,18 @@
-FROM python:3.12.4-slim-bookworm
+FROM python:3.12.5-slim-bookworm
 
 # ENV CRAWLER wirlernenonline_spider
 
 WORKDIR /
 
 COPY entrypoint.sh entrypoint.sh
-COPY requirements.txt requirements.txt
-RUN pip3 install -r requirements.txt
+COPY edu_sharing_openapi/ edu_sharing_openapi/
+COPY pyproject.toml poetry.lock ./
+RUN pip3 install poetry
+RUN poetry install
 COPY scrapy.cfg scrapy.cfg
 COPY setup.cfg setup.cfg
 COPY converter/ converter/
 COPY csv/ csv/
-COPY edu_sharing_openapi/ edu_sharing_openapi/
 COPY valuespace_converter/ valuespace_converter/
 
 

pyproject.toml

@@ -7,7 +7,7 @@ maintainers = [
     "Andreas Schnäpp <[email protected]>"
 ]
 readme = "README.md"
-python = "^3.11"
+python = "^3.12"
 homepage = "https://github.com/openeduhub/oeh-search-etl"
 repository = "https://github.com/openeduhub/oeh-search-etl"
 documentation = "https://github.com/openeduhub/oeh-search-etl"
@@ -17,15 +17,14 @@ keywords = ["metadata", "oer", "crawl", "wirlernenonline"]
 classifiers = [
     "Framework :: Scrapy",
     "Development Status :: 4 - Beta",
-    "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Topic :: Education :: Testing",
     "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
 ]
 
 [tool.black]
 line-length = 120
-target-version = ['py311']
+target-version = ['py312']
 include = '\.pyi?$'
 exclude = '''
 
@@ -59,7 +58,7 @@ readme = "Readme.md"
 packages = [{include = "converter"}]
 
 [tool.poetry.dependencies]
-python = "^3.10"
+python = "^3.12"
 wheel = "0.44.0"
 black = "24.8.0"
 certifi="2024.8.30"
@@ -81,7 +80,6 @@ requests="2.32.3"
 six="1.16.0"
 Scrapy="2.11.2"
 scrapy-splash="0.9.0"
-urllib3="2.2.2"
 vobject="0.9.7"
 w3lib="2.2.1"
 xmltodict="0.13.0"
@@ -90,12 +88,14 @@ babel = "2.15.0"
 langcodes = {extras = ["data"], version = "^3.3.0"}
 httpx = "0.27.2"
 async-lru = "2.0.4"
+urllib3 = "^2.2.2"
 
 [tool.poetry.group.edu_sharing_client.dependencies]
 # these dependencies are used (and automatically generated) by the "openapi-generator-cli"-generated client
 # see: /edu_sharing_openapi/pyproject.toml
 pydantic = ">=2.8.2"
 typing-extensions = ">=4.12.2"
+edu-sharing-client = {path = "edu_sharing_openapi"}
 
 [build-system]
 requires = ["poetry-core"]