From f65641ab1e4d0e7c215a480bcacfa8539c1c4182 Mon Sep 17 00:00:00 2001 From: Marco Donadoni Date: Tue, 9 Jul 2024 16:34:03 +0200 Subject: [PATCH] build(docker): upgrade to Ubuntu 24.04 and Python 3.12 (#458) Closes reanahub/reana#808 --- .github/workflows/ci.yml | 34 ++++++++-------- .readthedocs.yaml | 4 +- Dockerfile | 36 ++++++++-------- requirements.in | 2 +- requirements.txt | 88 +++++++++++++++++++++------------------- setup.py | 12 +++--- 6 files changed, 89 insertions(+), 87 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 61271025..e2b3baab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ on: [push, pull_request] jobs: lint-commitlint: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Checkout uses: actions/checkout@v4 @@ -36,7 +36,7 @@ jobs: ./run-tests.sh --check-commitlint ${{ github.event.pull_request.head.sha }}~${{ github.event.pull_request.commits }} ${{ github.event.pull_request.head.sha }} ${{ github.event.pull_request.number }} lint-shellcheck: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Checkout uses: actions/checkout@v4 @@ -47,7 +47,7 @@ jobs: ./run-tests.sh --check-shellcheck lint-black: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Checkout uses: actions/checkout@v4 @@ -55,7 +55,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.12" - name: Check Python code formatting run: | @@ -64,7 +64,7 @@ jobs: ./run-tests.sh --check-black lint-flake8: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Checkout uses: actions/checkout@v4 @@ -72,7 +72,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.12" - name: Check compliance with pep8, pyflakes and circular complexity run: | @@ -81,7 +81,7 @@ jobs: ./run-tests.sh --check-flake8 lint-pydocstyle: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Checkout uses: actions/checkout@v4 @@ -89,7 +89,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.12" - name: Check compliance with Python docstring conventions run: | @@ -98,7 +98,7 @@ jobs: ./run-tests.sh --check-pydocstyle lint-check-manifest: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Checkout uses: actions/checkout@v4 @@ -106,7 +106,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.12" - name: Check Python manifest completeness run: | @@ -115,7 +115,7 @@ jobs: ./run-tests.sh --check-manifest docs-sphinx: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Checkout uses: actions/checkout@v4 @@ -123,7 +123,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.12" - name: Install system dependencies run: | @@ -142,7 +142,7 @@ jobs: run: ./run-tests.sh --check-sphinx python-tests: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Checkout uses: actions/checkout@v4 @@ -155,7 +155,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v5 with: - python-version: "3.8" + python-version: "3.12" - name: Install Python dependencies run: | @@ -177,7 +177,7 @@ jobs: files: coverage.xml lint-dockerfile: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Checkout uses: actions/checkout@v4 @@ -186,7 +186,7 @@ jobs: run: ./run-tests.sh --check-dockerfile docker-build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - name: Checkout uses: actions/checkout@v4 @@ -195,7 +195,7 @@ jobs: run: ./run-tests.sh --check-docker-build release-docker: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 if: > vars.RELEASE_DOCKER == 'true' && github.event_name == 'push' && diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 1240941f..32f7a2d0 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,9 +7,9 @@ version: 2 build: - os: ubuntu-22.04 + os: ubuntu-24.04 tools: - python: "3.8" + python: "3.12" sphinx: configuration: docs/conf.py diff --git a/Dockerfile b/Dockerfile index 19231f6a..f60ad704 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ # This file is part of REANA. -# Copyright (C) 2017, 2018, 2019, 2020, 2021, 2022, 2023 CERN. +# Copyright (C) 2017, 2018, 2019, 2020, 2021, 2022, 2023, 2024 CERN. # # REANA is free software; you can redistribute it and/or modify it # under the terms of the MIT License; see LICENSE file for more details. # Use Ubuntu LTS base image -FROM docker.io/library/ubuntu:20.04 +FROM docker.io/library/ubuntu:24.04 # Recognise target architecture ARG TARGETARCH @@ -13,10 +13,8 @@ ARG TARGETARCH # Use default answers in installation commands ENV DEBIAN_FRONTEND=noninteractive -# Use distutils provided by the standard Python library instead of the vendored one in -# setuptools, so that editable installations are stored in the right directory. -# See https://github.com/pypa/setuptools/issues/3301 -ENV SETUPTOOLS_USE_DISTUTILS=stdlib +# Allow pip to install packages in the system site-packages dir +ENV PIP_BREAK_SYSTEM_PACKAGES=true # Prepare list of Python dependencies COPY requirements.txt /code/ @@ -28,15 +26,16 @@ RUN apt-get update -y && \ gcc \ krb5-config \ krb5-user \ - libauthen-krb5-perl \ + libauthen-krb5-simple-perl \ libkrb5-dev \ openssh-client \ - python3-gssapi \ + # matches version in setup.py/requirements.in + python3-gssapi=1.8.2-1ubuntu1 \ python3-pip \ - python3.8 \ - python3.8-dev \ + python3.12 \ + python3.12-dev \ vim-tiny && \ - pip install --no-cache-dir --upgrade pip 'setuptools<71' && \ + pip install --no-cache-dir --upgrade setuptools && \ pip install --no-cache-dir -r /code/requirements.txt && \ apt-get remove -y \ gcc && \ @@ -52,17 +51,12 @@ ARG COMPUTE_BACKENDS=kubernetes RUN if echo "$COMPUTE_BACKENDS" | grep -q "htcondorcern"; then \ set -e; \ apt-get update -y; \ - apt-get install --no-install-recommends -y wget alien gnupg2 rand; \ - wget -q -O ngbauth-submit.rpm https://linuxsoft.cern.ch/internal/repos/batch8s-stable/x86_64/os/Packages/n/ngbauth-submit-0.26-2.el8s.noarch.rpm; \ - wget -q -O myschedd.rpm https://linuxsoft.cern.ch/internal/repos/batch8s-stable/x86_64/os/Packages/m/myschedd-1.9-2.el8s.x86_64.rpm; \ + apt-get install --no-install-recommends -y wget alien gnupg2 condor; \ + wget -q -O ngbauth-submit.rpm https://linuxsoft.cern.ch/internal/repos/batch9al-stable/x86_64/os/Packages/n/ngbauth-submit-0.31-1.al9.cern.noarch.rpm; \ + wget -q -O myschedd.rpm https://linuxsoft.cern.ch/internal/repos/batch9al-stable/x86_64/os/Packages/m/myschedd-1.9-2.al9.cern.x86_64.rpm; \ yes | alien -i myschedd.rpm; \ yes | alien -i ngbauth-submit.rpm; \ rm -rf myschedd.rpm ngbauth-submit.rpm; \ - wget -qO - https://research.cs.wisc.edu/htcondor/repo/keys/HTCondor-9.0-Key | apt-key add -; \ - echo "deb https://research.cs.wisc.edu/htcondor/repo/ubuntu/9.0 focal main" >> /etc/apt/sources.list; \ - echo "deb-src https://research.cs.wisc.edu/htcondor/repo/ubuntu/9.0 focal main" >> /etc/apt/sources.list; \ - apt-get update -y; \ - apt-get install --no-install-recommends -y condor; \ apt-get remove -y gnupg2 wget alien; \ apt-get autoremove -y; \ apt-get clean; \ @@ -131,6 +125,10 @@ ENV COMPUTE_BACKENDS=$COMPUTE_BACKENDS \ FLASK_APP=reana_job_controller/app.py \ TERM=xterm +# Delete default `ubuntu` user, as its UID (1000) clashes with REANA's default one +# See https://bugs.launchpad.net/cloud-images/+bug/2005129 +RUN userdel -r ubuntu + # Expose ports to clients EXPOSE 5000 diff --git a/requirements.in b/requirements.in index 07d5b913..1d153fd7 100644 --- a/requirements.in +++ b/requirements.in @@ -5,4 +5,4 @@ # under the terms of the MIT License; see LICENSE file for more details. paramiko[gssapi]==3.0.0 -gssapi==1.6.1 +gssapi==1.8.2 # matches version in Dockerfile diff --git a/requirements.txt b/requirements.txt index b42af956..4c089f7c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,82 +1,86 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile --annotation-style=line --output-file=requirements.txt requirements.in setup.py # -alembic==1.13.1 # via reana-db +alembic==1.13.2 # via reana-db amqp==5.2.0 # via kombu apispec[yaml]==3.3.2 # via apispec-webframeworks, reana-job-controller (setup.py) apispec-webframeworks==0.5.2 # via reana-job-controller (setup.py) appdirs==1.4.4 # via fs -attrs==23.2.0 # via jsonschema -backports-zoneinfo[tzdata]==0.2.1 # via backports-zoneinfo, kombu -bcrypt==4.1.2 # via paramiko +arrow==1.3.0 # via isoduration +attrs==23.2.0 # via jsonschema, referencing +bcrypt==4.2.0 # via paramiko bracex==2.4 # via wcmatch bravado==10.3.2 # via reana-commons bravado-core==6.1.0 # via bravado, reana-commons -cachetools==5.3.3 # via google-auth -certifi==2024.2.2 # via kubernetes, requests +cachetools==5.4.0 # via google-auth +certifi==2024.7.4 # via kubernetes, requests cffi==1.16.0 # via cryptography, pynacl charset-normalizer==3.3.2 # via requests checksumdir==1.1.9 # via reana-commons click==8.1.7 # via flask, reana-commons -cryptography==42.0.5 # via paramiko, sqlalchemy-utils +cryptography==43.0.0 # via paramiko, sqlalchemy-utils decorator==5.1.1 # via gssapi -flask==2.1.3 # via reana-job-controller (setup.py) +flask==2.2.5 # via reana-job-controller (setup.py) +fqdn==1.5.1 # via jsonschema fs==2.4.16 # via reana-commons, reana-job-controller (setup.py) -google-auth==2.28.1 # via kubernetes -gssapi==1.6.1 # via -r requirements.in, paramiko -idna==3.6 # via jsonschema, requests -importlib-metadata==7.0.1 # via alembic, flask -importlib-resources==6.1.2 # via alembic -itsdangerous==2.1.2 # via flask +google-auth==2.32.0 # via kubernetes +greenlet==3.0.3 # via sqlalchemy +gssapi==1.8.2 # via -r requirements.in, paramiko +idna==3.7 # via jsonschema, requests +importlib-resources==6.4.0 # via swagger-spec-validator +isoduration==20.11.0 # via jsonschema +itsdangerous==2.2.0 # via flask jinja2==3.0.3 # via flask, reana-job-controller (setup.py) -jsonpointer==2.4 # via jsonschema +jsonpointer==3.0.0 # via jsonschema jsonref==1.1.0 # via bravado-core -jsonschema[format]==3.2.0 # via bravado-core, reana-commons, swagger-spec-validator -kombu==5.3.5 # via reana-commons +jsonschema[format]==4.23.0 # via bravado-core, reana-commons, swagger-spec-validator +jsonschema-specifications==2023.12.1 # via jsonschema +kombu==5.3.7 # via reana-commons kubernetes==22.6.0 # via reana-commons -mako==1.3.2 # via alembic +mako==1.3.5 # via alembic markupsafe==2.1.5 # via jinja2, mako, werkzeug -marshmallow==2.20.1 # via reana-job-controller (setup.py) +marshmallow==2.21.0 # via reana-job-controller (setup.py) mock==3.0.5 # via reana-commons monotonic==1.6 # via bravado -msgpack==1.0.7 # via bravado-core +msgpack==1.0.8 # via bravado-core msgpack-python==0.5.6 # via bravado oauthlib==3.2.2 # via requests-oauthlib paramiko[gssapi]==3.0.0 # via -r requirements.in psycopg2-binary==2.9.9 # via reana-db -pyasn1==0.5.1 # via paramiko, pyasn1-modules, rsa -pyasn1-modules==0.3.0 # via google-auth -pycparser==2.21 # via cffi +pyasn1==0.6.0 # via paramiko, pyasn1-modules, rsa +pyasn1-modules==0.4.0 # via google-auth +pycparser==2.22 # via cffi pynacl==1.5.0 # via paramiko -pyrsistent==0.20.0 # via jsonschema -python-dateutil==2.9.0 # via bravado, bravado-core, kubernetes +python-dateutil==2.9.0.post0 # via arrow, bravado, bravado-core, kubernetes pytz==2024.1 # via bravado-core pyyaml==6.0.1 # via apispec, bravado, bravado-core, kubernetes, reana-commons, swagger-spec-validator -reana-commons[kubernetes]==0.95.0a2 # via reana-db, reana-job-controller (setup.py) -reana-db==0.95.0a2 # via reana-job-controller (setup.py) -requests==2.31.0 # via bravado, bravado-core, kubernetes, requests-oauthlib -requests-oauthlib==1.3.1 # via kubernetes +reana-commons[kubernetes]==0.95.0a3 # via reana-db, reana-job-controller (setup.py) +reana-db==0.95.0a3 # via reana-job-controller (setup.py) +referencing==0.35.1 # via jsonschema, jsonschema-specifications +requests==2.32.3 # via bravado, bravado-core, kubernetes, requests-oauthlib +requests-oauthlib==2.0.0 # via kubernetes retrying==1.3.4 # via reana-job-controller (setup.py) +rfc3339-validator==0.1.4 # via jsonschema rfc3987==1.3.8 # via jsonschema +rpds-py==0.19.0 # via jsonschema, referencing rsa==4.9 # via google-auth simplejson==3.19.2 # via bravado, bravado-core -six==1.16.0 # via bravado, bravado-core, fs, gssapi, jsonschema, kubernetes, mock, python-dateutil, retrying -sqlalchemy==1.3.24 # via alembic, reana-db, sqlalchemy-utils -sqlalchemy-utils[encrypted]==0.41.1 # via reana-db, sqlalchemy-utils -strict-rfc3339==0.7 # via jsonschema -swagger-spec-validator==3.0.3 # via bravado-core -typing-extensions==4.10.0 # via alembic, bravado, kombu, swagger-spec-validator -tzdata==2024.1 # via backports-zoneinfo -urllib3==2.2.1 # via kubernetes, requests +six==1.16.0 # via bravado, bravado-core, fs, kubernetes, mock, python-dateutil, retrying, rfc3339-validator +sqlalchemy==1.4.52 # via alembic, reana-db, sqlalchemy-utils +sqlalchemy-utils[encrypted]==0.41.2 # via reana-db, sqlalchemy-utils +swagger-spec-validator==3.0.4 # via bravado-core +types-python-dateutil==2.9.0.20240316 # via arrow +typing-extensions==4.12.2 # via alembic, bravado, swagger-spec-validator +uri-template==1.3.0 # via jsonschema +urllib3==2.2.2 # via kubernetes, requests vine==5.1.0 # via amqp, kombu wcmatch==8.4.1 # via reana-commons -webcolors==1.13 # via jsonschema -websocket-client==1.7.0 # via kubernetes -werkzeug==2.3.8 # via flask, reana-commons, reana-job-controller (setup.py) -zipp==3.17.0 # via importlib-metadata, importlib-resources +webcolors==24.6.0 # via jsonschema +websocket-client==1.8.0 # via kubernetes +werkzeug==2.2.3 # via flask, reana-commons, reana-job-controller (setup.py) # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/setup.py b/setup.py index 30063c58..b52c1132 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ "sphinxcontrib-redoc>=1.5.1", ], "htcondor": [ - "htcondor==9.0.17", + "htcondor>=9.0.17", ], "tests": [ "pytest-reana>=0.95.0a2,<0.96.0", @@ -55,12 +55,12 @@ # apispec>=4.0 drops support for marshmallow<3 "apispec[yaml]>=3.0,<4.0", "apispec-webframeworks", - "Flask>=2.1.1,<2.2.0", + "Flask>=2.1.1,<2.3.0", # same upper pin as invenio-base/reana-server + "Werkzeug>=2.1.0,<2.3.0", # same upper pin as invenio-base "jinja2<3.1.0", - "Werkzeug>=2.1.0,<3.0", "fs>=2.0", - "marshmallow>2.13.0,<=2.20.1", - "reana-commons[kubernetes]>=0.95.0a2,<0.96.0", + "marshmallow>2.13.0,<3.0.0", # same upper pin as reana-server + "reana-commons[kubernetes]>=0.95.0a3,<0.96.0", "reana-db>=0.95.0a2,<0.96.0", "retrying>=1.3.3", ] @@ -102,7 +102,7 @@ "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python", "Topic :: Internet :: WWW/HTTP :: Dynamic Content",