From 399cb24ff905270ea18cf119a13347b146e0e92a Mon Sep 17 00:00:00 2001 From: CKrawczyk Date: Mon, 24 Feb 2025 14:38:01 +0000 Subject: [PATCH] Update docker containers to use the latest version of python For the server to work on python 3.13 and use `panoptes-client` two new dependencies needed to be added, `setuptools` and `standard-imghdr` as these are no longer included in Python. Ideally these should be in the client's dependency file, but for now this works. Using some carful version lower bounds we are able to keep python 3.9 support with all the latest code changes. There is only one numpy import that is in a `try` block to make the code work with either version of numpy. Finally, due to some odd things with folder mounting into the docker container on my mac, I am truing off the code folder mounting for the server container. I am not sure if this is just local folder permissions, or some new change to docker, either way it is likely better to be on the safe side with the container that is deployed. --- Dockerfile | 4 ++-- Dockerfile.bin_cmds | 4 ++-- docker-compose.yml | 4 ++-- .../extractors/poly_line_text_extractor.py | 11 +++++++++-- .../batch_aggregation/test_batch_aggregation.py | 16 +++++++++++----- pyproject.toml | 11 ++++++++--- 6 files changed, 34 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index d9e84c94..b0136b99 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.10-slim +FROM python:3.13-slim ENV LANG=C.UTF-8 WORKDIR /usr/src/aggregation @@ -10,7 +10,7 @@ RUN apt-get update && apt-get -y upgrade && \ # install dependencies RUN mkdir -p panoptes_aggregation/version -COPY pyproject.toml README.md ./ +COPY pyproject.toml README.md LICENSE ./ COPY panoptes_aggregation/__init__.py ./panoptes_aggregation/ COPY panoptes_aggregation/version/__init__.py ./panoptes_aggregation/version/ RUN pip install --upgrade pip diff --git a/Dockerfile.bin_cmds b/Dockerfile.bin_cmds index abab9037..adc862c3 100644 --- a/Dockerfile.bin_cmds +++ b/Dockerfile.bin_cmds @@ -1,4 +1,4 @@ -FROM python:3.10 +FROM python:3.13 ENV LANG=C.UTF-8 @@ -9,7 +9,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y libgeos-dev gi # install dependencies RUN mkdir -p panoptes_aggregation/version -COPY pyproject.toml README.md ./ +COPY pyproject.toml README.md LICENSE ./ COPY panoptes_aggregation/__init__.py ./panoptes_aggregation/ COPY panoptes_aggregation/version/__init__.py ./panoptes_aggregation/version/ RUN pip install --upgrade pip diff --git a/docker-compose.yml b/docker-compose.yml index 414ad5ba..e503b4d3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: REVISION: fake-git-sha-id image: aggregation-for-caesar:local volumes: - - ./:/usr/src/aggregation + # - ./:/usr/src/aggregation - ~/.aws:/root/.aws environment: - AWS_REGION=${AWS_REGION} @@ -59,4 +59,4 @@ services: redis: image: redis - command: redis-server --appendonly yes \ No newline at end of file + command: redis-server --appendonly yes diff --git a/panoptes_aggregation/extractors/poly_line_text_extractor.py b/panoptes_aggregation/extractors/poly_line_text_extractor.py index 7348cd34..18f6b91a 100644 --- a/panoptes_aggregation/extractors/poly_line_text_extractor.py +++ b/panoptes_aggregation/extractors/poly_line_text_extractor.py @@ -12,6 +12,13 @@ from .tool_wrapper import tool_wrapper import warnings +try: + # above numpy 2.0 + from numpy.exceptions import RankWarning +except ImportError: + # below numpy 2.0 + from numpy import RankWarning + @extractor_wrapper(gold_standard=True) @tool_wrapper @@ -95,7 +102,7 @@ def poly_line_text_extractor(classification, dot_freq='line', gold_standard=Fals dx = x[-1] - x[0] dy = y_fit[-1] - y_fit[0] slope = np.rad2deg(np.arctan2(dy, dx)) - except np.exceptions.RankWarning: + except RankWarning: try: # rotate by 90 before fitting x_tmp = -np.array(y) @@ -109,7 +116,7 @@ def poly_line_text_extractor(classification, dot_freq='line', gold_standard=Fals dy = 0.0 # rotate by -90 to bring back into correct coordinates slope = np.rad2deg(np.arctan2(dy, dx)) - 90 - except np.exceptions.RankWarning: + except RankWarning: # this is the case where dx = dy = 0 (a line of zero length) slope = 0 if dot_freq == 'word': diff --git a/panoptes_aggregation/tests/batch_aggregation/test_batch_aggregation.py b/panoptes_aggregation/tests/batch_aggregation/test_batch_aggregation.py index d9b5ac3e..0e420d1a 100644 --- a/panoptes_aggregation/tests/batch_aggregation/test_batch_aggregation.py +++ b/panoptes_aggregation/tests/batch_aggregation/test_batch_aggregation.py @@ -1,17 +1,23 @@ +try: + from panoptes_aggregation.scripts import batch_utils + from panoptes_aggregation.batch_aggregation import run_aggregation + from panoptes_aggregation import batch_aggregation as batch_agg + + batch_agg.celery.conf.update(CELERY_BROKER_URL='memory://') + batch_agg.celery.conf.update(CELERY_RESULT_BACKEND='cache+memory://') + OFFLINE = False +except ImportError: + OFFLINE = True import unittest import os from unittest.mock import patch, MagicMock, call -from panoptes_aggregation.scripts import batch_utils -from panoptes_aggregation.batch_aggregation import run_aggregation -from panoptes_aggregation import batch_aggregation as batch_agg -batch_agg.celery.conf.update(CELERY_BROKER_URL='memory://') -batch_agg.celery.conf.update(CELERY_RESULT_BACKEND='cache+memory://') wf_export = 'panoptes_aggregation/tests/batch_aggregation/wf_export.csv' cls_export = 'panoptes_aggregation/tests/batch_aggregation/cls_export.csv' +@unittest.skipIf(OFFLINE, 'Installed in offline mode') @patch("panoptes_aggregation.batch_aggregation.BatchAggregator._connect_api_client", new=MagicMock()) class TestBatchAggregation(unittest.TestCase): @patch("panoptes_aggregation.batch_aggregation.BatchAggregator") diff --git a/pyproject.toml b/pyproject.toml index 6ee41830..7c642768 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,19 +15,19 @@ classifiers = [ "License :: OSI Approved :: Apache Software License" ] dynamic = ["version"] -requires-python = ">=3.10,<3.12" +requires-python = ">=3.9,<3.14" dependencies = [ "beautifulsoup4>=4.8.1,<4.13", "collatex>=2.3,<2.4", "lxml>=4.4,<5.4", - "numpy>=2.0", + "numpy>=1.26.4,<2.3", "packaging>=20.1,<24.3", "pandas>=2.0", "progressbar2>=3.39,<4.6", "python-levenshtein>=0.21.0,<0.27", "python-slugify>=7.0.0,<8.1", "pyyaml>=6.0,<6.1", - "scikit-learn>=1.5.0", + "scikit-learn>=1.3.2,<1.7", "scipy>=1.10.0", "werkzeug>=2.3.0,<3.2", "shapely>=2.0,<2.1" @@ -46,6 +46,11 @@ online = [ "requests>=2.28,<2.33", "gunicorn>=20.0,<24.0", "sentry-sdk[flask]>=2.19,<2.20", + # to work on python >= 3.12 panoptes-client needs + # setuptools and standard-imghdr installed directly + # as they are no longer in the standard python build + "setuptools>=75.8,<76.0", + "standard-imghdr>=3.13,<4.0", "newrelic>=8.4.0,<10.4", "gitpython>=3.0.0,<3.2" ]