Update docker containers to use the latest version of python

For the server to work on python 3.13 and use `panoptes-client` two new dependencies needed to be added, `setuptools` and `standard-imghdr` as these are no longer included in Python.

Ideally these should be in the client's dependency file, but for now this works.

Using some carful version lower bounds we are able to keep python 3.9 support with all the latest code changes.  There is only one numpy import that is in a `try` block to make the code work with either version of numpy.

Finally, due to some odd things with folder mounting into the docker container on my mac, I am truing off the code folder mounting for the server container.  I am not sure if this is just local folder permissions, or some new change to docker, either way it is likely better to be on the safe side with the container that is deployed.

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.13-slim
 ENV LANG=C.UTF-8
 
 WORKDIR /usr/src/aggregation
@@ -10,7 +10,7 @@ RUN apt-get update && apt-get -y upgrade && \
 
 # install dependencies
 RUN mkdir -p panoptes_aggregation/version
-COPY pyproject.toml README.md ./
+COPY pyproject.toml README.md LICENSE ./
 COPY panoptes_aggregation/__init__.py ./panoptes_aggregation/
 COPY panoptes_aggregation/version/__init__.py ./panoptes_aggregation/version/
 RUN pip install --upgrade pip

Dockerfile.bin_cmds

@@ -1,4 +1,4 @@
-FROM python:3.10
+FROM python:3.13
 
 
 ENV LANG=C.UTF-8
@@ -9,7 +9,7 @@ RUN apt-get update && apt-get install  --no-install-recommends -y libgeos-dev gi
 
 # install dependencies
 RUN mkdir -p panoptes_aggregation/version
-COPY pyproject.toml README.md ./
+COPY pyproject.toml README.md LICENSE ./
 COPY panoptes_aggregation/__init__.py ./panoptes_aggregation/
 COPY panoptes_aggregation/version/__init__.py ./panoptes_aggregation/version/
 RUN pip install --upgrade pip

docker-compose.yml

@@ -7,7 +7,7 @@ services:
         REVISION: fake-git-sha-id
     image: aggregation-for-caesar:local
     volumes:
-      - ./:/usr/src/aggregation
+      # - ./:/usr/src/aggregation
       - ~/.aws:/root/.aws
     environment:
       - AWS_REGION=${AWS_REGION}
@@ -59,4 +59,4 @@ services:
 
   redis:
     image: redis
-    command: redis-server --appendonly yes
+    command: redis-server --appendonly yes

panoptes_aggregation/extractors/poly_line_text_extractor.py

@@ -12,6 +12,13 @@
 from .tool_wrapper import tool_wrapper
 import warnings
 
+try:
+    # above numpy 2.0
+    from numpy.exceptions import RankWarning
+except ImportError:
+    # below numpy 2.0
+    from numpy import RankWarning
+
 
 @extractor_wrapper(gold_standard=True)
 @tool_wrapper
@@ -95,7 +102,7 @@ def poly_line_text_extractor(classification, dot_freq='line', gold_standard=Fals
                         dx = x[-1] - x[0]
                         dy = y_fit[-1] - y_fit[0]
                         slope = np.rad2deg(np.arctan2(dy, dx))
-                    except np.exceptions.RankWarning:
+                    except RankWarning:
                         try:
                             # rotate by 90 before fitting
                             x_tmp = -np.array(y)
@@ -109,7 +116,7 @@ def poly_line_text_extractor(classification, dot_freq='line', gold_standard=Fals
                                 dy = 0.0
                             # rotate by -90 to bring back into correct coordinates
                             slope = np.rad2deg(np.arctan2(dy, dx)) - 90
-                        except np.exceptions.RankWarning:
+                        except RankWarning:
                             # this is the case where dx = dy = 0 (a line of zero length)
                             slope = 0
                 if dot_freq == 'word':

panoptes_aggregation/tests/batch_aggregation/test_batch_aggregation.py

@@ -1,17 +1,23 @@
+try:
+    from panoptes_aggregation.scripts import batch_utils
+    from panoptes_aggregation.batch_aggregation import run_aggregation
+    from panoptes_aggregation import batch_aggregation as batch_agg
+
+    batch_agg.celery.conf.update(CELERY_BROKER_URL='memory://')
+    batch_agg.celery.conf.update(CELERY_RESULT_BACKEND='cache+memory://')
+    OFFLINE = False
+except ImportError:
+    OFFLINE = True
 import unittest
 import os
 from unittest.mock import patch, MagicMock, call
-from panoptes_aggregation.scripts import batch_utils
-from panoptes_aggregation.batch_aggregation import run_aggregation
-from panoptes_aggregation import batch_aggregation as batch_agg
 
-batch_agg.celery.conf.update(CELERY_BROKER_URL='memory://')
-batch_agg.celery.conf.update(CELERY_RESULT_BACKEND='cache+memory://')
 
 wf_export = 'panoptes_aggregation/tests/batch_aggregation/wf_export.csv'
 cls_export = 'panoptes_aggregation/tests/batch_aggregation/cls_export.csv'
 
 
+@unittest.skipIf(OFFLINE, 'Installed in offline mode')
 @patch("panoptes_aggregation.batch_aggregation.BatchAggregator._connect_api_client", new=MagicMock())
 class TestBatchAggregation(unittest.TestCase):
     @patch("panoptes_aggregation.batch_aggregation.BatchAggregator")

pyproject.toml

@@ -15,19 +15,19 @@ classifiers = [
     "License :: OSI Approved :: Apache Software License"
 ]
 dynamic = ["version"]
-requires-python = ">=3.10,<3.12"
+requires-python = ">=3.9,<3.14"
 dependencies = [
     "beautifulsoup4>=4.8.1,<4.13",
     "collatex>=2.3,<2.4",
     "lxml>=4.4,<5.4",
-    "numpy>=2.0",
+    "numpy>=1.26.4,<2.3",
     "packaging>=20.1,<24.3",
     "pandas>=2.0",
     "progressbar2>=3.39,<4.6",
     "python-levenshtein>=0.21.0,<0.27",
     "python-slugify>=7.0.0,<8.1",
     "pyyaml>=6.0,<6.1",
-    "scikit-learn>=1.5.0",
+    "scikit-learn>=1.3.2,<1.7",
     "scipy>=1.10.0",
     "werkzeug>=2.3.0,<3.2",
     "shapely>=2.0,<2.1"
@@ -46,6 +46,11 @@ online = [
     "requests>=2.28,<2.33",
     "gunicorn>=20.0,<24.0",
     "sentry-sdk[flask]>=2.19,<2.20",
+    # to work on python >= 3.12 panoptes-client needs
+    # setuptools and standard-imghdr installed directly
+    # as they are no longer in the standard python build
+    "setuptools>=75.8,<76.0",
+    "standard-imghdr>=3.13,<4.0",
     "newrelic>=8.4.0,<10.4",
     "gitpython>=3.0.0,<3.2"
 ]