DRAFT - DO NOT MERGE Convert python 3 - DO NOT MERGE

.coveragerc

@@ -0,0 +1,18 @@
+# This config file is used by coverage.py (and supposedly pytest-cov, although it does not seem to).
+
+# The protip for generating a coverage report on idb-backend is to
+# change into the top level of idb-backend and then run:
+#
+# $ coverage run -m pytest
+#
+# which will use this config file to access the sources listed below.
+#
+# To view the coverage report, run:
+#
+# $ coverage report
+
+[run]
+source = 
+    ./idb
+    ./idigbio_ingestion
+    ./idigbio_workers

.gitignore

@@ -86,6 +86,7 @@ venv/
 ENV/
 .venv/
 .virtualenv/
+.venv*/
 
 # Spyder project settings
 .spyderproject

.travis.yml

@@ -1,19 +1,24 @@
 sudo: required
-dist: xenial
+dist: focal
 language: python
-python: 2.7 
+python: 3.8.10
 services:
 - docker
-- postgresql
 addons:
-  postgresql: "9.5"
+  postgresql: "12"
+  apt:
+    packages:
+    - postgresql-12
+    - postgresql-client-12
+
 git:
   depth: 1
 
 install:
 - docker build -t $BUILDING .
-- psql -U postgres -c 'create database test_idigbio;'
-- psql -U postgres -c "DROP SCHEMA public CASCADE;" test_idigbio
+- echo "docker build complete!"
+- psql -U travis -c 'create database test_idigbio;'
+- psql -U travis -c "DROP SCHEMA public CASCADE;" test_idigbio
 - docker run -d --net host --name minio-test -e "MINIO_ACCESS_KEY=${ACCESS_KEY}" -e "MINIO_SECRET_KEY=${SECRET_KEY=}"  minio/minio:RELEASE.2017-07-24T18-27-35Z server /export
 
 script:
@@ -22,7 +27,7 @@ script:
   -e "IDB_STORAGE_SECRET_KEY=${SECRET_KEY}"
   -e "IDB_STORAGE_HOST=localhost:9000"
   $BUILDING
-  pytest -p no:cacheprovider --pguser=postgres --pgpass="" tests/idb
+  pytest -p no:cacheprovider --pguser=travis --pgpass="" tests/idb
 
 after_success:
 # upload new docker images
@@ -42,6 +47,8 @@ env:
   - BUILDING=$DOCKER_IMAGE:$TRAVIS_COMMIT
   - ACCESS_KEY="testAccessKey"
   - SECRET_KEY="testSecretKey"
+  - PGPORT=5433
+  - PGUSER=travis
 notifications:
   slack:
     secure: "GVJcqmhKww+74MaAKiSPIf2oPxEFp0PoYHzdzL0xw5xd62OiJkrcdzOsiZ5DQxLlgU+MgWrxcxJmunMXOUvR7gt1V1WjuvYsmJ6+bC4/AU81Qr0z5BDANZRX7O8yA/WLvPyE5hsJQWAyBL0NFvkKbxzw/qKlaaoh+uUXI2FLA+LMq6kuW1BYyQuFsoX/14842ppRETrxK5uVAft7c+Egh68Tet5xO5lry3bBFZ0F75Jg5nuzgAgq3OcOe8Luo8FuekX75tBjT0hBNSM+Kn9LOPx25O6tRfivCJn1quOPNSNhet8zbeYecnSevW49VLtWi34fLX6rIsPcvGZg1Z5q1MjvxqDtuIEnhLBeR/8+hK+dLu6MP0YGcG4jBELEH13WlkmJJcH1X+DmRLK1tkgwWhJlW1RJ/QHPoqsMKMNFxS4aDenFGiYXSvv7skX8pr+i2QW1j55V6AU55OCvhsrgyHwFXlA+1ItU5geCVvmLscg8RXF7HuTrZrN/UxLjDVPAjY4rHS2ccZx1g3vo+K0OWafKk6yOi9J7rfwT+OGCMGv3Q57i1ShDIHvmfzfJmsjhtRCvKowwUrw2/Udean/ncdcRLvTTv0Xn+ioxnWAyvWo7UMdXj7YeHvp4bT9pY2eSjfHjB6Fe7S3vWC5zzLc0BDYouxkysv+j0EvP1vx84t4="

Dockerfile

@@ -1,5 +1,6 @@
-FROM python:2.7
-RUN apt-get update && apt-get install -y --no-install-recommends \
+FROM python:3.8.10
+# python:3.6 and later are based on "Debian GNU/Linux"
+RUN apt-get update > /dev/null && apt-get install -y --no-install-recommends \
     gfortran \
     libatlas-base-dev \
     ffmpeg \
@@ -12,9 +13,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 #    fonts-dejavu-core \
 #    libxml2 \
 #    libxslt1-dev \
-  && rm -rf /var/lib/apt/lists/*
+  > /dev/null && rm -rf /var/lib/apt/lists/*
 
 COPY . /opt/idb-backend/
 WORKDIR /opt/idb-backend/
-RUN pip --no-cache-dir install -e .[test]
+# RUN python3 -m virtualenv venv
+RUN pip install -q --upgrade pip
+RUN pip -q --no-cache-dir install -e .[test]
 USER www-data

README.md

@@ -8,27 +8,23 @@ iDigBio server and backend code for data ingestion and data API.
 
 ### System Dependencies
 
-Currently this project only works in python2.7 and is not compatible with Python 3.
-
-The following library packages will need to be installed to run the api:
-
-In Ubuntu 16.04:
-
-    apt-get install python2.7-dev libblas-dev liblapack-dev \
-      libatlas-base-dev gfortran libgdal-dev libpq-dev libgeos-c1v5 \
-      libsystemd-dev
+#### Python 3 instructions:
 
 In Ubuntu 18.04:
 
-    apt install python-dev libblas-dev liblapack-dev \
+    apt install python3-dev python3-all-dev libblas-dev liblapack-dev \
       libatlas-base-dev gfortran libgdal-dev libpq-dev libgeos-c1v5 \
       libsystemd-dev
 
 For Ingestion and Development, the following are also needed:
 
-In Ubuntu 16.04, 18.04:
+In 18.04:
+
+    apt install libxml2 libxslt1-dev ffmpeg fonts-dejavu-core libfreetype6-dev python-systemd
 
-    apt-get install libxml2 libxslt1-dev ffmpeg fonts-dejavu-core libfreetype6-dev python-systemd
+#### Python 2.7 instructions:
+
+This version of idb-backend will not work under Python 2.
 
 
 ### Package installation
@@ -70,6 +66,16 @@ no need to set the `PYTHONPATH`.
 In any invocation an `idigbio.json` must be present in either `$PWD`,
 `$HOME`, or `/etc/idigbio`.
 
+For convenience we can create links to the virtual environment entrypoints in `/usr/local/bin/` so we do not need to activate the virtual environment explicitly or use the full path.
+
+```
+$ sudo ln -s /home/idigbio-ingestion/idb-backend/venv/bin/idb  /usr/local/bin/idb
+$ ls -la /usr/local/bin/idb
+lrwxrwxrwx 1 root root 48 Aug 16 20:23 /usr/local/bin/idb -> /home/idigbio-ingestion/idb-backend/venv/bin/idb
+$ sudo ln -s /home/idigbio-ingestion/idb-backend/venv/bin/idigbio-ingestion  /usr/local/bin/idigbio-ingestion
+$ ls -la /usr/local/bin/idigbio-ingestion
+lrwxrwxrwx 1 root root 62 Aug 16 20:24 /usr/local/bin/idigbio-ingestion -> /home/idigbio-ingestion/idb-backend/venv/bin/idigbio-ingestion
+```
 ### Data API
 
 This serves the `api.idigbio.org` interface
@@ -82,7 +88,7 @@ This package can also be run as a [celery worker]; this is used by the
 data api that launches some tasks (most notably download building) via
 celery to a background worker.
 
-    celery worker --app=idigbio_workers -l WARNING
+    celery worker --app=idigbio_workers -l INFO
 
 [celery worker]: http://docs.celeryproject.org/en/latest/userguide/workers.html
 
@@ -91,111 +97,47 @@ celery to a background worker.
 You probably want to run in a virtual environment.   You may wish to disable the pip cache
 to verify package builds are working properly.
 
+
+### Python 3
+
 ```bash
-$ virtualenv -p python2.7 .venv
+
+$ python --version
+Python 2.7.17
+
+$ python3 -m virtualenv -p python3 .venv
+Already using interpreter /usr/bin/python3
+Using base prefix '/usr'
+New python executable in /tmp/venvtest/.venv/bin/python3
+Also creating executable in /tmp/venvtest/.venv/bin/python
+Installing setuptools, pkg_resources, pip, wheel...done.
 
 $ source .venv/bin/activate
 
 $ python --version
-Python 2.7.17
+Python 3.6.9
 
 $ pip --no-cache-dir install -e .
 
 $ pip --no-cache-dir install -r requirements.txt
 ```
 
-It is possible in the future that this project will be runnable using "Open in container" features of Microsoft Visual Studio Code (aka vscode or just `code`).
+### Python 2 (obsolete)
 
-### Testing Dependencies
+This project is no longer compatible with Python 2.
 
-Some idb-backend tests depend on external resources, such as a database or Elasticsearch.
 
-A local postgresql DB named `test_idigbio` with user/pass `test` / `test` must exist for many of the tests to run.  Note: The data in the DB will be destroyed during the testing.
+### Develop in Container
 
-Database tests will be SKIPPED if the database is not available.
+It is possible in the future that this project will be runnable using "Open in container" features of Microsoft Visual Studio Code (aka vscode or just `code`).
 
-Tests that depend on Elasticsearch will FAIL if the computer running the tests cannot reach the Elasticsearch cluster (fail very slowly in fact), or if there is some other failure.
 
-### Running tests
+### Running the Test Suite
 
 The test suite can be run by executing `py.test` (or `pytest`).
 
-However due to the dependencies mentioned above, you may wish to run the database in docker each time.  The sleep is needed to allow postgres time to start accepting connections.
-
-    docker run --rm --name postgres_test_idigbio --network host \
-      -e POSTGRES_PASSWORD=test -e POSTGRES_USER=test -e POSTGRES_DB=test_idigbio  \
-      -d postgres:9.5 && \
-      sleep 5; \
-      py.test ; \
-      docker stop postgres_test_idigbio
-
-
-To exclude a single set of tests that are failing (or Seg Faulting!), add the `--deselect` option to the pytest command:
+See the [README.md in the `tests` subdirectory](tests/README.md) for more information.
 
-    py.test --deselect=tests/idigbio_ingestion/mediaing/test_derivatives.py
+There are a number of important dependencies noted there.
 
-To find out why tests are being Skipped, add the `-rxs` options.
-
-A "what the heck is going on with the tests and skip the one that is Seg Faulting" example command:
-
-    docker run --rm --name postgres_test_idigbio --network host  \
-      -e POSTGRES_PASSWORD=test -e POSTGRES_USER=test -e POSTGRES_DB=test_idigbio \
-      -d postgres:9.5 && \
-      sleep 5; \
-      py.test -rxs --deselect=tests/idigbio_ingestion/mediaing/test_derivatives.py ; \
-      docker stop postgres_test_idigbio
-
-### Create a local postgres DB
-
-The recommended approach is to run postgres via docker (see above).
-
-If you have a full installation of postgres running locally, the db can be manually created with: 
-
-    createuser -l -e test -P
-    createdb -l 'en_US.UTF-8' -E UTF8 -O test -e test_idigbio;
-
-    # The schema obj is still owned by the user of the above
-    # statement, not the owner 'test'. Drop it so it will be recreated
-    # by the script appropriately
-    psql -c "DROP SCHEMA public CASCADE;" test_idigbio
-
-
-### Schema
-
-The live production db schema is copied into `tests/data/schema.sql` by periodically running this command:
-
-    pg_dump --host c18node8.acis.ufl.edu --username idigbio \
-        --format plain --schema-only --schema=public \
-        --clean --if-exists \
-        --no-owner --no-privileges --no-tablespaces --no-unlogged-table-data \
-        --file tests/data/schema.sql \
-        idb_api_prod
-
-Except not yet because there are lots of differences between the existing file and one created by running that command (due to fixes for https://wiki.postgresql.org/wiki/A_Guide_to_CVE-2018-1058:_Protect_Your_Search_Path).
-
-
-### Data
-
-A trimmed down set of data has been manually curated to support the test suite. It is provided in `tests/data/testdata.sql`
-
-The full dump / original was created with something like:
-
-    pg_dump --port 5432 --host c18node8.acis.ufl.edu --username idigbio \
-      --format plain --data-only --encoding UTF8 \
-      --inserts --column-inserts --no-privileges --no-tablespaces \
-      --verbose --no-unlogged-table-data  \
-      --exclude-table-data=ceph_server_files \
-      --file tests/data/testdata.sql idb_api_prod
-
-Such a dump is huge and un-usable and un-editable by normal means. It is not clear how the dump was transformed / curated into its current state.
-
-If running the dump again, consider adding multiple `--exclude-table-data=TABLE` for some of the bigger tables that are not materially relevant to test suite such as:
-
-```plaintext
-annotations
-data
-corrections
-ceph_server_files
-```
 
-We likely need to find a new way to refresh the test dataset.

etc/systemd/system/idb-api-stats.service → etc/systemd/system/idb-api-stats.service-deprecated

@@ -1,3 +1,5 @@
+# deprecated for both naming and the call.  see: idb-recordset-stats--aggregator.*
+
 [Unit]
 Description="Runs `idb api-stats`"
 

etc/systemd/system/idb-api-stats.timer → etc/systemd/system/idb-api-stats.timer-deprecated

@@ -1,3 +1,5 @@
+# deprecated for both naming and the call.  see: idb-recordset-stats--aggregator.*
+
 [Unit]
 Description="Daily timer for idb api-stats"
 

etc/systemd/system/idb-collect-stats.service → etc/systemd/system/idb-collect-stats.service-deprecated

@@ -1,3 +1,5 @@
+# deprecated for both naming and the call.  see: idb-interaction-telemetry-aggregator.*
+
 [Unit]
 Description="Runs `idb collect-stats`"
 

etc/systemd/system/idb-collect-stats.timer → etc/systemd/system/idb-collect-stats.timer-deprecated

@@ -1,3 +1,5 @@
+# deprecated for both naming and the call.  see: idb-interaction-telemetry-aggregator.*
+
 [Unit]
 Description="daily timer for idb collect-stats"
 

etc/systemd/system/idb-recordset-stats-aggregator.service

@@ -0,0 +1,12 @@
+# https://redmine.idigbio.org/projects/infrastructure/wiki/Portal_and_api_stats_overview
+
+[Unit]
+Description="Runs `idb api-stats` (recordset stats aggregator) via non-root user and virtualenv"
+
+[Service]
+Type=oneshot
+User=stats
+Environment=LANG=en_US.UTF-8
+
+# the virtualenv is "baked in" and does not need to be activated first
+ExecStart=/home/stats/.virtualenvs/idb-backend/bin/python /home/stats/.virtualenvs/idb-backend/bin/idb --journal -vv --config /etc/idigbio/idigbio-recordset-stats-aggregator-overrides.json api-stats

etc/systemd/system/idigbio-ingestion-derivatives.timer

@@ -3,8 +3,8 @@ Description="Timer for idigbio-ingestion derivatives"
 
 [Timer]
 OnBootSec=13min
-OnUnitInactiveSec=10min
-RandomizedDelaySec=4m
+OnUnitInactiveSec=1min
+RandomizedDelaySec=15
 
 [Install]
 WantedBy=timers.target

idb/__init__.py

@@ -1 +1,2 @@
-__version__ = "3.0.7"
+# This sets the version for the generated idb-backend PyPi package also.
+__version__ = "4.0.0"