Add support for Python 3.11 and MySQL 5.7

.devcontainer/Dockerfile

@@ -1,12 +1,13 @@
-# Note: You can use any Debian/Ubuntu based image you want. 
-FROM mcr.microsoft.com/devcontainers/python:3.7-bullseye
-
+ARG PY_VER
+ARG DISTRO
+FROM mcr.microsoft.com/devcontainers/python:${PY_VER}-${DISTRO}
 RUN \
     apt update && \
     apt-get install bash-completion graphviz default-mysql-client -y && \
     pip install flake8 black faker ipykernel pytest pytest-cov nose nose-cov datajoint && \
     pip uninstall datajoint -y
 
+USER root
 ENV DJ_HOST fakeservices.datajoint.io
 ENV DJ_USER root
-ENV DJ_PASS simple
+ENV DJ_PASS password

.devcontainer/devcontainer.json

@@ -1,34 +1,48 @@
 // For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-docker-compose
 {
-	"name": "Development",
-	"dockerComposeFile": "docker-compose.yaml",
+	"name": "Existing Docker Compose (Extend)",
+	// Update the 'dockerComposeFile' list if you have more compose files or use different names.
+	// The .devcontainer/docker-compose.yml file contains any overrides you need/want to make.
+	"dockerComposeFile": [
+		"../LNX-docker-compose.yml",
+		"docker-compose.yml"
+	],
+	// The 'service' property is the name of the service for the container that VS Code should
+	// use. Update this value and .devcontainer/docker-compose.yml to the real service name.
 	"service": "app",
+	// The optional 'workspaceFolder' property is the path VS Code should open by default when
+	// connected. This is typically a file mount in .devcontainer/docker-compose.yml
 	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
-	// Use this environment variable if you need to bind mount your local source code into a new container.
-	"remoteEnv": {
-		"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
-	},
-	// https://containers.dev/features
-	"features": {
-		"ghcr.io/devcontainers/features/docker-in-docker:2": {},
-		"ghcr.io/devcontainers/features/git:1": {},
-		"ghcr.io/eitsupi/devcontainer-features/jq-likes:1": {},
-		"ghcr.io/guiyomh/features/vim:0": {}
-	},
-	"onCreateCommand": "pip install -e .",
-	"postStartCommand": "MYSQL_VER=8.0 MINIO_VER=RELEASE.2022-08-11T04-37-28Z docker compose -f local-docker-compose.yml down && docker volume prune -f && MYSQL_VER=8.0 MINIO_VER=RELEASE.2022-08-11T04-37-28Z docker compose -f local-docker-compose.yml up --build --wait",
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
 	"forwardPorts": [
 		80,
 		443,
 		3306,
 		8080,
 		9000
 	],
+	// Uncomment the next line if you want start specific services in your Docker Compose config.
+	// "runServices": [],
+	// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
+	"shutdownAction": "stopCompose",
+	"onCreateCommand": "python3 -m pip install -e .",
+	"features": {
+		"ghcr.io/devcontainers/features/git:1": {},
+	},
+	// Configure tool-specific properties.
 	"customizations": {
 		"vscode": {
 			"extensions": [
 				"ms-python.python"
 			]
 		}
+	},
+	"remoteEnv": {
+		"LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}"
 	}
+	// Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "devcontainer"
 }

.devcontainer/docker-compose.yml

@@ -0,0 +1,32 @@
+version: '2.4'
+services:
+  # Update this to the name of the service you want to work with in your docker-compose.yml file
+  app:
+    # Uncomment if you want to override the service's Dockerfile to one in the .devcontainer
+    # folder. Note that the path of the Dockerfile and context is relative to the *primary*
+    # docker-compose.yml file (the first in the devcontainer.json "dockerComposeFile"
+    # array). The sample below assumes your primary file is in the root of your project.
+    container_name: devcontainer
+    image: devcontainer
+    build:
+      context: .
+      dockerfile: .devcontainer/Dockerfile
+      args:
+        - PY_VER=${PY_VER:-3.8}
+        - DISTRO=${DISTRO:-buster}
+
+    volumes:
+      # Update this to wherever you want VS Code to mount the folder of your project
+      - ..:/workspaces:cached
+
+    # Uncomment the next four lines if you will use a ptrace-based debugger like C++, Go, and Rust.
+    # cap_add:
+    #   - SYS_PTRACE
+    # security_opt:
+    #   - seccomp:unconfined
+
+    user: root
+
+    # Overrides default command so things don't shut down after the process ends.
+    command: /bin/sh -c "while sleep 1000; do :; done"
+

.github/workflows/development.yaml

@@ -58,6 +58,10 @@ jobs:
         py_ver: ["3.9"]
         mysql_ver: ["8.0", "5.7"]
         include:
+          - py_ver: "3.11"
+            mysql_ver: "8.0"
+          - py_ver: "3.10"
+            mysql_ver: "8.0"
           - py_ver: "3.8"
             mysql_ver: "5.7"
           - py_ver: "3.7"
@@ -77,7 +81,7 @@ jobs:
       - name: Run primary tests
         env:
           PY_VER: ${{matrix.py_ver}}
-          DJ_PASS: simple
+          DJ_PASS: password
           MYSQL_VER: ${{matrix.mysql_ver}}
           DISTRO: alpine
           MINIO_VER: RELEASE.2021-09-03T03-56-13Z

.gitignore

@@ -30,4 +30,6 @@ docs/site
 
 !.vscode/settings.json
 !.vscode/launch.json
-!.devcontainer/devcontainer.json
+!.devcontainer/devcontainer.json
+!.devcontainer/docker-compose.yml
+

LNX-docker-compose.yml

@@ -7,9 +7,10 @@ x-net:
 services:
   db:
     <<: *net
-    image: datajoint/mysql:${MYSQL_VER}
+    image: datajoint/mysql:${MYSQL_VER:-5.7}
     environment:
-      - MYSQL_ROOT_PASSWORD=${DJ_PASS}
+      - MYSQL_ROOT_PASSWORD=${DJ_PASS:-password}
+    command: mysqld --default-authentication-plugin=mysql_native_password
     # ports:
     #   - "3306:3306"
     # volumes:
@@ -21,7 +22,7 @@ services:
       interval: 15s
   minio:
     <<: *net
-    image: minio/minio:${MINIO_VER}
+    image: minio/minio:${MINIO_VER:-RELEASE.2022-08-11T04-37-28Z}
     environment:
       - MINIO_ACCESS_KEY=datajoint
       - MINIO_SECRET_KEY=datajoint
@@ -44,7 +45,7 @@ services:
       interval: 15s
   fakeservices.datajoint.io:
     <<: *net
-    image: datajoint/nginx:v0.2.6
+    image: datajoint/nginx:v0.2.8
     environment:
       - ADD_db_TYPE=DATABASE
       - ADD_db_ENDPOINT=db:3306
@@ -58,7 +59,7 @@ services:
     #   - "3306:3306"
   app:
     <<: *net
-    image: datajoint/djtest:py${PY_VER}-${DISTRO}
+    image: datajoint/djtest:py${PY_VER:-3.11}-${DISTRO:-alpine}
     depends_on:
       db:
         condition: service_healthy
@@ -69,7 +70,7 @@ services:
     environment:
       - DJ_HOST=fakeservices.datajoint.io
       - DJ_USER=root
-      - DJ_PASS
+      - DJ_PASS=password
       - DJ_TEST_HOST=fakeservices.datajoint.io
       - DJ_TEST_USER=datajoint
       - DJ_TEST_PASSWORD=datajoint
@@ -86,14 +87,12 @@ services:
       - -c
       - |
         set -e
-        pip install --user nose nose-cov
         pip install -e .
         pip list --format=freeze | grep datajoint
         pytest -sv --cov-report term-missing --cov=datajoint tests
-        nosetests -vsw tests_old --with-coverage --cover-package=datajoint
     # ports:
     #   - "8888:8888"
-    user: ${HOST_UID}:anaconda
+    user: ${HOST_UID:-1000}:anaconda
     volumes:
       - .:/src
       - /tmp/.X11-unix:/tmp/.X11-unix:rw

datajoint/connection.py

@@ -182,6 +182,8 @@ def __init__(self, host, user, password, port=None, init_fun=None, use_tls=None)
         elif port is None:
             port = config["database.port"]
         self.conn_info = dict(host=host, port=port, user=user, passwd=password)
+        if use_tls is None:
+            use_tls = config["database.use_tls"]
         if use_tls is not False:
             self.conn_info["ssl"] = (
                 use_tls if isinstance(use_tls, dict) else {"ssl": {}}

docs/src/develop.md

@@ -39,24 +39,16 @@ The following will verify there are no regression errors by running our test sui
 
 - Entire test suite:
   ```
-  nosetests -vw tests_old
+  pytest -sv --cov-report term-missing --cov=datajoint tests
   ```
-  > Note: We are in the process of upgrading to `pytest` tests. To run those, use:
-  > ```
-  > pytest -sv --cov-report term-missing --cov=datajoint tests
-  > ```
 
 - A single functional test:
   ```
-  nosetests -vs --tests=tests_old.test_external_class:test_insert_and_fetch
+  pytest -sv tests/test_connection.py::test_dj_conn
   ```
-  > Note: We are in the process of upgrading to `pytest` tests. To run those, use:
-  > ```
-  > pytest -sv tests/test_connection.py::test_dj_conn
-  > ```
 - A single class test:
   ```
-  nosetests -vs --tests=tests_old.test_fetch:TestFetch.test_getattribute_for_fetch1
+  pytest -sv tests/test_aggr_regressions.py::TestIssue558
   ```
 
 ### Style Tests
@@ -104,7 +96,7 @@ It is often useful in development to connect to DataJoint's relational database
 Connect as follows to the database running within your developer environment:
 
 ```
-mysql -hfakeservices.datajoint.io -uroot -psimple
+mysql -hfakeservices.datajoint.io -uroot -ppassword
 ```
 
 ### Documentation

tests/__init__.py

@@ -1,69 +0,0 @@
-import datajoint as dj
-from packaging import version
-import pytest
-import os
-
-PREFIX = "djtest"
-
-CONN_INFO_ROOT = dict(
-    host=os.getenv("DJ_HOST"),
-    user=os.getenv("DJ_USER"),
-    password=os.getenv("DJ_PASS"),
-)
-
-
-@pytest.fixture
-def connection_root():
-    """Root user database connection."""
-    dj.config["safemode"] = False
-    connection = dj.Connection(
-        host=os.getenv("DJ_HOST"),
-        user=os.getenv("DJ_USER"),
-        password=os.getenv("DJ_PASS"),
-    )
-    yield connection
-    dj.config["safemode"] = True
-    connection.close()
-
-
-@pytest.fixture
-def connection_test(connection_root):
-    """Test user database connection."""
-    database = f"{PREFIX}%%"
-    credentials = dict(
-        host=os.getenv("DJ_HOST"), user="datajoint", password="datajoint"
-    )
-    permission = "ALL PRIVILEGES"
-
-    # Create MySQL users
-    if version.parse(
-        connection_root.query("select @@version;").fetchone()[0]
-    ) >= version.parse("8.0.0"):
-        # create user if necessary on mysql8
-        connection_root.query(
-            f"""
-            CREATE USER IF NOT EXISTS '{credentials["user"]}'@'%%'
-            IDENTIFIED BY '{credentials["password"]}';
-            """
-        )
-        connection_root.query(
-            f"""
-            GRANT {permission} ON `{database}`.*
-            TO '{credentials["user"]}'@'%%';
-            """
-        )
-    else:
-        # grant permissions. For MySQL 5.7 this also automatically creates user
-        # if not exists
-        connection_root.query(
-            f"""
-            GRANT {permission} ON `{database}`.*
-            TO '{credentials["user"]}'@'%%'
-            IDENTIFIED BY '{credentials["password"]}';
-            """
-        )
-
-    connection = dj.Connection(**credentials)
-    yield connection
-    connection_root.query(f"""DROP USER `{credentials["user"]}`""")
-    connection.close()