Merge branch 'master' into v1.10.3

.dockerfiles/entrypoint-dev.sh

@@ -3,7 +3,7 @@
 set -e
 
 if [ ! -f "${HOME}/.installed" ]; then
-  /app/bin/install.sh -p '3.8' --docker --all-testers
+  /app/bin/install.sh -p '3.9' --docker --all-testers
 
   echo "export REDIS_URL=${REDIS_URL}
         export PGHOST=${PGHOST}

.travis.yml

@@ -4,6 +4,7 @@ python:
   - "3.6"
   - "3.7"
   - "3.8"
+  - "3.9"
 # command to run tests
 script:
   - python setup.py test

Changelog.md

@@ -1,6 +1,13 @@
 # CHANGELOG
 All notable changes to this project will be documented here.
 
+## [v1.10.3]
+- Fix bug where zip archive was unpacked two levels deep instead of just one (#271) 
+- Pass PGHOST and PGINFO environment variables to tests (#272)
+- Update to new version of markus-api that supports uploading binary files (#273)
+- Fix bug where environment variables were not string types (#274)
+- Add python3.9 as a tester option for the py and pyta testers, as well as the installer (#275)
+
 ## [v1.10.2]
 - Updated java tester to support configurable classpaths and source files (#268)
 

README.md

@@ -39,7 +39,7 @@ $ bin/install.sh [-p|--python-version python-version] [--non-interactive] [--doc
 
 options: 
 
-- `--python_version` : version of python to install/use to run the autotester (default is 3.8).
+- `--python_version` : version of python to install/use to run the autotester (default is 3.9).
 - `--non-interactive` : run the installer in non-interactive mode (all confirmations will be accepted without prompting the user).
 - `--docker` : run the installer for installing in docker. This installs in non-interactive mode and iptables, postgresql debian packages will not be installed.
 - `--all-testers` : install all testers as well as the server. See [Testers](#testers).

bin/install.sh

@@ -22,7 +22,7 @@ _check_python_version() {
 set_python_version() {
   # get the python version from the argument passed to this script or use python3.8 by default
   if [ -z "${PYTHON_VERSION}" ]; then
-    PYTHON_VERSION=3.8
+    PYTHON_VERSION=3.9
   else
     # check if both a major and minor version have been specified
     if [[ ${PYTHON_VERSION} != $(echo "${PYTHON_VERSION}" | grep -ow '^[0-9].[0-9]$') ]]; then

bin/uninstall.sh

@@ -147,7 +147,7 @@ BINDIR=$(dirname ${THISSCRIPT})
 SERVERDIR=$(dirname ${BINDIR})
 TESTERSDIR=$(dirname ${SERVERDIR})/testers
 THISUSER=$(whoami)
-PYTHONVERSION="3.8"
+PYTHONVERSION="3.9"
 
 SERVERUSER=$(get_config_param SERVER_USER)
 if [[ -n ${SERVERUSER} ]]; then

setup.py

@@ -15,12 +15,11 @@
     zip_safe=False,
     install_requires=[
         "redis==3.3.8",
-        "requests==2.22.0",
         "rq==1.1.0",
         "supervisor==4.1.0",
         "PyYAML==5.1.2",
         "psycopg2-binary==2.8.4",
-        "markusapi==0.1.1",
+        "markusapi==0.2.0",
         "jsonschema==3.0.2",
     ],
     tests_require=["pytest==5.3.1", "hypothesis==4.47.3", "fakeredis==1.1.0"],

src/autotester/resources/postgresql/__init__.py

@@ -10,6 +10,7 @@
 POSTGRES_PREFIX = config["resources", "postgresql", "_prefix"]
 PGPASSFILE = os.path.join(config["workspace"], config["_workspace_contents", "_logs"], ".pgpass")
 PGHOST = config["resources", "postgresql", "host"]
+PGPORT = config["resources", "postgresql", "port"]
 
 
 def setup_database(test_username: str) -> Dict[str, str]:
@@ -27,7 +28,7 @@ def setup_database(test_username: str) -> Dict[str, str]:
     with open(PGPASSFILE) as f:
         password = f.read().strip()
 
-    with psycopg2.connect(database=database, user=user, password=password, host=PGHOST) as conn:
+    with psycopg2.connect(database=database, user=user, password=password, host=PGHOST, port=PGPORT) as conn:
         with conn.cursor() as cursor:
             cursor.execute("DROP OWNED BY CURRENT_USER;")
             if test_username != user:
@@ -39,5 +40,7 @@ def setup_database(test_username: str) -> Dict[str, str]:
         "PGDATABASE": database,
         "PGPASSWORD": password,
         "PGUSER": user,
+        "PGHOST": PGHOST,
+        "PGPORT": str(PGPORT),
         "AUTOTESTENV": "true",
     }

src/autotester/server/client_customizations/markus.py

@@ -36,7 +36,7 @@ def write_student_files(self, destination: str) -> None:
         zip_content = self._api.get_files_from_repo(self.assignment_id, self.group_id, collected=collected)
         if zip_content is None:
             raise TestScriptFilesError("No test files found")
-        extract_zip_stream(zip_content, destination, ignore_root_dirs=2)
+        extract_zip_stream(zip_content, destination, ignore_root_dirs=1)
 
     def send_test_results(self, results_data: Dict) -> None:
         """ Send test results to the client """
@@ -73,7 +73,7 @@ def upload_feedback_to_repo(self, feedback_file: str) -> None:
         Upload the feedback file to the group's repo.
         """
         if os.path.isfile(feedback_file):
-            with open(feedback_file) as feedback_open:
+            with open(feedback_file, 'rb') as feedback_open:
                 self._api.upload_file_to_repo(
                     self.assignment_id, self.group_id, os.path.basename(feedback_file), feedback_open.read()
                 )
@@ -83,7 +83,7 @@ def upload_feedback_file(self, feedback_file: str) -> None:
         Upload the feedback file using MarkUs' api.
         """
         if os.path.isfile(feedback_file):
-            with open(feedback_file) as feedback_open:
+            with open(feedback_file, 'rb') as feedback_open:
                 self._api.upload_feedback_file(
                     self.assignment_id, self.group_id, os.path.basename(feedback_file), feedback_open.read()
                 )

src/autotester/testers/py/lib/sql_helper.py

@@ -40,7 +40,8 @@ def connection(*args, **kwargs):
             "database": os.environ.get("PGDATABASE"),
             "password": os.environ.get("PGPASSWORD"),
             "user": os.environ.get("PGUSER"),
-            "host": "localhost",
+            "host": os.environ.get("PGHOST"),
+            "port": os.environ.get("PGPORT")
         }
     return _unmockable_psycopg2_connect(*args, **kwargs)
 
@@ -99,7 +100,9 @@ def execute_psql_file(
     *args: str,
     database: Optional[str] = None,
     password: Optional[str] = None,
-    user: Optional[str] = None
+    user: Optional[str] = None,
+    host: Optional[str] = None,
+    port: Optional[str] = None,
 ) -> subprocess.CompletedProcess:
     """
     Return a CompletedProcess object returned after calling:
@@ -140,6 +143,8 @@ def execute_psql_file(
             "PGUSER": user or os.environ.get("PGUSER"),
             "PGPASSWORD": password or os.environ.get("PGPASSWORD"),
             "PGDATABASE": database or os.environ.get("PGDATABASE"),
+            "PGHOST": host or os.environ.get("PGHOST"),
+            "PGPORT": port or os.environ.get("PGPORT")
         }
         env = {**os.environ, **db_vars}
     return subprocess.run(["psql", "-f", filename] + list(args), env=env, capture_output=True)

src/autotester/testers/py/specs/settings_schema.json

@@ -21,11 +21,12 @@
           "title": "Python version",
           "type": "string",
           "enum": [
+            "3.9",
             "3.8",
             "3.7",
             "3.6"
           ],
-          "default": "3.8"
+          "default": "3.9"
         },
         "pip_requirements": {
           "title": "Package requirements",

src/autotester/testers/py/tests/specs.json

@@ -3,7 +3,7 @@
     {
       "tester_type": "py",
       "env_data": {
-        "python_version": "3.8",
+        "python_version": "3.9",
         "pip_requirements": "hypothesis pytest-timeout"
       },
       "test_data": [

src/autotester/testers/pyta/specs/settings_schema.json

@@ -21,11 +21,12 @@
           "title": "Python version",
           "type": "string",
           "enum": [
+            "3.9",
             "3.8",
             "3.7",
             "3.6"
           ],
-          "default": "3.8"
+          "default": "3.9"
         },
         "pip_requirements": {
           "title": "Package requirements",

src/autotester/testers/pyta/tests/specs.json

@@ -17,7 +17,7 @@
         }
       ],
       "env_data": {
-        "python_version": "3.8"
+        "python_version": "3.9"
       }
     }
   ]