diff --git a/.dockerfiles/entrypoint-dev.sh b/.dockerfiles/entrypoint-dev.sh index 64406cd2..323fcfd4 100755 --- a/.dockerfiles/entrypoint-dev.sh +++ b/.dockerfiles/entrypoint-dev.sh @@ -3,7 +3,7 @@ set -e if [ ! -f "${HOME}/.installed" ]; then - /app/bin/install.sh -p '3.8' --docker --all-testers + /app/bin/install.sh -p '3.9' --docker --all-testers echo "export REDIS_URL=${REDIS_URL} export PGHOST=${PGHOST} diff --git a/.travis.yml b/.travis.yml index 14b06eaa..bbf063f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ python: - "3.6" - "3.7" - "3.8" + - "3.9" # command to run tests script: - python setup.py test diff --git a/Changelog.md b/Changelog.md index 3dedf658..aa3025af 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,6 +1,13 @@ # CHANGELOG All notable changes to this project will be documented here. +## [v1.10.3] +- Fix bug where zip archive was unpacked two levels deep instead of just one (#271) +- Pass PGHOST and PGINFO environment variables to tests (#272) +- Update to new version of markus-api that supports uploading binary files (#273) +- Fix bug where environment variables were not string types (#274) +- Add python3.9 as a tester option for the py and pyta testers, as well as the installer (#275) + ## [v1.10.2] - Updated java tester to support configurable classpaths and source files (#268) diff --git a/README.md b/README.md index af835175..17e6dfe2 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ $ bin/install.sh [-p|--python-version python-version] [--non-interactive] [--doc options: -- `--python_version` : version of python to install/use to run the autotester (default is 3.8). +- `--python_version` : version of python to install/use to run the autotester (default is 3.9). - `--non-interactive` : run the installer in non-interactive mode (all confirmations will be accepted without prompting the user). - `--docker` : run the installer for installing in docker. This installs in non-interactive mode and iptables, postgresql debian packages will not be installed. - `--all-testers` : install all testers as well as the server. See [Testers](#testers). diff --git a/bin/install.sh b/bin/install.sh index 10580809..101bbf05 100755 --- a/bin/install.sh +++ b/bin/install.sh @@ -22,7 +22,7 @@ _check_python_version() { set_python_version() { # get the python version from the argument passed to this script or use python3.8 by default if [ -z "${PYTHON_VERSION}" ]; then - PYTHON_VERSION=3.8 + PYTHON_VERSION=3.9 else # check if both a major and minor version have been specified if [[ ${PYTHON_VERSION} != $(echo "${PYTHON_VERSION}" | grep -ow '^[0-9].[0-9]$') ]]; then diff --git a/bin/uninstall.sh b/bin/uninstall.sh index cfa2c36b..014e07af 100755 --- a/bin/uninstall.sh +++ b/bin/uninstall.sh @@ -147,7 +147,7 @@ BINDIR=$(dirname ${THISSCRIPT}) SERVERDIR=$(dirname ${BINDIR}) TESTERSDIR=$(dirname ${SERVERDIR})/testers THISUSER=$(whoami) -PYTHONVERSION="3.8" +PYTHONVERSION="3.9" SERVERUSER=$(get_config_param SERVER_USER) if [[ -n ${SERVERUSER} ]]; then diff --git a/setup.py b/setup.py index 8a6985ae..80a5fc75 100644 --- a/setup.py +++ b/setup.py @@ -15,12 +15,11 @@ zip_safe=False, install_requires=[ "redis==3.3.8", - "requests==2.22.0", "rq==1.1.0", "supervisor==4.1.0", "PyYAML==5.1.2", "psycopg2-binary==2.8.4", - "markusapi==0.1.1", + "markusapi==0.2.0", "jsonschema==3.0.2", ], tests_require=["pytest==5.3.1", "hypothesis==4.47.3", "fakeredis==1.1.0"], diff --git a/src/autotester/resources/postgresql/__init__.py b/src/autotester/resources/postgresql/__init__.py index 688b2eb7..8239c7a7 100644 --- a/src/autotester/resources/postgresql/__init__.py +++ b/src/autotester/resources/postgresql/__init__.py @@ -10,6 +10,7 @@ POSTGRES_PREFIX = config["resources", "postgresql", "_prefix"] PGPASSFILE = os.path.join(config["workspace"], config["_workspace_contents", "_logs"], ".pgpass") PGHOST = config["resources", "postgresql", "host"] +PGPORT = config["resources", "postgresql", "port"] def setup_database(test_username: str) -> Dict[str, str]: @@ -27,7 +28,7 @@ def setup_database(test_username: str) -> Dict[str, str]: with open(PGPASSFILE) as f: password = f.read().strip() - with psycopg2.connect(database=database, user=user, password=password, host=PGHOST) as conn: + with psycopg2.connect(database=database, user=user, password=password, host=PGHOST, port=PGPORT) as conn: with conn.cursor() as cursor: cursor.execute("DROP OWNED BY CURRENT_USER;") if test_username != user: @@ -39,5 +40,7 @@ def setup_database(test_username: str) -> Dict[str, str]: "PGDATABASE": database, "PGPASSWORD": password, "PGUSER": user, + "PGHOST": PGHOST, + "PGPORT": str(PGPORT), "AUTOTESTENV": "true", } diff --git a/src/autotester/server/client_customizations/markus.py b/src/autotester/server/client_customizations/markus.py index c361ed79..6b7bf92f 100644 --- a/src/autotester/server/client_customizations/markus.py +++ b/src/autotester/server/client_customizations/markus.py @@ -36,7 +36,7 @@ def write_student_files(self, destination: str) -> None: zip_content = self._api.get_files_from_repo(self.assignment_id, self.group_id, collected=collected) if zip_content is None: raise TestScriptFilesError("No test files found") - extract_zip_stream(zip_content, destination, ignore_root_dirs=2) + extract_zip_stream(zip_content, destination, ignore_root_dirs=1) def send_test_results(self, results_data: Dict) -> None: """ Send test results to the client """ @@ -73,7 +73,7 @@ def upload_feedback_to_repo(self, feedback_file: str) -> None: Upload the feedback file to the group's repo. """ if os.path.isfile(feedback_file): - with open(feedback_file) as feedback_open: + with open(feedback_file, 'rb') as feedback_open: self._api.upload_file_to_repo( self.assignment_id, self.group_id, os.path.basename(feedback_file), feedback_open.read() ) @@ -83,7 +83,7 @@ def upload_feedback_file(self, feedback_file: str) -> None: Upload the feedback file using MarkUs' api. """ if os.path.isfile(feedback_file): - with open(feedback_file) as feedback_open: + with open(feedback_file, 'rb') as feedback_open: self._api.upload_feedback_file( self.assignment_id, self.group_id, os.path.basename(feedback_file), feedback_open.read() ) diff --git a/src/autotester/testers/py/lib/sql_helper.py b/src/autotester/testers/py/lib/sql_helper.py index 6adc5d0c..22774146 100644 --- a/src/autotester/testers/py/lib/sql_helper.py +++ b/src/autotester/testers/py/lib/sql_helper.py @@ -40,7 +40,8 @@ def connection(*args, **kwargs): "database": os.environ.get("PGDATABASE"), "password": os.environ.get("PGPASSWORD"), "user": os.environ.get("PGUSER"), - "host": "localhost", + "host": os.environ.get("PGHOST"), + "port": os.environ.get("PGPORT") } return _unmockable_psycopg2_connect(*args, **kwargs) @@ -99,7 +100,9 @@ def execute_psql_file( *args: str, database: Optional[str] = None, password: Optional[str] = None, - user: Optional[str] = None + user: Optional[str] = None, + host: Optional[str] = None, + port: Optional[str] = None, ) -> subprocess.CompletedProcess: """ Return a CompletedProcess object returned after calling: @@ -140,6 +143,8 @@ def execute_psql_file( "PGUSER": user or os.environ.get("PGUSER"), "PGPASSWORD": password or os.environ.get("PGPASSWORD"), "PGDATABASE": database or os.environ.get("PGDATABASE"), + "PGHOST": host or os.environ.get("PGHOST"), + "PGPORT": port or os.environ.get("PGPORT") } env = {**os.environ, **db_vars} return subprocess.run(["psql", "-f", filename] + list(args), env=env, capture_output=True) diff --git a/src/autotester/testers/py/specs/settings_schema.json b/src/autotester/testers/py/specs/settings_schema.json index 0e5a841f..b989cf54 100644 --- a/src/autotester/testers/py/specs/settings_schema.json +++ b/src/autotester/testers/py/specs/settings_schema.json @@ -21,11 +21,12 @@ "title": "Python version", "type": "string", "enum": [ + "3.9", "3.8", "3.7", "3.6" ], - "default": "3.8" + "default": "3.9" }, "pip_requirements": { "title": "Package requirements", diff --git a/src/autotester/testers/py/tests/specs.json b/src/autotester/testers/py/tests/specs.json index 8e042b85..96908f76 100644 --- a/src/autotester/testers/py/tests/specs.json +++ b/src/autotester/testers/py/tests/specs.json @@ -3,7 +3,7 @@ { "tester_type": "py", "env_data": { - "python_version": "3.8", + "python_version": "3.9", "pip_requirements": "hypothesis pytest-timeout" }, "test_data": [ diff --git a/src/autotester/testers/pyta/specs/settings_schema.json b/src/autotester/testers/pyta/specs/settings_schema.json index 1c74a922..2b3a50cc 100644 --- a/src/autotester/testers/pyta/specs/settings_schema.json +++ b/src/autotester/testers/pyta/specs/settings_schema.json @@ -21,11 +21,12 @@ "title": "Python version", "type": "string", "enum": [ + "3.9", "3.8", "3.7", "3.6" ], - "default": "3.8" + "default": "3.9" }, "pip_requirements": { "title": "Package requirements", diff --git a/src/autotester/testers/pyta/tests/specs.json b/src/autotester/testers/pyta/tests/specs.json index b77bcc6f..ad9af6c7 100644 --- a/src/autotester/testers/pyta/tests/specs.json +++ b/src/autotester/testers/pyta/tests/specs.json @@ -17,7 +17,7 @@ } ], "env_data": { - "python_version": "3.8" + "python_version": "3.9" } } ]