Merge pull request #387 from mishaschwartz/v2.2.0

v2.2.0

.flake8

@@ -0,0 +1,16 @@
+[flake8]
+max-line-length = 120
+ignore = E266, BLK100
+# flake8 does not parse the .gitignore file so we need to re-specify what to ignore
+exclude =
+    .git,
+    .idea,
+    __pycache__.
+    .DS_Store,
+    .hypothesis,
+    .pytest_cache,
+    *.egg-info,
+    .eggs,
+    venv,
+    venv2,
+    build

.git-blame-ignore-revs

@@ -0,0 +1,2 @@
+# Initial pre-commit hooks: flake8 and black
+3ed4c4c62ae4785cb3d38db49a223dec1f1fa41f

.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: "/client"
+    schedule:
+      interval: "monthly"
+  - package-ecosystem: pip
+    directory: "/server"
+    schedule:
+      interval: "monthly"

.github/workflows/test_ci.yml

@@ -5,14 +5,14 @@ on:
 jobs:
   test:
     if: github.event.pull_request.draft == false
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         python-version:
-          - 3.6
           - 3.7
           - 3.8
           - 3.9
+          - '3.10'
         test-dir:
           - client
           - server

.pre-commit-config.yaml

@@ -0,0 +1,9 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: 22.6.0
+    hooks:
+    - id: black
+  - repo: https://github.com/pycqa/flake8
+    rev: 4.0.1
+    hooks:
+    - id: flake8

Changelog.md

@@ -1,6 +1,14 @@
 # CHANGELOG
 All notable changes to this project will be documented here.
 
+## [v2.2.0]
+- Support dependencies on specific package versions and non-CRAN sources for R tester (#323)
+- Testers no longer generate automated content for feedback files (#375)
+- Multiple feedback files can now be created per test run (#375)
+- PyTA plaintext reports are sent as part of the test result, not as a feedback file (#375)
+- Allow client to define environment variables to pass to individual test runs (#370)
+- Add ability to clean up test scripts that haven't been used for X days (#379)
+
 ## [v2.1.2]
 - Support dependencies on specific package versions and non-CRAN sources for R tester (#323) 
 

README.md

@@ -1,5 +1,3 @@
-[![Acceptance tests](https://layerci.com/badge/github/MarkUsProject/markus-autotesting)](https://layerci.com/jobs/github/MarkUsProject/markus-autotesting)
-
 Autotesting
 ===========
 
@@ -20,7 +18,7 @@ Both the autotester and the API are designed to be run on Ubuntu 20.04 (or suffi
 
 #### Installing up the autotester
 
-1. Make sure that your system has python3 installed (at least version 3.6, but we recommend the latest version if 
+1. Make sure that your system has python3 installed (at least version 3.7, but we recommend the latest version if 
    possible).
 2. Create or assign one user to run the autotester. For example, you may create a user named `autotest` for this purpose.
 3. Create or assign at least one user to run test code. For example, you may create 3 users named `test1`, `test2`, `test3`
@@ -68,7 +66,7 @@ Both the autotester and the API are designed to be run on Ubuntu 20.04 (or suffi
 6. [Configure the autotester](#autotester-configuration-options)
 7. Optionally install additional python versions.
    
-   The `py` (python3) and `pyta` testers can be run using any version of python between versions 3.6 and 3.10. When
+   The `py` (python3) and `pyta` testers can be run using any version of python between versions 3.7 and 3.10. When
    these testers are installed the autotester will search the PATH for available python executables. If you want users
    to be able to run tests with a specific python version, ensure that it is visible in the PATH of both the user running
    the autotester and all users who run tests.
@@ -102,7 +100,7 @@ Both the autotester and the API are designed to be run on Ubuntu 20.04 (or suffi
 
 #### Installing the API
 
-1. Make sure that your system has python3 installed (at least version 3.6, but we recommend the latest version if 
+1. Make sure that your system has python3 installed (at least version 3.7, but we recommend the latest version if 
    possible).
 2. Install the python requirements:
 
@@ -293,3 +291,21 @@ where:
         - if `0 < points_earned < points_total` then `status == "partial"`
         - `status == "error"` if some error occurred that meant the number of points for this test could not be determined
 - `time` is optional (can be null) and is the amount of time it took to run the test (in ms)
+
+## Managing files on disk
+
+Test settings files and virtual environments are created in the `workspace` directory. These files can build up over
+time and are not automatically cleaned up. In order to safely clean up these files, you can use the `clean` command 
+with the `start_stop.sh` script. By calling this script with the optional `--age` argument, settings files and virtual
+environments older than X days will be deleted safely. For example
+
+```shell
+autotest:/$ python3 markus-autotesting/server/start_stop.py clean --age 30
+```
+
+will delete settings that have not been accessed (updated or used to run a test) in the last 30 days.
+
+To see which settings *would be* deleted without actually deleting them, use the optional `--dry-run` flag as well.
+
+Users who try to run tests after the settings have been cleaned up in this manner will see an error message telling them
+that the test settings have expired and prompting them to upload more.

client/.dockerfiles/Dockerfile

@@ -7,6 +7,7 @@ RUN apt-get update -y && apt-get install -y python3 python3-venv
 COPY ./requirements.txt /requirements.txt
 
 RUN python3 -m venv /markus_venv && \
+    /markus_venv/bin/pip install wheel && \
     /markus_venv/bin/pip install -r /requirements.txt
 
 WORKDIR /app

client/.env

@@ -3,4 +3,4 @@ FLASK_HOST=localhost
 FLASK_PORT=5000
 ACCESS_LOG=
 ERROR_LOG=
-SETTINGS_JOB_TIMEOUT=60
+SETTINGS_JOB_TIMEOUT=600

client/autotest_client/__init__.py

@@ -20,7 +20,7 @@
 
 ERROR_LOG = os.environ.get("ERROR_LOG")
 ACCESS_LOG = os.environ.get("ACCESS_LOG")
-SETTINGS_JOB_TIMEOUT = os.environ.get("SETTINGS_JOB_TIMEOUT", 60)
+SETTINGS_JOB_TIMEOUT = os.environ.get("SETTINGS_JOB_TIMEOUT", 600)
 REDIS_URL = os.environ["REDIS_URL"]
 
 app = Flask(__name__)
@@ -60,19 +60,21 @@ def _handle_error(e):
         code = e.code
     with _open_log(ERROR_LOG, fallback=sys.stderr) as f:
         try:
-            user = _authorize_user()
+            api_key = request.headers.get("Api-Key")
         except Exception:
-            user = "ERROR: user not found"
-        f.write(f"{datetime.now()}\n\tuser: {user}\n\t{traceback.format_exc()}\n")
+            api_key = "ERROR: user not found"
+        f.write(f"{datetime.now()}\n\tuser: {api_key}\n\t{traceback.format_exc()}\n")
         f.flush()
+    if not app.debug:
+        error = str(e).replace(api_key, "[client-api-key]")
     return jsonify(message=error), code
 
 
-def _check_rate_limit(user_name):
+def _check_rate_limit(api_key):
     conn = _redis_connection()
-    key = f"autotest:ratelimit:{user_name}:{datetime.now().minute}"
+    key = f"autotest:ratelimit:{api_key}:{datetime.now().minute}"
     n_requests = conn.get(key) or 0
-    user_limit = conn.get(f"autotest:ratelimit:{user_name}:limit") or 20  # TODO: make default limit configurable
+    user_limit = conn.get(f"autotest:ratelimit:{api_key}:limit") or 20  # TODO: make default limit configurable
     if int(n_requests) > int(user_limit):
         abort(make_response(jsonify(message="Too many requests"), 429))
     else:
@@ -87,7 +89,7 @@ def _authorize_user():
     user_name = (_redis_connection().hgetall("autotest:user_credentials") or {}).get(api_key)
     if user_name is None:
         abort(make_response(jsonify(message="Unauthorized"), 401))
-    _check_rate_limit(user_name)
+    _check_rate_limit(api_key)
     return api_key
 
 
@@ -228,24 +230,33 @@ def update_settings(settings_id, user):
 @app.route("/settings/<settings_id>/test", methods=["PUT"])
 @authorize
 def run_tests(settings_id, user):
-    file_urls = request.json["file_urls"]
+    test_data = request.json["test_data"]
     categories = request.json["categories"]
     high_priority = request.json.get("request_high_priority")
-    queue_name = "batch" if len(file_urls) > 1 else ("high" if high_priority else "low")
+    queue_name = "batch" if len(test_data) > 1 else ("high" if high_priority else "low")
     queue = rq.Queue(queue_name, connection=_rq_connection())
 
     timeout = 0
 
     for settings_ in settings(settings_id)["testers"]:
-        for test_data in settings_["test_data"]:
-            timeout += test_data["timeout"]
+        for data in settings_["test_data"]:
+            timeout += data["timeout"]
 
     ids = []
-    for url in file_urls:
+    for data in test_data:
+        url = data["file_url"]
+        test_env_vars = data.get("env_vars", {})
         id_ = _redis_connection().incr("autotest:tests_id")
         _redis_connection().hset("autotest:tests", key=id_, value=settings_id)
         ids.append(id_)
-        data = {"settings_id": settings_id, "test_id": id_, "files_url": url, "categories": categories, "user": user}
+        data = {
+            "settings_id": settings_id,
+            "test_id": id_,
+            "files_url": url,
+            "categories": categories,
+            "user": user,
+            "test_env_vars": test_env_vars,
+        }
         queue.enqueue_call(
             "autotest_server.run_test",
             kwargs=data,
@@ -285,9 +296,7 @@ def get_feedback_file(settings_id, tests_id, feedback_id, **_kw):
     if data is None:
         abort(make_response(jsonify(message="File doesn't exist"), 404))
     _redis_connection().delete(key)
-    return send_file(
-        io.BytesIO(data), mimetype="application/gzip", as_attachment=True, attachment_filename=str(feedback_id)
-    )
+    return send_file(io.BytesIO(data), mimetype="application/gzip", as_attachment=True, download_name=str(feedback_id))
 
 
 @app.route("/settings/<settings_id>/tests/status", methods=["GET"])

client/autotest_client/tests/test_flask_app.py

@@ -28,10 +28,9 @@ def fake_redis_db(monkeypatch, fake_redis_conn):
 
 
 class TestRegister:
-
     @pytest.fixture
     def credentials(self):
-        return {'auth_type': 'test', 'credentials': '12345'}
+        return {"auth_type": "test", "credentials": "12345"}
 
     @pytest.fixture
     def response(self, client, credentials):
@@ -41,7 +40,7 @@ def test_status(self, response):
         assert response.status_code == 200
 
     def test_api_key_set(self, response):
-        assert response.json['api_key']
+        assert response.json["api_key"]
 
     def test_credentials_set(self, response, fake_redis_conn, credentials):
-        assert json.loads(fake_redis_conn.hget('autotest:user_credentials', response.json['api_key'])) == credentials
+        assert json.loads(fake_redis_conn.hget("autotest:user_credentials", response.json["api_key"])) == credentials

client/requirements.txt

@@ -1,6 +1,5 @@
-flask==1.1.4
-markupsafe==2.0.1
-python-dotenv==0.15.0
-rq==1.7.0
-redis==3.5.3
-jsonschema==3.2.0
+flask==2.1.2
+python-dotenv==0.20.0
+rq==1.10.1
+redis==4.3.4
+jsonschema==4.7.2

docker-compose.yml

@@ -6,10 +6,10 @@ services:
       context: ./server
       dockerfile: ./.dockerfiles/Dockerfile
       args:
-        UBUNTU_VERSION: '18.04'
+        UBUNTU_VERSION: '20.04'
         LOGIN_USER: 'docker'
         WORKSPACE: '/home/docker/.autotesting'
-    image: markus-autotest-server-dev:1.0.1
+    image: markus-autotest-server-dev:1.1.0
     volumes:
       - ./server:/app:cached
       - workspace:/home/docker/.autotesting:rw
@@ -27,8 +27,8 @@ services:
       context: ./client
       dockerfile: ./.dockerfiles/Dockerfile
       args:
-        UBUNTU_VERSION: '18.04'
-    image: markus-autotest-client-dev:1.0.0
+        UBUNTU_VERSION: '20.04'
+    image: markus-autotest-client-dev:1.1.0
     container_name: 'autotest-client'
     volumes:
       - ./client:/app:cached
@@ -64,7 +64,7 @@ services:
   rq_dashboard:
     image: eoranged/rq-dashboard
     environment:
-      - RQ_DASHBOARD_REDIS_URL=redis://redis_autotest:6379/
+      - RQ_DASHBOARD_REDIS_URL=redis://redis:6379/
     ports:
       - '9181:9181'
     depends_on:

pyproject.toml

@@ -0,0 +1,4 @@
+[tool.black]
+line-length = 120
+include = '\.pyi?$'
+extend-exclude = '\.git'

server/.dockerfiles/Dockerfile

@@ -10,14 +10,14 @@ ARG WORKSPACE
 RUN apt-get update -y && \
     apt-get -y install software-properties-common && \
     add-apt-repository -y ppa:deadsnakes/ppa && \
-    apt-get -y install python3.6 \
-                       python3.6-venv \
-                       python3.7 \
+    apt-get -y install python3.7 \
                        python3.7-venv \
                        python3.8 \
                        python3.8-venv \
                        python3.9 \
                        python3.9-venv \
+                       python3.10 \
+                       python3.10-venv \
                        redis-server \
                        postgresql-client \
                        libpq-dev \
@@ -34,7 +34,8 @@ RUN useradd -ms /bin/bash $LOGIN_USER && \
 
 COPY . /app
 
-RUN python3.9 -m venv /markus_venv && \
+RUN python3.10 -m venv /markus_venv && \
+    /markus_venv/bin/pip install wheel && \
     /markus_venv/bin/pip install -r /app/requirements.txt && \
     find /app/autotest_server/testers -name requirements.system -exec {} \; && \
     rm -rf /app/*