Add ability to maintain quotas and file logging

README.rst

@@ -44,10 +44,22 @@ The configuration file must contain the following keys:
   Allow cross-origin access to all endpoints unconditionally. This is needed
   to allow web clients to use the upload feature.
 
+Setting an upload quota
+-----------------------
+
+A quota will be enforced if it's received from the XMPP server in the PUT
+request as a `q` URL parameter.
+
+The quota is enforced on the parent directory of the UUID directory containing
+the file and metadata.
+
+If this parent directory is global to all users, then the quota is also global,
+otherwise if this parent directory is specific to each user, then the quota is
+also per user.
+
 Issues, Bugs, Limitations
 =========================
 
-* This service **does not handle any kind of quota**.
 * The format in which the files are stored is **not** compatible with ``mod_http_upload`` -- so you'll lose all uploaded files when switching.
 * This blindly trusts the clients Content-Type. I don't think this is a major issue, because we also tell the browser to blindly trust the clients MIME type. This, in addition with forcing all but a white list of MIME types to be downloaded instead of shown inline, should provide safety against any type of XSS attacks.
 * I have no idea about web security. The headers I set may be subtly wrong and circumvent all security measures I intend this to have. Please double-check for yourself and report if you find anything amiss.
@@ -87,4 +99,4 @@ Enable systemd service::
 Configure your webserver:
 
 As final step you need to point your external webserver to your xmpp-http-upload flask app.
-Check the ``contrib`` directory, there is an example for nginx there.
+Check the ``contrib`` directory, there is an example for nginx there.

xhu.py

@@ -22,10 +22,14 @@
 import contextlib
 import errno
 import fnmatch
-import json
 import hashlib
 import hmac
+import json
+import logging
+import os
 import pathlib
+import shutil
+import stat
 import typing
 
 import flask
@@ -34,6 +38,14 @@
 app.config.from_envvar("XMPP_HTTP_UPLOAD_CONFIG")
 application = app
 
+if app.config['LOGDIR']:
+    logging.basicConfig(
+        filename='{}/xhu.log'.format(app.config['LOGDIR']),
+        filemode='a',
+        level=logging.INFO
+    )
+logger = logging.getLogger(__name__)
+
 if app.config['ENABLE_CORS']:
     from flask_cors import CORS
     CORS(app)
@@ -49,7 +61,6 @@ def sanitized_join(path: str, root: pathlib.Path) -> pathlib.Path:
 def get_paths(base_path: pathlib.Path):
     data_file = pathlib.Path(str(base_path) + ".data")
     metadata_file = pathlib.Path(str(base_path) + ".meta")
-
     return data_file, metadata_file
 
 
@@ -65,12 +76,45 @@ def get_info(path: str, root: pathlib.Path) -> typing.Tuple[
         path,
         pathlib.Path(app.config["DATA_ROOT"]),
     )
-
     data_file, metadata_file = get_paths(dest_path)
-
     return data_file, load_metadata(metadata_file)
 
 
+def apply_quota(root: pathlib.Path, quota: int):
+    """ Get the files, sorted by last modification date and the sum of their
+        sizes.
+    """
+    if not quota:
+        return
+
+    file_list = []
+    total_size = 0
+    # We assume a file structure whereby files are are stored inside
+    # uuid() directories inside the root dir and that there aren't any files in
+    # the root dir itself.
+    for uuid_dir in os.listdir(root):
+        for path, dirnames, filenames in os.walk(root/uuid_dir):
+            for name in [n for n in filenames if n.endswith('.data')]:
+                fp = os.path.join(path, name)
+                size = os.path.getsize(fp)
+                total_size += size
+                modified = os.stat(fp)[stat.ST_MTIME]
+                file_list.append((modified, path, name, size))
+
+    bytes = total_size - quota
+    if (bytes > 0):
+        # Remove files (oldest first) until we're under our quota
+        file_list.sort(key=lambda a: a[0])
+        while (bytes >= 0):
+            modified, path, name, size = file_list.pop()
+            logger.info(
+                "Removing file {}/{} to maintain quota of {}"
+                .format(path, name, quota)
+            )
+            shutil.rmtree(path)
+            bytes -= size
+
+
 @contextlib.contextmanager
 def write_file(at: pathlib.Path):
     with at.open("xb") as f:
@@ -135,6 +179,11 @@ def put_file(path):
     )
 
     dest_path.parent.mkdir(parents=True, exist_ok=True, mode=0o770)
+
+    quota = flask.request.args.get("q", "")
+    if (quota):
+        apply_quota(dest_path.parent.parent, int(quota))
+
     data_file, metadata_file = get_paths(dest_path)
 
     try:
@@ -183,7 +232,8 @@ def generate_headers(response_headers, metadata_headers):
 
     response_headers["X-Content-Type-Options"] = "nosniff"
     response_headers["X-Frame-Options"] = "DENY"
-    response_headers["Content-Security-Policy"] = "default-src 'none'; frame-ancestors 'none'; sandbox"
+    response_headers["Content-Security-Policy"] = \
+        "default-src 'none'; frame-ancestors 'none'; sandbox"
 
 
 @app.route("/<path:path>", methods=["HEAD"])