Re-architect backend tornado http server. Adds supporting decoupled library packages. (Large PR)

.gitignore

@@ -137,7 +137,7 @@ develop-eggs/
 downloads/
 eggs/
 .eggs/
-lib/
+/lib/
 lib64/
 parts/
 sdist/

MANIFEST.in

@@ -1 +1,2 @@
 include hydroshare_jupyter_sync/assets/*
+include hydroshare_jupyter_sync/labextension/*

hydroshare_jupyter_sync/__init__.py

@@ -1,47 +1,58 @@
-'''
-This file sets up the jupyter server extension to launch our backend
-server when jupyter is launched.
+"""
+Setup jupyterlab server and lab extensions. Add tornado HTTP handlers to jupyter session.
+"""
+import json
+from pathlib import Path
 
-Author: 2019-20 CUAHSI Olin SCOPE Team
-Vicky McDermott, Kyle Combes, Emily Lepert, and Charlie Weiss
-'''
-# !/usr/bin/python
-# -*- coding: utf-8
+# local imports
+from .config_setup import ConfigFile
+from .handlers import get_route_handlers
 
-import logging
-from hydroshare_jupyter_sync.config_reader_writer import get_config_values
-from hydroshare_jupyter_sync.index_html import (set_backend_url,
-                                                set_frontend_url)
-from .server import get_route_handlers
-from notebook.utils import url_path_join
+# Constants
+FRONTEND_PATH = "/sync"
+BACKEND_PATH = "/syncApi"
+
+MODULE_NAME = "hydroshare_jupyter_sync"
+EXTENSION_DIRNAME = "labextension"
+
+PARENT_DIR = Path(__file__).parent.resolve()
+EXTENSION_METADATA_PATH = PARENT_DIR / f"{EXTENSION_DIRNAME}/package.json"
+
+# read metadata from js extension package metadata file, `package.json`
+extension_metadata = json.loads(EXTENSION_METADATA_PATH.read_text())
+
+
+def _jupyter_labextension_paths():
+    return [{"src": EXTENSION_DIRNAME, "dest": extension_metadata["name"]}]
 
 
 def _jupyter_server_extension_paths():
-    """Creates the path to load the jupyter server extension.
-    """
-    return [{
-        "module": "hydroshare_jupyter_sync"
-    }]
+    return [{"module": MODULE_NAME}]
 
 
-def load_jupyter_server_extension(nb_server_app):
-    """Sets up logging to a specific file, sets frontend & backend urls,
-    and loads up the server extension.
+def _load_jupyter_server_extension(server_app):
+    """Registers the API handler to receive HTTP requests from the frontend extension.
+
+    Parameters
+    ----------
+    server_app: jupyterlab.labapp.LabApp
+        JupyterLab application instance
     """
-    nb_server_app.log.info("Successfully loaded hydroshare_jupyter_sync server "
-                           "extension.")
-
-    config = get_config_values(['logPath'])
-    log_file_path = None
-    if config:
-        log_file_path = config.get('logPath')
-    logging.basicConfig(level=logging.DEBUG, filename=log_file_path)
-
-    web_app = nb_server_app.web_app
-
-    frontend_base_url = url_path_join(web_app.settings['base_url'], 'sync')
-    backend_base_url = url_path_join(web_app.settings['base_url'], 'syncApi')
-    set_backend_url(backend_base_url)
-    set_frontend_url(frontend_base_url)
-    handlers = get_route_handlers(frontend_base_url, backend_base_url)
-    web_app.add_handlers('.*$', handlers)
+    handlers = get_route_handlers(FRONTEND_PATH, BACKEND_PATH)
+
+    # `cookie_secret` inherited from `server_app`
+    server_app.web_app.add_handlers(".*$", handlers)
+    server_app.log.info(f"Registered {MODULE_NAME} extension")
+
+    # parse config file. if env variables present, they take precedence.
+    # looks for config in following order:
+    # 1. "~/.config/hydroshare_jupyter_sync/config"
+    # 2. "~/.hydroshare_jupyter_sync_config"
+    config = ConfigFile()
+
+    # pass config file settings to Tornado Application (web app)
+    server_app.web_app.settings.update(config.dict())
+
+
+# For backward compatibility with the classical notebook
+load_jupyter_server_extension = _load_jupyter_server_extension

hydroshare_jupyter_sync/__main__.py

@@ -0,0 +1,72 @@
+import argparse
+import logging
+import signal
+import sys
+
+from tornado import options
+from tornado import ioloop
+from tornado.web import Application
+from pathlib import Path
+from .handlers import get_route_handlers
+from .cli import parse
+from .config_setup import ConfigFile
+
+
+class TestApp(Application):
+    """Class for setting up the server & making sure it can exit cleanly"""
+
+    is_closing = False
+
+    def signal_handler(self, signum, frame):
+        logging.info("Shutting down")
+        self.is_closing = True
+
+    def try_exit(self):
+        if self.is_closing:
+            ioloop.IOLoop.instance().stop()
+            logging.info("Exit successful")
+
+
+def get_test_app(**settings) -> Application:
+    """Thin wrapper returning web.Application instance. Written in this way for use in
+    unit tests.
+    """
+    return TestApp(
+        get_route_handlers("/", "/syncApi"),
+        cookie_secret="__TODO:_GENERATE_YOUR_OWN_RANDOM_VALUE_HERE__",
+        login_url="/syncApi/login",
+        template_path=Path(__file__).resolve().parent / "templates",
+        **settings,
+    )
+
+
+def main(parser: argparse.Namespace):
+    # clear argv. options parse command line somehow sets up logging
+    # tornados logging setup is pretty broken. Do not want to pass any command line args
+    # here
+    sys.argv = []
+    options.parse_command_line()
+    debug_enabled = not parser.no_debug
+    # TODO: write logs to file in config.log
+
+    # parse config file
+    config = (
+        ConfigFile() if parser.config is None else ConfigFile(_env_file=parser.config)
+    )
+
+    app = get_test_app(
+        default_hostname=parser.hostname, debug=debug_enabled, **config.dict()
+    )
+
+    logging.info(f"Server starting on {parser.hostname}:{parser.port}")
+    logging.info(f"Debugging mode {'enabled' if debug_enabled else 'disabled'}")
+
+    signal.signal(signal.SIGINT, app.signal_handler)
+    app.listen(parser.port)
+    ioloop.PeriodicCallback(app.try_exit, 100).start()
+    ioloop.IOLoop.instance().start()
+
+
+if __name__ == "__main__":
+    parser = parse()
+    main(parser)

hydroshare_jupyter_sync/cli.py

@@ -0,0 +1,73 @@
+import argparse
+from pathlib import Path
+from typing import Union
+from .utilities.pathlib_utils import expand_and_resolve
+
+
+def parse() -> Union[argparse.Namespace, None]:
+    parser = argparse.ArgumentParser(
+        prog="hydroshare_jupyter_sync",
+        description=(
+            """HydroShare Jupyter Sync:\n\t
+            A Jupyter server extension enabling management of HydroShare resource
+            files within Jupyter. Open HydroShare resources, work on their files,
+            and then sync those changes back to HydroShare using a drag-and-drop
+            interface.
+
+            Note:
+                Debug mode is enabled by default when starting the server via this CLI.
+            """
+        ),
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,  # This adds defaults to help page
+    )
+
+    parser.add_argument(
+        "-p",
+        "--port",
+        type=int,
+        nargs="?",
+        help="Port number to listen on",
+        default=8080,
+    )
+
+    parser.add_argument(
+        "-n",
+        "--hostname",
+        type=str,
+        nargs="?",
+        help="HTTP Server hostname",
+        default="127.0.0.1",  # localhost
+    )
+
+    parser.add_argument(
+        "-d",
+        "--no-debug",
+        action="store_true",
+        default=False,
+        help="Disable debugging mode",
+    )
+
+    parser.add_argument(
+        "-c",
+        "--config",
+        nargs="?",
+        type=absolute_file_path,
+        help="Path to configuration file. By default read from ~/.config/hydroshare_jupyter_sync/config then ~/.hydroshare_jupyter_sync_config if either exist.",
+        required=False,
+    )
+
+    return parser.parse_args()
+
+
+def is_file_and_exists(f: Union[str, Path]) -> bool:
+    """Expand and resolve path and return if it is a file."""
+    f = expand_and_resolve(f)
+    return f.is_file() and f.exists()
+
+
+def absolute_file_path(f: Union[str, Path]) -> str:
+    """Return absolute path to file, if exists."""
+    f = expand_and_resolve(f)
+    if is_file_and_exists(f):
+        return str(f)
+    raise FileNotFoundError(f"File: {f}, does not exist.")

hydroshare_jupyter_sync/config_setup.py

@@ -0,0 +1,40 @@
+from pydantic import BaseSettings, Field, root_validator
+from pathlib import Path
+from typing import Union
+from .utilities.pathlib_utils import first_existing_file, expand_and_resolve
+
+_DEFAULT_CONFIG_FILE_LOCATIONS = (
+    "~/.config/hydroshare_jupyter_sync/config",
+    "~/.hydroshare_jupyter_sync_config",
+)
+
+_DEFAULT_DATA_PATH = expand_and_resolve("~/hydroshare")
+_DEFAULT_LOG_PATH = expand_and_resolve("~/hydroshare/logs")
+
+
+class FileNotDirectoryError(Exception):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+
+
+class ConfigFile(BaseSettings):
+    # case-insensitive alias values DATA and LOG
+    data_path: Path = Field(_DEFAULT_DATA_PATH, env="data")
+    log_path: Path = Field(_DEFAULT_LOG_PATH, env="log")
+
+    class Config:
+        env_file: Union[str, None] = first_existing_file(_DEFAULT_CONFIG_FILE_LOCATIONS)
+        env_file_encoding = "utf-8"
+
+    @root_validator
+    def create_paths_if_do_not_exist(cls, values: dict):
+        for key, path in values.items():
+            path = expand_and_resolve(path)
+            if path.is_file():
+                raise FileNotDirectoryError(
+                    f"Configuration setting: {key}={str(path)} is a file not a directory."
+                )
+            elif not path.exists():
+                path.mkdir()
+
+        return values

hydroshare_jupyter_sync/fs_event_handler.py

@@ -0,0 +1,93 @@
+from watchdog.events import (
+    FileSystemEventHandler,
+    PatternMatchingEventHandler,
+    FileCreatedEvent,
+    FileModifiedEvent,
+    FileDeletedEvent,
+    FileMovedEvent,
+    FileClosedEvent,
+)
+
+from .fs_events import Events
+from .lib.filesystem.fs_resource_map import LocalFSResourceMap
+from .lib.events.event_broker import EventBroker
+
+from functools import wraps
+import logging
+
+# module level log
+logger = logging.getLogger(__name__)
+
+
+def log_event(fn):
+    @wraps(fn)
+    def wrapper(self, event):
+        logger.debug(event)
+        return fn(self, event)
+
+    return wrapper
+
+
+def fs_event_handler_factory(event_broker: EventBroker) -> FileSystemEventHandler:
+    """Wrap FSEventHandler in event_broker context. There should be only one resource per
+    FSEventHandler instance."""
+
+    class FSEventHandler(PatternMatchingEventHandler):
+        def __init__(self, local_fs_map: LocalFSResourceMap):
+            # TODO: use pattern kwarg to ignore certain files/file extensions. it would be nice if
+            # this were a configurable.
+            super().__init__(ignore_directories=True)
+
+            # dependency inject local filesystem map
+            self._res_map = local_fs_map
+
+        @log_event
+        def on_any_event(self, event):
+            # log all events
+            ...
+
+        def on_created(self, event: FileCreatedEvent) -> None:
+            # add file to local fs map
+            self._res_map.add_file(event.src_path)
+
+            # dispatch new state
+            event_broker.dispatch(Events.STATUS, self.resource_id)
+
+        def on_modified(self, event: FileModifiedEvent) -> None:
+            # update file in local fs map
+            self._res_map.update_file(event.src_path)
+
+            # dispatch new state
+            event_broker.dispatch(Events.STATUS, self.resource_id)
+
+        def on_deleted(self, event: FileDeletedEvent) -> None:
+            # remove file from local fs map
+            self._res_map.delete_file(event.src_path)
+
+            # dispatch new state
+            event_broker.dispatch(Events.STATUS, self.resource_id)
+
+        def on_moved(self, event: FileMovedEvent) -> None:
+            # update/add file in local fs map, remove file from local fs map
+            self._res_map.delete_file(event.src_path)
+
+            # NOTE: change in the future. Right now, this covers all cases.
+            self._res_map.add_file(event.dest_path)
+            self._res_map.update_file(event.dest_path)
+
+            # dispatch new state
+            event_broker.dispatch(Events.STATUS, self.resource_id)
+
+        def on_closed(self, event: FileClosedEvent) -> None:
+            # update file in local fs map
+            self._res_map.update_file(event.src_path)
+
+            # dispatch new state
+            event_broker.dispatch(Events.STATUS, self.resource_id)
+
+        # properties
+        @property
+        def resource_id(self) -> str:
+            return self._res_map.resource_id
+
+    return FSEventHandler