[wip] Add API server and SysML v2 Kernel

.gitignore

@@ -89,6 +89,8 @@ ipython_config.py
 # OpenMDAO
 cases.sql
 n2.html
+connections.html
+openmdao_checks.out
 
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
@@ -156,3 +158,4 @@ envs/
 api/
 db/
 downloads/
+raw.json

README.md

@@ -87,6 +87,24 @@ Copy the URL where JupyterLab is running into your preferred browser, and you sh
 ## Widgets
 
 You can interact with the SysML v2 data using widgets, as illustrated below:
-![Composed Widget](https://user-images.githubusercontent.com/1438114/113528145-bb494280-958d-11eb-8d9f-5b8f7d2b1dbe.gif)
+![Composed Widget](https://user-images.githubusercontent.com/1438114/132993186-858063a7-1bb7-483b-b3be-0d61d3d27fca.gif)
 
 > If you can't see the animation clearly, click on it to see it in higher resolution.
+
+## Setup SysML Kernel
+
+If you want to experiment with the SysML v2 kernel, you can install it on the environment you want by running the following command:
+
+```bash
+anaconda-project run sysml:kernel:install
+```
+
+## Setup SysML API Kernel
+
+You can setup and run a local SysML Pilot Implementation API server by running:
+
+```bash
+anaconda-project run sysml:api:setup  # to install the service
+anaconda-project run sysml:api:start  # to start the server
+anaconda-project run sysml:api:stop   # to stop the server
+```

_scripts/_paths.py

@@ -0,0 +1,24 @@
+import sys
+from pathlib import Path
+from shutil import which
+
+ROOT = (Path(__file__) / "../..").resolve()
+
+DOWNLOADS = ROOT / "downloads"
+
+ENV_ROOT = Path(sys.executable).parent
+
+DOT_LOCATION = which("dot")
+if DOT_LOCATION:
+    DOT_LOCATION = Path(DOT_LOCATION)
+
+POSTGRES_USER = "postgres"
+POSTGRES_PASSWORD = "mysecretpassword"
+POSTGRES_DB = "sysml2"
+
+PSQL_DATA = ROOT / "db"
+PSQL_LOGS = PSQL_DATA / "logs"
+PSQL_PWFILE = ROOT / "_pwfile"
+
+API = ROOT / "api"
+SBT_BIN = API / "sbt/bin"

_scripts/_variables.py

@@ -0,0 +1,28 @@
+import platform
+from os import environ as ENV_VARS
+
+OK = 0
+ERR = 1
+
+# system
+PLATFORM = platform.system()
+WIN = PLATFORM == "Windows"
+OSX = PLATFORM == "Darwin"
+CI = ENV_VARS.get("JENKINS_HOME")
+
+# installer versions
+SBT_VERSION = ENV_VARS.get("SBT_VERSION", "1.2.8")
+SYSML2_API_RELEASE = ENV_VARS.get("SYSML2_API_RELEASE", "2021-06")
+SYSML2_RELEASE = ENV_VARS.get("SYSML2_RELEASE", "2021-06.1")
+
+# API Configuration
+API_PORT = ENV_VARS.get("API_PORT", 9000)
+LOCAL_API_SERVER_URL = ENV_VARS.get("LOCAL_API_SERVER_URL", "http://localhost")
+REMOTE_API_SERVER_URL = ENV_VARS.get("REMOTE_API_SERVER_URL", "http://sysml2.intercax.com")
+SBT_GITHUB = "https://github.com/sbt/sbt"
+SYSML_RELEASE_GITHUB = "https://github.com/Systems-Modeling/SysML-v2-Release"
+SYSML_API_GITHUB = "https://github.com/Systems-Modeling/SysML-v2-API-Services"
+
+PSQL_DBNAME = ENV_VARS.get("PSQL_DBNAME", "sysml2")
+PSQL_USERNAME = ENV_VARS.get("PSQL_USERNAME", "postgres")
+PSQL_PASSWORD = ENV_VARS.get("PSQL_PASSWORD", "pUtY0uR$eCr3tP@$sW0rDh3R3")

_scripts/api_server.py

@@ -0,0 +1,282 @@
+import re
+import sys
+import traceback
+from argparse import ArgumentParser
+from dataclasses import dataclass
+from pathlib import Path
+from zipfile import ZipFile
+
+from . import _paths as P
+from . import _variables as V
+from .utils import COLOR as C
+from .utils import _check_output, _run, download_file
+
+API_ZIP_FILE = P.DOWNLOADS / "api_server.zip"
+API_DOWNLOAD_URL = f"{V.SYSML_API_GITHUB}/archive/refs/tags/{V.SYSML2_API_RELEASE}.zip"
+
+SBT_ZIP_FILE = P.DOWNLOADS / "sbt.zip"
+SBT_DOWNLOAD_URL = f"{V.SBT_GITHUB}/releases/download/v{V.SBT_VERSION}/sbt-{V.SBT_VERSION}.zip"
+
+# Executables
+PG_CTL = "pg_ctl"
+PSQL = "psql"
+SBT = P.SBT_BIN / ("sbt" + (".bat" if V.WIN else ""))
+
+PARSER = ArgumentParser()
+
+PARSER.add_argument(
+    "--host",
+    default="127.0.0.1",
+    type=str,
+    help="Hostname on which to run PosgreSQL (default='127.0.0.1')",
+)
+PARSER.add_argument(
+    "--port",
+    default=5432,
+    type=int,
+    help="Port on which to run PosgreSQL (default=5432)",
+)
+PARSER.add_argument("--setup", action="store_true")
+PARSER.add_argument("--silent", action="store_true")
+PARSER.add_argument("--start", action="store_true")
+PARSER.add_argument("--stop", action="store_true")
+PARSER.add_argument("--tear-down", action="store_true")
+PARSER.add_argument("--no-autostart", action="store_true")
+
+
+# Commands:
+# 1. Create User `postgres`
+# 2. Create database 'sysml2'
+# 3.
+
+
+def setup_api(force=False, skip_clean=True):
+    """Setup the SysML v2 Pilot API and the Scala Build Tools (SBT)"""
+    if not P.API.exists() or not list(P.API.iterdir()):
+        if force or not API_ZIP_FILE.exists():
+            if (
+                download_file(
+                    url=API_DOWNLOAD_URL,
+                    filename=API_ZIP_FILE,
+                )
+                != 0
+            ):
+                print(
+                    f"{C.FAIL} Failed to download the SysML v2 Pilot API!"
+                    f"  (url={API_DOWNLOAD_URL} {C.ENDC}"
+                )
+        # unzip the contents and remove the zip file
+        try:
+            with ZipFile(API_ZIP_FILE) as zip_file:
+                zip_file.extractall(path=P.API)
+            if not skip_clean:
+                API_ZIP_FILE.unlink()
+        except:  # pylint: disable=bare-except; # noqa: 722
+            print(f"{C.FAIL} Could not unzip file '{API_ZIP_FILE}' to '{P.API}' {C.ENDC}")
+            print(f"{C.FAIL} {traceback.format_exc()} {C.ENDC}")
+
+    if not P.SBT_BIN.exists():
+        # download and extract file
+        if force or not SBT_ZIP_FILE.exists():
+            if (
+                download_file(
+                    url=SBT_DOWNLOAD_URL,
+                    filename=SBT_ZIP_FILE,
+                )
+                != 0
+            ):
+                print(f"{C.FAIL} Failed to download SBT!  (url={SBT_DOWNLOAD_URL} {C.ENDC}")
+        # unzip the contents and remove the zip file
+        try:
+            with ZipFile(SBT_ZIP_FILE) as zip_file:
+                zip_file.extractall(path=P.API)
+            if not skip_clean:
+                SBT_ZIP_FILE.unlink()
+        except:  # pylint: disable=bare-except; # noqa: 722
+            print(f"{C.FAIL} Could not unzip file '{SBT_ZIP_FILE}' to '{P.API}' {C.ENDC}")
+            print(f"{C.FAIL} {traceback.format_exc()} {C.ENDC}")
+
+
+def tear_down():
+    try:
+        P.PSQL_DATA.unlink(missing_ok=True)
+    except:  # pylint: disable=bare-except; # noqa: 722
+        print(f"{C.FAIL} Failed to delete the PostgreSQL data folder {C.ENDC}")
+        print(f"{C.FAIL} {traceback.format_exc()} {C.ENDC}")
+    try:
+        P.API.unlink(missing_ok=True)
+    except:  # pylint: disable=bare-except; # noqa: 722
+        print(f"{C.FAIL} Failed to delete the API folder {C.ENDC}")
+        print(f"{C.FAIL} {traceback.format_exc()} {C.ENDC}")
+
+
+@dataclass
+class PostgreSQLProcess:  # pylint: disable=too-many-instance-attributes
+    """
+    A context manager for a psql process.
+
+    IMPORTANT: This configuration is not intended to be used in production!
+
+    """
+
+    pid_regex = re.compile(r"\(PID: ?(\d+)\)")
+
+    host: str = "127.0.0.1"
+    port: int = 5432
+    autostart: bool = True
+    silent: bool = False
+
+    _pid: int = None
+
+    _dbname: str = V.PSQL_DBNAME
+    _datafile: Path = P.PSQL_DATA.resolve().absolute()
+    _logsfile: Path = P.PSQL_LOGS.resolve().absolute()
+    _pwfile: Path = P.PSQL_PWFILE.resolve().absolute()
+    _username: str = V.PSQL_USERNAME
+
+    def __post_init__(self):
+        if "0.0.0.0" in self.host:
+            print(
+                f"{C.WARNING} You are binding to PostgreSQL to 0.0.0.0, "
+                "this exposes serious risks! {C.ENDC}"
+            )
+
+        if not self._datafile.exists() or not list(self._datafile.iterdir()):
+            self.initialize_db()
+
+        if not self._logsfile.exists():
+            self._logsfile.parent.mkdir(parents=True, exist_ok=True)
+
+        if self.autostart:
+            self.start_proc()
+
+    def clear(self):
+        self.server("stop")
+        self._datafile.unlink()
+
+    def write_pwfile(self):
+        """
+        Write password to pwfile
+
+        IMPORTANT: This is NOT secure!!!  It should NOT be used in production!
+        """
+        if self._pwfile.exists():
+            self._pwfile.unlink()
+        self._pwfile.write_text(f"{V.PSQL_PASSWORD}\n")
+
+    def initialize_db(self):
+        P.PSQL_DATA.mkdir(parents=True, exist_ok=True)
+
+        # Initialize the database
+        silent_args = ["--silent"] if self.silent else []
+        result_code = _run(
+            [
+                PG_CTL,
+                "initdb",
+                f"--pgdata={self._datafile}",
+                f"--pwfile={self._pwfile}",
+                f"--username={self._username}",
+            ]
+            + silent_args,
+            cwd=P.ROOT,
+            wait=True,
+        )
+        if result_code != 0:
+            print(f"{C.FAIL} Database initialization (initdb) returned a non-zero value {C.ENDC}")
+
+    def create_db(self):
+        # Create the sysml database
+        is_running = self.is_running()
+        if not is_running:
+            self.server("start")
+
+        result_code = _run(
+            [
+                "createdb",
+                f"--owner={self._username}",
+                f"--host={self.host}",
+                f"--port={self.port}",
+                "--no-password",
+                self._dbname,
+            ],
+            cwd=self._datafile,
+            wait=True,
+        )
+
+        if result_code != 0:
+            print(f"{C.FAIL} Database creation (createdb) returned a non-zero value {C.ENDC}")
+        if not is_running:
+            self.server("stop")
+
+    def server(self, command: str, wait=False):
+        func = _run if wait else _check_output
+        print(f"{C.OKBLUE} running '{command}' server command {C.ENDC}")
+
+        os_dep_args = ["-U", V.PSQL_USERNAME] if V.WIN else []
+        return func(
+            [
+                PG_CTL,
+                f"--pgdata={self._datafile}",
+                f"--log={self._logsfile}",
+                command,
+            ]
+            + os_dep_args,
+            cwd=P.ROOT,
+            wait=wait,
+        )
+
+    @property
+    def server_status(self):
+        return self.server("status", wait=True)  # .decode()
+
+    def start_proc(self):
+        if self.is_running():
+            print(f"{C.WARNING} PostgreSQL is already running {C.ENDC}")
+            return
+
+        self.server("start")
+
+        if not self.is_running():
+            print(f"{C.FAIL} Could not start PostgreSQL {C.ENDC}")
+        else:
+            print(f"{C.OKGREEN} Serving PostgreSQL on {self.host}:{self.port} {C.ENDC}")
+
+    def is_running(self):
+        pids = self.pid_regex.findall(self.server_status)
+        self._pid = pid = int(pids[0]) if pids else None
+        return pid
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        print(f"{C.OKBLUE} Shutting down psql on {self.host}:{self.port} {C.ENDC}")
+        self.server("stop")
+        print(f"{C.OKGREEN} Finished shutting down psql on {self.host}:{self.port} {C.ENDC}")
+
+
+def setup():
+    pass
+
+
+if __name__ == "__main__":
+    args, extra_args = PARSER.parse_known_args(sys.argv[1:])
+
+    result_codes = []
+    if args.tear_down:
+        result_codes.append(tear_down())
+
+    if args.setup:
+        result_codes.append(setup())
+
+    with PostgreSQLProcess(
+        host=args.host,
+        port=args.port,
+        autostart=not args.no_autostart,
+    ) as psql_proc:
+        if args.start:
+            result_codes.append(
+                # launch(env=env, port=args.mbee_port)
+            )
+
+    sys.exit(max(result_codes))