Merge remote-tracking branch 'origin/master' into musicode

.readthedocs.yaml

@@ -1,9 +1,13 @@
 # https://docs.readthedocs.io/en/stable/config-file/v2.html
+# https://blog.readthedocs.com/migrate-configuration-v2/
 version: 2
 build:
   os: ubuntu-22.04
   tools:
     python: "3.11"
+sphinx:
+  configuration: doc/source/conf.py
 python:
-   install:
-   - requirements: requirements.txt
+  install:
+    - requirements: requirements-mupif.txt
+    - requirements: requirements.txt

doc/source/conf.py

@@ -17,7 +17,7 @@
 
 # -- Project information -----------------------------------------------------
 
-project = 'MuPIF'
+project = 'MuPIF services' 
 copyright = '2023, Bořek Patzák, Václav Šmilauer'
 author = 'Bořek Patzák, Václav Šmilauer'
 # (Czech Technical University, Faculty of Civil Engineering, Department of Mechanics, Thákurova 7, 16629, Prague, Czech Republic.)'
@@ -34,6 +34,9 @@
 extensions = [
     # 'sphinx.ext.autodoc',
     # 'sphinxcontrib.apidoc'
+    'sphinxcontrib.openapi',
+    'myst_nb',
+    'sphinx_rtd_theme',
 ]
 
 import sys, os.path
@@ -46,6 +49,14 @@
 #apidoc_excluded_paths=[]
 #apidoc_module_first=True
 
+source_suffix={
+    '.rst':'restructuredtext',
+    '.ipynb':'myst-nb',
+}
+# don't run notebooks at readthedocs, without REST API server
+# just put it inline as it is
+nb_execution_mode='off'
+
 
 sys.path.append(thisDir+'/../..')
 import mupif
@@ -72,17 +83,7 @@
 
 # -- Options for HTML output -------------------------------------------------
 
-# https://readthedocsorg.readthedocs.io/en/latest/theme.html#how-do-i-use-this-locally-and-on-read-the-docs
-
-# use ReadTheDocs theme both locally and when building docs at readthedocs.io
-import os
-on_rtd=os.environ.get('READTHEDOCS',None)=='True'
-if not on_rtd:
-    import sphinx_rtd_theme
-    html_theme='sphinx_rtd_theme'
-    html_theme_path=[sphinx_rtd_theme.get_html_theme_path()]
-else:
-    html_theme='default'
+html_theme='sphinx_rtd_theme'
 
 #html_theme_options=dict(
 #    github_banner=True,
@@ -103,3 +104,28 @@
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ['_static']
+
+# for readthedocs.org, don't try to connect to the DB when importing mupifDB.api.main
+import os
+os.environ['MUPIFDB_DRY_RUN']='1'
+
+# Generate OpenAPI json description prior to running sphinx
+#
+# https://github.com/tiangolo/fastapi/issues/1173
+import fastapi.openapi.utils
+import mupifDB.api.main
+import json
+with open('mupifdb-rest-api.openapi.json', 'w') as f:
+    app=mupifDB.api.main.app
+    json.dump(fastapi.openapi.utils.get_openapi(
+        title='MupifDB REST API',
+        version=app.version,
+        openapi_version=app.openapi_version,
+        description=app.description,
+        routes=app.routes,
+        # openapi_prefix=app.openapi_prefix,
+    ),f)
+
+# copy files from outside of the doc subdirectory here so that they can be included in the docs
+import shutil
+shutil.copyfile('../../mupifDB/api/edm/jupyter/04-dms-data.ipynb','04-dms-data.ipynb')

doc/source/index.rst

@@ -16,6 +16,7 @@ This is a preliminary MupifDB documentation.
    :numbered:
 
    infrastructure.rst
+   rest-api.rst
 
 
 .. .. toctree::

doc/source/infrastructure.rst

@@ -23,9 +23,6 @@ Wireguard VPN
 ==============
 
 
-REST API
-=========
-
 Database
 ---------
 

doc/source/rest-api.rst

@@ -0,0 +1,22 @@
+REST API
+##########
+
+MupifDB
+========
+
+.. openapi:: mupifdb-rest-api.openapi.json
+   :exclude: /EDM/
+
+Entity Data Model (EDM)
+========================
+
+.. openapi:: mupifdb-rest-api.openapi.json
+   :include: /EDM/
+
+
+Demonstration
+---------------
+
+.. toctree::
+
+   04-dms-data.ipynb

mupifDB/api/main.py

@@ -73,7 +73,10 @@
 # import and initialize EDM
 if 1:
     import mupifDB.api.edm as edm
-    edm.initializeEdm(client)
+    # when imported at readthedocs, don't try to connect to the DB (no DB running there)
+    if not 'MUPIFDB_DRY_RUN' in os.environ:
+        edm.initializeEdm(client)
+    else: print('MUPIFDB_DRY_RUN defined, not initializing EDM DB connection.')
     app.include_router(edm.dms3.router)
 
 

mupifDB/api/safeapi.py

@@ -0,0 +1,63 @@
+import fastapi
+import pymongo
+import Pyro5.api
+import mupif as mp
+import os
+
+app = fastapi.FastAPI(openapi_tags=[{'name':'Stats'}])
+
+mongoClient = pymongo.MongoClient("mongodb://localhost:27017")
+
+
+
+@app.get("/status2/", tags=["Stats"])
+def get_status2():
+    ns = None
+    try:
+        ns = mp.pyroutil.connectNameserver();
+        nameserverStatus = 'OK'
+    except:
+        nameserverStatus = 'Failed'
+
+    # get Scheduler status
+    schedulerStatus = 'Failed'
+    query = ns.yplookup(meta_any={"type:scheduler"})
+    try:
+        for name, (uri, metadata) in query.items():
+            s = Pyro5.api.Proxy(uri)
+            st = s.getStatistics()
+            schedulerStatus = 'OK'
+    except Exception as e:
+        print(str(e))
+
+    # get DMS status
+    if mongoClient: DMSStatus = 'OK'
+    else: DMSStatus = 'Failed'
+
+    return {'nameserver': nameserverStatus, 'dms': DMSStatus, 'scheduler': schedulerStatus, 'name':os.environ["MUPIF_VPN_NAME"]}
+
+
+@app.get("/scheduler-status2/", tags=["Stats"])
+def get_scheduler_status2():
+    ns = mp.pyroutil.connectNameserver();
+    return mp.monitor.schedulerInfo(ns)
+
+@app.get("/ns-status2/", tags=["Stats"])
+def get_ns_status2():
+    ns = mp.pyroutil.connectNameserver();
+    return mp.monitor.nsInfo(ns)
+
+@app.get("/vpn-status2/", tags=["Stats"])
+def get_vpn_status2():
+    return mp.monitor.vpnInfo(hidePriv=False)
+
+@app.get("/jobmans-status2/", tags=["Stats"])
+def get_jobmans_status2():
+    ns = mp.pyroutil.connectNameserver();
+    return mp.monitor.jobmanInfo(ns)
+
+
+if __name__ == '__main__':
+    import uvicorn
+    uvicorn.run('safeapi:app', host='0.0.0.0', port=8081, reload=True)
+

mupifDB/restApiControl.py

@@ -87,7 +87,7 @@ def getAuthToken():
     global bearer_token_expires_at
     time_now = datetime.datetime.now()
     time_secs = time_now.timestamp()
-    if time_secs > bearer_token_expires_at - 3:
+    if time_secs > bearer_token_expires_at - 10:
         URL = 'https://auth.musicode.cloud/realms/musicode/protocol/openid-connect/token'
         CLIENT_ID = granta_credentials['username']
         CLIENT_SECRET = granta_credentials['password']
@@ -280,30 +280,33 @@ def ObjIDIsIterable(val):
     token = getAuthToken()
     headers = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8', 'Authorization': f'Bearer {token["access_token"]}'}
     r = rGet(url=url, headers=headers)
+
     for inp in r.json():
         if name == inp['name']:
             if inp['type'] == 'float':
                 # fint units first :(
                 execution_record = getExecutionRecord(eid)
                 w_inputs = _getGrantaWorkflowMetadataFromDatabase(execution_record['WorkflowID']).get('Inputs', [])
                 units = ''
+                data_id = 'Unknown'
                 for w_i in w_inputs:
                     if w_i['Name'] == name:
                         units = w_i['Units']
+                        data_id = w_i['Type_ID']
                 return {
                     'Compulsory': True,
                     'Description': '',
                     'Name': inp['name'],
                     'ObjID': inp['name'],
                     'Type': 'mupif.Property',
-                    'TypeID': w_i['Type_ID'],
+                    'TypeID': data_id,
                     'Units': units,  # todo
                     'ValueType': 'Scalar',
                     'Link': {},
                     'Object': {
                         'ClassName': 'ConstantProperty',
                         'ValueType': 'Scalar',
-                        'DataID': w_i['Type_ID'].replace('mupif.DataID.', ''),
+                        'DataID': data_id.replace('mupif.DataID.', ''),
                         'Unit': units,  # todo
                         'Value': inp['value'],
                         'Time': None

mupifDB/workflowmanager.py

@@ -1047,6 +1047,15 @@ def _getGrantaOutput(app, eid, name, obj_id, data_id, time, object_type):
                     "type": "hyperlink"
                 }
 
+    elif object_type == 'mupif.PiecewiseLinFunction':
+        obj = app.get(mupif.DataID[data_id], time, obj_id)
+        return {
+            "name": str(name),
+            "value": {"x": obj.x, "y": obj.y},
+            "type": "series",
+            "unit": str(obj.y_unit)
+        }
+
     return None
 
 

requirements-mupif.txt

@@ -0,0 +1 @@
+mupif @ git+https://github.com/mupif/mupif@master

requirements.txt

@@ -1,6 +1,6 @@
 pygal
 python-pidfile
-git+https://github.com/mupif/mupif/@master#egg=mupif
+mupif
 pymongo
 flask_pymongo
 flask_cors
@@ -9,6 +9,9 @@ isodate
 # flask 2.3 does not provide flask.json.JSONEncoder anymore
 # (can be replaced by vanilla json encoder, but our code is not updated (yet?)
 flask==2.2.3
+# https://stackoverflow.com/a/77217971
+# current version 3.0 does not work with (older) Flask
+Werkzeug==2.2.2
 psutil
 fastapi
 uvicorn[standard]
@@ -28,3 +31,6 @@ rotate-backups
 pydantic==1.10.0
 oauthlib
 requests_oauthlib
+sphinxcontrib-openapi
+sphinx-rtd-theme
+myst-nb