Pydantic

.github/workflows/tests.yml

@@ -13,7 +13,7 @@ jobs:
       fail-fast: false
       matrix:
         # needs quotes so that float 3.10 does not get interpreted as 3.1
-        python-version: [ '3.10' ]
+        python-version: [ '3.10', '3.11', '3.12', '3.13' ]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
@@ -42,3 +42,13 @@ jobs:
       timeout-minutes: 2
       run: |
         python -m pytest .
+    - name: webapi test
+      timeout-minutes: 2
+      run: |
+        make & # runs all services, a separate instance
+        sleep 10
+        # cd as webtest contains relative paths to uploaded files
+        # run twice to also test workflow version bumping
+        cd tools && bash webtest.sh && bash webtest.sh
+        #
+        kill $! # terminate make

.gitignore

@@ -1,4 +1,4 @@
 doc/build
 __pycache__
 .ipynb_checkpoints
-
+.pytest_cache

Makefile

@@ -1,13 +1,24 @@
-MAKEFLAGS+=-j3
-TASKS=rest web browse
+MAKEFLAGS+=-j6
+TASKS=ns mongo rest web browse scheduler
 
 .PHONY: $(TASKS)
 run: $(TASKS)
 
 
+# if any task fails, use kill -TERM $(MAKPID) to teminate everything immediately
+DIE := kill -TERM $(shell echo $$PPID)
+
+
+ns:
+	python3 -m Pyro5.nameserver --port 11001 || $(DIE)
+mongo:
+	mkdir -p mongodb-tmp~ && /usr/bin/mongod --port 11002 --noauth --dbpath=./mongodb-tmp~ --logpath=/dev/null --logappend || $(DIE)
 rest:
-	cd mupifDB && FLASK_APP=mupifdbRestApi.py PYTHONPATH=.. python3 -m flask run --host 127.0.0.1 --port 5000
+	# MUPIFDB_DRY_RUN=1
+	sleep 2 && cd mupifDB/api && MUPIFDB_MONGODB_PORT=11002 MUPIFDB_REST_SERVER=http://127.0.0.1:11003 MUPIFDB_LOG_LEVEL=DEBUG MUPIFDB_RESTAPI_HOST=localhost MUPIFDB_RESTAPI_PORT=11003 PYTHONPATH=../.. python3 main.py || $(DIE)
 web:
-	cd webapi && MUPIFDB_REST_SERVER=http://127.0.0.1:5000 FLASK_APP=index.py PYTHONPATH=.. python3 -m flask run --host 127.0.0.1 --port 5555
+	sleep 7 && cd webapi &&  MUPIFDB_MONGODB_PORT=11002 MUPIFDB_REST_SERVER=http://127.0.0.1:11003 MUPIFDB_WEB_FAKE_AUTH=1 FLASK_APP=index.py PYTHONPATH=.. python3 -m flask run --debug --no-reload --host 127.0.0.1 --port 11004 || $(DIE)
 browse:
-	sleep 2 && xdg-open http://127.0.0.1:5555
+	sleep 9 # && xdg-open http://127.0.0.1:11004
+scheduler:
+	sleep 7 && MUPIF_LOG_LEVEL=DEBUG MUPIFDB_REST_SERVER=http://127.0.0.1:11003 MUPIF_NS=localhost:11001 PYTHONPATH=.. python3 -c 'from mupifDB import workflowscheduler as ws; ws.LOOP_SLEEP_SEC=5; ws.schedulerStatFile="./sched-stat.json"; ws.main()' || $(DIE)

mupifDB/__init__.py

@@ -27,8 +27,10 @@
 __version__ = '0.0.1'
 __author__ = 'Borek Patzak'
 
+__package__ = __name__
+
 # List all submodules, so they can all be imported: from mupif import *
-__all__ = ['workflowmanager', 'schedulerstat', 'error', 'restApiControl', 'my_email']
+# __all__ = ['workflowmanager', 'schedulerstat', 'error', 'restApiControl', 'my_email']
 
 from . import workflowmanager
 from . import schedulerstat

mupifDB/api/client.py

@@ -0,0 +1,26 @@
+import requests
+import json
+import datetime
+import sys
+import os
+import tempfile
+import importlib
+import re
+import logging
+
+import pydantic
+from typing import List,Optional,Literal
+
+from .. import models
+
+from rich import print_json
+from rich.pretty import pprint
+
+from .client_util import api_type, NotFoundResponse
+
+if api_type=='granta':
+    from .client_granta import *
+else:
+    from .client_mupif import *
+
+from .client_edm import *

mupifDB/api/client_edm.py

@@ -0,0 +1,51 @@
+from .client_util import *
+import json
+import re
+
+def getEDMDataArray(DBName, Type):
+    response = rGet(f"EDM/{DBName}/{Type}")
+    return response.json()
+
+def getEDMData(DBName, Type, ID, path):
+    if ID == '' or ID is None:
+        return None
+    response = rGet(f"EDM/{DBName}/{Type}/{ID}/?path={path}")
+    return response.json()
+
+
+def setEDMData(DBName, Type, ID, path, data):
+    response = rPatch(f"EDM/{DBName}/{Type}/{ID}", data=json.dumps({"path": str(path), "data": data}))
+    return response.json()
+
+
+def createEDMData(DBName, Type, data):
+    response = rPost(f"EDM/{DBName}/Type", data=json.dumps(data))
+    return response.json()
+
+
+def cloneEDMData(DBName, Type, ID, shallow=[]):
+    response = rGet(f"EDM/{DBName}/{Type}/{ID}/clone", params={"shallow": ' '.join(shallow)})
+    return response.json()
+
+
+def getSafeLinks(DBName, Type, ID, paths=[]):
+    response = rGet(f"EDM/{DBName}/{Type}/{ID}/safe-links", params={"paths": ' '.join(paths)})
+    return response.json()
+
+
+def getEDMEntityIDs(DBName, Type, filter=None):
+    response = rPut(f"EDM/{DBName}/{Type}/find", data=json.dumps({"filter": (filter if filter else {})}))
+    return response.json()
+
+
+def uploadEDMBinaryFile(DBName, binary_data):
+    response = rPost(f"EDM/{DBName}/blob/upload", files={"blob": binary_data})
+    return response.json()
+
+
+def getEDMBinaryFileByID(DBName, fid):
+    response = rGet(f"EDM/{DBName}/blob/{fid}")
+    d = response.headers['Content-Disposition']
+    filename = re.findall("filename=(.+)", d)[0]
+    return response.content, filename
+