diff --git a/CMakeLists.txt b/CMakeLists.txt index d6ab73b..30fd09f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,14 +2,18 @@ cmake_minimum_required(VERSION 2.8.9) project(DatabaseInteractor) +set(DatabaseInteractor_VERSION_MAJOR 1) +set(DatabaseInteractor_VERSION_MINOR 1) +set(DatabaseInteractor_VERSION_PATCH 0) + #----------------------------------------------------------------------------- # Extension meta-information set(EXTENSION_HOMEPAGE "http://slicer.org/slicerWiki/index.php/Documentation/Nightly/Extensions/DatabaseInteractor") set(EXTENSION_CATEGORY "Web System Tools") set(EXTENSION_CONTRIBUTORS "Clement Mirabel (University of Michigan), Juan Carlos Prieto (UNC)") set(EXTENSION_DESCRIPTION "This extension can interact with online data in a database and local folders.") -set(EXTENSION_ICONURL "https://www.slicer.org/wiki/File:DatabaseInteractor_Logo.png") -set(EXTENSION_SCREENSHOTURLS "https://www.slicer.org/wiki/File:FullView_DatabaseInteractor.png") +set(EXTENSION_ICONURL "https://www.slicer.org/w/images/7/7f/DatabaseInteractor_Logo.png") +set(EXTENSION_SCREENSHOTURLS "https://www.slicer.org/w/images/1/1f/FullView_DatabaseInteractor.png") set(EXTENSION_DEPENDS "NA") # Specified as a space separated string, a list or 'NA' if any #----------------------------------------------------------------------------- diff --git a/DatabaseInteractor/.gitignore b/DatabaseInteractor/.gitignore new file mode 100644 index 0000000..d4a80c8 --- /dev/null +++ b/DatabaseInteractor/.gitignore @@ -0,0 +1,3 @@ +.DS_Store +*.pyc + diff --git a/DatabaseInteractor/ClusterpostLib.py b/DatabaseInteractor/ClusterpostLib.py new file mode 100644 index 0000000..94372ef --- /dev/null +++ b/DatabaseInteractor/ClusterpostLib.py @@ -0,0 +1,184 @@ +import os,sys +#sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)),'requests/')) +import requests +import vtk, qt, ctk, slicer +from slicer.ScriptedLoadableModule import * + +import json + +class ClusterpostLib(): + def __init__(self, parent=None): + if parent: + parent.title = " " + self.server = "" + self.verify = False + self.auth = {} + self.user = None + + def setServerUrl(self, server): + self.server = server + + def setVerifyHttps(self, verify): + self.verify = verify + + + def getUser(self): + if(self.user == None): + + r = requests.get(url=self.server + "/auth/user", + auth=self.auth, + verify=False) + + self.user = r.json() + return self.user + + def userLogin(self, user): + + r = requests.post(url=self.server + "/auth/login", + json=user, + verify=self.verify) + + self.auth = JWTAuth(r.json()["token"]) + + def setToken(self, token): + self.auth = JWTAuth(token) + + + def getExecutionServers(self): + r = requests.get(url=self.server + "/executionserver", + auth=self.auth, + verify=self.verify) + + return r.json() + + + def createJob(self, job): + r = requests.post(url=self.server + "/dataprovider", + auth=self.auth, + verify=self.verify, + json=job) + + return r.json() + + def addAttachment(self, jobid, filename): + + basefn = os.path.basename(filename) + data = open(filename, 'rb') + + r = requests.put(url=self.server + "/dataprovider/" + jobid + "/" + basefn, + auth=self.auth, + verify=self.verify, + headers={"Content-Type": "application/octet-stream"}, + data=data) + + data.close() + + return r.json() + + def getAttachment(self, jobid, name, filename="", responseType=None): + + stream=filename!="" + + r = requests.get(url=self.server + "/dataprovider/" + jobid + "/" + name, + auth=self.auth, + verify=self.verify, + stream=stream, + headers={ + 'responseType': responseType, + 'Content-Type': 'application/octet-stream' + }) + + if(stream): + with open(filename, 'wb') as fd: + for chunk in r.iter_content(chunk_size=2048): + fd.write(chunk) + + return r + + def executeJob(self, jobid, force=False): + r = requests.post(url=self.server + "/executionserver/" + jobid, + auth=self.auth, + verify=self.verify, + json={"force":force}) + + return r.json() + + def getJobs(self, executable=None, jobstatus=None, email=None): + + payload={} + + if(executable): + payload["executable"] = executable + + if(jobstatus): + payload["jobstatus"] = jobstatus + + if(email): + payload["email"] = email + + r = requests.get(url=self.server + "/dataprovider/user", + auth=self.auth, + verify=self.verify, + params=payload) + + return r.json() + + def getJob(self, id): + r = requests.get(url=self.server + "/dataprovider/" + id, + auth=self.auth, + verify=self.verify) + + return r.json() + + def createAndSubmitJob(self, job, files): + res = self.createJob(job) + + jobid = res["id"] + + for file in files: + self.addAttachment(jobid, file) + + return self.executeJob(jobid) + + def getJobsDone(self, outdir): + res = self.getJobs(jobstatus="DONE") + for job in res: + outputs = job["outputs"] + + jobname = job["_id"] + + key = "name" + + if(key in job): + jobname = job["name"] + + outputdir = os.path.join(outdir, jobname) + + if(not os.path.exists(outputdir)): + os.mkdir(outputdir) + + for output in outputs: + if(output["type"] == "file"): + self.getAttachment(job["_id"], output["name"], os.path.join(outputdir, output["name"]), "blob") + + def updateJobStatus(self, jobid, status): + res = self.getJob(jobid) + res["jobstatus"] = { + "status": status + } + r = requests.put(url=self.server + "/dataprovider", + auth=self.auth, + verify=self.verify, + data=json.dumps(res)) + + return r.json() + + +class JWTAuth(requests.auth.AuthBase): + def __init__(self, token): + self.token = token + + def __call__(self, r): + # Implement my authentication + r.headers['Authorization'] = 'Bearer ' + self.token + return r diff --git a/DatabaseInteractor/DatabaseInteractor.py b/DatabaseInteractor/DatabaseInteractor.py index 5eddb38..625e96c 100644 --- a/DatabaseInteractor/DatabaseInteractor.py +++ b/DatabaseInteractor/DatabaseInteractor.py @@ -2,8 +2,9 @@ from slicer.ScriptedLoadableModule import * import json import logging -import os - +import os, shutil, zipfile +import subprocess +from urlparse import urlparse # # DatabaseInteractor @@ -31,15 +32,22 @@ def __init__(self, parent): class DatabaseInteractorWidget(slicer.ScriptedLoadableModule.ScriptedLoadableModuleWidget): def setup(self): + """ + Function used to setup the UI, global variables and libraries used in this extension + """ slicer.ScriptedLoadableModule.ScriptedLoadableModuleWidget.setup(self) - # ---------------------------------------------------------------- # # ------------------------ Global variables ---------------------- # # ---------------------------------------------------------------- # + import DatabaseInteractorLib -# reload(DatabaseInteractorLib) self.DatabaseInteractorLib = DatabaseInteractorLib.DatabaseInteractorLib() + import ClusterpostLib + self.ClusterpostLib = ClusterpostLib.ClusterpostLib() + + self.logic = DatabaseInteractorLogic() + self.connected = False self.collections = dict() self.morphologicalData = dict() @@ -49,6 +57,12 @@ def setup(self): self.moduleName = 'DatabaseInteractor' scriptedModulesPath = eval('slicer.modules.%s.path' % self.moduleName.lower()) scriptedModulesPath = os.path.dirname(scriptedModulesPath) + self.modulesLoaded = dict() + + # Timer definition + self.timer = qt.QTimer() + self.timer.timeout.connect(self.overflow) + self.timerPeriod = 60 * 60 * 1000 # ---------------------------------------------------------------- # # ---------------- Definition of the UI interface ---------------- # @@ -317,6 +331,84 @@ def setup(self): self.createButton.enabled = False self.managementFormLayout.addRow(self.createButton) + # -------------------------------------------------------------- # + # -------- Definition of task creator collapsible button ------- # + # -------------------------------------------------------------- # + # Collapsible button + self.taskCreatorCollapsibleButton = ctk.ctkCollapsibleButton() + self.taskCreatorCollapsibleButton.text = "Create a task" + self.layout.addWidget(self.taskCreatorCollapsibleButton) + self.taskCreatorFormLayout = qt.QFormLayout(self.taskCreatorCollapsibleButton) + + self.executableSelector = qt.QComboBox() + self.taskCreatorFormLayout.addRow("Select an executable:", self.executableSelector) + + self.widgetSelectedGroupBox = qt.QGroupBox() + self.widgetSelectedGroupBoxLayout = qt.QVBoxLayout(self.widgetSelectedGroupBox) + self.taskCreatorFormLayout.addRow(self.widgetSelectedGroupBox) + self.widgetSelectedGroupBox.hide() + + self.currentExecutable = None + # self.condyleClassificationCollapsibleButton.hide() + + self.taskCreatorCollapsibleButton.hide() + + # -------------------------------------------------------------- # + # ------------- Job computing collapsible button --------------- # + # -------------------------------------------------------------- # + # Collapsible button + self.jobComputerCollapsibleButton = ctk.ctkCollapsibleButton() + self.jobComputerCollapsibleButton.text = "Auto compute tasks" + self.layout.addWidget(self.jobComputerCollapsibleButton) + self.jobComputerCollapsibleFormLayout = qt.QFormLayout(self.jobComputerCollapsibleButton) + + self.jobComputerHostInput = qt.QLineEdit() + self.jobComputerHostInput.setReadOnly(True) + self.jobComputerPortInput = qt.QLineEdit() + self.jobComputerPortInput.setReadOnly(True) + self.jobComputerPortInput = qt.QLineEdit() + self.jobComputerPortInput.setReadOnly(True) + self.timePeriodSelector = qt.QDoubleSpinBox() + self.timePeriodSelector.setSuffix(' min') + self.timePeriodSelector.setRange(1,1440) + self.timePeriodSelector.setDecimals(0) + + self.jobComputerParametersLayout = qt.QHBoxLayout() + self.jobComputerParametersLayout.addWidget(qt.QLabel("Host: ")) + self.jobComputerParametersLayout.addWidget(self.jobComputerHostInput) + self.jobComputerParametersLayout.addWidget(qt.QLabel("Port: ")) + self.jobComputerParametersLayout.addWidget(self.jobComputerPortInput) + self.jobComputerParametersLayout.addWidget(qt.QLabel("Period: ")) + self.jobComputerParametersLayout.addWidget(self.timePeriodSelector) + + self.jobComputerCollapsibleFormLayout.addRow(self.jobComputerParametersLayout) + + # Connect Socket Button + self.connectListenerButton = qt.QPushButton("Connect") + + # Disconnect Socket Button + self.disconnectListenerButton = qt.QPushButton("Disconnect") + self.disconnectListenerButton.enabled = False + + # HBox Layout for (dis)connection buttons + self.connectionHBoxLayout = qt.QHBoxLayout() + self.connectionHBoxLayout.addWidget(self.connectListenerButton) + self.connectionHBoxLayout.addWidget(self.disconnectListenerButton) + + self.jobComputerCollapsibleFormLayout.addRow(self.connectionHBoxLayout) + + # Websocket Console + self.displayConsole = qt.QTextEdit() + self.displayConsole.readOnly = True + self.displayConsole.setStyleSheet( + "color: white;" + "background-color: black;" + "font-family: Courier;font-style: normal;font-size: 12pt;" + ) + self.jobComputerCollapsibleFormLayout.addRow(self.displayConsole) + + self.jobComputerCollapsibleButton.hide() + # Add vertical spacer self.layout.addStretch(1) @@ -331,6 +423,8 @@ def setup(self): self.downloadCollectionButton.connect('clicked(bool)', self.onDownloadCollectionButton) self.uploadButton.connect('clicked(bool)', self.onUploadButton) self.createButton.connect('clicked(bool)', self.onCreateButton) + self.connectListenerButton.connect('clicked(bool)', self.onConnectListenerButton) + self.disconnectListenerButton.connect('clicked(bool)', self.onDiconnectListenerButton) # Radio Buttons self.downloadRadioButtonPatientOnly.toggled.connect(self.onRadioButtontoggled) @@ -349,6 +443,7 @@ def setup(self): self.uploadPatientSelector.connect('currentIndexChanged(const QString&)', self.onUploadPatientChosen) self.uploadDateSelector.connect('currentIndexChanged(const QString&)', self.onUploadDateChosen) self.managementPatientSelector.connect('currentIndexChanged(const QString&)', self.isPossibleAddDate) + self.executableSelector.connect('currentIndexChanged(const QString&)', self.executableSelected) # Calendar self.downloadDate.connect('clicked(const QDate&)', self.fillSelectorWithAttachments) @@ -365,6 +460,8 @@ def setup(self): self.DatabaseInteractorLib.getServer(self.serverFilePath) if first_line != "": # self.token = first_line + self.ClusterpostLib.setToken(first_line) + self.ClusterpostLib.setServerUrl(self.DatabaseInteractorLib.server[:-1]) self.DatabaseInteractorLib.token = first_line self.connected = True self.connectionGroupBox.hide() @@ -373,19 +470,39 @@ def setup(self): self.fillSelectorWithCollections() self.downloadCollapsibleButton.show() self.uploadCollapsibleButton.show() + self.taskCreatorCollapsibleButton.show() + self.jobComputerCollapsibleButton.show() + self.fillExecutableSelector() + server = self.DatabaseInteractorLib.server + self.jobComputerHostInput.text = urlparse(server).hostname + self.jobComputerPortInput.text = urlparse(server).port if "admin" in self.DatabaseInteractorLib.getUserScope(): self.managementCollapsibleButton.show() file.close() - - def cleanup(self): - pass + + + def exit(self): + """ + Function used to reset the CLI widget when exiting the module + """ + print("-----------EXIT------------") + for moduleName, slot in self.modulesLoaded.iteritems(): + node = getattr(slicer.modules, moduleName) + applyButton = node.widgetRepresentation().ApplyPushButton + applyButton.disconnect(applyButton, 'clicked(bool)') + applyButton.connect('clicked(bool)', slot) + self.modulesLoaded = dict() # ------------ Buttons -------------- # - # Function used to connect user to the database and store token in a file def onConnectionButton(self): + """ + Function used to connect user to the database and store token in a file + """ self.DatabaseInteractorLib.setServer(self.serverInput.text, self.serverFilePath) + self.ClusterpostLib.setServerUrl(self.serverInput.text[:-1]) token, error = self.DatabaseInteractorLib.connect(self.emailInput.text, self.passwordInput.text) + print(error) if token != -1: userScope = self.DatabaseInteractorLib.getUserScope() if len(userScope) != 1 or "default" not in userScope: @@ -393,6 +510,7 @@ def onConnectionButton(self): file = open(self.tokenFilePath, 'w+') file.write(token) file.close() + self.ClusterpostLib.setToken(token) self.connected = True self.connectionGroupBox.hide() self.connectionButton.hide() @@ -401,6 +519,11 @@ def onConnectionButton(self): self.downloadCollapsibleButton.show() self.uploadCollapsibleButton.show() self.managementCollapsibleButton.show() + self.taskCreatorCollapsibleButton.show() + self.jobComputerCollapsibleButton.show() + self.fillExecutableSelector() + self.jobComputerHostInput.text = urlparse(self.serverInput.text).hostname + self.jobComputerPortInput.text = urlparse(self.serverInput.text).port if "admin" not in userScope: self.managementCollapsibleButton.hide() elif token == -1: @@ -410,8 +533,12 @@ def onConnectionButton(self): self.errorLoginText.text = "Insufficient scope ! Email luciacev@umich.edu for access." self.errorLoginText.show() - # Function used to disconnect user to the database def onDisconnectionButton(self): + """ + Function used to disconnect user to the database + """ + self.serverInput.text = self.DatabaseInteractorLib.server + self.emailInput.text = self.DatabaseInteractorLib.getUserEmail() self.DatabaseInteractorLib.disconnect() self.connected = False self.passwordInput.text = '' @@ -422,12 +549,16 @@ def onDisconnectionButton(self): self.downloadCollapsibleButton.hide() self.uploadCollapsibleButton.hide() self.managementCollapsibleButton.hide() + self.taskCreatorCollapsibleButton.hide() + self.jobComputerCollapsibleButton.hide() # Erase token from file with open(self.tokenFilePath, "w"): pass - # Function used to download data with information provided def onDownloadButton(self): + """ + Function used to download data with information provided + """ for items in self.morphologicalData: if "_attachments" in items: for attachments in items["_attachments"].keys(): @@ -443,11 +574,12 @@ def onDownloadButton(self): file.close() # Load the file - self.fileLoader(filePath) - # slicer.util.loadModel(filePath) + self.logic.fileLoader(filePath) - # Function used to download an entire collection and organise it with folders def onDownloadCollectionButton(self): + """ + Function used to download an entire collection and organise it with folders + """ collectionPath = os.path.join(self.downloadFilepathSelector.directory, self.downloadCollectionSelector.currentText) # Check if folder already exists @@ -496,11 +628,10 @@ def onDownloadCollectionButton(self): file.close() # Save the attachment - if data != -1: - filePath = os.path.join(collectionPath, patientId, date[:10], attachments) - with open(filePath, 'wb+') as file: - for chunk in data.iter_content(2048): - file.write(chunk) + filePath = os.path.join(collectionPath, patientId, date[:10], attachments) + with open(filePath, 'wb+') as file: + for chunk in data: + file.write(chunk) file.close() file = open(os.path.join(collectionPath, '.DBIDescriptor'), 'w+') @@ -509,24 +640,24 @@ def onDownloadCollectionButton(self): self.uploadFilepathSelector.directory = collectionPath self.managementFilepathSelector.directory = collectionPath - # Function used to upload a data to the correct patient def onUploadButton(self): + """ + Function used to upload a data to the correct patient + """ + collection = self.uploadFilepathSelector.directory self.checkBoxesChecked() for patient in self.checkedList.keys(): for date in self.checkedList[patient].keys(): for attachment in self.checkedList[patient][date]["items"]: + with open(os.path.join(collection, patient, date, '.DBIDescriptor'), 'r') as file: + descriptor = json.load(file) + documentId = descriptor["_id"] # Add new attachments to patient - collection = self.uploadFilepathSelector.directory path = os.path.join(collection,patient,date,attachment) - for items in self.morphologicalData: - if not "date" in items: - items["date"] = "NoDate" - if items["patientId"] == patient and items["date"][:10] == date: - documentId = items["_id"] - data = open(path, 'r') - self.DatabaseInteractorLib.addAttachment(documentId, attachment, data) + with open(path, 'r') as file: + self.DatabaseInteractorLib.addAttachment(documentId, attachment, file) # Update descriptor - data = self.DatabaseInteractorLib.getMorphologicalDataByPatientId(patient).json() + data = self.DatabaseInteractorLib.getMorphologicalDataByPatientId(patient).json()[0] file = open(os.path.join(self.uploadFilepathSelector.directory, patient, date, '.DBIDescriptor'), 'w+') json.dump(data, file, indent=3, sort_keys=True) @@ -537,8 +668,10 @@ def onUploadButton(self): self.morphologicalData = self.DatabaseInteractorLib.getMorphologicalData(items["_id"]).json() self.uploadLabel.show() - # Function used to create the architecture for a new patient or new date, updating descriptors def onCreateButton(self): + """ + Function used to create the architecture for a new patient or new date, updating descriptors + """ collectionPath = self.managementFilepathSelector.directory patientId = self.managementPatientSelector.currentText date = str(self.createDate.selectedDate) @@ -586,9 +719,28 @@ def onCreateButton(self): file.close() self.fillSelectorWithPatients() + def onConnectListenerButton(self): + """ + Function used to create a timer with the period selected in the UI + """ + self.timerPeriod = int(self.timePeriodSelector.value) * 60 * 1000 # Timer period is in ms + self.timer.start(self.timerPeriod) + self.connectListenerButton.enabled = False + self.disconnectListenerButton.enabled = True + + def onDiconnectListenerButton(self): + """ + Function used to stop the timer and the job listening + """ + self.timer.stop() + self.connectListenerButton.enabled = True + self.disconnectListenerButton.enabled = False + # ---------- Radio Buttons ---------- # - # Function used to display interface corresponding to the query checked def onRadioButtontoggled(self): + """ + Function used to display interface corresponding to the query checked + """ self.downloadErrorText.hide() self.fillSelectorWithAttachments() if self.downloadRadioButtonPatientOnly.isChecked(): @@ -600,8 +752,10 @@ def onRadioButtontoggled(self): self.downloadDateLabel.show() self.downloadDate.show() - # Function used to display interface corresponding to the management action checked def onManagementRadioButtontoggled(self): + """ + Function used to display interface corresponding to the management action checked + """ if self.managementRadioButtonPatient.isChecked(): self.managementPatientSelector.hide() self.managementPatientSelectorLabel.hide() @@ -620,20 +774,27 @@ def onManagementRadioButtontoggled(self): self.isPossibleAddDate() # ------------- Inputs -------------- # - # Function used to enable the connection button if userlogin and password are provided def onInputChanged(self): + """ + Function used to enable the connection button if userlogin and password are provided + """ self.connectionButton.enabled = (len(self.emailInput.text) != 0 and len(self.passwordInput.text) != 0) - # Function used to enable the creation button if path contains a descriptor and is a name is given def isPossibleCreatePatient(self): + """ + Function used to enable the creation button if path contains a descriptor and is a name is given + """ directoryPath = self.managementFilepathSelector.directory self.createButton.enabled = False + # Check if the folder is a collection with a DBIDescriptor file if self.newPatientIdInput.text != '' and os.path.exists(os.path.join(directoryPath, '.DBIDescriptor')): self.createButton.enabled = True # ----------- Combo Boxes ----------- # - # Function used to fill the comboBoxes with morphologicalCollections def fillSelectorWithCollections(self): + """ + Function used to fill the comboBoxes with morphologicalCollections + """ self.collections = self.DatabaseInteractorLib.getMorphologicalDataCollections().json() self.downloadCollectionSelector.clear() for items in self.collections: @@ -641,8 +802,10 @@ def fillSelectorWithCollections(self): if self.downloadCollectionSelector.count == 0: self.downloadCollectionSelector.addItem("None") - # Function used to fill the comboBox with patientId corresponding to the collection selected def fillSelectorWithPatients(self): + """ + Function used to fill the comboBox with patientId corresponding to the collection selected + """ for items in self.morphologicalData: if "date" in items: date = items["date"] @@ -651,7 +814,6 @@ def fillSelectorWithPatients(self): else: date = "NoDate" - text = self.downloadCollectionSelector.currentText self.downloadButton.enabled = text if text != "None": @@ -669,8 +831,10 @@ def fillSelectorWithPatients(self): self.downloadPatientSelector.model().sort(0) self.downloadPatientSelector.setCurrentIndex(0) - # Function used to fille the comboBox with a list of attachment corresponding to the query results def fillSelectorWithDescriptorPatients(self): + """ + Function used to fill the comboBox with a list of attachment corresponding to the query results + """ directoryPath = self.managementFilepathSelector.directory self.managementPatientSelector.clear() if os.path.exists(os.path.join(directoryPath, '.DBIDescriptor')): @@ -683,16 +847,31 @@ def fillSelectorWithDescriptorPatients(self): self.managementPatientSelector.model().sort(0) self.managementPatientSelector.setCurrentIndex(0) - # Function used to enable creation button if path contains a descriptor and is a patient is chosen + def fillExecutableSelector(self): + """ + Function used to fill the comboBox with a list of executable in the current Slicer + """ + self.modules = {} + modules = slicer.modules.__dict__.keys() + for moduleName in modules: + module = getattr(slicer.modules, moduleName) + if hasattr(module, "cliModuleLogic"): + self.executableSelector.addItem(module.name) + def isPossibleAddDate(self): + """ + Function used to enable creation button if path contains a descriptor and is a patient is chosen + """ directoryPath = self.managementFilepathSelector.directory self.createButton.enabled = False if self.managementPatientSelector.currentText != 'None' and os.path.exists( os.path.join(directoryPath, '.DBIDescriptor')): self.createButton.enabled = True - # Function used to enable the download button when everything is ok def onDownloadPatientChosen(self): + """ + Function used to enable the download button when everything is ok + """ collectionName = self.downloadCollectionSelector.currentText patientId = self.downloadPatientSelector.currentText if collectionName != "None" and patientId != "None": @@ -700,8 +879,10 @@ def onDownloadPatientChosen(self): self.fillSelectorWithAttachments() self.highlightDates() - # Function used to show in a list the new documents for a patient def onUploadPatientChosen(self): + """ + Function used to show in a list the new documents for a patient + """ self.uploadDateSelector.clear() self.uploadLabel.hide() if self.uploadPatientSelector.currentText != "" and self.uploadPatientSelector.currentText != "None": @@ -710,8 +891,10 @@ def onUploadPatientChosen(self): if self.uploadDateSelector.count == 0: self.uploadDateSelector.addItem("None") - # Function used to display the checkboxes corresponding on the patient and timepoint selected def onUploadDateChosen(self): + """ + Function used to display the checkboxes corresponding on the patient and timepoint selected + """ self.clearCheckBoxList() # Display new attachments in the layout if self.uploadDateSelector.currentText != "" and self.uploadDateSelector.currentText != "None": @@ -727,10 +910,124 @@ def onUploadDateChosen(self): self.uploadListLayout.addWidget(self.noneLabel) self.noneLabel.setText("None") + def executableSelected(self): + """ + Function used to display the widget corresponding to the CLI selected + """ + if hasattr(slicer.modules, self.executableSelector.currentText.lower()): + self.modulesLoaded[str(self.executableSelector.currentText.lower())] = getattr(slicer.modules, self.executableSelector.currentText.lower()).widgetRepresentation().apply + self.widgetSelectedGroupBox.show() + if self.currentExecutable: + self.layout.removeWidget(self.currentExecutable.widgetRepresentation()) + self.currentExecutable.widgetRepresentation().hide() + self.currentExecutable = getattr(slicer.modules, self.executableSelector.currentText.lower()) + self.widgetSelectedGroupBoxLayout.addWidget(self.currentExecutable.widgetRepresentation()) + + # Adjust height + self.currentExecutable.widgetRepresentation().setFixedHeight(350) + + # Change Apply connection to send a remote task + applyButton = self.currentExecutable.widgetRepresentation().ApplyPushButton + self.currentExecutable.widgetRepresentation().show() + applyButton.disconnect(applyButton,'clicked()') + applyButton.connect('clicked(bool)', self.createJobFromModule) + + def createJobFromModule(self): + """ + Function used to create and submit a job based on the CLI interface + """ + cli = {} + attachments = [] + if self.currentExecutable.widgetRepresentation().currentCommandLineModuleNode(): + node = self.currentExecutable.widgetRepresentation().currentCommandLineModuleNode() + executionServer = "" + if slicer.app.applicationName == "Slicer": + executionServer = "Slicer" + slicer.app.applicationVersion[:5] + else: + executionServer = slicer.app.applicationName + cli = { + "executable": self.executableSelector.currentText, + "parameters": [], + "inputs": [], + "outputs": [], + "type": "job", + "userEmail": self.DatabaseInteractorLib.getUserEmail(), + "executionserver": executionServer + } + for groupIndex in xrange(0, node.GetNumberOfParameterGroups()): + for parameterIndex in xrange(0, node.GetNumberOfParametersInGroup(groupIndex)): + if node.GetParameterLongFlag(groupIndex, parameterIndex): + flag = node.GetParameterLongFlag(groupIndex, parameterIndex) + if flag: + while flag[0] == "-": + flag = flag[1:] + flag = "--" + flag + else: + flag = node.GetParameterFlag(groupIndex, parameterIndex) + if flag: + while flag[0] == "-": + flag = flag[1:] + flag = "-" + flag + + value = node.GetParameterAsString(node.GetParameterName(groupIndex, parameterIndex)) + path = '' + tag = node.GetParameterTag(groupIndex, parameterIndex) + + if tag == "image" or tag == "geometry" or tag == "transform": + # Write file in a temporary + IOnode = slicer.util.getNode(node.GetParameterAsString(node.GetParameterName(groupIndex, parameterIndex))) + if IOnode: + path = self.logic.nodeWriter(IOnode, slicer.app.temporaryPath) + value = os.path.basename(path) + + if tag == "file": + path = node.GetParameterAsString(node.GetParameterName(groupIndex, parameterIndex)) + value = os.path.basename(path) + + if tag == "table" and node.GetParameterType(groupIndex, parameterIndex) == "color": + i = 1 + while parameterIndex - i >= 0 : + if node.GetParameterType(groupIndex, parameterIndex-i) == 'label': + labelNode = slicer.util.getNode(node.GetParameterAsString(node.GetParameterName(groupIndex, parameterIndex-i))) + path = self.logic.nodeWriter(labelNode.GetDisplayNode().GetColorNode(), slicer.app.temporaryPath) + value = os.path.basename(path) + i = 100 + i += 1 + + channel = node.GetParameterChannel(groupIndex, parameterIndex) + if channel and path: + if channel == "input": + attachments.append(path) + cli["inputs"].append({ + "name": value + }) + else: + if os.path.isdir(path) or os.path.basename(path).find('.') == -1: + type = 'directory' + else: + type = 'file' + cli["outputs"].append({ + "type": type, + "name": value + }) + cli['parameters'].append({ + "flag": flag, + "name": value + # "name": node.GetParameterName(groupIndex, parameterIndex), + # "value": value, + # "type": node.GetParameterTag(groupIndex, parameterIndex) + }) + import pprint + pprint.pprint(cli) + if not self.ClusterpostLib.server: + self.ClusterpostLib.setServerUrl(self.DatabaseInteractorLib.server[:-1]) + self.ClusterpostLib.createAndSubmitJob(cli, attachments) # ----------- Calendars ----------- # - # Function used to fill a comboBox with attachments retrieved by queries def fillSelectorWithAttachments(self): + """ + Function used to fill a comboBox with attachments retrieved by queries + """ self.downloadAttachmentSelector.clear() self.downloadErrorText.hide() self.downloadButton.enabled = True @@ -757,8 +1054,10 @@ def fillSelectorWithAttachments(self): self.downloadButton.enabled = False # ------ Filepath selectors ------- # - # Function used to create a dictionary corresponding to the collection architecture def createFilesDictionary(self): + """ + Function used to create a dictionary corresponding to the collection architecture + """ directoryPath = self.uploadFilepathSelector.directory # Check if the directory selected is a valid collection if not os.path.exists(os.path.join(directoryPath, '.DBIDescriptor')): @@ -795,8 +1094,10 @@ def createFilesDictionary(self): self.uploadButton.enabled = False self.uploadPatientSelector.addItem("None") - # Function used to chose what signal to connect depending on management action checked def onManagementDirectorySelected(self): + """ + Function used to chose what signal to connect depending on management action checked + """ if self.managementRadioButtonPatient.isChecked(): self.isPossibleCreatePatient() else: @@ -806,43 +1107,10 @@ def onManagementDirectorySelected(self): # ---------------------------------------------------- # # ------------------ Other functions ----------------- # # ---------------------------------------------------- # - # Function used to check the downloaded file extension in order to load it with the correct loader - def fileLoader(self, filepath): - # Documentation : - # http://wiki.slicer.org/slicerWiki/index.php/Documentation/4.5/SlicerApplication/SupportedDataFormat - sceneExtensions = ["mrml","mrb","xml","xcat"] - volumeExtensions = ["dcm","nrrd","nhdr","mhd","mha","vtk","hdr","img","nia","nii","bmp","pic","mask","gipl","jpg","jpeg","lsm","png","spr","tif","tiff","mgz","mrc","rec"] - modelExtensions = ["vtk","vtp","stl","obj","orig","inflated","sphere","white","smoothwm","pial","g","byu"] - fiducialExtensions = ["fcsv","txt"] - rulerExtensions = ["acsv","txt"] - transformExtensions = ["tfm","mat","txt","nrrd","nhdr","mha","mhd","nii"] - volumeRenderingExtensions = ["vp","txt"] - colorsExtensions = ["ctbl","txt"] - extension = "" - - if filepath.rfind(".") != -1: - extension = filepath[filepath.rfind(".") + 1:] - if extension == "gz": - extension = filepath[filepath[:filepath.rfind(".")].rfind(".") + 1:filepath.rfind(".")] - if extension in sceneExtensions: - slicer.util.loadScene(filepath) - if extension in volumeExtensions: - slicer.util.loadVolume(filepath) - if extension in modelExtensions: - slicer.util.loadModel(filepath) - if extension in fiducialExtensions: - if not slicer.util.loadFiducialList(filepath): - if not slicer.util.loadAnnotationFiducial(filepath): - slicer.util.loadNodeFromFile(filepath) - # if extension in rulerExtensions: - if extension in transformExtensions: - slicer.util.loadTrandform(filepath) - # if extension in volumeRenderingExtensions: - if extension in colorsExtensions: - slicer.util.loadColorTable(filepath) - - # Function used to clear the layout which displays the checkboxes for upload def clearCheckBoxList(self): + """ + Function used to clear the layout which displays the checkboxes for upload + """ for patient in self.attachmentsList.keys(): for date in self.attachmentsList[patient].keys(): for items in self.attachmentsList[patient][date]["checkbox"].keys(): @@ -853,8 +1121,10 @@ def clearCheckBoxList(self): self.noneLabel.setText("") self.uploadListLayout.removeWidget(self.noneLabel) - # Function used to get in a dictionnary attachments selected to be uploaded def checkBoxesChecked(self): + """ + Function used to store in a dictionary the attachments selected to be uploaded + """ self.checkedList = {} for patient in self.attachmentsList.keys(): self.checkedList[patient] = {} @@ -865,8 +1135,10 @@ def checkBoxesChecked(self): if str(self.attachmentsList[patient][date]["checkbox"][items].checkState()) == "2" : self.checkedList[patient][date]["items"].append(items) - # Function used to color the dates which contain one or multiple attachments for a given patientId def highlightDates(self): + """ + Function used to color the dates which contain one or multiple attachments for a given patientId + """ for items in self.morphologicalData: if "date" in items: date = items["date"] @@ -878,34 +1150,225 @@ def highlightDates(self): self.downloadDate.setDateTextFormat(qt.QDate(int(date[:4]), int(date[5:7]), int(date[8:10])), self.normalDateFormat) + def overflow(self): + """ + Function triggered by the overflow of the timer + """ + self.timer.stop() + print(">>>>>>>>>>>>>>>>") + # Retrieve the jobs to be computed in the database + jobs = self.ClusterpostLib.getJobs(jobstatus='QUEUE') + if jobs: + self.runJob(jobs[0]) + self.timer.start(self.timerPeriod) + + def runJob(self, job): + """ + Function used to run a job and send it back to the server + """ + # Creates a folder to store IO for the job. Folder is in Slicer temporary path and named with job id + jobpath = os.path.join(slicer.app.temporaryPath, job["_id"]) + if os.path.exists(jobpath): + shutil.rmtree(jobpath) + os.makedirs(jobpath) + # Check if the executable needed is in the current instance of Slicer + if hasattr(slicer.modules, job["executable"].lower()): + executableNode = getattr(slicer.modules, job["executable"].lower()) + command = list() + # Depending on Slicer version, the executable path is pointing to the library or the executable + if os.path.basename(executableNode.path).find('.') == -1: + command.append(executableNode.path) + else: + command.append(os.path.join(os.path.dirname(executableNode.path), executableNode.name)) + # Parse all the parameters to create de CLI command + for parameter in job["parameters"]: + if not parameter["name"] == "" and not parameter["flag"] == "": + if parameter["name"] == "true": + command.append(parameter["flag"]) + elif not parameter["name"] == "false": + command.append(parameter["flag"]) + command.append(parameter["name"]) + elif parameter["flag"] == "": + command.append(parameter["name"]) + self.ClusterpostLib.updateJobStatus(job["_id"], "DOWNLOADING") + # Downloads the attachments and store them in the previously created folder + for attachment in job["_attachments"]: + self.ClusterpostLib.getAttachment(job["_id"], attachment, os.path.join(jobpath, attachment)) + if attachment in command: + i = command.index(attachment) + command[i] = os.path.join(jobpath, attachment) + # This value is set as true if there is at least one output with type folder + directory = False + # Formating the ouput content (file/directory) + for output in job["outputs"]: + if output['type'] == 'directory': + if os.path.basename(output["name"]): + if output["name"] in command: + i = command.index(output["name"]) + command[i] = os.path.join(jobpath, os.path.basename(output["name"])) + folderName = os.path.basename(output["name"]) + else: + if output["name"] in command: + i = command.index(output["name"]) + command[i] = os.path.join(jobpath) + folderName = os.path.basename(os.path.dirname(output["name"])) + directory = True + else: + file = open(os.path.join(jobpath, output["name"]), 'w+') + file.close() + if output["name"] in command: + i = command.index(output["name"]) + command[i] = os.path.join(jobpath, output["name"]) + # Check the files before computation + filesBeforeComputation = os.listdir(jobpath) + print(command) + self.ClusterpostLib.updateJobStatus(job["_id"], "RUN") + try: + # Subprocess is a way to run a cli from python + p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, err = p.communicate() + self.displayConsole.append(out) + self.displayConsole.append(err) + # Check files differences before and after computation for output with type directory + filesAfterComputation = os.listdir(jobpath) + filesDifference = list(set(filesAfterComputation) - set(filesBeforeComputation)) + self.ClusterpostLib.updateJobStatus(job["_id"], "UPLOADING") + # Upload stdout and stderr + with open(os.path.join(jobpath, 'stdout.out'), 'w') as f: + f.write(out) + with open(os.path.join(jobpath, 'stderr.err'), 'w') as f: + f.write(err) + self.ClusterpostLib.addAttachment(job["_id"], os.path.join(jobpath, 'stdout.out')) + self.ClusterpostLib.addAttachment(job["_id"], os.path.join(jobpath, 'stderr.err')) + outputSize = [] + for output in job["outputs"]: + if output['type'] == 'file': + self.ClusterpostLib.addAttachment(job["_id"], + os.path.join(jobpath, output["name"])) + outputSize.append(os.stat(os.path.join(jobpath, output["name"])).st_size) + # Add all the new files in a zip file + if not directory: + folderName = "computationFiles" + print filesDifference + if len(filesDifference) > 0: + with zipfile.ZipFile(os.path.join(jobpath, folderName + '.zip'), 'w') as myzip: + for file in filesDifference: + myzip.write(os.path.join(jobpath, file), file) + self.ClusterpostLib.addAttachment(job["_id"], + os.path.join(jobpath, folderName + '.zip')) + outputSize.append(os.stat(os.path.join(jobpath, folderName + '.zip')).st_size) + # Check if the output file is not empty + if 0 in outputSize: + self.ClusterpostLib.updateJobStatus(job["_id"], "FAIL") + else: + self.ClusterpostLib.updateJobStatus(job["_id"], "DONE") + except Exception as e: + with open(os.path.join(jobpath, 'stderr.err'), 'w') as f: + f.write(str(e)) + self.ClusterpostLib.addAttachment(job["_id"], os.path.join(jobpath, 'stderr.err')) + self.ClusterpostLib.updateJobStatus(job["_id"], "FAIL") + # # DatabaseInteractorLogic # class DatabaseInteractorLogic(slicer.ScriptedLoadableModule.ScriptedLoadableModuleLogic): - """This class should implement all the actual - computation done by your module. The interface - should be such that other python code can import - this class and make use of the functionality without - requiring an instance of the Widget. - Uses ScriptedLoadableModuleLogic base class, available at: - https://github.com/Slicer/Slicer/blob/master/Base/Python/slicer/ScriptedLoadableModule.py - """ + def fileLoader(self, filepath): + """ + Function used to load a file in the node corresponding to the file extension + Documentation : http://wiki.slicer.org/slicerWiki/index.php/Documentation/4.5/SlicerApplication/SupportedDataFormat + """ + sceneExtensions = ["mrml","mrb","xml","xcat"] + volumeExtensions = ["dcm","nrrd","nhdr","mhd","mha","vtk","hdr","img","nia","nii","bmp","pic","mask","gipl","jpg","jpeg","lsm","png","spr","tif","tiff","mgz","mrc","rec"] + modelExtensions = ["vtk","vtp","stl","obj","orig","inflated","sphere","white","smoothwm","pial","g","byu"] + fiducialExtensions = ["fcsv","txt"] + rulerExtensions = ["acsv","txt"] + transformExtensions = ["tfm","mat","txt","nrrd","nhdr","mha","mhd","nii"] + volumeRenderingExtensions = ["vp","txt"] + colorsExtensions = ["ctbl","txt"] + extension = "" - def run(self, email, password): - return self.DatabaseInteractorLib.connect(email, password) + if filepath.rfind(".") != -1: + extension = filepath[filepath.rfind(".") + 1:] + if extension == "gz": + extension = filepath[filepath[:filepath.rfind(".")].rfind(".") + 1:filepath.rfind(".")] + if extension in sceneExtensions: + slicer.util.loadScene(filepath) + if extension in volumeExtensions: + slicer.util.loadVolume(filepath) + if extension in modelExtensions: + slicer.util.loadModel(filepath) + if extension in fiducialExtensions: + if not slicer.util.loadFiducialList(filepath): + if not slicer.util.loadAnnotationFiducial(filepath): + slicer.util.loadNodeFromFile(filepath) + # if extension in rulerExtensions: + if extension in transformExtensions: + slicer.util.loadTrandform(filepath) + # if extension in volumeRenderingExtensions: + if extension in colorsExtensions: + slicer.util.loadColorTable(filepath) + + def nodeWriter(self, node, dirpath): + """ + Function used to write the content of a node in a file given the node and the path + """ + fileName = node.GetName() + + # Window UI to choose file extension + extensionBox = qt.QDialog() + extensionBox.setWindowTitle("Select file extension") + + extensionBoxLayout = qt.QFormLayout() + extensionComboBox = qt.QComboBox() + extensionBoxLayout.addRow(fileName, extensionComboBox) + buttonBox = qt.QDialogButtonBox(qt.QDialogButtonBox.Ok) + extensionBoxLayout.addWidget(buttonBox) + extensionBox.setLayout(extensionBoxLayout) + + buttonBox.accepted.connect(extensionBox.accept) + buttonBox.rejected.connect(extensionBox.reject) + + if "LabelMap" in node.GetClassName() or "ScalarVolume" in node.GetClassName(): + extensionComboBox.addItems([".nrrd",".nii",".gipl.gz"]) + extensionBox.exec_() + extension = extensionComboBox.currentText + if "ColorTable" in node.GetClassName(): + extension = '.txt' + if "ModelHierarchy" in node.GetClassName(): + extension = '.mrml' + if "ModelNode" in node.GetClassName(): + extensionComboBox.addItems([".vtk", ".stl", ".obj"]) + extensionBox.exec_() + extension = extensionComboBox.currentText + if "Transform" in node.GetClassName(): + extension = ".mat" + + write = slicer.util.saveNode(node,os.path.join(dirpath,fileName + extension)) + if not write and extension == ".mrml": + slicer.util.saveScene(os.path.join(dirpath, fileName + extension)) + return os.path.join(dirpath,fileName + extension) class DatabaseInteractorTest(slicer.ScriptedLoadableModule.ScriptedLoadableModuleTest): - # Reset the scene def setUp(self): + """ + Function used to reset the scene for the tests + """ self.widget = slicer.modules.DatabaseInteractorWidget self.DatabaseInteractorLib = self.widget.DatabaseInteractorLib slicer.mrmlScene.Clear(0) self.DatabaseInteractorLib.disconnect() - # Run the tests def runTest(self): + """ + Function used to run some tests about the extension behaviour + """ + self.runTestDbInteractor() + self.runTestClusterpost() + + # Run the tests + def runTestDbInteractor(self): self.setUp() self.delayDisplay(' Starting tests ') @@ -933,9 +1396,9 @@ def runTest(self): def test_Login(self): - # ---------------------------------------------------------------- # - # ------------------------ Login to server ----------------------- # - # ---------------------------------------------------------------- # + """ ---------------------------------------------------------------- + ---------------------- Login to the server --------------------- + ---------------------------------------------------------------- """ server = 'http://localhost:8180/' user = 'clement.mirabel@gmail.com' password = 'Password1234' @@ -949,9 +1412,9 @@ def test_Login(self): return True def test_createCollection(self): - # ---------------------------------------------------------------- # - # ------------------- Creating a test collection ----------------- # - # ---------------------------------------------------------------- # + """ ---------------------------------------------------------------- + ------------------ Creating a test collection ------------------ + ---------------------------------------------------------------- """ data = {"items": "[]", "type": "morphologicalDataCollection", "name": "CollectionTest"} @@ -963,9 +1426,9 @@ def test_createCollection(self): return True def test_getCollection(self): - # ---------------------------------------------------------------- # - # ------------------ Getting the test collection ----------------- # - # ---------------------------------------------------------------- # + """ ---------------------------------------------------------------- + ------------------ Getting the test collection ----------------- + ---------------------------------------------------------------- """ rep = self.DatabaseInteractorLib.getMorphologicalDataCollections() for items in rep.json(): if items["name"]=="CollectionTest": @@ -976,9 +1439,9 @@ def test_getCollection(self): return False def test_createPatient(self): - # ---------------------------------------------------------------- # - # ---------------------- Creating a patient ---------------------- # - # ---------------------------------------------------------------- # + """ ---------------------------------------------------------------- + ---------------------- Creating a patient ---------------------- + ---------------------------------------------------------------- """ data = {"type": "morphologicalData", "patientId": "PatientTest"} rep = self.DatabaseInteractorLib.createMorphologicalData(data) if rep == -1: @@ -997,9 +1460,9 @@ def test_createPatient(self): def test_uploadAttachment(self): - # ---------------------------------------------------------------- # - # -------------------- Uploading an attachment ------------------- # - # ---------------------------------------------------------------- # + """ ---------------------------------------------------------------- + -------------------- Uploading an attachment ------------------- + ---------------------------------------------------------------- """ self.moduleName = 'DatabaseInteractor' filePath = slicer.app.temporaryPath + '/FA.nrrd' file = open(filePath,'rb') @@ -1012,9 +1475,9 @@ def test_uploadAttachment(self): return True def test_getAttachment(self): - # ---------------------------------------------------------------- # - # --------------------- Getting an attachment -------------------- # - # ---------------------------------------------------------------- # + """ ---------------------------------------------------------------- + -------------------- Getting an attachment ------------------- + ---------------------------------------------------------------- """ rep = self.DatabaseInteractorLib.getAttachment(self.patientId,'attachmentTest.nrrd', 'blob') if rep == -1: print("Getting attachment Failed!") @@ -1028,9 +1491,9 @@ def test_getAttachment(self): return True def test_deletePatient(self): - # ---------------------------------------------------------------- # - # --------------------- Delete the test patient ------------------ # - # ---------------------------------------------------------------- # + """ ---------------------------------------------------------------- + -------------------- Delete the test patient ------------------- + ---------------------------------------------------------------- """ rep = self.DatabaseInteractorLib.deleteMorphologicalData(self.patientId) if rep == -1: print("Patient deletion Failed!") @@ -1039,9 +1502,9 @@ def test_deletePatient(self): return True def test_deleteCollection(self): - # ---------------------------------------------------------------- # - # ------------------- Delete the test collection ----------------- # - # ---------------------------------------------------------------- # + """ ---------------------------------------------------------------- + ------------------ Delete the test collection ------------------ + ---------------------------------------------------------------- """ rep = self.DatabaseInteractorLib.deleteMorphologicalDataCollection(self.collectionTestId) if rep == -1: print("Collection deletion Failed!") @@ -1050,6 +1513,9 @@ def test_deleteCollection(self): return True def test_importData(self): + """ ---------------------------------------------------------------- + ------------------- Download some data online ------------------ + ---------------------------------------------------------------- """ import urllib downloads = ( ('http://slicer.kitware.com/midas3/download?items=5767', 'FA.nrrd'), @@ -1060,4 +1526,169 @@ def test_importData(self): if not os.path.exists(filePath) or os.stat(filePath).st_size == 0: logging.info('Requesting download %s from %s...\n' % (name, url)) urllib.urlretrieve(url, filePath) - return True \ No newline at end of file + return True + + def runTestClusterpost(self): + import ClusterpostLib + import urllib + + self.testfile = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../DatabaseInteractor.png") + + self.setUp() + + self.clusterpost = ClusterpostLib.ClusterpostLib() + + self.clusterpost.setServerUrl("http://localhost:8180") + + self.delayDisplay(' Starting tests ') + + self.assertTrue(self.testClusterpostLogin()) + + self.assertTrue(self.testGetExecutionServers()) + + self.assertTrue(self.testCreateJob()) + + self.assertTrue(self.testAddAttachment()) + + self.assertTrue(self.testExecuteJob()) + + self.assertTrue(self.testGetJob()) + + self.assertTrue(self.testGetJobs()) + + self.assertTrue(self.testGetDocumentAttachment()) + + self.assertTrue(self.testCreateAndSubmitJob()) + + self.assertTrue(self.testGetJobsDone()) + + def testClusterpostLogin(self): + self.clusterpost.userLogin({ + "email": "algiedi85@gmail.com", + "password": "123Algiedi!" + }) + + return True + + def testGetExecutionServers(self): + servers = self.clusterpost.getExecutionServers() + print servers + self.executionserver = servers[0]["name"] + return True + + def testCreateJob(self): + job = { + "executable": "cksum", + "parameters": [ + { + "flag": "", + "name": "DatabaseInteractor.png" + } + ], + "inputs": [ + { + "name": "DatabaseInteractor.png" + } + ], + "outputs": [ + { + "type": "directory", + "name": "./" + }, + { + "type": "tar.gz", + "name": "./" + }, + { + "type": "file", + "name": "stdout.out" + }, + { + "type": "file", + "name": "stderr.err" + } + ], + "type": "job", + "userEmail": "algiedi85@gmail.com", + "executionserver": self.executionserver + } + + res = self.clusterpost.createJob(job) + + self.jobid = res["id"] + + return True + + def testAddAttachment(self): + res = self.clusterpost.addAttachment(self.jobid, self.testfile) + + return True + + def testExecuteJob(self): + + res = self.clusterpost.executeJob(self.jobid) + + return True + + def testGetJob(self): + res = self.clusterpost.getJob(self.jobid) + + return True + + def testGetJobs(self): + res = self.clusterpost.getJobs("cksum") + + return True + + def testGetDocumentAttachment(self): + res = self.clusterpost.getAttachment(self.jobid, self.testfile, "/tmp/out.png", "blob") + + return True + + def testCreateAndSubmitJob(self): + job = { + "executable": "cksum", + "parameters": [ + { + "flag": "", + "name": "DatabaseInteractor.png" + } + ], + "inputs": [ + { + "name": "DatabaseInteractor.png" + } + ], + "outputs": [ + { + "type": "directory", + "name": "./" + }, + { + "type": "tar.gz", + "name": "./" + }, + { + "type": "file", + "name": "stdout.out" + }, + { + "type": "file", + "name": "stderr.err" + } + ], + "type": "job", + "userEmail": "algiedi85@gmail.com", + "executionserver": self.executionserver + } + + files = [self.testfile] + res = self.clusterpost.createAndSubmitJob(job, [self.testfile]) + + return True + + def testGetJobsDone(self): + outdir = "/tmp/" + self.clusterpost.getJobsDone(outdir) + + return True diff --git a/README.md b/README.md index 9a2b095..1a8b290 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ DatabaseInteractor ================== -##What is it? +## What is it? This extension contains multiple panels that allow the user to manage data from a CouchDB database stored on a server. It has been developed to work with this [website](https://ec2-52-42-49-63.us-west-2.compute.amazonaws.com:8180/DCBIA-OrthoLab/public/), so the user will need to have an account on the website. The data currently stored on this website is for a pilot study which needs to federate biological, morphological and clinical data. At the end of the development of the website, this should be available for other projects. @@ -12,15 +12,17 @@ Main functionalities are: - Download one or multiple attachments from online database - Upload data stored in a local folder to the database - Manage database architecture +- Create a job based on a Slicer CLI to be run remotely +- Use your Slicer instance as a remote computer -##Technologies used +## Technologies used This extensions uses **requests** library for python (*http://docs.python-requests.org/en/master/*) that ease doing http requests. For more information about the website linked to this project, visit [Github](https://github.com/NIRALUser/shiny-tooth). -##License +## License See License.txt for information on using and contributing. -##Source code +## Source code Find the source code on [Github](https://github.com/DCBIA-OrthoLab/DatabaseInteractorExtension).