From ad926d3ab371c4be702086db6db828d16780bf80 Mon Sep 17 00:00:00 2001 From: phborba Date: Wed, 21 Feb 2024 16:41:54 -0300 Subject: [PATCH 01/23] primeiro commit do refactor do workflow --- DsgTools/core/DSGToolsWorkflow/__init__.py | 0 .../core/DSGToolsWorkflow/workflowItem.py | 156 ++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 DsgTools/core/DSGToolsWorkflow/__init__.py create mode 100644 DsgTools/core/DSGToolsWorkflow/workflowItem.py diff --git a/DsgTools/core/DSGToolsWorkflow/__init__.py b/DsgTools/core/DSGToolsWorkflow/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/DsgTools/core/DSGToolsWorkflow/workflowItem.py b/DsgTools/core/DSGToolsWorkflow/workflowItem.py new file mode 100644 index 000000000..91e44b9a7 --- /dev/null +++ b/DsgTools/core/DSGToolsWorkflow/workflowItem.py @@ -0,0 +1,156 @@ +# -*- coding: utf-8 -*- +""" +/*************************************************************************** + DsgTools + A QGIS plugin + Brazilian Army Cartographic Production Tools + ------------------- + begin : 2024-02-21 + git sha : $Format:%H$ + copyright : (C) 2024 by Philipe Borba - Cartographic Engineer @ Brazilian Army + email : borba.philipe@eb.mil.br + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +""" + +import copy +from dataclasses import asdict, dataclass, field +from typing import Callable, Dict, List +import os, json +from time import time, sleep + +from qgis.core import ( + QgsTask, + QgsProject, + QgsMapLayer, + QgsLayerTreeLayer, + QgsProcessingFeedback, + QgsProcessingModelAlgorithm, + QgsVectorLayer, + QgsProcessingUtils, + QgsProcessingContext, +) +from qgis.PyQt.QtCore import pyqtSignal, QCoreApplication +from qgis.utils import iface +from processing.tools import dataobjects +import processing + +@dataclass +class FlagSettings: + onFlagsRaised: str + modelCanHaveFalsePositiveFlags: bool + loadOutput: bool + flagLayerNames: List[str] = field(default_factory=[]) + +@dataclass +class ModelSource: + type: str + data: str + + @staticmethod + def modelFromFile(filepath): + """ + Initiates a model from a filepath. + :param filepath: (str) filepath for target file. + :return: (QgsProcessingModelAlgorithm) model as a processing algorithm. + """ + alg = QgsProcessingModelAlgorithm() + alg.fromFile(filepath) + alg.initAlgorithm() + return alg + + def modelFromXml(self) -> QgsProcessingModelAlgorithm: + """ + Creates a processing model object from XML text. + :param xml: (str) XML file contents. + :return: (QgsProcessingModelAlgorithm) model as a processing algorithm. + """ + temp = os.path.join( + os.path.dirname(__file__), "temp_model_{0}.model3".format(hash(time())) + ) + with open(temp, "w+", encoding="utf-8") as xmlFile: + xmlFile.write(self.data) + alg = ModelSource.modelFromFile(temp) + os.remove(temp) + return alg + +@dataclass +class Metadata: + originalName: str + +@dataclass +class DSGToolsWorkflowItem: + displayName: str + flags: FlagSettings + pauseAfterExecution: bool + source: ModelSource + metadata: Metadata + + def __post_init__(self): + self.output = { + "result": dict(), + "status": False, + "executionTime": 0.0, + "errorMessage": "Thread not started yet.", + "finishStatus": "initial", + } + self.model = self.getModel() + + def as_dict(self) -> Dict[str, str]: + return {k: str(v) for k, v in asdict(self).items()} + + def getModel(self) -> QgsProcessingModelAlgorithm: + return self.source.modelFromXml() + + def getModelParameters(self) -> List[str]: + if self.model is None: + return [] + return [param.name() for param in self.model.parameterDefinitions()] + + def getFlagNames(self) -> List[str]: + return self.flags.flagLayerNames + + def getOutputFlags(self): + pass + + def getTask(self, feedback: QgsProcessingFeedback) -> QgsTask: + func = self.getTaskRunningFunction(feedback) + on_finished_func = self.getOnFinishedFunction() + return QgsTask.fromFunction( + func, + on_finished=on_finished_func, + ) + + def getTaskRunningFunction(self, feedback: QgsProcessingFeedback) -> Callable: + model = copy.deepcopy(self.model) + modelParameters = self.getModelParameters() + def func(): + context = dataobjects.createContext(feedback=feedback) + out = processing.run( + model, + {param: "memory:" for param in modelParameters}, + feedback=feedback, + context=context, + ) + out.pop("CHILD_INPUTS", None) + out.pop("CHILD_RESULTS", None) + return out + return func + + def getOnFinishedFunction(self): + return + +def load_from_json(input_dict: dict) -> DSGToolsWorkflowItem: + params = copy.deepcopy(input_dict) + params["flags"] = FlagSettings(**params["flags"]) + params["source"] = ModelSource(**params["source"]) + params["metadata"] = Metadata(**params["metadata"]) + return DSGToolsWorkflowItem(**params) From a7648f2fb89b03898a36300bae1da4a667c77807 Mon Sep 17 00:00:00 2001 From: phborba Date: Mon, 4 Mar 2024 17:48:58 -0300 Subject: [PATCH 02/23] end of the day commit --- DsgTools/core/DSGToolsWorkflow/workflow.py | 119 +++++++++++ .../core/DSGToolsWorkflow/workflowItem.py | 194 ++++++++++++++++-- 2 files changed, 296 insertions(+), 17 deletions(-) create mode 100644 DsgTools/core/DSGToolsWorkflow/workflow.py diff --git a/DsgTools/core/DSGToolsWorkflow/workflow.py b/DsgTools/core/DSGToolsWorkflow/workflow.py new file mode 100644 index 000000000..c970c6e8d --- /dev/null +++ b/DsgTools/core/DSGToolsWorkflow/workflow.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +""" +/*************************************************************************** + DsgTools + A QGIS plugin + Brazilian Army Cartographic Production Tools + ------------------- + begin : 2024-03-04 + git sha : $Format:%H$ + copyright : (C) 2024 by Philipe Borba - Cartographic Engineer @ Brazilian Army + email : borba.philipe@eb.mil.br + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +""" + +from dataclasses import dataclass, field +from typing import List + +from qgis.core import (QgsApplication, QgsProcessingFeedback, + QgsProcessingMultiStepFeedback, QgsTask) +from qgis.PyQt.QtCore import pyqtSignal, QObject + +from DsgTools.core.DSGToolsWorkflow.workflowItem import DSGToolsWorkflowItem, ExecutionStatus + + +@dataclass +class WorkflowMetadata: + author: str + version: str + lastModified: str + +@dataclass +class DSGToolsWorkflow(QObject): + workflowItemList: List[DSGToolsWorkflowItem] + displayName: str + metadata: WorkflowMetadata + + def __post_init__(self): + self.currentStepIndex = 0 + self.feedback = QgsProcessingFeedback() + self.multiStepFeedback = QgsProcessingMultiStepFeedback( + len(self.workflowItemList), self.feedback + ) + self.currentWorkflowItemStatusChanged = pyqtSignal(int, DSGToolsWorkflowItem) + self.currentTaskChanged = pyqtSignal(int, QgsTask) + if not self.validateWorkflowItems(): + raise Exception("Invalid workflow") + + def validateWorkflowItems(self): + # TODO + return True + + def getCurrentWorkflowStepIndex(self) -> int: + return self.currentStepIndex + + def getNextWorkflowStep(self) -> int: + nextIndex = self.currentStepIndex + 1 + return nextIndex if nextIndex <= len(self.workflowItemList) - 1 else None + + def getCurrentWorkflowItem(self) -> DSGToolsWorkflowItem: + idx = self.getCurrentWorkflowStepIndex() + return self.workflowItemList[idx] + + def setCurrentWorkflowItem(self, idx) -> None: + if idx < 0 or idx >= len(self.workflowItemList): + return + self.currentStepIndex = idx + + def connectSignals(self) -> None: + for workflowItem in self.workflowItemList: + workflowItem.workflowItemExecutionFinished.connect(self.postProcessWorkflowItem) + + def resetWorkflowItems(self): + for workflowItem in self.workflowItemList: + workflowItem.resetItem() + + def prepareTask(self) -> QgsTask: + currentWorkflowItem: DSGToolsWorkflowItem = self.getCurrentWorkflowItem() + return currentWorkflowItem.getTask(self.feedback) + + def postProcessWorkflowItem(self, workflowItem: DSGToolsWorkflowItem): + self.currentTask = None + self.currentWorkflowItemStatusChanged.emit(self.currentStepIndex, workflowItem) + if workflowItem.getStatus() in [ExecutionStatus.FAILED, ExecutionStatus.FINISHED_WITH_FLAGS, ExecutionStatus.CANCELED]: + self.multiStepFeedback.setCurrentStep(self.currentStepIndex) + return + self.currentStepIndex = self.getNextWorkflowStep() + if self.currentStepIndex is None: + self.multiStepFeedback.setProgress(100) + return + self.run(resumeFromStart=False) + + def run(self, resumeFromStart=True): + if resumeFromStart: + self.resetWorkflowItems() + self.setCurrentWorkflowItem(0) + currentTask: QgsTask = self.prepareTask() + self.multiStepFeedback.setCurrentStep(self.currentStepIndex) + self.currentWorkflowItemStatusChanged.emit(self.currentStepIndex, self.getCurrentWorkflowItem()) + self.currentTaskChanged.emmit(self.currentStepIndex, currentTask) + QgsApplication.taskManager().addTask(currentTask) + + def cancelCurrentRun(self): + currentWorkflowItem = self.getCurrentWorkflowItem() + currentWorkflowItem.cancelCurrentTask() + self.currentWorkflowItemStatusChanged.emit(self.currentStepIndex, currentWorkflowItem) + + def pauseCurrentRun(self): + currentWorkflowItem = self.getCurrentWorkflowItem() + currentWorkflowItem.pauseCurrentTask() + self.currentWorkflowItemStatusChanged.emit(self.currentStepIndex, currentWorkflowItem) \ No newline at end of file diff --git a/DsgTools/core/DSGToolsWorkflow/workflowItem.py b/DsgTools/core/DSGToolsWorkflow/workflowItem.py index 91e44b9a7..e4ed452d7 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflowItem.py +++ b/DsgTools/core/DSGToolsWorkflow/workflowItem.py @@ -23,26 +23,38 @@ import copy from dataclasses import asdict, dataclass, field -from typing import Callable, Dict, List -import os, json -from time import time, sleep +from enum import Enum +from typing import Any, Callable, Dict, List +import os +from time import time from qgis.core import ( QgsTask, QgsProject, QgsMapLayer, - QgsLayerTreeLayer, QgsProcessingFeedback, QgsProcessingModelAlgorithm, QgsVectorLayer, QgsProcessingUtils, QgsProcessingContext, + QgsMessageLog, + Qgis, ) -from qgis.PyQt.QtCore import pyqtSignal, QCoreApplication +from qgis.PyQt.QtCore import pyqtSignal, QObject from qgis.utils import iface from processing.tools import dataobjects import processing + +class ExecutionStatus(Enum): + INITIAL = "initial" + FAILED = "failed" + CANCELED = "canceled" + FINISHED = "finished" + FINISHED_WITH_FLAGS = "finished with flags" + SKIPPED = "skipped" + ON_HOLD = "on hold" + @dataclass class FlagSettings: onFlagsRaised: str @@ -87,7 +99,14 @@ class Metadata: originalName: str @dataclass -class DSGToolsWorkflowItem: +class ModelExecutionOutput: + result: Dict[str, Any] = dict() + executionTime: float = 0.0 + executionMessage: str = "" + status: ExecutionStatus = ExecutionStatus.INITIAL + +@dataclass +class DSGToolsWorkflowItem(QObject): displayName: str flags: FlagSettings pauseAfterExecution: bool @@ -95,14 +114,13 @@ class DSGToolsWorkflowItem: metadata: Metadata def __post_init__(self): - self.output = { - "result": dict(), - "status": False, - "executionTime": 0.0, - "errorMessage": "Thread not started yet.", - "finishStatus": "initial", - } + self.resetItem() self.model = self.getModel() + self.currentTask = None + self.workflowItemExecutionFinished = pyqtSignal(DSGToolsWorkflowItem) + + def resetItem(self): + self.executionOutput = ModelExecutionOutput() def as_dict(self) -> Dict[str, str]: return {k: str(v) for k, v in asdict(self).items()} @@ -123,17 +141,36 @@ def getOutputFlags(self): def getTask(self, feedback: QgsProcessingFeedback) -> QgsTask: func = self.getTaskRunningFunction(feedback) - on_finished_func = self.getOnFinishedFunction() - return QgsTask.fromFunction( + on_finished_func = self.getOnFinishedFunction(feedback) + self.currentTask = QgsTask.fromFunction( func, on_finished=on_finished_func, ) + return self.currentTask + + def cancelCurrentTask(self): + if self.currentTask is None: + return + self.currentTask.cancel() + self.currentTask = None + + def pauseCurrentTask(self): + if self.currentTask is None: + return + self.currentTask.hold() + + def resumeCurrentTask(self): + if self.currentTask is None: + return + self.currentTask.unhold() def getTaskRunningFunction(self, feedback: QgsProcessingFeedback) -> Callable: model = copy.deepcopy(self.model) modelParameters = self.getModelParameters() def func(): + start = time() context = dataobjects.createContext(feedback=feedback) + context.setProject(QgsProject.instance()) out = processing.run( model, {param: "memory:" for param in modelParameters}, @@ -142,11 +179,134 @@ def func(): ) out.pop("CHILD_INPUTS", None) out.pop("CHILD_RESULTS", None) + out["start_time"] = start return out return func - def getOnFinishedFunction(self): - return + def getOnFinishedFunction(self, feedback: QgsProcessingFeedback) -> Callable: + def on_finished_func(exception, result=None): + if exception is not None: + QgsMessageLog.logMessage( + f"Exception: {exception}", + "DSGTools Plugin", + Qgis.Critical + ) + self.executionOutput = ModelExecutionOutput( + executionMessage=self.tr(f"Model execution has failed:\n {str(exception)}"), + status=ExecutionStatus.FAILED, + ) + self.workflowItemExecutionFinished.emit(self) + return + if result is not None: + self.handleOutputs(result, feedback) + self.loadOutputs(feedback) + else: + self.executionOutput = ModelExecutionOutput( + executionMessage=self.tr(f"Model execution was canceled by the user."), + status=ExecutionStatus.CANCELED, + ) + self.workflowItemExecutionFinished.emit(self) + self.currentTask = None + + return on_finished_func + + def handleOutputs(self, result, feedback): + start = result.pop("start_time") + context = dataobjects.createContext(feedback=feedback) + context.setProject(QgsProject.instance()) + for paramName, vl in result.items(): + baseName = paramName.rsplit(":", 1)[-1] + name = baseName + idx = 1 + while name in self.executionOutput.result: + name = "{0} ({1})".format(baseName, idx) + idx += 1 + if isinstance(vl, str): + vl = QgsProcessingUtils.mapLayerFromString(vl, context) + vl.setName(name) + self.executionOutput.result[name] = vl + self.executionOutput.executionTime = time() - start + self.executionOutput.executionMessage = self.tr("Model execution finished.") + + def loadOutput(self) -> bool: + return self.flags.loadOutput + + def getStatus(self) -> ExecutionStatus: + return self.executionOutput.status + + def loadOutputs(self, feedback): + loadOutput = self.loadOutput() + if not loadOutput: + return + flagLayerNames = self.flagLayerNames() + context = QgsProcessingContext() + iface.mapCanvas().freeze(True) + for name, vl in self.executionOutput.result.items(): + if vl is None: + continue + if vl.name() not in flagLayerNames and not loadOutput: + continue + if isinstance(vl, str): + vl = QgsProcessingUtils.mapLayerFromString(vl, context) + self.executionOutput.result[name] = vl + if not isinstance(vl, QgsMapLayer) or not vl.isValid(): + continue + vl.setName(name.split(":", 2)[-1]) + if vl.name() in flagLayerNames and vl.featureCount() == 0: + continue + cloneVl = vl.clone() + self.executionOutput.result[name] = cloneVl + self.addLayerToGroup(cloneVl, self.displayName(), clearGroupBeforeAdding=True) + self.enableFeatureCount(cloneVl) + iface.mapCanvas().freeze(False) + + def addLayerToGroup( + self, layer, subgroupname, clearGroupBeforeAdding=False + ): + """ + Adds a layer to a group into layer panel. + :param layer: (QgsMapLayer) layer to be added to canvas. + :param subgroupname: (str) name for the subgroup to be added. + """ + subGroup = self.createGroups(subgroupname) + if clearGroupBeforeAdding: + self.clearGroup(subGroup) + layer = ( + layer + if isinstance(layer, QgsMapLayer) + else QgsProcessingUtils.mapLayerFromString(layer) + ) + QgsProject.instance().addMapLayer(layer, addToLegend=False) + subGroup.addLayer(layer) + + def createGroups(self, subgroupname): + rootNode = QgsProject.instance().layerTreeRoot() + parentGroupName = "DSGTools_QA_Toolbox" + parentGroupNode = rootNode.findGroup(parentGroupName) + parentGroupNode = ( + parentGroupNode + if parentGroupNode + else rootNode.insertGroup(0, parentGroupName) + ) + subGroup = self.createGroup(subgroupname, parentGroupNode) + return subGroup + + def createGroup(self, groupName, rootNode): + groupNode = rootNode.findGroup(groupName) + return groupNode if groupNode else rootNode.addGroup(groupName) + + def prepareGroup(self, model): + subGroup = self.createGroups( + self.model().model.displayName() + ) + self.clearGroup(subGroup) + + def clearGroup(self, group): + for lyrGroup in group.findLayers(): + lyr = lyrGroup.layer() + if isinstance(lyr, QgsVectorLayer): + lyr.rollBack() + group.removeAllChildren() def load_from_json(input_dict: dict) -> DSGToolsWorkflowItem: params = copy.deepcopy(input_dict) From 5f0f689bde2d91661662736945384093a80142e0 Mon Sep 17 00:00:00 2001 From: phborba Date: Tue, 5 Mar 2024 11:40:34 -0300 Subject: [PATCH 03/23] improve signals --- DsgTools/core/DSGToolsWorkflow/workflow.py | 16 +++++++++++++--- DsgTools/core/DSGToolsWorkflow/workflowItem.py | 5 +++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/DsgTools/core/DSGToolsWorkflow/workflow.py b/DsgTools/core/DSGToolsWorkflow/workflow.py index c970c6e8d..b6349e6f4 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflow.py +++ b/DsgTools/core/DSGToolsWorkflow/workflow.py @@ -50,6 +50,7 @@ def __post_init__(self): len(self.workflowItemList), self.feedback ) self.currentWorkflowItemStatusChanged = pyqtSignal(int, DSGToolsWorkflowItem) + self.workflowHasBeenReset = pyqtSignal() self.currentTaskChanged = pyqtSignal(int, QgsTask) if not self.validateWorkflowItems(): raise Exception("Invalid workflow") @@ -102,18 +103,27 @@ def run(self, resumeFromStart=True): if resumeFromStart: self.resetWorkflowItems() self.setCurrentWorkflowItem(0) + self.workflowHasBeenReset.emit() currentTask: QgsTask = self.prepareTask() + currentWorkflowItem = self.getCurrentWorkflowItem() + currentWorkflowItem.changeCurrentStatus(status=ExecutionStatus.RUNNING, executionMessage=self.tr("Execution started")) self.multiStepFeedback.setCurrentStep(self.currentStepIndex) - self.currentWorkflowItemStatusChanged.emit(self.currentStepIndex, self.getCurrentWorkflowItem()) - self.currentTaskChanged.emmit(self.currentStepIndex, currentTask) + self.currentWorkflowItemStatusChanged.emit(self.currentStepIndex, currentWorkflowItem) + self.currentTaskChanged.emit(self.currentStepIndex, currentTask) QgsApplication.taskManager().addTask(currentTask) def cancelCurrentRun(self): currentWorkflowItem = self.getCurrentWorkflowItem() currentWorkflowItem.cancelCurrentTask() + self.multiStepFeedback.setCurrentStep(self.currentStepIndex) self.currentWorkflowItemStatusChanged.emit(self.currentStepIndex, currentWorkflowItem) def pauseCurrentRun(self): currentWorkflowItem = self.getCurrentWorkflowItem() currentWorkflowItem.pauseCurrentTask() - self.currentWorkflowItemStatusChanged.emit(self.currentStepIndex, currentWorkflowItem) \ No newline at end of file + self.currentWorkflowItemStatusChanged.emit(self.currentStepIndex, currentWorkflowItem) + + def resumeCurrentRun(self): + currentWorkflowItem = self.getCurrentWorkflowItem() + currentWorkflowItem.pauseCurrentTask() + self.currentWorkflowItemStatusChanged.emit(self.currentStepIndex, currentWorkflowItem) diff --git a/DsgTools/core/DSGToolsWorkflow/workflowItem.py b/DsgTools/core/DSGToolsWorkflow/workflowItem.py index e4ed452d7..c0c3b8077 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflowItem.py +++ b/DsgTools/core/DSGToolsWorkflow/workflowItem.py @@ -48,6 +48,7 @@ class ExecutionStatus(Enum): INITIAL = "initial" + RUNNING = "running" FAILED = "failed" CANCELED = "canceled" FINISHED = "finished" @@ -147,6 +148,10 @@ def getTask(self, feedback: QgsProcessingFeedback) -> QgsTask: on_finished=on_finished_func, ) return self.currentTask + + def changeCurrentStatus(self, status: ExecutionStatus, executionMessage: str) -> None: + self.executionOutput.status = status + self.executionOutput.executionMessage = executionMessage def cancelCurrentTask(self): if self.currentTask is None: From 6a1516542edb4ab0f97881efa082ad0732f8eb74 Mon Sep 17 00:00:00 2001 From: phborba Date: Tue, 5 Mar 2024 16:57:54 -0300 Subject: [PATCH 04/23] adding pause after execution --- DsgTools/core/DSGToolsWorkflow/workflow.py | 27 ++++++++++++-- .../core/DSGToolsWorkflow/workflowItem.py | 37 ++++++++++++++++--- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/DsgTools/core/DSGToolsWorkflow/workflow.py b/DsgTools/core/DSGToolsWorkflow/workflow.py index b6349e6f4..ba74258cb 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflow.py +++ b/DsgTools/core/DSGToolsWorkflow/workflow.py @@ -21,8 +21,8 @@ ***************************************************************************/ """ -from dataclasses import dataclass, field -from typing import List +from dataclasses import asdict, dataclass, field +from typing import Dict, List from qgis.core import (QgsApplication, QgsProcessingFeedback, QgsProcessingMultiStepFeedback, QgsTask) @@ -39,9 +39,9 @@ class WorkflowMetadata: @dataclass class DSGToolsWorkflow(QObject): - workflowItemList: List[DSGToolsWorkflowItem] displayName: str metadata: WorkflowMetadata + workflowItemList: List[DSGToolsWorkflowItem] def __post_init__(self): self.currentStepIndex = 0 @@ -51,10 +51,14 @@ def __post_init__(self): ) self.currentWorkflowItemStatusChanged = pyqtSignal(int, DSGToolsWorkflowItem) self.workflowHasBeenReset = pyqtSignal() + self.workflowPaused = pyqtSignal() self.currentTaskChanged = pyqtSignal(int, QgsTask) if not self.validateWorkflowItems(): raise Exception("Invalid workflow") + def as_dict(self) -> Dict[str, str]: + return {k: v for k, v in asdict(self).items()} + def validateWorkflowItems(self): # TODO return True @@ -97,6 +101,11 @@ def postProcessWorkflowItem(self, workflowItem: DSGToolsWorkflowItem): if self.currentStepIndex is None: self.multiStepFeedback.setProgress(100) return + if workflowItem.pauseAfterExecution: + currentWorkflowItem = self.getCurrentWorkflowItem() + currentWorkflowItem.pauseBeforeRunning() + self.currentWorkflowItemStatusChanged.emit(self.currentStepIndex, currentWorkflowItem) + return self.run(resumeFromStart=False) def run(self, resumeFromStart=True): @@ -104,14 +113,24 @@ def run(self, resumeFromStart=True): self.resetWorkflowItems() self.setCurrentWorkflowItem(0) self.workflowHasBeenReset.emit() - currentTask: QgsTask = self.prepareTask() currentWorkflowItem = self.getCurrentWorkflowItem() + if currentWorkflowItem.getStatus() == ExecutionStatus.IGNORE_FLAGS: + self.currentStepIndex = self.getNextWorkflowStep() + if self.currentStepIndex is None: + self.multiStepFeedback.setProgress(100) + return + currentWorkflowItem = self.getCurrentWorkflowItem() + currentTask: QgsTask = self.prepareTask() currentWorkflowItem.changeCurrentStatus(status=ExecutionStatus.RUNNING, executionMessage=self.tr("Execution started")) self.multiStepFeedback.setCurrentStep(self.currentStepIndex) self.currentWorkflowItemStatusChanged.emit(self.currentStepIndex, currentWorkflowItem) self.currentTaskChanged.emit(self.currentStepIndex, currentTask) QgsApplication.taskManager().addTask(currentTask) + def setIgnoreFlagsStatusOnCurrentStep(self): + currentWorkflowItem = self.getCurrentWorkflowItem() + currentWorkflowItem.setCurrentStateToIgnoreFlags() + def cancelCurrentRun(self): currentWorkflowItem = self.getCurrentWorkflowItem() currentWorkflowItem.cancelCurrentTask() diff --git a/DsgTools/core/DSGToolsWorkflow/workflowItem.py b/DsgTools/core/DSGToolsWorkflow/workflowItem.py index c0c3b8077..e3094c48c 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflowItem.py +++ b/DsgTools/core/DSGToolsWorkflow/workflowItem.py @@ -53,8 +53,9 @@ class ExecutionStatus(Enum): CANCELED = "canceled" FINISHED = "finished" FINISHED_WITH_FLAGS = "finished with flags" - SKIPPED = "skipped" ON_HOLD = "on hold" + PAUSED_BEFORE_RUNNING = "paused before running" + IGNORE_FLAGS = "ignore flags" @dataclass class FlagSettings: @@ -63,6 +64,10 @@ class FlagSettings: loadOutput: bool flagLayerNames: List[str] = field(default_factory=[]) + def __post_init__(self): + if self.onFlagsRaised not in ("halt", "warn", "ignore"): + raise ValueError("Invalid on flags raised flag.") + @dataclass class ModelSource: type: str @@ -101,7 +106,7 @@ class Metadata: @dataclass class ModelExecutionOutput: - result: Dict[str, Any] = dict() + result: Dict[str, Any] = field(default_factory=dict) executionTime: float = 0.0 executionMessage: str = "" status: ExecutionStatus = ExecutionStatus.INITIAL @@ -124,7 +129,7 @@ def resetItem(self): self.executionOutput = ModelExecutionOutput() def as_dict(self) -> Dict[str, str]: - return {k: str(v) for k, v in asdict(self).items()} + return {k: v for k, v in asdict(self).items()} def getModel(self) -> QgsProcessingModelAlgorithm: return self.source.modelFromXml() @@ -149,6 +154,21 @@ def getTask(self, feedback: QgsProcessingFeedback) -> QgsTask: ) return self.currentTask + def pauseBeforeRunning(self): + self.executionOutput = ModelExecutionOutput( + executionMessage=self.tr(f"Workflow item {self.displayName} execution paused by previous step."), + status=ExecutionStatus.PAUSED_BEFORE_RUNNING, + ) + + def setCurrentStateToIgnoreFlags(self): + if not self.flags.modelCanHaveFalsePositiveFlags: + return + self.changeCurrentStatus( + status=ExecutionStatus.IGNORE_FLAGS, + executionMessage=self.tr(f"Workflow item {self.displayName} flags were ignored by the user.") + ) + # não emite sinal pois esse passo é feito fora da execução. + def changeCurrentStatus(self, status: ExecutionStatus, executionMessage: str) -> None: self.executionOutput.status = status self.executionOutput.executionMessage = executionMessage @@ -197,7 +217,7 @@ def on_finished_func(exception, result=None): Qgis.Critical ) self.executionOutput = ModelExecutionOutput( - executionMessage=self.tr(f"Model execution has failed:\n {str(exception)}"), + executionMessage=self.tr(f"Workflow item {self.displayName} execution has failed:\n {str(exception)}"), status=ExecutionStatus.FAILED, ) self.workflowItemExecutionFinished.emit(self) @@ -205,9 +225,15 @@ def on_finished_func(exception, result=None): if result is not None: self.handleOutputs(result, feedback) self.loadOutputs(feedback) + status = ExecutionStatus.FINISHED_WITH_FLAGS if any(lyr.featureCount() > 0 for k, lyr in self.executionOutput.result.items() if lyr.name() in self.flagLayerNames()) else ExecutionStatus.FINISHED + statusMsg = self.tr("finished with flags.") if status == ExecutionStatus.FINISHED_WITH_FLAGS else self.tr("finished.") + self.changeCurrentStatus( + status=status, + executionMessage=self.tr(f"Workflow item {self.displayName} {statusMsg}") + ) else: self.executionOutput = ModelExecutionOutput( - executionMessage=self.tr(f"Model execution was canceled by the user."), + executionMessage=self.tr(f"Workflow item {self.displayName} execution was canceled by the user."), status=ExecutionStatus.CANCELED, ) self.workflowItemExecutionFinished.emit(self) @@ -231,7 +257,6 @@ def handleOutputs(self, result, feedback): vl.setName(name) self.executionOutput.result[name] = vl self.executionOutput.executionTime = time() - start - self.executionOutput.executionMessage = self.tr("Model execution finished.") def loadOutput(self) -> bool: return self.flags.loadOutput From 716494c49b09f8df03cee3d5c0026d20406faa66 Mon Sep 17 00:00:00 2001 From: phborba Date: Tue, 5 Mar 2024 16:58:52 -0300 Subject: [PATCH 05/23] fix --- DsgTools/core/DSGToolsWorkflow/workflow.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/DsgTools/core/DSGToolsWorkflow/workflow.py b/DsgTools/core/DSGToolsWorkflow/workflow.py index ba74258cb..2fa8264ba 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflow.py +++ b/DsgTools/core/DSGToolsWorkflow/workflow.py @@ -53,16 +53,10 @@ def __post_init__(self): self.workflowHasBeenReset = pyqtSignal() self.workflowPaused = pyqtSignal() self.currentTaskChanged = pyqtSignal(int, QgsTask) - if not self.validateWorkflowItems(): - raise Exception("Invalid workflow") def as_dict(self) -> Dict[str, str]: return {k: v for k, v in asdict(self).items()} - def validateWorkflowItems(self): - # TODO - return True - def getCurrentWorkflowStepIndex(self) -> int: return self.currentStepIndex From 39ca7e8c229f79f028bce83b85665fdef4bf61c5 Mon Sep 17 00:00:00 2001 From: phborba Date: Wed, 6 Mar 2024 11:58:48 -0300 Subject: [PATCH 06/23] fix --- DsgTools/core/DSGToolsWorkflow/workflow.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/DsgTools/core/DSGToolsWorkflow/workflow.py b/DsgTools/core/DSGToolsWorkflow/workflow.py index 2fa8264ba..2ccf2ee9a 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflow.py +++ b/DsgTools/core/DSGToolsWorkflow/workflow.py @@ -22,7 +22,7 @@ """ from dataclasses import asdict, dataclass, field -from typing import Dict, List +from typing import Any, Dict, List from qgis.core import (QgsApplication, QgsProcessingFeedback, QgsProcessingMultiStepFeedback, QgsTask) @@ -54,7 +54,7 @@ def __post_init__(self): self.workflowPaused = pyqtSignal() self.currentTaskChanged = pyqtSignal(int, QgsTask) - def as_dict(self) -> Dict[str, str]: + def as_dict(self) -> Dict[str, Any]: return {k: v for k, v in asdict(self).items()} def getCurrentWorkflowStepIndex(self) -> int: From c384432ee60d3ce63d78211c9f175cb41748c94a Mon Sep 17 00:00:00 2001 From: phborba Date: Wed, 6 Mar 2024 17:28:40 -0300 Subject: [PATCH 07/23] more changes --- .../qualityAssuranceDockWidget.py | 298 +++--------------- 1 file changed, 45 insertions(+), 253 deletions(-) diff --git a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py index 3f392d3e2..e5403f799 100644 --- a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py +++ b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py @@ -54,6 +54,7 @@ from DsgTools.core.DSGToolsProcessingAlgs.Models.qualityAssuranceWorkflow import ( QualityAssuranceWorkflow, ) +from DsgTools.core.DSGToolsWorkflow.workflowItem import ExecutionStatus FORM_CLASS, _ = uic.loadUiType( @@ -63,17 +64,6 @@ class QualityAssuranceDockWidget(QDockWidget, FORM_CLASS): # current execution status - ( - INITIAL, - RUNNING, - PAUSED, - HALTED, - CANCELED, - FAILED, - FINISHED, - FINISHED_WITH_FLAGS, - IGNORE_FLAGS, - ) = range(9) def __init__(self, iface, parent=None): """ @@ -91,47 +81,47 @@ def __init__(self, iface, parent=None): self._showButtons = True self.parent = parent self.statusMap = { - self.INITIAL: self.tr("Not yet run"), - self.RUNNING: self.tr("Running..."), - self.PAUSED: self.tr("On hold. Check data and resume."), - self.HALTED: self.tr("Halted on flags"), - self.CANCELED: self.tr("Canceled"), - self.FAILED: self.tr("Failed"), - self.FINISHED: self.tr("Completed"), - self.FINISHED_WITH_FLAGS: self.tr("Completed (raised flags)"), - self.IGNORE_FLAGS: self.tr("Completed (false positive flags)"), + ExecutionStatus.INITIAL: self.tr("Not yet run"), + ExecutionStatus.RUNNING: self.tr("Running..."), + ExecutionStatus.PAUSED_BEFORE_RUNNING: self.tr("On hold. Check data and resume."), + ExecutionStatus.ON_HOLD: self.tr("Task paused."), + ExecutionStatus.CANCELED: self.tr("Canceled"), + ExecutionStatus.FAILED: self.tr("Failed"), + ExecutionStatus.FINISHED: self.tr("Completed"), + ExecutionStatus.FINISHED_WITH_FLAGS: self.tr("Completed (raised flags)"), + ExecutionStatus.IGNORE_FLAGS: self.tr("Completed (false positive flags)"), } self.colorForeground = { - self.INITIAL: (0, 0, 0), - self.RUNNING: (0, 0, 125), - self.PAUSED: (187, 201, 25), - self.HALTED: (187, 201, 25), - self.CANCELED: (200, 0, 0), - self.FAILED: (169, 18, 28), - self.FINISHED: (0, 125, 0), - self.FINISHED_WITH_FLAGS: (100, 150, 20), - self.IGNORE_FLAGS: (0, 0, 0), + ExecutionStatus.INITIAL: (0, 0, 0), + ExecutionStatus.RUNNING: (0, 0, 125), + ExecutionStatus.PAUSED_BEFORE_RUNNING: (187, 201, 25), + ExecutionStatus.ON_HOLD: (187, 201, 25), + ExecutionStatus.CANCELED: (200, 0, 0), + ExecutionStatus.FAILED: (169, 18, 28), + ExecutionStatus.FINISHED: (0, 125, 0), + ExecutionStatus.FINISHED_WITH_FLAGS: (100, 150, 20), + ExecutionStatus.IGNORE_FLAGS: (0, 0, 0), } self.colorBackground = { - self.INITIAL: (255, 255, 255, 75), - self.RUNNING: (0, 0, 125, 90), - self.PAUSED: (187, 201, 25, 20), - self.HALTED: (200, 215, 40, 20), - self.CANCELED: (200, 0, 0, 85), - self.FAILED: (169, 18, 28, 85), - self.FINISHED: (0, 125, 0, 90), - self.FINISHED_WITH_FLAGS: (100, 150, 20, 45), - self.IGNORE_FLAGS: (255, 230, 1), + ExecutionStatus.INITIAL: (255, 255, 255, 75), + ExecutionStatus.RUNNING: (0, 0, 125, 90), + ExecutionStatus.PAUSED_BEFORE_RUNNING: (187, 201, 25, 20), + ExecutionStatus.ON_HOLD: (200, 215, 40, 20), + ExecutionStatus.CANCELED: (200, 0, 0, 85), + ExecutionStatus.FAILED: (169, 18, 28, 85), + ExecutionStatus.FINISHED: (0, 125, 0, 90), + ExecutionStatus.FINISHED_WITH_FLAGS: (100, 150, 20, 45), + ExecutionStatus.IGNORE_FLAGS: (255, 230, 1), } self.qgisStatusDict = { - self.RUNNING: Qgis.Info, - self.PAUSED: Qgis.Info, - self.HALTED: Qgis.Critical, - self.CANCELED: Qgis.Warning, - self.FAILED: Qgis.Critical, - self.FINISHED: Qgis.Info, - self.FINISHED_WITH_FLAGS: Qgis.Warning, - self.IGNORE_FLAGS: Qgis.Warning, + ExecutionStatus.RUNNING: Qgis.Info, + ExecutionStatus.PAUSED_BEFORE_RUNNING: Qgis.Info, + ExecutionStatus.ON_HOLD: Qgis.Critical, + ExecutionStatus.CANCELED: Qgis.Warning, + ExecutionStatus.FAILED: Qgis.Critical, + ExecutionStatus.FINISHED: Qgis.Info, + ExecutionStatus.FINISHED_WITH_FLAGS: Qgis.Warning, + ExecutionStatus.IGNORE_FLAGS: Qgis.Warning, } self.workflowStatusDict = defaultdict(OrderedDict) self.ignoreFlagsMenuDict = defaultdict(dict) @@ -140,13 +130,13 @@ def __init__(self, iface, parent=None): self.workflows = dict() self.resetComboBox() self.resetTable() - self.loadState() + # self.loadState() self.prepareProgressBar() # make sure workflows are loaded as per project instances - QgsProject.instance().projectSaved.connect(self.saveState) + # QgsProject.instance().projectSaved.connect(self.saveState) # self.iface.newProjectCreated.connect(self.saveState) # self.iface.newProjectCreated.connect(self.loadState) - self.iface.projectRead.connect(self.loadState) + # self.iface.projectRead.connect(self.loadState) def generateMenu(self, pos, idx, widget, modelName, workflow): workflowName = workflow.name() @@ -465,7 +455,7 @@ def currentWorkflow(self): :return: (QualityAssuranceWorkflow) current workflow. """ name = self.currentWorkflowName() - return self.workflows[name] if name in self.workflows else None + return self.workflows.get(name, None) def setRowColor(self, row, backgroundColor, foregroundColor): """ @@ -604,34 +594,21 @@ def setWorkflow(self, workflow): ) ) self.tableWidget.setCellWidget(row, 1, statusWidget) - code = currentStatusDict.get(modelName, self.INITIAL) + code = currentStatusDict.get(modelName, ExecutionStatus.INITIAL) self.setModelStatus(row, code, modelName) pb = self.progressWidget( value=100 if code in [ - self.FINISHED, - self.FINISHED_WITH_FLAGS, - self.IGNORE_FLAGS, + ExecutionStatus.FINISHED, + ExecutionStatus.FINISHED_WITH_FLAGS, + ExecutionStatus.IGNORE_FLAGS, ] else 0 ) pb.setContextMenuPolicy(Qt.CustomContextMenu) self.tableWidget.setCellWidget(row, 2, pb) - def preProcessing(self, firstModel=None): - """ - Clears all progresses and set status to intial state. - :param firstModel: (str) first model to be run. - """ - isAfter = False - for row in range(self.tableWidget.rowCount()): - modelName = self.tableWidget.cellWidget(row, 0).text() - if firstModel is not None and modelName != firstModel and not isAfter: - continue - isAfter = True - self.setModelStatus(row, self.INITIAL, modelName) - self.tableWidget.cellWidget(row, 2).setValue(0) def saveState(self): """ @@ -718,191 +695,6 @@ def runWorkflow(self): # these methods are defined locally as they are not supposed to be # outside thread execution setup and should all be handled from # within this method - at runtime - def refreshFeedback(): - # refresh feedback track to not carry "bias" to next execution - workflow.feedback.progressChanged.disconnect(self.setProgress) - del workflow.feedback - workflow.feedback = QgsProcessingFeedback() - workflow.feedback.progressChanged.connect(self.setProgress) - - def intWrapper(pb, v): - pb.setValue(int(v)) - - def statusChangedWrapper(row, model, status): - """status: (QgsTask.Enum) status enum""" - if row is None: - for row in range(self.tableWidget.rowCount()): - if self.tableWidget.cellWidget(row, 0).text() == model.name(): - break - code = { - model.Queued: self.INITIAL, - model.OnHold: self.PAUSED, - model.Running: self.RUNNING, - model.Complete: self.FINISHED, - model.Terminated: self.FAILED, - model.WarningFlags: self.HALTED, - model.HaltedOnFlags: self.FINISHED_WITH_FLAGS, - model.HaltedOnPossibleFalsePositiveFlags: self.IGNORE_FLAGS, - }[status] - if ( - status == model.Complete - and model.output.get("finishStatus", None) == "halt" - ): - code = self.FINISHED_WITH_FLAGS - if ( - status == model.Terminated - and model.output.get("finishStatus", None) != "halt" - ): - if self.__workflowCanceled: - code = self.CANCELED - # if workflow was canceled (through the cancel push button), - # workflowFinished signal will not be emited... - postProcessing() - self.__workflowCanceled = False - if code != self.INITIAL: - self.setModelStatus(row, code, model.displayName(), raiseMessage=True) - - def begin(model): - for row in range(self.tableWidget.rowCount()): - if self.tableWidget.cellWidget(row, 0).text() != model.name(): - continue - self.__progressFunc = partial( - intWrapper, self.tableWidget.cellWidget(row, 2) - ) - model.feedback.progressChanged.connect(self.__progressFunc) - self.__statusFunc = partial(statusChangedWrapper, row, model) - model.statusChanged.connect(self.__statusFunc) - self.setModelStatus(row, self.RUNNING, model.displayName()) - return - - def end(model): - for row in range(self.tableWidget.rowCount()): - if self.tableWidget.cellWidget(row, 0).text() != model.name(): - continue - model.feedback.progressChanged.disconnect(self.__progressFunc) - model.statusChanged.disconnect(self.__statusFunc) - return - - def pause(model): - for row in range(self.tableWidget.rowCount()): - if self.tableWidget.cellWidget(row, 0).text() != model.name(): - continue - self.setModelStatus(row, self.PAUSED, model.displayName()) - postProcessing() - - def stopOnFlags(model): - refreshFeedback() - isAfter = False - for row in range(self.tableWidget.rowCount()): - if ( - self.tableWidget.cellWidget(row, 0).text() != model.name() - and not isAfter - ): - continue - if isAfter: - code = self.INITIAL - self.tableWidget.cellWidget(row, 2).setValue(0) - else: - model.feedback.progressChanged.disconnect(self.__progressFunc) - model.statusChanged.disconnect(self.__statusFunc) - code = self.FINISHED_WITH_FLAGS - isAfter = True - self.setModelStatus(row, code, model.displayName()) - postProcessing() - - def warningFlags(model): - for row in range(self.tableWidget.rowCount()): - if self.tableWidget.cellWidget(row, 0).text() != model.name(): - continue - model.feedback.progressChanged.disconnect(self.__progressFunc) - model.statusChanged.disconnect(self.__statusFunc) - self.setModelStatus(row, self.FINISHED_WITH_FLAGS, model.displayName()) - return - - def postProcessing(): - """ - When workflow finishes, its signals are kept connected and that - might cause missbehaviour on next executions. - """ - workflow.modelStarted.disconnect(begin) - workflow.modelFinished.disconnect(end) - workflow.haltedOnFlags.disconnect(stopOnFlags) - workflow.modelFinishedWithFlags.disconnect(warningFlags) - workflow.workflowFinished.disconnect(postProcessing) - workflow.workflowPaused.disconnect(pause) - refreshFeedback() - self.setGuiState(False) - self.pausePushButton.show() - self.continuePushButton.hide() - for m in workflow.validModels().values(): - if m.hasFlags(): - msg = self.tr("workflow {0} finished with flags.") - lvl = Qgis.Warning - break - else: - msg = self.tr("workflow {0} finished.") - lvl = Qgis.Success - self.iface.messageBar().pushMessage( - self.tr("DSGTools Q&A Toolbox"), - msg.format(workflow.displayName()), - lvl, - duration=3, - ) - - sender = self.sender() - isFirstModel = sender is None or sender.objectName() == "runPushButton" - idx, lastModelDisplayName = workflow.lastModelName(returnIdx=True) - if idx != 0 and sender.objectName() == "runPushButton": - if not self.confirmAction(msg=self.tr("The workflow has already started running. Would you like to start over?"), showCancel=True): - refreshFeedback() - self.setGuiState(False) - self.pausePushButton.show() - self.continuePushButton.hide() - return - workflow.modelStarted.connect(begin) - workflow.modelFinished.connect(end) - workflow.haltedOnFlags.connect(stopOnFlags) - workflow.modelFinishedWithFlags.connect(warningFlags) - workflow.workflowFinished.connect(postProcessing) - workflow.workflowPaused.connect(pause) - self.prepareOutputTreeNodes( - lastModelDisplayName=lastModelDisplayName, - clearBeforeRunning=True, - ) - self.preProcessing(firstModel=None if isFirstModel else lastModelDisplayName) - workflow.run(firstModelName=None if isFirstModel else lastModelDisplayName) - - def prepareOutputTreeNodes(self, lastModelDisplayName, clearBeforeRunning=False): - self.iface.mapCanvas().freeze(True) - rootNode = QgsProject.instance().layerTreeRoot() - parentGroupName = "DSGTools_QA_Toolbox" - parentGroupNode = rootNode.findGroup(parentGroupName) - parentGroupNode = ( - parentGroupNode - if parentGroupNode - else rootNode.insertGroup(0, parentGroupName) - ) - if lastModelDisplayName is None: - return parentGroupName - lyrsToRemoveIds = [] - lastModelGroup = parentGroupNode.findGroup(lastModelDisplayName) - if lastModelGroup is None: - self.iface.mapCanvas().freeze(False) - return parentGroupName - if not clearBeforeRunning: - self.iface.mapCanvas().freeze(False) - return parentGroupName - for lyrGroup in lastModelGroup.findLayers(): - lyr = lyrGroup.layer() - if isinstance(lyr, QgsVectorLayer): - lyr.rollBack() - lyrsToRemoveIds.append(lyr.id()) - for lyrId in lyrsToRemoveIds: - QgsProject.instance().removeMapLayer(lyrId) - if len(lastModelGroup.children()) == 0: - parentGroupNode.removeChildrenGroupWithoutLayers() - self.iface.mapCanvas().freeze(False) - return parentGroupName @pyqtSlot(bool, name="on_importPushButton_clicked") def importWorkflow(self): @@ -989,7 +781,7 @@ def workflowIsFinished(self, modelName=None) -> bool: if modelName not in self.workflowStatusDict: return False return all( - value in (self.FINISHED, self.IGNORE_FLAGS) + value in (ExecutionStatus.FINISHED, ExecutionStatus.IGNORE_FLAGS) for _, value in self.workflowStatusDict[modelName].items() ) From ddb6c1f78bbc1c35a3a0ff90a83c1b85fa04d0ca Mon Sep 17 00:00:00 2001 From: phborba Date: Thu, 7 Mar 2024 21:20:04 -0300 Subject: [PATCH 08/23] format files --- .../loadThemesAlgorithm.py | 2 +- ...tifyUncoveredStartAndEndPointsAlgorithm.py | 1 - DsgTools/core/DSGToolsWorkflow/workflow.py | 69 +++++++++----- .../core/DSGToolsWorkflow/workflowItem.py | 90 ++++++++++++------- DsgTools/dsg_tools.py | 1 + .../qualityAssuranceDockWidget.py | 5 +- .../Toolboxes/toolBoxesGuiManager.py | 6 +- 7 files changed, 117 insertions(+), 57 deletions(-) diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Algs/LayerManagementAlgs/loadThemesAlgorithm.py b/DsgTools/core/DSGToolsProcessingAlgs/Algs/LayerManagementAlgs/loadThemesAlgorithm.py index 3e3ff4c8b..0945c65d1 100644 --- a/DsgTools/core/DSGToolsProcessingAlgs/Algs/LayerManagementAlgs/loadThemesAlgorithm.py +++ b/DsgTools/core/DSGToolsProcessingAlgs/Algs/LayerManagementAlgs/loadThemesAlgorithm.py @@ -88,7 +88,7 @@ def processAlgorithm(self, parameters, context, feedback): elif len(inputJSONData) > 0: self.loadThemes(inputJSONData) return {self.OUTPUT: []} - + def loadExpressionFieldFromJSONFile(self, inputJSONFile): with open(inputJSONFile, "r") as f: inputJSONData = json.load(f) diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/identifyUncoveredStartAndEndPointsAlgorithm.py b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/identifyUncoveredStartAndEndPointsAlgorithm.py index 8133b5244..66ed11cae 100644 --- a/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/identifyUncoveredStartAndEndPointsAlgorithm.py +++ b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/identifyUncoveredStartAndEndPointsAlgorithm.py @@ -123,7 +123,6 @@ def processAlgorithm(self, parameters, context, feedback): parameters, self.GEOGRAPHIC_BOUNDARY, context ) - nSteps = 9 if geographicBoundaryLyr is None else 10 multiStepFeedback = QgsProcessingMultiStepFeedback(nSteps, feedback) currentStep = 0 diff --git a/DsgTools/core/DSGToolsWorkflow/workflow.py b/DsgTools/core/DSGToolsWorkflow/workflow.py index 2ccf2ee9a..21c5960d1 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflow.py +++ b/DsgTools/core/DSGToolsWorkflow/workflow.py @@ -24,11 +24,18 @@ from dataclasses import asdict, dataclass, field from typing import Any, Dict, List -from qgis.core import (QgsApplication, QgsProcessingFeedback, - QgsProcessingMultiStepFeedback, QgsTask) +from qgis.core import ( + QgsApplication, + QgsProcessingFeedback, + QgsProcessingMultiStepFeedback, + QgsTask, +) from qgis.PyQt.QtCore import pyqtSignal, QObject -from DsgTools.core.DSGToolsWorkflow.workflowItem import DSGToolsWorkflowItem, ExecutionStatus +from DsgTools.core.DSGToolsWorkflow.workflowItem import ( + DSGToolsWorkflowItem, + ExecutionStatus, +) @dataclass @@ -37,6 +44,7 @@ class WorkflowMetadata: version: str lastModified: str + @dataclass class DSGToolsWorkflow(QObject): displayName: str @@ -59,28 +67,30 @@ def as_dict(self) -> Dict[str, Any]: def getCurrentWorkflowStepIndex(self) -> int: return self.currentStepIndex - + def getNextWorkflowStep(self) -> int: nextIndex = self.currentStepIndex + 1 return nextIndex if nextIndex <= len(self.workflowItemList) - 1 else None - + def getCurrentWorkflowItem(self) -> DSGToolsWorkflowItem: idx = self.getCurrentWorkflowStepIndex() return self.workflowItemList[idx] - + def setCurrentWorkflowItem(self, idx) -> None: if idx < 0 or idx >= len(self.workflowItemList): return self.currentStepIndex = idx - + def connectSignals(self) -> None: for workflowItem in self.workflowItemList: - workflowItem.workflowItemExecutionFinished.connect(self.postProcessWorkflowItem) - + workflowItem.workflowItemExecutionFinished.connect( + self.postProcessWorkflowItem + ) + def resetWorkflowItems(self): for workflowItem in self.workflowItemList: workflowItem.resetItem() - + def prepareTask(self) -> QgsTask: currentWorkflowItem: DSGToolsWorkflowItem = self.getCurrentWorkflowItem() return currentWorkflowItem.getTask(self.feedback) @@ -88,7 +98,11 @@ def prepareTask(self) -> QgsTask: def postProcessWorkflowItem(self, workflowItem: DSGToolsWorkflowItem): self.currentTask = None self.currentWorkflowItemStatusChanged.emit(self.currentStepIndex, workflowItem) - if workflowItem.getStatus() in [ExecutionStatus.FAILED, ExecutionStatus.FINISHED_WITH_FLAGS, ExecutionStatus.CANCELED]: + if workflowItem.getStatus() in [ + ExecutionStatus.FAILED, + ExecutionStatus.FINISHED_WITH_FLAGS, + ExecutionStatus.CANCELED, + ]: self.multiStepFeedback.setCurrentStep(self.currentStepIndex) return self.currentStepIndex = self.getNextWorkflowStep() @@ -98,10 +112,12 @@ def postProcessWorkflowItem(self, workflowItem: DSGToolsWorkflowItem): if workflowItem.pauseAfterExecution: currentWorkflowItem = self.getCurrentWorkflowItem() currentWorkflowItem.pauseBeforeRunning() - self.currentWorkflowItemStatusChanged.emit(self.currentStepIndex, currentWorkflowItem) + self.currentWorkflowItemStatusChanged.emit( + self.currentStepIndex, currentWorkflowItem + ) return self.run(resumeFromStart=False) - + def run(self, resumeFromStart=True): if resumeFromStart: self.resetWorkflowItems() @@ -115,12 +131,17 @@ def run(self, resumeFromStart=True): return currentWorkflowItem = self.getCurrentWorkflowItem() currentTask: QgsTask = self.prepareTask() - currentWorkflowItem.changeCurrentStatus(status=ExecutionStatus.RUNNING, executionMessage=self.tr("Execution started")) + currentWorkflowItem.changeCurrentStatus( + status=ExecutionStatus.RUNNING, + executionMessage=self.tr("Execution started"), + ) self.multiStepFeedback.setCurrentStep(self.currentStepIndex) - self.currentWorkflowItemStatusChanged.emit(self.currentStepIndex, currentWorkflowItem) + self.currentWorkflowItemStatusChanged.emit( + self.currentStepIndex, currentWorkflowItem + ) self.currentTaskChanged.emit(self.currentStepIndex, currentTask) QgsApplication.taskManager().addTask(currentTask) - + def setIgnoreFlagsStatusOnCurrentStep(self): currentWorkflowItem = self.getCurrentWorkflowItem() currentWorkflowItem.setCurrentStateToIgnoreFlags() @@ -129,14 +150,20 @@ def cancelCurrentRun(self): currentWorkflowItem = self.getCurrentWorkflowItem() currentWorkflowItem.cancelCurrentTask() self.multiStepFeedback.setCurrentStep(self.currentStepIndex) - self.currentWorkflowItemStatusChanged.emit(self.currentStepIndex, currentWorkflowItem) - + self.currentWorkflowItemStatusChanged.emit( + self.currentStepIndex, currentWorkflowItem + ) + def pauseCurrentRun(self): currentWorkflowItem = self.getCurrentWorkflowItem() currentWorkflowItem.pauseCurrentTask() - self.currentWorkflowItemStatusChanged.emit(self.currentStepIndex, currentWorkflowItem) - + self.currentWorkflowItemStatusChanged.emit( + self.currentStepIndex, currentWorkflowItem + ) + def resumeCurrentRun(self): currentWorkflowItem = self.getCurrentWorkflowItem() currentWorkflowItem.pauseCurrentTask() - self.currentWorkflowItemStatusChanged.emit(self.currentStepIndex, currentWorkflowItem) + self.currentWorkflowItemStatusChanged.emit( + self.currentStepIndex, currentWorkflowItem + ) diff --git a/DsgTools/core/DSGToolsWorkflow/workflowItem.py b/DsgTools/core/DSGToolsWorkflow/workflowItem.py index e3094c48c..fed83fa5d 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflowItem.py +++ b/DsgTools/core/DSGToolsWorkflow/workflowItem.py @@ -57,6 +57,7 @@ class ExecutionStatus(Enum): PAUSED_BEFORE_RUNNING = "paused before running" IGNORE_FLAGS = "ignore flags" + @dataclass class FlagSettings: onFlagsRaised: str @@ -68,6 +69,7 @@ def __post_init__(self): if self.onFlagsRaised not in ("halt", "warn", "ignore"): raise ValueError("Invalid on flags raised flag.") + @dataclass class ModelSource: type: str @@ -100,10 +102,12 @@ def modelFromXml(self) -> QgsProcessingModelAlgorithm: os.remove(temp) return alg + @dataclass class Metadata: originalName: str + @dataclass class ModelExecutionOutput: result: Dict[str, Any] = field(default_factory=dict) @@ -111,6 +115,7 @@ class ModelExecutionOutput: executionMessage: str = "" status: ExecutionStatus = ExecutionStatus.INITIAL + @dataclass class DSGToolsWorkflowItem(QObject): displayName: str @@ -124,10 +129,10 @@ def __post_init__(self): self.model = self.getModel() self.currentTask = None self.workflowItemExecutionFinished = pyqtSignal(DSGToolsWorkflowItem) - + def resetItem(self): self.executionOutput = ModelExecutionOutput() - + def as_dict(self) -> Dict[str, str]: return {k: v for k, v in asdict(self).items()} @@ -138,10 +143,10 @@ def getModelParameters(self) -> List[str]: if self.model is None: return [] return [param.name() for param in self.model.parameterDefinitions()] - + def getFlagNames(self) -> List[str]: return self.flags.flagLayerNames - + def getOutputFlags(self): pass @@ -153,23 +158,29 @@ def getTask(self, feedback: QgsProcessingFeedback) -> QgsTask: on_finished=on_finished_func, ) return self.currentTask - + def pauseBeforeRunning(self): self.executionOutput = ModelExecutionOutput( - executionMessage=self.tr(f"Workflow item {self.displayName} execution paused by previous step."), + executionMessage=self.tr( + f"Workflow item {self.displayName} execution paused by previous step." + ), status=ExecutionStatus.PAUSED_BEFORE_RUNNING, ) - + def setCurrentStateToIgnoreFlags(self): if not self.flags.modelCanHaveFalsePositiveFlags: return self.changeCurrentStatus( status=ExecutionStatus.IGNORE_FLAGS, - executionMessage=self.tr(f"Workflow item {self.displayName} flags were ignored by the user.") + executionMessage=self.tr( + f"Workflow item {self.displayName} flags were ignored by the user." + ), ) # não emite sinal pois esse passo é feito fora da execução. - def changeCurrentStatus(self, status: ExecutionStatus, executionMessage: str) -> None: + def changeCurrentStatus( + self, status: ExecutionStatus, executionMessage: str + ) -> None: self.executionOutput.status = status self.executionOutput.executionMessage = executionMessage @@ -178,12 +189,12 @@ def cancelCurrentTask(self): return self.currentTask.cancel() self.currentTask = None - + def pauseCurrentTask(self): if self.currentTask is None: return self.currentTask.hold() - + def resumeCurrentTask(self): if self.currentTask is None: return @@ -192,6 +203,7 @@ def resumeCurrentTask(self): def getTaskRunningFunction(self, feedback: QgsProcessingFeedback) -> Callable: model = copy.deepcopy(self.model) modelParameters = self.getModelParameters() + def func(): start = time() context = dataobjects.createContext(feedback=feedback) @@ -206,18 +218,19 @@ def func(): out.pop("CHILD_RESULTS", None) out["start_time"] = start return out + return func def getOnFinishedFunction(self, feedback: QgsProcessingFeedback) -> Callable: def on_finished_func(exception, result=None): if exception is not None: QgsMessageLog.logMessage( - f"Exception: {exception}", - "DSGTools Plugin", - Qgis.Critical + f"Exception: {exception}", "DSGTools Plugin", Qgis.Critical ) self.executionOutput = ModelExecutionOutput( - executionMessage=self.tr(f"Workflow item {self.displayName} execution has failed:\n {str(exception)}"), + executionMessage=self.tr( + f"Workflow item {self.displayName} execution has failed:\n {str(exception)}" + ), status=ExecutionStatus.FAILED, ) self.workflowItemExecutionFinished.emit(self) @@ -225,20 +238,36 @@ def on_finished_func(exception, result=None): if result is not None: self.handleOutputs(result, feedback) self.loadOutputs(feedback) - status = ExecutionStatus.FINISHED_WITH_FLAGS if any(lyr.featureCount() > 0 for k, lyr in self.executionOutput.result.items() if lyr.name() in self.flagLayerNames()) else ExecutionStatus.FINISHED - statusMsg = self.tr("finished with flags.") if status == ExecutionStatus.FINISHED_WITH_FLAGS else self.tr("finished.") + status = ( + ExecutionStatus.FINISHED_WITH_FLAGS + if any( + lyr.featureCount() > 0 + for k, lyr in self.executionOutput.result.items() + if lyr.name() in self.flagLayerNames() + ) + else ExecutionStatus.FINISHED + ) + statusMsg = ( + self.tr("finished with flags.") + if status == ExecutionStatus.FINISHED_WITH_FLAGS + else self.tr("finished.") + ) self.changeCurrentStatus( status=status, - executionMessage=self.tr(f"Workflow item {self.displayName} {statusMsg}") + executionMessage=self.tr( + f"Workflow item {self.displayName} {statusMsg}" + ), ) else: self.executionOutput = ModelExecutionOutput( - executionMessage=self.tr(f"Workflow item {self.displayName} execution was canceled by the user."), + executionMessage=self.tr( + f"Workflow item {self.displayName} execution was canceled by the user." + ), status=ExecutionStatus.CANCELED, ) self.workflowItemExecutionFinished.emit(self) self.currentTask = None - + return on_finished_func def handleOutputs(self, result, feedback): @@ -257,13 +286,13 @@ def handleOutputs(self, result, feedback): vl.setName(name) self.executionOutput.result[name] = vl self.executionOutput.executionTime = time() - start - + def loadOutput(self) -> bool: return self.flags.loadOutput - + def getStatus(self) -> ExecutionStatus: return self.executionOutput.status - + def loadOutputs(self, feedback): loadOutput = self.loadOutput() if not loadOutput: @@ -286,13 +315,13 @@ def loadOutputs(self, feedback): continue cloneVl = vl.clone() self.executionOutput.result[name] = cloneVl - self.addLayerToGroup(cloneVl, self.displayName(), clearGroupBeforeAdding=True) + self.addLayerToGroup( + cloneVl, self.displayName(), clearGroupBeforeAdding=True + ) self.enableFeatureCount(cloneVl) iface.mapCanvas().freeze(False) - - def addLayerToGroup( - self, layer, subgroupname, clearGroupBeforeAdding=False - ): + + def addLayerToGroup(self, layer, subgroupname, clearGroupBeforeAdding=False): """ Adds a layer to a group into layer panel. :param layer: (QgsMapLayer) layer to be added to canvas. @@ -326,9 +355,7 @@ def createGroup(self, groupName, rootNode): return groupNode if groupNode else rootNode.addGroup(groupName) def prepareGroup(self, model): - subGroup = self.createGroups( - self.model().model.displayName() - ) + subGroup = self.createGroups(self.model().model.displayName()) self.clearGroup(subGroup) def clearGroup(self, group): @@ -338,6 +365,7 @@ def clearGroup(self, group): lyr.rollBack() group.removeAllChildren() + def load_from_json(input_dict: dict) -> DSGToolsWorkflowItem: params = copy.deepcopy(input_dict) params["flags"] = FlagSettings(**params["flags"]) diff --git a/DsgTools/dsg_tools.py b/DsgTools/dsg_tools.py index 501737914..239c62fb1 100644 --- a/DsgTools/dsg_tools.py +++ b/DsgTools/dsg_tools.py @@ -114,6 +114,7 @@ def initGui(self): Create the menu entries and toolbar icons inside the QGIS GUI """ from .gui.guiManager import GuiManager + self.dsgTools = QMenu(self.iface.mainWindow()) self.dsgTools.setObjectName("DsgTools") self.dsgTools.setTitle("DSGTools") diff --git a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py index e5403f799..890e6c6c7 100644 --- a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py +++ b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py @@ -83,7 +83,9 @@ def __init__(self, iface, parent=None): self.statusMap = { ExecutionStatus.INITIAL: self.tr("Not yet run"), ExecutionStatus.RUNNING: self.tr("Running..."), - ExecutionStatus.PAUSED_BEFORE_RUNNING: self.tr("On hold. Check data and resume."), + ExecutionStatus.PAUSED_BEFORE_RUNNING: self.tr( + "On hold. Check data and resume." + ), ExecutionStatus.ON_HOLD: self.tr("Task paused."), ExecutionStatus.CANCELED: self.tr("Canceled"), ExecutionStatus.FAILED: self.tr("Failed"), @@ -609,7 +611,6 @@ def setWorkflow(self, workflow): pb.setContextMenuPolicy(Qt.CustomContextMenu) self.tableWidget.setCellWidget(row, 2, pb) - def saveState(self): """ Makes sure all added workflows are stored to active instance of diff --git a/DsgTools/gui/ProductionTools/Toolboxes/toolBoxesGuiManager.py b/DsgTools/gui/ProductionTools/Toolboxes/toolBoxesGuiManager.py index 9502946ca..905030468 100644 --- a/DsgTools/gui/ProductionTools/Toolboxes/toolBoxesGuiManager.py +++ b/DsgTools/gui/ProductionTools/Toolboxes/toolBoxesGuiManager.py @@ -57,7 +57,11 @@ def __init__( self.stackButton = stackButton self.iconBasePath = ":/plugins/DsgTools/icons/" - self.acquisitionMenuCtrl = AcquisitionMenuCtrl() if acquisitionMenuCtrl is None else acquisitionMenuCtrl + self.acquisitionMenuCtrl = ( + AcquisitionMenuCtrl() + if acquisitionMenuCtrl is None + else acquisitionMenuCtrl + ) def initGui(self): self.qaToolBox = None From a185a27a18db97cdcee26295a3884120b1cdc892 Mon Sep 17 00:00:00 2001 From: phborba Date: Thu, 7 Mar 2024 21:30:23 -0300 Subject: [PATCH 09/23] adds docstrings --- DsgTools/core/DSGToolsWorkflow/workflow.py | 66 +++++++ .../core/DSGToolsWorkflow/workflowItem.py | 186 +++++++++++++++++- 2 files changed, 248 insertions(+), 4 deletions(-) diff --git a/DsgTools/core/DSGToolsWorkflow/workflow.py b/DsgTools/core/DSGToolsWorkflow/workflow.py index 21c5960d1..b05eb1c4c 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflow.py +++ b/DsgTools/core/DSGToolsWorkflow/workflow.py @@ -40,6 +40,14 @@ @dataclass class WorkflowMetadata: + """Dataclass representing metadata for a workflow. + + Attributes: + author (str): The author of the workflow. + version (str): The version of the workflow. + lastModified (str): The last modification date of the workflow. + """ + author: str version: str lastModified: str @@ -47,11 +55,26 @@ class WorkflowMetadata: @dataclass class DSGToolsWorkflow(QObject): + """Class representing a workflow in DSGTools. + + Attributes: + displayName (str): The display name of the workflow. + metadata (WorkflowMetadata): Metadata associated with the workflow. + workflowItemList (List[DSGToolsWorkflowItem]): List of workflow items. + + Signals: + currentWorkflowItemStatusChanged: Emitted when the status of the current workflow item changes. + workflowHasBeenReset: Emitted when the workflow has been reset. + workflowPaused: Emitted when the workflow is paused. + currentTaskChanged: Emitted when the current task changes. + """ + displayName: str metadata: WorkflowMetadata workflowItemList: List[DSGToolsWorkflowItem] def __post_init__(self): + """Initialize post dataclass creation.""" self.currentStepIndex = 0 self.feedback = QgsProcessingFeedback() self.multiStepFeedback = QgsProcessingMultiStepFeedback( @@ -63,39 +86,72 @@ def __post_init__(self): self.currentTaskChanged = pyqtSignal(int, QgsTask) def as_dict(self) -> Dict[str, Any]: + """Convert the workflow object to a dictionary.""" return {k: v for k, v in asdict(self).items()} def getCurrentWorkflowStepIndex(self) -> int: + """Get the index of the current workflow step. + + Returns: + int: The index of the current workflow step. + """ return self.currentStepIndex def getNextWorkflowStep(self) -> int: + """Get the index of the next workflow step. + + Returns: + int: The index of the next workflow step, or None if at the end of the workflow. + """ nextIndex = self.currentStepIndex + 1 return nextIndex if nextIndex <= len(self.workflowItemList) - 1 else None def getCurrentWorkflowItem(self) -> DSGToolsWorkflowItem: + """Get the current workflow item. + + Returns: + DSGToolsWorkflowItem: The current workflow item. + """ idx = self.getCurrentWorkflowStepIndex() return self.workflowItemList[idx] def setCurrentWorkflowItem(self, idx) -> None: + """Set the current workflow item by index. + + Args: + idx (int): The index of the workflow item to set as current. + """ if idx < 0 or idx >= len(self.workflowItemList): return self.currentStepIndex = idx def connectSignals(self) -> None: + """Connect signals for each workflow item.""" for workflowItem in self.workflowItemList: workflowItem.workflowItemExecutionFinished.connect( self.postProcessWorkflowItem ) def resetWorkflowItems(self): + """Reset all workflow items.""" for workflowItem in self.workflowItemList: workflowItem.resetItem() def prepareTask(self) -> QgsTask: + """Prepare the current task based on the current workflow item. + + Returns: + QgsTask: The prepared task. + """ currentWorkflowItem: DSGToolsWorkflowItem = self.getCurrentWorkflowItem() return currentWorkflowItem.getTask(self.feedback) def postProcessWorkflowItem(self, workflowItem: DSGToolsWorkflowItem): + """Handle post-processing for a completed workflow item. + + Args: + workflowItem (DSGToolsWorkflowItem): The completed workflow item. + """ self.currentTask = None self.currentWorkflowItemStatusChanged.emit(self.currentStepIndex, workflowItem) if workflowItem.getStatus() in [ @@ -119,6 +175,12 @@ def postProcessWorkflowItem(self, workflowItem: DSGToolsWorkflowItem): self.run(resumeFromStart=False) def run(self, resumeFromStart=True): + """Run the workflow. + + Args: + resumeFromStart (bool): Whether to resume the workflow from the start. + + """ if resumeFromStart: self.resetWorkflowItems() self.setCurrentWorkflowItem(0) @@ -143,10 +205,12 @@ def run(self, resumeFromStart=True): QgsApplication.taskManager().addTask(currentTask) def setIgnoreFlagsStatusOnCurrentStep(self): + """Set the status to ignore flags on the current workflow step.""" currentWorkflowItem = self.getCurrentWorkflowItem() currentWorkflowItem.setCurrentStateToIgnoreFlags() def cancelCurrentRun(self): + """Cancel the current run of the workflow.""" currentWorkflowItem = self.getCurrentWorkflowItem() currentWorkflowItem.cancelCurrentTask() self.multiStepFeedback.setCurrentStep(self.currentStepIndex) @@ -155,6 +219,7 @@ def cancelCurrentRun(self): ) def pauseCurrentRun(self): + """Pause the current run of the workflow.""" currentWorkflowItem = self.getCurrentWorkflowItem() currentWorkflowItem.pauseCurrentTask() self.currentWorkflowItemStatusChanged.emit( @@ -162,6 +227,7 @@ def pauseCurrentRun(self): ) def resumeCurrentRun(self): + """Resume the current run of the workflow.""" currentWorkflowItem = self.getCurrentWorkflowItem() currentWorkflowItem.pauseCurrentTask() self.currentWorkflowItemStatusChanged.emit( diff --git a/DsgTools/core/DSGToolsWorkflow/workflowItem.py b/DsgTools/core/DSGToolsWorkflow/workflowItem.py index fed83fa5d..adab816bf 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflowItem.py +++ b/DsgTools/core/DSGToolsWorkflow/workflowItem.py @@ -47,6 +47,20 @@ class ExecutionStatus(Enum): + """Enumeration representing the execution status of a workflow item. + + Attributes: + INITIAL: Initial state. + RUNNING: Currently running. + FAILED: Execution failed. + CANCELED: Execution canceled. + FINISHED: Execution finished successfully. + FINISHED_WITH_FLAGS: Execution finished with flags raised. + ON_HOLD: Execution on hold. + PAUSED_BEFORE_RUNNING: Execution paused before running. + IGNORE_FLAGS: Flags are ignored for this step. + """ + INITIAL = "initial" RUNNING = "running" FAILED = "failed" @@ -60,6 +74,15 @@ class ExecutionStatus(Enum): @dataclass class FlagSettings: + """Dataclass representing settings related to flags in a workflow item. + + Attributes: + onFlagsRaised (str): Action to take when flags are raised - "halt", "warn", or "ignore". + modelCanHaveFalsePositiveFlags (bool): Whether the model can have false positive flags. + loadOutput (bool): Whether to load the output. + flagLayerNames (List[str]): List of flag layer names. + """ + onFlagsRaised: str modelCanHaveFalsePositiveFlags: bool loadOutput: bool @@ -72,6 +95,20 @@ def __post_init__(self): @dataclass class ModelSource: + """Dataclass representing the source of a processing model. + + Attributes: + type (str): Type of the model source. + data (str): Data related to the model source. + + Methods: + modelFromFile(filepath: str) -> QgsProcessingModelAlgorithm: + Initiates a model from a filepath. + + modelFromXml() -> QgsProcessingModelAlgorithm: + Creates a processing model object from XML text. + """ + type: str data: str @@ -105,11 +142,26 @@ def modelFromXml(self) -> QgsProcessingModelAlgorithm: @dataclass class Metadata: + """Dataclass representing metadata for a workflow item. + + Attributes: + originalName (str): The original name of the workflow item. + """ + originalName: str @dataclass class ModelExecutionOutput: + """Dataclass representing the output of a model execution. + + Attributes: + result (Dict[str, Any]): Execution result. + executionTime (float): Execution time in seconds. + executionMessage (str): Message related to the execution. + status (ExecutionStatus): Execution status. + """ + result: Dict[str, Any] = field(default_factory=dict) executionTime: float = 0.0 executionMessage: str = "" @@ -118,6 +170,19 @@ class ModelExecutionOutput: @dataclass class DSGToolsWorkflowItem(QObject): + """Class representing a workflow item in DSGTools. + + Attributes: + displayName (str): Display name of the workflow item. + flags (FlagSettings): Settings related to flags in the workflow item. + pauseAfterExecution (bool): Whether to pause after execution. + source (ModelSource): Source of the processing model. + metadata (Metadata): Metadata related to the workflow item. + + Signals: + workflowItemExecutionFinished: Emitted when the workflow item execution is finished. + """ + displayName: str flags: FlagSettings pauseAfterExecution: bool @@ -125,32 +190,59 @@ class DSGToolsWorkflowItem(QObject): metadata: Metadata def __post_init__(self): + """Initialize post dataclass creation.""" self.resetItem() self.model = self.getModel() self.currentTask = None self.workflowItemExecutionFinished = pyqtSignal(DSGToolsWorkflowItem) def resetItem(self): + """Reset the workflow item.""" self.executionOutput = ModelExecutionOutput() def as_dict(self) -> Dict[str, str]: + """Convert the workflow item to a dictionary.""" return {k: v for k, v in asdict(self).items()} def getModel(self) -> QgsProcessingModelAlgorithm: + """Get the processing model from the source. + + Returns: + QgsProcessingModelAlgorithm: The processing model. + """ return self.source.modelFromXml() def getModelParameters(self) -> List[str]: + """Get the parameters of the processing model. + + Returns: + List[str]: List of parameter names. + """ if self.model is None: return [] return [param.name() for param in self.model.parameterDefinitions()] def getFlagNames(self) -> List[str]: + """Get the names of flag layers. + + Returns: + List[str]: List of flag layer names. + """ return self.flags.flagLayerNames def getOutputFlags(self): + """Get the output flags.""" pass def getTask(self, feedback: QgsProcessingFeedback) -> QgsTask: + """Prepare the task for the workflow item execution. + + Args: + feedback (QgsProcessingFeedback): Feedback for the task. + + Returns: + QgsTask: The prepared task. + """ func = self.getTaskRunningFunction(feedback) on_finished_func = self.getOnFinishedFunction(feedback) self.currentTask = QgsTask.fromFunction( @@ -160,6 +252,7 @@ def getTask(self, feedback: QgsProcessingFeedback) -> QgsTask: return self.currentTask def pauseBeforeRunning(self): + """Pause the workflow item before running.""" self.executionOutput = ModelExecutionOutput( executionMessage=self.tr( f"Workflow item {self.displayName} execution paused by previous step." @@ -168,6 +261,7 @@ def pauseBeforeRunning(self): ) def setCurrentStateToIgnoreFlags(self): + """Set the status to ignore flags on the current workflow step.""" if not self.flags.modelCanHaveFalsePositiveFlags: return self.changeCurrentStatus( @@ -181,26 +275,43 @@ def setCurrentStateToIgnoreFlags(self): def changeCurrentStatus( self, status: ExecutionStatus, executionMessage: str ) -> None: + """Change the current status of the workflow item. + + Args: + status (ExecutionStatus): The new status. + executionMessage (str): Message related to the status change. + """ self.executionOutput.status = status self.executionOutput.executionMessage = executionMessage def cancelCurrentTask(self): + """Cancel the current task.""" if self.currentTask is None: return self.currentTask.cancel() self.currentTask = None def pauseCurrentTask(self): + """Pause the current task.""" if self.currentTask is None: return self.currentTask.hold() def resumeCurrentTask(self): + """Resume the current task.""" if self.currentTask is None: return self.currentTask.unhold() def getTaskRunningFunction(self, feedback: QgsProcessingFeedback) -> Callable: + """Get the function to run for the task. + + Args: + feedback (QgsProcessingFeedback): Feedback for the task. + + Returns: + Callable: The function to run for the task. + """ model = copy.deepcopy(self.model) modelParameters = self.getModelParameters() @@ -222,6 +333,15 @@ def func(): return func def getOnFinishedFunction(self, feedback: QgsProcessingFeedback) -> Callable: + """Get the function to run when the task is finished. + + Args: + feedback (QgsProcessingFeedback): Feedback for the task. + + Returns: + Callable: The function to run when the task is finished. + """ + def on_finished_func(exception, result=None): if exception is not None: QgsMessageLog.logMessage( @@ -271,6 +391,12 @@ def on_finished_func(exception, result=None): return on_finished_func def handleOutputs(self, result, feedback): + """Handle the outputs of the task. + + Args: + result: The result of the task. + feedback (QgsProcessingFeedback): Feedback for the task. + """ start = result.pop("start_time") context = dataobjects.createContext(feedback=feedback) context.setProject(QgsProject.instance()) @@ -288,12 +414,27 @@ def handleOutputs(self, result, feedback): self.executionOutput.executionTime = time() - start def loadOutput(self) -> bool: + """Check if output loading is enabled. + + Returns: + bool: True if output loading is enabled, False otherwise. + """ return self.flags.loadOutput def getStatus(self) -> ExecutionStatus: + """Get the current status of the workflow item. + + Returns: + ExecutionStatus: The current status. + """ return self.executionOutput.status def loadOutputs(self, feedback): + """Load the outputs of the workflow item. + + Args: + feedback (QgsProcessingFeedback): Feedback for the task. + """ loadOutput = self.loadOutput() if not loadOutput: return @@ -322,10 +463,12 @@ def loadOutputs(self, feedback): iface.mapCanvas().freeze(False) def addLayerToGroup(self, layer, subgroupname, clearGroupBeforeAdding=False): - """ - Adds a layer to a group into layer panel. - :param layer: (QgsMapLayer) layer to be added to canvas. - :param subgroupname: (str) name for the subgroup to be added. + """Add a layer to a group in the layer panel. + + Args: + layer (QgsMapLayer): The layer to be added. + subgroupname (str): Name for the subgroup to be added. + clearGroupBeforeAdding (bool): Whether to clear the group before adding the layer. """ subGroup = self.createGroups(subgroupname) if clearGroupBeforeAdding: @@ -339,6 +482,14 @@ def addLayerToGroup(self, layer, subgroupname, clearGroupBeforeAdding=False): subGroup.addLayer(layer) def createGroups(self, subgroupname): + """Create groups in the layer panel. + + Args: + subgroupname (str): Name for the subgroup to be created. + + Returns: + QgsLayerTreeGroup: The created subgroup. + """ rootNode = QgsProject.instance().layerTreeRoot() parentGroupName = "DSGTools_QA_Toolbox" parentGroupNode = rootNode.findGroup(parentGroupName) @@ -351,14 +502,33 @@ def createGroups(self, subgroupname): return subGroup def createGroup(self, groupName, rootNode): + """Create a group in the layer panel. + + Args: + groupName (str): Name for the group. + rootNode (QgsLayerTree): The root node for the group. + + Returns: + QgsLayerTreeGroup: The created group. + """ groupNode = rootNode.findGroup(groupName) return groupNode if groupNode else rootNode.addGroup(groupName) def prepareGroup(self, model): + """Prepare the group for the layer panel. + + Args: + model: The model to be prepared. + """ subGroup = self.createGroups(self.model().model.displayName()) self.clearGroup(subGroup) def clearGroup(self, group): + """Clear a group in the layer panel. + + Args: + group (QgsLayerTreeGroup): The group to be cleared. + """ for lyrGroup in group.findLayers(): lyr = lyrGroup.layer() if isinstance(lyr, QgsVectorLayer): @@ -367,6 +537,14 @@ def clearGroup(self, group): def load_from_json(input_dict: dict) -> DSGToolsWorkflowItem: + """Load a DSGToolsWorkflowItem from a JSON-like dictionary. + + Args: + input_dict (dict): The dictionary containing the information for the workflow item. + + Returns: + DSGToolsWorkflowItem: The loaded DSGToolsWorkflowItem. + """ params = copy.deepcopy(input_dict) params["flags"] = FlagSettings(**params["flags"]) params["source"] = ModelSource(**params["source"]) From 8cb9c242b4059ac3454e77ba9e8e3ca5d934551a Mon Sep 17 00:00:00 2001 From: phborba Date: Tue, 30 Apr 2024 18:51:07 -0300 Subject: [PATCH 10/23] end of the day commit --- DsgTools/core/DSGToolsWorkflow/workflow.py | 32 +++++++++++ .../core/DSGToolsWorkflow/workflowItem.py | 1 + .../qualityAssuranceDockWidget.py | 1 + .../workflowSetupDialog.py | 56 +++++++++---------- 4 files changed, 59 insertions(+), 31 deletions(-) diff --git a/DsgTools/core/DSGToolsWorkflow/workflow.py b/DsgTools/core/DSGToolsWorkflow/workflow.py index b05eb1c4c..bdcb815ba 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflow.py +++ b/DsgTools/core/DSGToolsWorkflow/workflow.py @@ -22,6 +22,7 @@ """ from dataclasses import asdict, dataclass, field +import json from typing import Any, Dict, List from qgis.core import ( @@ -35,6 +36,7 @@ from DsgTools.core.DSGToolsWorkflow.workflowItem import ( DSGToolsWorkflowItem, ExecutionStatus, + load_from_json, ) @@ -75,6 +77,7 @@ class DSGToolsWorkflow(QObject): def __post_init__(self): """Initialize post dataclass creation.""" + super().__init__() self.currentStepIndex = 0 self.feedback = QgsProcessingFeedback() self.multiStepFeedback = QgsProcessingMultiStepFeedback( @@ -233,3 +236,32 @@ def resumeCurrentRun(self): self.currentWorkflowItemStatusChanged.emit( self.currentStepIndex, currentWorkflowItem ) + +def dsgtools_workflow_from_json(json_file): + """Create a DSGToolsWorkflow object from a JSON file.""" + with open(json_file, 'r') as f: + data = json.load(f) + return dsgtools_workflow_from_dict(data) + +def dsgtools_workflow_from_dict(data): + # Extract data for initialization + display_name = data.get('displayName') + metadata = WorkflowMetadata( + author=data['metadata'].get('author'), + version=data['metadata'].get('version'), + lastModified=data['metadata'].get('lastModified') + ) + + if display_name is None or None in (metadata.author, metadata.version, metadata.lastModified): + raise ValueError("Display name, author, version, and last modified are required fields.") + + workflow_item_list = [ + load_from_json(item_data) + for item_data in data.get('workflowItemList', []) + ] + + if not workflow_item_list: + raise ValueError("Workflow item list cannot be empty.") + + # Create and return DSGToolsWorkflow object + return DSGToolsWorkflow(displayName=display_name, metadata=metadata, workflowItemList=workflow_item_list) \ No newline at end of file diff --git a/DsgTools/core/DSGToolsWorkflow/workflowItem.py b/DsgTools/core/DSGToolsWorkflow/workflowItem.py index adab816bf..f587a9e95 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflowItem.py +++ b/DsgTools/core/DSGToolsWorkflow/workflowItem.py @@ -191,6 +191,7 @@ class DSGToolsWorkflowItem(QObject): def __post_init__(self): """Initialize post dataclass creation.""" + super().__init__() self.resetItem() self.model = self.getModel() self.currentTask = None diff --git a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py index 890e6c6c7..705f84611 100644 --- a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py +++ b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py @@ -55,6 +55,7 @@ QualityAssuranceWorkflow, ) from DsgTools.core.DSGToolsWorkflow.workflowItem import ExecutionStatus +from DsgTools.core.DSGToolsWorkflow.workflow import DSGToolsWorkflow FORM_CLASS, _ = uic.loadUiType( diff --git a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py index 6ee5a036e..158ef65d0 100644 --- a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py +++ b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py @@ -26,6 +26,7 @@ from time import time from datetime import datetime +from DsgTools.core.DSGToolsWorkflow.workflowItem import DSGToolsWorkflowItem from qgis.PyQt import uic from qgis.core import Qgis from qgis.gui import QgsMessageBar @@ -44,12 +45,7 @@ from DsgTools.gui.CustomWidgets.SelectionWidgets.selectFileWidget import ( SelectFileWidget, ) -from DsgTools.core.DSGToolsProcessingAlgs.Models.qualityAssuranceWorkflow import ( - QualityAssuranceWorkflow, -) -from DsgTools.core.DSGToolsProcessingAlgs.Models.dsgToolsProcessingModel import ( - DsgToolsProcessingModel, -) +from DsgTools.core.DSGToolsWorkflow.workflow import DSGToolsWorkflow, WorkflowMetadata, dsgtools_workflow_from_dict, dsgtools_workflow_from_json FORM_CLASS, _ = uic.loadUiType( os.path.join(os.path.dirname(__file__), "workflowSetupDialog.ui") @@ -472,7 +468,7 @@ def readRow(self, row): }, } - def setModelToRow(self, row, model): + def setModelToRow(self, row: int, workflowItem: DSGToolsWorkflowItem): """ Reads model's parameters from model parameters default map. :param row: (int) row to have its widgets filled with model's @@ -480,17 +476,17 @@ def setModelToRow(self, row, model): :param model: (DsgToolsProcessingModel) model object. """ # all model files handled by this tool are read/written on QGIS model dir - data = model.data() - if model.source() == "file" and os.path.exists(data): + data = workflowItem.source.data + if workflowItem.source.type == "file" and os.path.exists(data): with open(data, "r", encoding="utf-8") as f: xml = f.read() originalName = os.path.basename(data) - elif model.source() == "xml": + elif workflowItem.source.type == "xml": xml = data - meta = model.metadata() + meta = workflowItem.metadata originalName = ( - model.originalName() - if model.originalName() + meta.originalName + if os.path.exists(meta.originalName) else "temp_{0}.model3".format(hash(time())) ) else: @@ -506,19 +502,19 @@ def setModelToRow(self, row, model): f.write(xml) self.orderedTableWidget.addRow( contents={ - self.MODEL_NAME_HEADER: model.displayName(), + self.MODEL_NAME_HEADER: workflowItem.displayName, self.MODEL_SOURCE_HEADER: path, self.ON_FLAGS_HEADER: { "halt": self.ON_FLAGS_HALT, "warn": self.ON_FLAGS_WARN, "ignore": self.ON_FLAGS_IGNORE, - }[model.onFlagsRaised()], - self.FLAG_CAN_BE_FALSE_POSITIVE_HEADER: model.modelCanHaveFalsePositiveFlags(), - self.LOAD_OUT_HEADER: model.loadOutput(), + }[workflowItem.flags.onFlagsRaised], + self.FLAG_CAN_BE_FALSE_POSITIVE_HEADER: workflowItem.flags.modelCanHaveFalsePositiveFlags, + self.LOAD_OUT_HEADER: workflowItem.flags.loadOutput, self.FLAG_KEYS_HEADER: ",".join( - map(lambda x: str(x).strip(), model.flagLayerNames()) + map(lambda x: str(x).strip(), workflowItem.flags.flagLayerNames) ), - self.PAUSE_AFTER_EXECUTION: model.pauseAfterExecution(), + self.PAUSE_AFTER_EXECUTION: workflowItem.pauseAfterExecution, } ) return True @@ -535,7 +531,7 @@ def validateRowContents(self, contents): return self.tr("Model is empty or file was not found.") return "" - def models(self): + def workflowItems(self): """ Reads all table contents and sets it as a DsgToolsProcessingAlgorithm's set of parameters. @@ -556,7 +552,7 @@ def validateModels(self): msg = self.validateRowContents(self.readRow(row)) if msg: return "Row {row}: '{error}'".format(row=row + 1, error=msg) - if len(self.models()) != self.modelCount(): + if len(self.workflowItems()) != self.modelCount(): return self.tr("Check if no model name is repeated.") return "" @@ -566,7 +562,7 @@ def workflowParameterMap(self): """ return { "displayName": self.workflowName(), - "models": self.models(), + "workflowItemList": self.workflowItems(), "metadata": { "author": self.author(), "version": self.version(), @@ -580,7 +576,7 @@ def currentWorkflow(self): :return: (QualityAssuranceWorkflow) current workflow object. """ try: - return QualityAssuranceWorkflow(self.workflowParameterMap()) + return dsgtools_workflow_from_dict(self.workflowParameterMap()) except: return None @@ -661,15 +657,13 @@ def importWorkflow(self, filepath): Sets workflow contents from an imported DSGTools Workflow dump file. :param filepath: (str) workflow file to be imported. """ - with open(filepath, "r", encoding="utf-8") as f: - xml = json.load(f) - workflow = QualityAssuranceWorkflow(xml) + workflow = dsgtools_workflow_from_json(filepath) self.clear() - self.setWorkflowAuthor(workflow.author()) - self.setWorkflowVersion(workflow.version()) - self.setWorkflowName(workflow.displayName()) - for row, modelParam in enumerate(xml["models"].values()): - self.setModelToRow(row, DsgToolsProcessingModel(modelParam, "")) + self.setWorkflowAuthor(workflow.metadata.author) + self.setWorkflowVersion(workflow.metadata.version) + self.setWorkflowName(workflow.displayName) + for row, workflowItem in enumerate(workflow.workflowItemList): + self.setModelToRow(row, workflowItem) @pyqtSlot(bool, name="on_importPushButton_clicked") def import_(self): From 2aa863908c70dd5d36a6e93f6e77d5caadcedeba Mon Sep 17 00:00:00 2001 From: phborba Date: Thu, 23 May 2024 18:05:12 -0300 Subject: [PATCH 11/23] end of the day commit --- DsgTools/core/DSGToolsWorkflow/workflow.py | 88 +++++++++--- .../core/DSGToolsWorkflow/workflowItem.py | 53 +++++-- .../qualityAssuranceDockWidget.py | 131 ++++++++---------- .../workflowSetupDialog.py | 24 ++-- 4 files changed, 178 insertions(+), 118 deletions(-) diff --git a/DsgTools/core/DSGToolsWorkflow/workflow.py b/DsgTools/core/DSGToolsWorkflow/workflow.py index bdcb815ba..af096bd8d 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflow.py +++ b/DsgTools/core/DSGToolsWorkflow/workflow.py @@ -21,8 +21,10 @@ ***************************************************************************/ """ +import copy from dataclasses import asdict, dataclass, field import json +import os from typing import Any, Dict, List from qgis.core import ( @@ -75,7 +77,12 @@ class DSGToolsWorkflow(QObject): metadata: WorkflowMetadata workflowItemList: List[DSGToolsWorkflowItem] - def __post_init__(self): + currentWorkflowItemStatusChanged = pyqtSignal(int, DSGToolsWorkflowItem) + workflowHasBeenReset = pyqtSignal() + workflowPaused = pyqtSignal() + currentTaskChanged = pyqtSignal(int, QgsTask) + + def __post_init__(self) -> None: """Initialize post dataclass creation.""" super().__init__() self.currentStepIndex = 0 @@ -83,14 +90,45 @@ def __post_init__(self): self.multiStepFeedback = QgsProcessingMultiStepFeedback( len(self.workflowItemList), self.feedback ) - self.currentWorkflowItemStatusChanged = pyqtSignal(int, DSGToolsWorkflowItem) - self.workflowHasBeenReset = pyqtSignal() - self.workflowPaused = pyqtSignal() - self.currentTaskChanged = pyqtSignal(int, QgsTask) + self.connectSignals() def as_dict(self) -> Dict[str, Any]: """Convert the workflow object to a dictionary.""" - return {k: v for k, v in asdict(self).items()} + return { + k: v for k, v in asdict(self).items() if k not in [ + "currentWorkflowItemStatusChanged", + "workflowHasBeenReset", + "workflowPaused", + "currentTaskChanged", + ] + } + + def getCurrentWorkflowItemStatus(self) -> ExecutionStatus: + currentWorkflowItem = self.getCurrentWorkflowItem() + return currentWorkflowItem.getStatus() + + def setStatusDict(self, data: Dict[str, Dict[str, Any]]) -> None: + """ + Sets the status dict on each workflow item. + data has the following format: + { + "workflowItemName": { + "executionTime": int, + "executionMessage": str, + " + } + } + """ + for workflowItem in self.workflowItemList: + d = data.get(workflowItem.displayName, None) + if d is None: + continue + workflowItem.setStatusFromDict(d) + + def getStatusDict(self) -> Dict[str, Dict[str, Any]]: + return { + workflowItem.displayName: workflowItem.executionStatusAsDict() for workflowItem in self.workflowItemList + } def getCurrentWorkflowStepIndex(self) -> int: """Get the index of the current workflow step. @@ -116,7 +154,7 @@ def getCurrentWorkflowItem(self) -> DSGToolsWorkflowItem: DSGToolsWorkflowItem: The current workflow item. """ idx = self.getCurrentWorkflowStepIndex() - return self.workflowItemList[idx] + return self.workflowItemList[idx] if idx is not None else None def setCurrentWorkflowItem(self, idx) -> None: """Set the current workflow item by index. @@ -127,6 +165,12 @@ def setCurrentWorkflowItem(self, idx) -> None: if idx < 0 or idx >= len(self.workflowItemList): return self.currentStepIndex = idx + + def getWorklowItemFromName(self, name): + for workflowItem in self.workflowItemList: + if workflowItem.displayName == name: + return workflowItem + return None def connectSignals(self) -> None: """Connect signals for each workflow item.""" @@ -135,7 +179,7 @@ def connectSignals(self) -> None: self.postProcessWorkflowItem ) - def resetWorkflowItems(self): + def resetWorkflowItems(self) -> None: """Reset all workflow items.""" for workflowItem in self.workflowItemList: workflowItem.resetItem() @@ -149,7 +193,7 @@ def prepareTask(self) -> QgsTask: currentWorkflowItem: DSGToolsWorkflowItem = self.getCurrentWorkflowItem() return currentWorkflowItem.getTask(self.feedback) - def postProcessWorkflowItem(self, workflowItem: DSGToolsWorkflowItem): + def postProcessWorkflowItem(self, workflowItem: DSGToolsWorkflowItem) -> None: """Handle post-processing for a completed workflow item. Args: @@ -163,6 +207,7 @@ def postProcessWorkflowItem(self, workflowItem: DSGToolsWorkflowItem): ExecutionStatus.CANCELED, ]: self.multiStepFeedback.setCurrentStep(self.currentStepIndex) + self.workflowPaused.emit() return self.currentStepIndex = self.getNextWorkflowStep() if self.currentStepIndex is None: @@ -177,7 +222,7 @@ def postProcessWorkflowItem(self, workflowItem: DSGToolsWorkflowItem): return self.run(resumeFromStart=False) - def run(self, resumeFromStart=True): + def run(self, resumeFromStart: bool = True) -> None: """Run the workflow. Args: @@ -212,7 +257,7 @@ def setIgnoreFlagsStatusOnCurrentStep(self): currentWorkflowItem = self.getCurrentWorkflowItem() currentWorkflowItem.setCurrentStateToIgnoreFlags() - def cancelCurrentRun(self): + def cancelCurrentRun(self) -> None: """Cancel the current run of the workflow.""" currentWorkflowItem = self.getCurrentWorkflowItem() currentWorkflowItem.cancelCurrentTask() @@ -221,7 +266,7 @@ def cancelCurrentRun(self): self.currentStepIndex, currentWorkflowItem ) - def pauseCurrentRun(self): + def pauseCurrentRun(self) -> None: """Pause the current run of the workflow.""" currentWorkflowItem = self.getCurrentWorkflowItem() currentWorkflowItem.pauseCurrentTask() @@ -229,21 +274,32 @@ def pauseCurrentRun(self): self.currentStepIndex, currentWorkflowItem ) - def resumeCurrentRun(self): + def resumeCurrentRun(self) -> None: """Resume the current run of the workflow.""" currentWorkflowItem = self.getCurrentWorkflowItem() currentWorkflowItem.pauseCurrentTask() self.currentWorkflowItemStatusChanged.emit( self.currentStepIndex, currentWorkflowItem ) + + def export(self, filepath: str) -> bool: + """ + Dumps workflow's parameters as a JSON file. + :param filepath: (str) path to JSON file. + :return: (bool) operation success. + """ + with open(filepath, "w", encoding="utf-8") as fp: + fp.write(json.dumps(self.as_dict(), indent=4)) + return os.path.exists(filepath) + -def dsgtools_workflow_from_json(json_file): +def dsgtools_workflow_from_json(json_file: str) -> DSGToolsWorkflow: """Create a DSGToolsWorkflow object from a JSON file.""" with open(json_file, 'r') as f: data = json.load(f) return dsgtools_workflow_from_dict(data) -def dsgtools_workflow_from_dict(data): +def dsgtools_workflow_from_dict(data: Dict[str, Any]) -> DSGToolsWorkflow: # Extract data for initialization display_name = data.get('displayName') metadata = WorkflowMetadata( @@ -264,4 +320,4 @@ def dsgtools_workflow_from_dict(data): raise ValueError("Workflow item list cannot be empty.") # Create and return DSGToolsWorkflow object - return DSGToolsWorkflow(displayName=display_name, metadata=metadata, workflowItemList=workflow_item_list) \ No newline at end of file + return DSGToolsWorkflow(displayName=display_name, metadata=metadata, workflowItemList=workflow_item_list) diff --git a/DsgTools/core/DSGToolsWorkflow/workflowItem.py b/DsgTools/core/DSGToolsWorkflow/workflowItem.py index f587a9e95..5609658f3 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflowItem.py +++ b/DsgTools/core/DSGToolsWorkflow/workflowItem.py @@ -23,8 +23,8 @@ import copy from dataclasses import asdict, dataclass, field -from enum import Enum -from typing import Any, Callable, Dict, List +from enum import Enum, unique +from typing import Any, Callable, Dict, List, Optional import os from time import time @@ -45,8 +45,8 @@ from processing.tools import dataobjects import processing - -class ExecutionStatus(Enum): +@unique +class ExecutionStatus(str, Enum): """Enumeration representing the execution status of a workflow item. Attributes: @@ -167,6 +167,10 @@ class ModelExecutionOutput: executionMessage: str = "" status: ExecutionStatus = ExecutionStatus.INITIAL + def __post_init__(self): + if isinstance(self.status, str): + self.status = ExecutionStatus(self.status) + @dataclass class DSGToolsWorkflowItem(QObject): @@ -189,13 +193,15 @@ class DSGToolsWorkflowItem(QObject): source: ModelSource metadata: Metadata + workflowItemExecutionFinished = pyqtSignal(object) + def __post_init__(self): """Initialize post dataclass creation.""" super().__init__() self.resetItem() self.model = self.getModel() self.currentTask = None - self.workflowItemExecutionFinished = pyqtSignal(DSGToolsWorkflowItem) + self.executionOutput = ModelExecutionOutput() def resetItem(self): """Reset the workflow item.""" @@ -203,7 +209,15 @@ def resetItem(self): def as_dict(self) -> Dict[str, str]: """Convert the workflow item to a dictionary.""" - return {k: v for k, v in asdict(self).items()} + return {k: v for k, v in asdict(self).items() if k not in ["workflowItemExecutionFinished"]} + + def setStatusFromDict(self, data: dict[str, Any]): + self.executionOutput = ModelExecutionOutput(**data) + + def executionStatusAsDict(self): + d = asdict(self.executionOutput) + d.pop("result") + return d def getModel(self) -> QgsProcessingModelAlgorithm: """Get the processing model from the source. @@ -230,6 +244,12 @@ def getFlagNames(self) -> List[str]: List[str]: List of flag layer names. """ return self.flags.flagLayerNames + + def flagsCanHaveFalsePositiveResults(self) -> bool: + return self.flags.modelCanHaveFalsePositiveFlags + + def getDescription(self) -> str: + return self.model.shortDescription() def getOutputFlags(self): """Get the output flags.""" @@ -247,6 +267,7 @@ def getTask(self, feedback: QgsProcessingFeedback) -> QgsTask: func = self.getTaskRunningFunction(feedback) on_finished_func = self.getOnFinishedFunction(feedback) self.currentTask = QgsTask.fromFunction( + self.model.name(), func, on_finished=on_finished_func, ) @@ -263,7 +284,7 @@ def pauseBeforeRunning(self): def setCurrentStateToIgnoreFlags(self): """Set the status to ignore flags on the current workflow step.""" - if not self.flags.modelCanHaveFalsePositiveFlags: + if not self.flagsCanHaveFalsePositiveResults(): return self.changeCurrentStatus( status=ExecutionStatus.IGNORE_FLAGS, @@ -274,7 +295,10 @@ def setCurrentStateToIgnoreFlags(self): # não emite sinal pois esse passo é feito fora da execução. def changeCurrentStatus( - self, status: ExecutionStatus, executionMessage: str + self, + status: ExecutionStatus, + executionMessage: Optional[str] = None, + executionTime: Optional[float] = None ) -> None: """Change the current status of the workflow item. @@ -282,8 +306,10 @@ def changeCurrentStatus( status (ExecutionStatus): The new status. executionMessage (str): Message related to the status change. """ - self.executionOutput.status = status - self.executionOutput.executionMessage = executionMessage + self.executionOutput.status = status if isinstance(status, ExecutionStatus) else ExecutionStatus(status) + self.executionOutput.executionMessage = executionMessage if executionMessage is not None else "" + if executionTime is not None: + self.executionOutput.executionTime = executionTime def cancelCurrentTask(self): """Cancel the current task.""" @@ -313,15 +339,14 @@ def getTaskRunningFunction(self, feedback: QgsProcessingFeedback) -> Callable: Returns: Callable: The function to run for the task. """ - model = copy.deepcopy(self.model) modelParameters = self.getModelParameters() - def func(): + def func(obj=None): start = time() context = dataobjects.createContext(feedback=feedback) context.setProject(QgsProject.instance()) out = processing.run( - model, + self.model, {param: "memory:" for param in modelParameters}, feedback=feedback, context=context, @@ -364,7 +389,7 @@ def on_finished_func(exception, result=None): if any( lyr.featureCount() > 0 for k, lyr in self.executionOutput.result.items() - if lyr.name() in self.flagLayerNames() + if lyr.name() in self.flags.flagLayerNames ) else ExecutionStatus.FINISHED ) diff --git a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py index 705f84611..a44c1f729 100644 --- a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py +++ b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py @@ -55,7 +55,7 @@ QualityAssuranceWorkflow, ) from DsgTools.core.DSGToolsWorkflow.workflowItem import ExecutionStatus -from DsgTools.core.DSGToolsWorkflow.workflow import DSGToolsWorkflow +from DsgTools.core.DSGToolsWorkflow.workflow import DSGToolsWorkflow, dsgtools_workflow_from_dict, dsgtools_workflow_from_json FORM_CLASS, _ = uic.loadUiType( @@ -143,16 +143,19 @@ def __init__(self, iface, parent=None): def generateMenu(self, pos, idx, widget, modelName, workflow): workflowName = workflow.name() - currentStatusDict = self.workflowStatusDict.get(workflowName, {}) - if idx == -1 or currentStatusDict.get(modelName, self.INITIAL) not in [ + currentWorkflowItem = workflow.getWorklowItemFromName(modelName) + if currentWorkflowItem is None: + return + currentWorkflowStatus = currentWorkflowItem.getStatus() + if idx == -1 or currentWorkflowStatus not in [ self.FINISHED_WITH_FLAGS, self.IGNORE_FLAGS, ]: return + nextWorkflowItem = workflow.getCurrentWorkflowItem() if ( - currentStatusDict.get(modelName, self.INITIAL) == self.IGNORE_FLAGS - and currentStatusDict.get(workflow.getNextModelName(idx), self.INITIAL) - != self.INITIAL + currentWorkflowStatus == ExecutionStatus.IGNORE_FLAGS + and (nextWorkflowItem is not None and nextWorkflowItem.getStatus() != ExecutionStatus.INITIAL) ): return if idx not in self.ignoreFlagsMenuDict[workflowName]: @@ -161,7 +164,7 @@ def generateMenu(self, pos, idx, widget, modelName, workflow): out = self.ignoreFlagsMenuDict[workflowName][idx].exec_(widget.mapToGlobal(pos)) def prepareIgnoreFlagMenuDictItem(self, idx, modelName, workflow): - workflowName = workflow.name() + workflowName = workflow.displayName self.ignoreFlagsMenuDict[workflowName][idx] = QMenu(self) action = QAction( self.tr(f"Ignore false positive flags on model {modelName}"), @@ -199,7 +202,7 @@ def workflowOnHold(self, currentModel=None): """ Sets workflow to be on hold. """ - self.currentWorkflow().hold() + self.currentWorkflow().pauseCurrentRun() self.pausePushButton.hide() self.continuePushButton.show() self.runPushButton.setEnabled(False) @@ -210,7 +213,7 @@ def continueWorkflow(self): """ Sets workflow to be on hold. """ - self.currentWorkflow().unhold() + self.currentWorkflow().resumeCurrentRun() self.pausePushButton.show() self.continuePushButton.hide() self.runPushButton.setEnabled(False) @@ -221,7 +224,7 @@ def cancelWorkflow(self): """ Cancels current workflow's execution. """ - self.currentWorkflow().feedback.cancel() + self.currentWorkflow().cancelCurrentRun() self.pausePushButton.show() self.continuePushButton.hide() self.setGuiState(False) @@ -245,9 +248,9 @@ def prepareProgressBar(self): self.progressBar.setValue(0) if self._previousWorkflow is not None: self._previousWorkflow.feedback.progressChanged.disconnect(self.setProgress) - self._previousWorkflow = self.currentWorkflow() - if self._previousWorkflow is not None: - self._previousWorkflow.feedback.progressChanged.connect(self.setProgress) + currentWorkflow = self.currentWorkflow() + if currentWorkflow is not None: + currentWorkflow.feedback.progressChanged.connect(self.setProgress) def showEditionButton(self, show=False): """ @@ -321,10 +324,10 @@ def setWorkflowTooltip(self, idx, metadata): self.comboBox.setItemData( idx, self.tr( - "Workflow author: {author}\n" - "Workflow version: {version}\n" - "Last modification: {lastModified}" - ).format(**metadata), + f"Workflow author: {metadata.author}\n" + f"Workflow version: {metadata.version}\n" + f"Last modification: {metadata.lastModified}" + ), Qt.ToolTipRole, ) @@ -478,7 +481,7 @@ def setRowColor(self, row, backgroundColor, foregroundColor): self.tableWidget.cellWidget(row, 0).setStyleSheet(styleSheet) self.tableWidget.cellWidget(row, 1).setStyleSheet(styleSheet) - def setModelStatus(self, row, code, modelName, raiseMessage=False): + def setModelStatus(self, row, workflowItem): """ Sets model execution status to its cell. :param row: (int) model's row on GUI. @@ -487,53 +490,30 @@ def setModelStatus(self, row, code, modelName, raiseMessage=False): should be passed only through dynamic changes in order to avoid polluting QGIS main window. """ + code = workflowItem.getStatus() status = self.statusMap[code] self.setRowStatus(row, code) self.tableWidget.cellWidget(row, 1).setText(status) - if raiseMessage or code in [self.HALTED, self.FAILED, self.FINISHED_WITH_FLAGS]: + if code in [ExecutionStatus.FAILED, ExecutionStatus.FINISHED_WITH_FLAGS]: # advise user a model status has changed only if it came from a # signal call self.iface.messageBar().pushMessage( self.tr("DSGTools Q&A Toolbox"), - self.tr("model {0} status changed to {1}.").format(modelName, status), + self.tr(f"model {workflowItem.displayName} status changed to {status}."), self.qgisStatusDict[code], duration=3, ) - elif code != self.INITIAL: + elif code != ExecutionStatus.INITIAL: QgsMessageLog.logMessage( - self.tr("Model {0} status changed to {1}.").format(modelName, status), + self.tr(f"Model {workflowItem.displayName} status changed to {status}."), "DSGTools Plugin", self.qgisStatusDict[code], ) - if ( - modelName in self.workflowStatusDict[self.comboBox.currentText()] - and code == self.workflowStatusDict[self.comboBox.currentText()][modelName] - ): - return - if code == self.FINISHED and self.workflowStatusDict[ - self.comboBox.currentText() - ][modelName] in [self.FAILED, self.HALTED, self.FINISHED_WITH_FLAGS]: - return - if ( - modelName in self.workflowStatusDict[self.comboBox.currentText()] - and code == self.INITIAL - and self.workflowStatusDict[self.comboBox.currentText()][modelName] - in [self.FAILED, self.FINISHED_WITH_FLAGS] - ): - return - if code == self.IGNORE_FLAGS: - workflow = self.currentWorkflow() - outputStatusDict = workflow.getOutputStatusDict() - outputStatusDict[modelName]["finishStatus"] = "finished" - workflow.setOutputStatusDict(outputStatusDict) - self.workflowStatusDict[self.comboBox.currentText()][modelName] = code - return - self.workflowStatusDict[self.comboBox.currentText()][modelName] = code def setRowStatus(self, row, code): colorForeground = self.colorForeground[code] colorBackground = self.colorBackground[code] - if code == self.INITIAL: + if code == ExecutionStatus.INITIAL: # dark mode does not look good with this color pallete... self.tableWidget.cellWidget(row, 0).setStyleSheet("") self.tableWidget.cellWidget(row, 1).setStyleSheet("") @@ -563,24 +543,21 @@ def setWorkflow(self, workflow): self.clearTable() if workflow is None: return - models = workflow.validModels() - self.tableWidget.setRowCount(len(models)) - - currentStatusDict = self.workflowStatusDict.get(workflow.name(), {}) - for row, (modelName, model) in enumerate(models.items()): + self.tableWidget.setRowCount(len(workflow.workflowItemList)) + for row, workflowItem in enumerate(workflow.workflowItemList): tooltip = self.tr( - f"Model name: {model.displayName()}\n{model.description()}" + f"Model name: {workflowItem.displayName}\n{workflowItem.getDescription()}" ) - if model.modelCanHaveFalsePositiveFlags(): - self.prepareIgnoreFlagMenuDictItem(row, modelName, workflow) - nameWidget = self.customLineWidget(modelName, tooltip) + if workflowItem.flagsCanHaveFalsePositiveResults(): + self.prepareIgnoreFlagMenuDictItem(row, workflowItem.displayName, workflow) + nameWidget = self.customLineWidget(workflowItem.displayName, tooltip) nameWidget.setContextMenuPolicy(Qt.CustomContextMenu) nameWidget.customContextMenuRequested.connect( partial( self.generateMenu, idx=row, widget=nameWidget, - modelName=modelName, + modelName=workflowItem.displayName, workflow=workflow, ) ) @@ -592,13 +569,13 @@ def setWorkflow(self, workflow): self.generateMenu, idx=row, widget=statusWidget, - modelName=modelName, + modelName=workflowItem.displayName, workflow=workflow, ) ) self.tableWidget.setCellWidget(row, 1, statusWidget) - code = currentStatusDict.get(modelName, ExecutionStatus.INITIAL) - self.setModelStatus(row, code, modelName) + code = workflowItem.getStatus() + self.setModelStatus(row, workflowItem) pb = self.progressWidget( value=100 if code @@ -611,6 +588,7 @@ def setWorkflow(self, workflow): ) pb.setContextMenuPolicy(Qt.CustomContextMenu) self.tableWidget.setCellWidget(row, 2, pb) + workflow.currentWorkflowItemStatusChanged.connect(self.setModelStatus) def saveState(self): """ @@ -619,7 +597,11 @@ def saveState(self): """ # workflow objects cannot be serialized, so they must be passed as dict workflows = { - w.displayName(): w.asDict(withOutputDict=True) + w.displayName: w.as_dict() + for w in self.workflows.values() + } + workflowStatusDict = { + w.displayName: w.getStatusDict() for w in self.workflows.values() } @@ -631,7 +613,7 @@ def saveState(self): "workflows": workflows, "current_workflow": self.comboBox.currentIndex(), "show_buttons": self._showButtons, - "workflow_status_dict": self.workflowStatusDict, + "workflow_status_dict": workflowStatusDict, } ), ) @@ -654,12 +636,10 @@ def loadState(self, state=None): workflow_status_dict = state.get("workflow_status_dict", {}) self.resetComboBox() for idx, (name, workflowMap) in enumerate(workflows.items()): - self.workflows[name] = QualityAssuranceWorkflow(workflowMap) + self.workflows[name] = dsgtools_workflow_from_dict(workflowMap) self.comboBox.addItem(name) - self.setWorkflowTooltip(idx + 1, self.workflows[name].metadata()) - self.workflowStatusDict[name] = OrderedDict( - workflow_status_dict.get(name, {}) - ) + self.setWorkflowTooltip(idx + 1, self.workflows[name].metadata) + self.workflows[name].setStatusDict(workflow_status_dict[name]) currentIdx = state["current_workflow"] if "current_workflow" in state else 0 self.comboBox.setCurrentIndex(currentIdx) showButtons = state["show_buttons"] if "show_buttons" in state else True @@ -697,9 +677,12 @@ def runWorkflow(self): # these methods are defined locally as they are not supposed to be # outside thread execution setup and should all be handled from # within this method - at runtime + sender = self.sender() + resumeFromStart = sender is None or sender.objectName() == "runPushButton" + workflow.run(resumeFromStart=resumeFromStart) @pyqtSlot(bool, name="on_importPushButton_clicked") - def importWorkflow(self): + def importWorkflow(self) -> None: """ Directly imports an workflow instead of going through the Workflow Setup Dialog. @@ -714,9 +697,7 @@ def importWorkflow(self): return for wPath in paths: try: - with open(wPath, "r", encoding="utf-8") as f: - wMap = json.load(f) - workflow = QualityAssuranceWorkflow(wMap) + workflow = dsgtools_workflow_from_json(wPath) except Exception as e: self.iface.messageBar().pushMessage( self.tr("DSGTools Q&A Tool Box"), @@ -729,8 +710,8 @@ def importWorkflow(self): continue self.addWorkflowItem(workflow) - def addWorkflowItem(self, workflow: QualityAssuranceWorkflow): - name = workflow.displayName() + def addWorkflowItem(self, workflow: DSGToolsWorkflow) -> None: + name = workflow.displayName idx = self.comboBox.findText(name) if idx < 0: self.comboBox.addItem(name) @@ -740,7 +721,7 @@ def addWorkflowItem(self, workflow: QualityAssuranceWorkflow): else: self.comboBox.setCurrentIndex(idx) # what should we do? check version/last modified? replace model? - self.setWorkflowTooltip(self.comboBox.currentIndex(), workflow.metadata()) + self.setWorkflowTooltip(self.comboBox.currentIndex(), workflow.metadata) self.saveState() QgsMessageLog.logMessage( self.tr("Model {model} imported.").format(model=name), @@ -748,7 +729,7 @@ def addWorkflowItem(self, workflow: QualityAssuranceWorkflow): Qgis.Info, ) - def importWorkflowFromJsonPayload(self, data: List[Dict]): + def importWorkflowFromJsonPayload(self, data: List[Dict]) -> None: for workflow_dict in data: try: workflow = QualityAssuranceWorkflow(workflow_dict) diff --git a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py index 158ef65d0..235cc162c 100644 --- a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py +++ b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py @@ -537,11 +537,9 @@ def workflowItems(self): set of parameters. :return: (dict) map to each model's set of parameters. """ - models = dict() - for row in range(self.modelCount()): - contents = self.readRow(row) - models[contents["displayName"]] = contents - return models + return [ + self.readRow(row) for row in range(self.modelCount()) + ] def validateModels(self): """ @@ -596,12 +594,13 @@ def validate(self): return msg return "" - def exportWorkflow(self, filepath): + def exportWorkflow(self, filepath: str) -> bool: """ Exports current data to a JSON file. :param filepath: (str) output file directory. """ - QualityAssuranceWorkflow(self.workflowParameterMap()).export(filepath) + workflow = dsgtools_workflow_from_dict(self.workflowParameterMap()) + return workflow.export(filepath=filepath) @pyqtSlot(bool, name="on_exportPushButton_clicked") def export(self): @@ -629,7 +628,7 @@ def export(self): else "{0}.workflow".format(filename) ) try: - self.exportWorkflow(filename) + result = self.exportWorkflow(filename) except Exception as e: self.messageBar.pushMessage( self.tr("Invalid workflow"), @@ -640,7 +639,6 @@ def export(self): duration=5, ) return False - result = os.path.exists(filename) msg = ( self.tr("Workflow exported to {fp}") if result @@ -652,7 +650,7 @@ def export(self): ) return result - def importWorkflow(self, filepath): + def importWorkflow(self, filepath: str) -> None: """ Sets workflow contents from an imported DSGTools Workflow dump file. :param filepath: (str) workflow file to be imported. @@ -666,7 +664,7 @@ def importWorkflow(self, filepath): self.setModelToRow(row, workflowItem) @pyqtSlot(bool, name="on_importPushButton_clicked") - def import_(self): + def import_(self) -> None: """ Request a file for Workflow importation and sets it to GUI. :return: (bool) operation status. @@ -700,7 +698,7 @@ def import_(self): return True @pyqtSlot(bool, name="on_okPushButton_clicked") - def ok(self): + def ok(self) -> None: """ Closes dialog and checks if current workflow is valid. """ @@ -716,7 +714,7 @@ def ok(self): ) @pyqtSlot(bool, name="on_cancelPushButton_clicked") - def cancel(self): + def cancel(self) -> None: """ Restores GUI to last state and closes it. """ From 369dc6c3f6528587b6a9cd08203218c0fdd36fe1 Mon Sep 17 00:00:00 2001 From: phborba Date: Thu, 23 May 2024 20:44:12 -0300 Subject: [PATCH 12/23] fix --- DsgTools/core/DSGToolsWorkflow/workflow.py | 51 ++++++++++++------- .../core/DSGToolsWorkflow/workflowItem.py | 40 ++++++++++----- .../qualityAssuranceDockWidget.py | 49 +++++++++++------- 3 files changed, 92 insertions(+), 48 deletions(-) diff --git a/DsgTools/core/DSGToolsWorkflow/workflow.py b/DsgTools/core/DSGToolsWorkflow/workflow.py index af096bd8d..a08348b84 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflow.py +++ b/DsgTools/core/DSGToolsWorkflow/workflow.py @@ -95,7 +95,10 @@ def __post_init__(self) -> None: def as_dict(self) -> Dict[str, Any]: """Convert the workflow object to a dictionary.""" return { - k: v for k, v in asdict(self).items() if k not in [ + k: v + for k, v in asdict(self).items() + if k + not in [ "currentWorkflowItemStatusChanged", "workflowHasBeenReset", "workflowPaused", @@ -106,12 +109,12 @@ def as_dict(self) -> Dict[str, Any]: def getCurrentWorkflowItemStatus(self) -> ExecutionStatus: currentWorkflowItem = self.getCurrentWorkflowItem() return currentWorkflowItem.getStatus() - + def setStatusDict(self, data: Dict[str, Dict[str, Any]]) -> None: """ Sets the status dict on each workflow item. data has the following format: - { + { "workflowItemName": { "executionTime": int, "executionMessage": str, @@ -124,10 +127,11 @@ def setStatusDict(self, data: Dict[str, Dict[str, Any]]) -> None: if d is None: continue workflowItem.setStatusFromDict(d) - + def getStatusDict(self) -> Dict[str, Dict[str, Any]]: return { - workflowItem.displayName: workflowItem.executionStatusAsDict() for workflowItem in self.workflowItemList + workflowItem.displayName: workflowItem.executionStatusAsDict() + for workflowItem in self.workflowItemList } def getCurrentWorkflowStepIndex(self) -> int: @@ -165,7 +169,7 @@ def setCurrentWorkflowItem(self, idx) -> None: if idx < 0 or idx >= len(self.workflowItemList): return self.currentStepIndex = idx - + def getWorklowItemFromName(self, name): for workflowItem in self.workflowItemList: if workflowItem.displayName == name: @@ -181,8 +185,9 @@ def connectSignals(self) -> None: def resetWorkflowItems(self) -> None: """Reset all workflow items.""" - for workflowItem in self.workflowItemList: + for idx, workflowItem in enumerate(self.workflowItemList): workflowItem.resetItem() + self.currentWorkflowItemStatusChanged.emit(idx, workflowItem) def prepareTask(self) -> QgsTask: """Prepare the current task based on the current workflow item. @@ -234,7 +239,7 @@ def run(self, resumeFromStart: bool = True) -> None: self.setCurrentWorkflowItem(0) self.workflowHasBeenReset.emit() currentWorkflowItem = self.getCurrentWorkflowItem() - if currentWorkflowItem.getStatus() == ExecutionStatus.IGNORE_FLAGS: + if currentWorkflowItem.getStatus() in [ExecutionStatus.IGNORE_FLAGS]: self.currentStepIndex = self.getNextWorkflowStep() if self.currentStepIndex is None: self.multiStepFeedback.setProgress(100) @@ -281,7 +286,7 @@ def resumeCurrentRun(self) -> None: self.currentWorkflowItemStatusChanged.emit( self.currentStepIndex, currentWorkflowItem ) - + def export(self, filepath: str) -> bool: """ Dumps workflow's parameters as a JSON file. @@ -295,29 +300,37 @@ def export(self, filepath: str) -> bool: def dsgtools_workflow_from_json(json_file: str) -> DSGToolsWorkflow: """Create a DSGToolsWorkflow object from a JSON file.""" - with open(json_file, 'r') as f: + with open(json_file, "r") as f: data = json.load(f) return dsgtools_workflow_from_dict(data) + def dsgtools_workflow_from_dict(data: Dict[str, Any]) -> DSGToolsWorkflow: # Extract data for initialization - display_name = data.get('displayName') + display_name = data.get("displayName") metadata = WorkflowMetadata( - author=data['metadata'].get('author'), - version=data['metadata'].get('version'), - lastModified=data['metadata'].get('lastModified') + author=data["metadata"].get("author"), + version=data["metadata"].get("version"), + lastModified=data["metadata"].get("lastModified"), ) - if display_name is None or None in (metadata.author, metadata.version, metadata.lastModified): - raise ValueError("Display name, author, version, and last modified are required fields.") + if display_name is None or None in ( + metadata.author, + metadata.version, + metadata.lastModified, + ): + raise ValueError( + "Display name, author, version, and last modified are required fields." + ) workflow_item_list = [ - load_from_json(item_data) - for item_data in data.get('workflowItemList', []) + load_from_json(item_data) for item_data in data.get("workflowItemList", []) ] if not workflow_item_list: raise ValueError("Workflow item list cannot be empty.") # Create and return DSGToolsWorkflow object - return DSGToolsWorkflow(displayName=display_name, metadata=metadata, workflowItemList=workflow_item_list) + return DSGToolsWorkflow( + displayName=display_name, metadata=metadata, workflowItemList=workflow_item_list + ) diff --git a/DsgTools/core/DSGToolsWorkflow/workflowItem.py b/DsgTools/core/DSGToolsWorkflow/workflowItem.py index 5609658f3..1371e626b 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflowItem.py +++ b/DsgTools/core/DSGToolsWorkflow/workflowItem.py @@ -45,6 +45,7 @@ from processing.tools import dataobjects import processing + @unique class ExecutionStatus(str, Enum): """Enumeration representing the execution status of a workflow item. @@ -209,11 +210,15 @@ def resetItem(self): def as_dict(self) -> Dict[str, str]: """Convert the workflow item to a dictionary.""" - return {k: v for k, v in asdict(self).items() if k not in ["workflowItemExecutionFinished"]} - + return { + k: v + for k, v in asdict(self).items() + if k not in ["workflowItemExecutionFinished"] + } + def setStatusFromDict(self, data: dict[str, Any]): self.executionOutput = ModelExecutionOutput(**data) - + def executionStatusAsDict(self): d = asdict(self.executionOutput) d.pop("result") @@ -244,7 +249,7 @@ def getFlagNames(self) -> List[str]: List[str]: List of flag layer names. """ return self.flags.flagLayerNames - + def flagsCanHaveFalsePositiveResults(self) -> bool: return self.flags.modelCanHaveFalsePositiveFlags @@ -281,6 +286,7 @@ def pauseBeforeRunning(self): ), status=ExecutionStatus.PAUSED_BEFORE_RUNNING, ) + self.workflowItemExecutionFinished.emit(self) def setCurrentStateToIgnoreFlags(self): """Set the status to ignore flags on the current workflow step.""" @@ -298,7 +304,7 @@ def changeCurrentStatus( self, status: ExecutionStatus, executionMessage: Optional[str] = None, - executionTime: Optional[float] = None + executionTime: Optional[float] = None, ) -> None: """Change the current status of the workflow item. @@ -306,8 +312,12 @@ def changeCurrentStatus( status (ExecutionStatus): The new status. executionMessage (str): Message related to the status change. """ - self.executionOutput.status = status if isinstance(status, ExecutionStatus) else ExecutionStatus(status) - self.executionOutput.executionMessage = executionMessage if executionMessage is not None else "" + self.executionOutput.status = ( + status if isinstance(status, ExecutionStatus) else ExecutionStatus(status) + ) + self.executionOutput.executionMessage = ( + executionMessage if executionMessage is not None else "" + ) if executionTime is not None: self.executionOutput.executionTime = executionTime @@ -315,7 +325,10 @@ def cancelCurrentTask(self): """Cancel the current task.""" if self.currentTask is None: return - self.currentTask.cancel() + try: + self.currentTask.cancel() + except: + pass self.currentTask = None def pauseCurrentTask(self): @@ -464,7 +477,7 @@ def loadOutputs(self, feedback): loadOutput = self.loadOutput() if not loadOutput: return - flagLayerNames = self.flagLayerNames() + flagLayerNames = self.flags.flagLayerNames context = QgsProcessingContext() iface.mapCanvas().freeze(True) for name, vl in self.executionOutput.result.items(): @@ -482,9 +495,7 @@ def loadOutputs(self, feedback): continue cloneVl = vl.clone() self.executionOutput.result[name] = cloneVl - self.addLayerToGroup( - cloneVl, self.displayName(), clearGroupBeforeAdding=True - ) + self.addLayerToGroup(cloneVl, self.displayName, clearGroupBeforeAdding=True) self.enableFeatureCount(cloneVl) iface.mapCanvas().freeze(False) @@ -561,6 +572,11 @@ def clearGroup(self, group): lyr.rollBack() group.removeAllChildren() + def enableFeatureCount(self, lyr): + root = QgsProject.instance().layerTreeRoot() + lyrNode = root.findLayer(lyr.id()) + lyrNode.setCustomProperty("showFeatureCount", True) + def load_from_json(input_dict: dict) -> DSGToolsWorkflowItem: """Load a DSGToolsWorkflowItem from a JSON-like dictionary. diff --git a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py index a44c1f729..a64ca3825 100644 --- a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py +++ b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py @@ -55,7 +55,11 @@ QualityAssuranceWorkflow, ) from DsgTools.core.DSGToolsWorkflow.workflowItem import ExecutionStatus -from DsgTools.core.DSGToolsWorkflow.workflow import DSGToolsWorkflow, dsgtools_workflow_from_dict, dsgtools_workflow_from_json +from DsgTools.core.DSGToolsWorkflow.workflow import ( + DSGToolsWorkflow, + dsgtools_workflow_from_dict, + dsgtools_workflow_from_json, +) FORM_CLASS, _ = uic.loadUiType( @@ -142,20 +146,20 @@ def __init__(self, iface, parent=None): # self.iface.projectRead.connect(self.loadState) def generateMenu(self, pos, idx, widget, modelName, workflow): - workflowName = workflow.name() + workflowName = workflow.displayName currentWorkflowItem = workflow.getWorklowItemFromName(modelName) if currentWorkflowItem is None: return currentWorkflowStatus = currentWorkflowItem.getStatus() if idx == -1 or currentWorkflowStatus not in [ - self.FINISHED_WITH_FLAGS, - self.IGNORE_FLAGS, + ExecutionStatus.FINISHED_WITH_FLAGS, + ExecutionStatus.IGNORE_FLAGS, ]: return nextWorkflowItem = workflow.getCurrentWorkflowItem() - if ( - currentWorkflowStatus == ExecutionStatus.IGNORE_FLAGS - and (nextWorkflowItem is not None and nextWorkflowItem.getStatus() != ExecutionStatus.INITIAL) + if currentWorkflowStatus == ExecutionStatus.IGNORE_FLAGS and ( + nextWorkflowItem is not None + and nextWorkflowItem.getStatus() != ExecutionStatus.INITIAL ): return if idx not in self.ignoreFlagsMenuDict[workflowName]: @@ -174,7 +178,9 @@ def prepareIgnoreFlagMenuDictItem(self, idx, modelName, workflow): self.setModelStatus, row=idx, modelName=modelName, raiseMessage=True ) callback = lambda x: func( - code=self.IGNORE_FLAGS if x else self.FINISHED_WITH_FLAGS + code=ExecutionStatus.IGNORE_FLAGS + if x + else ExecutionStatus.FINISHED_WITH_FLAGS ) action.setCheckable(True) action.triggered.connect(callback) @@ -494,18 +500,23 @@ def setModelStatus(self, row, workflowItem): status = self.statusMap[code] self.setRowStatus(row, code) self.tableWidget.cellWidget(row, 1).setText(status) + self.setGuiState(code == ExecutionStatus.RUNNING) if code in [ExecutionStatus.FAILED, ExecutionStatus.FINISHED_WITH_FLAGS]: # advise user a model status has changed only if it came from a # signal call self.iface.messageBar().pushMessage( self.tr("DSGTools Q&A Toolbox"), - self.tr(f"model {workflowItem.displayName} status changed to {status}."), + self.tr( + f"model {workflowItem.displayName} status changed to {status}." + ), self.qgisStatusDict[code], duration=3, ) elif code != ExecutionStatus.INITIAL: QgsMessageLog.logMessage( - self.tr(f"Model {workflowItem.displayName} status changed to {status}."), + self.tr( + f"Model {workflowItem.displayName} status changed to {status}." + ), "DSGTools Plugin", self.qgisStatusDict[code], ) @@ -549,7 +560,9 @@ def setWorkflow(self, workflow): f"Model name: {workflowItem.displayName}\n{workflowItem.getDescription()}" ) if workflowItem.flagsCanHaveFalsePositiveResults(): - self.prepareIgnoreFlagMenuDictItem(row, workflowItem.displayName, workflow) + self.prepareIgnoreFlagMenuDictItem( + row, workflowItem.displayName, workflow + ) nameWidget = self.customLineWidget(workflowItem.displayName, tooltip) nameWidget.setContextMenuPolicy(Qt.CustomContextMenu) nameWidget.customContextMenuRequested.connect( @@ -589,6 +602,12 @@ def setWorkflow(self, workflow): pb.setContextMenuPolicy(Qt.CustomContextMenu) self.tableWidget.setCellWidget(row, 2, pb) workflow.currentWorkflowItemStatusChanged.connect(self.setModelStatus) + workflow.currentTaskChanged.connect(self.setupProgressBar) + + def setupProgressBar(self, idx, currentTask): + currentTask.progressChanged.connect( + lambda x: self.tableWidget.cellWidget(idx, 2).setValue(x) + ) def saveState(self): """ @@ -596,13 +615,9 @@ def saveState(self): QgsProject, making it "loadable" along with saved QGIS projects. """ # workflow objects cannot be serialized, so they must be passed as dict - workflows = { - w.displayName: w.as_dict() - for w in self.workflows.values() - } + workflows = {w.displayName: w.as_dict() for w in self.workflows.values()} workflowStatusDict = { - w.displayName: w.getStatusDict() - for w in self.workflows.values() + w.displayName: w.getStatusDict() for w in self.workflows.values() } QgsExpressionContextUtils.setProjectVariable( From 18b2afec63e12d949c8a6548588416b4effc2386 Mon Sep 17 00:00:00 2001 From: phborba Date: Thu, 23 May 2024 22:08:21 -0300 Subject: [PATCH 13/23] fix --- DsgTools/core/DSGToolsWorkflow/workflow.py | 2 +- DsgTools/core/DSGToolsWorkflow/workflowItem.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/DsgTools/core/DSGToolsWorkflow/workflow.py b/DsgTools/core/DSGToolsWorkflow/workflow.py index a08348b84..0f5875b35 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflow.py +++ b/DsgTools/core/DSGToolsWorkflow/workflow.py @@ -214,7 +214,6 @@ def postProcessWorkflowItem(self, workflowItem: DSGToolsWorkflowItem) -> None: self.multiStepFeedback.setCurrentStep(self.currentStepIndex) self.workflowPaused.emit() return - self.currentStepIndex = self.getNextWorkflowStep() if self.currentStepIndex is None: self.multiStepFeedback.setProgress(100) return @@ -225,6 +224,7 @@ def postProcessWorkflowItem(self, workflowItem: DSGToolsWorkflowItem) -> None: self.currentStepIndex, currentWorkflowItem ) return + self.currentStepIndex = self.getNextWorkflowStep() self.run(resumeFromStart=False) def run(self, resumeFromStart: bool = True) -> None: diff --git a/DsgTools/core/DSGToolsWorkflow/workflowItem.py b/DsgTools/core/DSGToolsWorkflow/workflowItem.py index 1371e626b..bc88aa6e4 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflowItem.py +++ b/DsgTools/core/DSGToolsWorkflow/workflowItem.py @@ -475,8 +475,6 @@ def loadOutputs(self, feedback): feedback (QgsProcessingFeedback): Feedback for the task. """ loadOutput = self.loadOutput() - if not loadOutput: - return flagLayerNames = self.flags.flagLayerNames context = QgsProcessingContext() iface.mapCanvas().freeze(True) From 9e2a1c169478442fa2ff95a935aa74851803a19d Mon Sep 17 00:00:00 2001 From: phborba Date: Fri, 24 May 2024 13:01:55 -0300 Subject: [PATCH 14/23] corrige problemas no ignorar falso positivo --- .../Algs/OtherAlgs/batchRunAlgorithm.py | 2 + ...AlgorithmWithGeographicBoundsConstraint.py | 2 + .../deaggregateGeometriesAlgorithm.py | 2 +- DsgTools/core/DSGToolsWorkflow/workflow.py | 19 +---- .../core/DSGToolsWorkflow/workflowItem.py | 54 +++++++------- .../qualityAssuranceDockWidget.py | 72 +++++++------------ .../qualityAssuranceDockWidget.ui | 48 ------------- 7 files changed, 61 insertions(+), 138 deletions(-) diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Algs/OtherAlgs/batchRunAlgorithm.py b/DsgTools/core/DSGToolsProcessingAlgs/Algs/OtherAlgs/batchRunAlgorithm.py index 5422fa3cc..0c8882ea4 100644 --- a/DsgTools/core/DSGToolsProcessingAlgs/Algs/OtherAlgs/batchRunAlgorithm.py +++ b/DsgTools/core/DSGToolsProcessingAlgs/Algs/OtherAlgs/batchRunAlgorithm.py @@ -125,6 +125,8 @@ def processAlgorithm(self, parameters, context, feedback): nSteps = len(layerList) multiStepFeedback = QgsProcessingMultiStepFeedback(nSteps, feedback) for idx, layer_id in enumerate(layerList): + if feedback.isCanceled(): + break layer = QgsProcessingUtils.mapLayerFromString(layer_id, context) layerName = layer.name() multiStepFeedback.setCurrentStep(idx) diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Algs/OtherAlgs/batchRunAlgorithmWithGeographicBoundsConstraint.py b/DsgTools/core/DSGToolsProcessingAlgs/Algs/OtherAlgs/batchRunAlgorithmWithGeographicBoundsConstraint.py index d4e85da43..87b10e3c0 100644 --- a/DsgTools/core/DSGToolsProcessingAlgs/Algs/OtherAlgs/batchRunAlgorithmWithGeographicBoundsConstraint.py +++ b/DsgTools/core/DSGToolsProcessingAlgs/Algs/OtherAlgs/batchRunAlgorithmWithGeographicBoundsConstraint.py @@ -168,6 +168,8 @@ def processAlgorithm(self, parameters, context, feedback): nSteps = len(layerList) multiStepFeedback = QgsProcessingMultiStepFeedback(nSteps, feedback) for idx, layer_id in enumerate(layerList): + if feedback.isCanceled(): + break layer = QgsProcessingUtils.mapLayerFromString(layer_id, context) layerName = layer.name() multiStepFeedback.setCurrentStep(idx) diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/deaggregateGeometriesAlgorithm.py b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/deaggregateGeometriesAlgorithm.py index a34da5f01..f5a205850 100644 --- a/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/deaggregateGeometriesAlgorithm.py +++ b/DsgTools/core/DSGToolsProcessingAlgs/Algs/ValidationAlgs/deaggregateGeometriesAlgorithm.py @@ -89,7 +89,7 @@ def processAlgorithm(self, parameters, context, feedback): for current, feature in enumerate(features): if feedback.isCanceled(): - break + return {} if not feature.geometry(): target.deleteFeature(feature.id()) feedback.setProgress(int(current * total)) diff --git a/DsgTools/core/DSGToolsWorkflow/workflow.py b/DsgTools/core/DSGToolsWorkflow/workflow.py index a08348b84..259bbc166 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflow.py +++ b/DsgTools/core/DSGToolsWorkflow/workflow.py @@ -196,7 +196,7 @@ def prepareTask(self) -> QgsTask: QgsTask: The prepared task. """ currentWorkflowItem: DSGToolsWorkflowItem = self.getCurrentWorkflowItem() - return currentWorkflowItem.getTask(self.feedback) + return currentWorkflowItem.getTask() def postProcessWorkflowItem(self, workflowItem: DSGToolsWorkflowItem) -> None: """Handle post-processing for a completed workflow item. @@ -266,27 +266,12 @@ def cancelCurrentRun(self) -> None: """Cancel the current run of the workflow.""" currentWorkflowItem = self.getCurrentWorkflowItem() currentWorkflowItem.cancelCurrentTask() + QgsApplication.taskManager().cancelAll() self.multiStepFeedback.setCurrentStep(self.currentStepIndex) self.currentWorkflowItemStatusChanged.emit( self.currentStepIndex, currentWorkflowItem ) - def pauseCurrentRun(self) -> None: - """Pause the current run of the workflow.""" - currentWorkflowItem = self.getCurrentWorkflowItem() - currentWorkflowItem.pauseCurrentTask() - self.currentWorkflowItemStatusChanged.emit( - self.currentStepIndex, currentWorkflowItem - ) - - def resumeCurrentRun(self) -> None: - """Resume the current run of the workflow.""" - currentWorkflowItem = self.getCurrentWorkflowItem() - currentWorkflowItem.pauseCurrentTask() - self.currentWorkflowItemStatusChanged.emit( - self.currentStepIndex, currentWorkflowItem - ) - def export(self, filepath: str) -> bool: """ Dumps workflow's parameters as a JSON file. diff --git a/DsgTools/core/DSGToolsWorkflow/workflowItem.py b/DsgTools/core/DSGToolsWorkflow/workflowItem.py index 1371e626b..aac6a4e2b 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflowItem.py +++ b/DsgTools/core/DSGToolsWorkflow/workflowItem.py @@ -68,7 +68,6 @@ class ExecutionStatus(str, Enum): CANCELED = "canceled" FINISHED = "finished" FINISHED_WITH_FLAGS = "finished with flags" - ON_HOLD = "on hold" PAUSED_BEFORE_RUNNING = "paused before running" IGNORE_FLAGS = "ignore flags" @@ -203,6 +202,9 @@ def __post_init__(self): self.model = self.getModel() self.currentTask = None self.executionOutput = ModelExecutionOutput() + self.feedback = QgsProcessingFeedback() + self.context = dataobjects.createContext(feedback=self.feedback) + self.context.setProject(QgsProject.instance()) def resetItem(self): """Reset the workflow item.""" @@ -260,7 +262,7 @@ def getOutputFlags(self): """Get the output flags.""" pass - def getTask(self, feedback: QgsProcessingFeedback) -> QgsTask: + def getTask(self) -> QgsTask: """Prepare the task for the workflow item execution. Args: @@ -269,8 +271,8 @@ def getTask(self, feedback: QgsProcessingFeedback) -> QgsTask: Returns: QgsTask: The prepared task. """ - func = self.getTaskRunningFunction(feedback) - on_finished_func = self.getOnFinishedFunction(feedback) + func = self.getTaskRunningFunction() + on_finished_func = self.getOnFinishedFunction() self.currentTask = QgsTask.fromFunction( self.model.name(), func, @@ -286,12 +288,19 @@ def pauseBeforeRunning(self): ), status=ExecutionStatus.PAUSED_BEFORE_RUNNING, ) - self.workflowItemExecutionFinished.emit(self) def setCurrentStateToIgnoreFlags(self): """Set the status to ignore flags on the current workflow step.""" if not self.flagsCanHaveFalsePositiveResults(): return + if self.executionOutput.status == ExecutionStatus.IGNORE_FLAGS: + self.changeCurrentStatus( + status=ExecutionStatus.FINISHED_WITH_FLAGS, + executionMessage=self.tr( + f"Workflow item {self.displayName} status changed from ignore flags to finished with flags." + ), + ) + return self.changeCurrentStatus( status=ExecutionStatus.IGNORE_FLAGS, executionMessage=self.tr( @@ -330,20 +339,14 @@ def cancelCurrentTask(self): except: pass self.currentTask = None + self.changeCurrentStatus( + status=ExecutionStatus.CANCELED, + executionMessage=self.tr( + f"Workflow item {self.displayName} canceled by user." + ), + ) - def pauseCurrentTask(self): - """Pause the current task.""" - if self.currentTask is None: - return - self.currentTask.hold() - - def resumeCurrentTask(self): - """Resume the current task.""" - if self.currentTask is None: - return - self.currentTask.unhold() - - def getTaskRunningFunction(self, feedback: QgsProcessingFeedback) -> Callable: + def getTaskRunningFunction(self) -> Callable: """Get the function to run for the task. Args: @@ -353,16 +356,15 @@ def getTaskRunningFunction(self, feedback: QgsProcessingFeedback) -> Callable: Callable: The function to run for the task. """ modelParameters = self.getModelParameters() + self.feedback.setProgress(0) def func(obj=None): start = time() - context = dataobjects.createContext(feedback=feedback) - context.setProject(QgsProject.instance()) out = processing.run( self.model, {param: "memory:" for param in modelParameters}, - feedback=feedback, - context=context, + feedback=self.feedback, + context=self.context, ) out.pop("CHILD_INPUTS", None) out.pop("CHILD_RESULTS", None) @@ -371,7 +373,7 @@ def func(obj=None): return func - def getOnFinishedFunction(self, feedback: QgsProcessingFeedback) -> Callable: + def getOnFinishedFunction(self) -> Callable: """Get the function to run when the task is finished. Args: @@ -395,8 +397,8 @@ def on_finished_func(exception, result=None): self.workflowItemExecutionFinished.emit(self) return if result is not None: - self.handleOutputs(result, feedback) - self.loadOutputs(feedback) + self.handleOutputs(result, self.feedback) + self.loadOutputs(self.feedback) status = ( ExecutionStatus.FINISHED_WITH_FLAGS if any( @@ -475,8 +477,6 @@ def loadOutputs(self, feedback): feedback (QgsProcessingFeedback): Feedback for the task. """ loadOutput = self.loadOutput() - if not loadOutput: - return flagLayerNames = self.flags.flagLayerNames context = QgsProcessingContext() iface.mapCanvas().freeze(True) diff --git a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py index a64ca3825..b24a9c944 100644 --- a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py +++ b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py @@ -54,7 +54,7 @@ from DsgTools.core.DSGToolsProcessingAlgs.Models.qualityAssuranceWorkflow import ( QualityAssuranceWorkflow, ) -from DsgTools.core.DSGToolsWorkflow.workflowItem import ExecutionStatus +from DsgTools.core.DSGToolsWorkflow.workflowItem import DSGToolsWorkflowItem, ExecutionStatus from DsgTools.core.DSGToolsWorkflow.workflow import ( DSGToolsWorkflow, dsgtools_workflow_from_dict, @@ -91,7 +91,6 @@ def __init__(self, iface, parent=None): ExecutionStatus.PAUSED_BEFORE_RUNNING: self.tr( "On hold. Check data and resume." ), - ExecutionStatus.ON_HOLD: self.tr("Task paused."), ExecutionStatus.CANCELED: self.tr("Canceled"), ExecutionStatus.FAILED: self.tr("Failed"), ExecutionStatus.FINISHED: self.tr("Completed"), @@ -102,7 +101,6 @@ def __init__(self, iface, parent=None): ExecutionStatus.INITIAL: (0, 0, 0), ExecutionStatus.RUNNING: (0, 0, 125), ExecutionStatus.PAUSED_BEFORE_RUNNING: (187, 201, 25), - ExecutionStatus.ON_HOLD: (187, 201, 25), ExecutionStatus.CANCELED: (200, 0, 0), ExecutionStatus.FAILED: (169, 18, 28), ExecutionStatus.FINISHED: (0, 125, 0), @@ -113,7 +111,6 @@ def __init__(self, iface, parent=None): ExecutionStatus.INITIAL: (255, 255, 255, 75), ExecutionStatus.RUNNING: (0, 0, 125, 90), ExecutionStatus.PAUSED_BEFORE_RUNNING: (187, 201, 25, 20), - ExecutionStatus.ON_HOLD: (200, 215, 40, 20), ExecutionStatus.CANCELED: (200, 0, 0, 85), ExecutionStatus.FAILED: (169, 18, 28, 85), ExecutionStatus.FINISHED: (0, 125, 0, 90), @@ -123,7 +120,6 @@ def __init__(self, iface, parent=None): self.qgisStatusDict = { ExecutionStatus.RUNNING: Qgis.Info, ExecutionStatus.PAUSED_BEFORE_RUNNING: Qgis.Info, - ExecutionStatus.ON_HOLD: Qgis.Critical, ExecutionStatus.CANCELED: Qgis.Warning, ExecutionStatus.FAILED: Qgis.Critical, ExecutionStatus.FINISHED: Qgis.Info, @@ -133,7 +129,6 @@ def __init__(self, iface, parent=None): self.workflowStatusDict = defaultdict(OrderedDict) self.ignoreFlagsMenuDict = defaultdict(dict) self.setGuiState() - self.continuePushButton.hide() self.workflows = dict() self.resetComboBox() self.resetTable() @@ -175,16 +170,20 @@ def prepareIgnoreFlagMenuDictItem(self, idx, modelName, workflow): self.ignoreFlagsMenuDict[workflowName][idx], ) func = partial( - self.setModelStatus, row=idx, modelName=modelName, raiseMessage=True - ) - callback = lambda x: func( - code=ExecutionStatus.IGNORE_FLAGS - if x - else ExecutionStatus.FINISHED_WITH_FLAGS + self.setCurrentWorkflowItemToIgnoreFlags, row=idx ) + callback = lambda x: func() action.setCheckable(True) action.triggered.connect(callback) self.ignoreFlagsMenuDict[workflowName][idx].addAction(action) + + def setCurrentWorkflowItemToIgnoreFlags(self, row): + workflow: DSGToolsWorkflow = self.currentWorkflow() + workflow.setIgnoreFlagsStatusOnCurrentStep() + code = workflow.getCurrentWorkflowItemStatus() + self.setRowStatus(row, code) + self.tableWidget.cellWidget(row, 1).setText(self.statusMap[code]) + def confirmAction(self, msg, showCancel=True): """ @@ -203,36 +202,12 @@ def confirmAction(self, msg, showCancel=True): == QMessageBox.Ok ) - @pyqtSlot(bool, name="on_pausePushButton_clicked") - def workflowOnHold(self, currentModel=None): - """ - Sets workflow to be on hold. - """ - self.currentWorkflow().pauseCurrentRun() - self.pausePushButton.hide() - self.continuePushButton.show() - self.runPushButton.setEnabled(False) - self.resumePushButton.setEnabled(False) - - @pyqtSlot(bool, name="on_continuePushButton_clicked") - def continueWorkflow(self): - """ - Sets workflow to be on hold. - """ - self.currentWorkflow().resumeCurrentRun() - self.pausePushButton.show() - self.continuePushButton.hide() - self.runPushButton.setEnabled(False) - self.resumePushButton.setEnabled(False) - @pyqtSlot(bool, name="on_cancelPushButton_clicked") def cancelWorkflow(self): """ Cancels current workflow's execution. """ self.currentWorkflow().cancelCurrentRun() - self.pausePushButton.show() - self.continuePushButton.hide() self.setGuiState(False) self.__workflowCanceled = True @@ -342,9 +317,7 @@ def setGuiState(self, isActive=False): Sets GUI to idle (not running a Workflow) or active state (running it). :param isActive: (bool) whether GUI is running a Workflow. """ - self.pausePushButton.setEnabled(isActive) self.cancelPushButton.setEnabled(isActive) - self.continuePushButton.setEnabled(isActive) self.runPushButton.setEnabled(not isActive) self.resumePushButton.setEnabled(not isActive) @@ -368,7 +341,7 @@ def addWorkflow(self): if dlg.exec_() != 1: return workflow = dlg.currentWorkflow() - name = workflow.displayName() + name = workflow.displayName idx = self.comboBox.findText(name) if idx < 0: self.comboBox.addItem(name) @@ -378,7 +351,7 @@ def addWorkflow(self): else: self.comboBox.setCurrentIndex(idx) # what should we do? check version/last modified? replace model? - self.setWorkflowTooltip(self.comboBox.currentIndex(), workflow.metadata()) + self.setWorkflowTooltip(self.comboBox.currentIndex(), workflow.metadata) self.saveState() @pyqtSlot(bool, name="on_removePushButton_clicked") @@ -407,7 +380,7 @@ def editCurrentWorkflow(self): Edits current workflow selection from combo box options. """ workflow = self.currentWorkflow() - previousName = workflow.displayName() + previousName = workflow.displayName if workflow is None: return dlg = WorkflowSetupDialog(self) @@ -423,7 +396,7 @@ def editCurrentWorkflow(self): return # block "if modifications are confirmed by user" newWorkflow = dlg.currentWorkflow() - newName = newWorkflow.displayName() + newName = newWorkflow.displayName if newName != previousName and newName in self.workflows: self.iface.messageBar().pushMessage( self.tr("DSGTools Q&A Tool Box"), @@ -445,7 +418,7 @@ def editCurrentWorkflow(self): "{1} renamed to {0} and updated (make sure you exported it)." ).format(newName, previousName) self.workflows[newName] = newWorkflow - self.setWorkflowTooltip(self.comboBox.currentIndex(), newWorkflow.metadata()) + self.setWorkflowTooltip(self.comboBox.currentIndex(), newWorkflow.metadata) self.setCurrentWorkflow() self.iface.messageBar().pushMessage( self.tr("DSGTools Q&A Tool Box"), msg, Qgis.Info, duration=3 @@ -501,7 +474,7 @@ def setModelStatus(self, row, workflowItem): self.setRowStatus(row, code) self.tableWidget.cellWidget(row, 1).setText(status) self.setGuiState(code == ExecutionStatus.RUNNING) - if code in [ExecutionStatus.FAILED, ExecutionStatus.FINISHED_WITH_FLAGS]: + if code in [ExecutionStatus.FAILED, ExecutionStatus.FINISHED_WITH_FLAGS, ExecutionStatus.IGNORE_FLAGS]: # advise user a model status has changed only if it came from a # signal call self.iface.messageBar().pushMessage( @@ -600,9 +573,14 @@ def setWorkflow(self, workflow): else 0 ) pb.setContextMenuPolicy(Qt.CustomContextMenu) + workflowItem.feedback.progressChanged.connect(partial(self.intWrapper, row)) self.tableWidget.setCellWidget(row, 2, pb) workflow.currentWorkflowItemStatusChanged.connect(self.setModelStatus) - workflow.currentTaskChanged.connect(self.setupProgressBar) + # workflow.currentTaskChanged.connect(self.setupProgressBar) + + def intWrapper(self, idx, v): + pb = self.tableWidget.cellWidget(idx, 2) + pb.setValue(int(v)) def setupProgressBar(self, idx, currentTask): currentTask.progressChanged.connect( @@ -694,6 +672,10 @@ def runWorkflow(self): # within this method - at runtime sender = self.sender() resumeFromStart = sender is None or sender.objectName() == "runPushButton" + if resumeFromStart: + workflow.resetWorkflowItems() + for row in range(self.tableWidget.rowCount()): + self.tableWidget.cellWidget(row, 2).setValue(0) workflow.run(resumeFromStart=resumeFromStart) @pyqtSlot(bool, name="on_importPushButton_clicked") diff --git a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.ui b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.ui index 47956a8b8..10df1ffbd 100644 --- a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.ui +++ b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.ui @@ -180,54 +180,6 @@ Qt::Horizontal - - - - 0 - 0 - - - - - 30 - 32 - - - - - - - - :/plugins/DsgTools/icons/runModel.png:/plugins/DsgTools/icons/runModel.png - - - true - - - - - - 0 - 0 - - - - - 30 - 32 - - - - - - - - :/plugins/DsgTools/icons/pause.png:/plugins/DsgTools/icons/pause.png - - - true - - true From a4bff67187148ecbab9bc0f013010490ce7e1fce Mon Sep 17 00:00:00 2001 From: phborba Date: Mon, 27 May 2024 18:02:46 -0300 Subject: [PATCH 15/23] removing old files --- .../DSGToolsProcessingAlgs/Models/__init__.py | 0 .../Models/dsgToolsProcessingModel.py | 551 ------------------ .../Models/qualityAssuranceWorkflow.py | 458 --------------- .../qualityAssuranceDockWidget.py | 3 - 4 files changed, 1012 deletions(-) delete mode 100644 DsgTools/core/DSGToolsProcessingAlgs/Models/__init__.py delete mode 100644 DsgTools/core/DSGToolsProcessingAlgs/Models/dsgToolsProcessingModel.py delete mode 100644 DsgTools/core/DSGToolsProcessingAlgs/Models/qualityAssuranceWorkflow.py diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Models/__init__.py b/DsgTools/core/DSGToolsProcessingAlgs/Models/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Models/dsgToolsProcessingModel.py b/DsgTools/core/DSGToolsProcessingAlgs/Models/dsgToolsProcessingModel.py deleted file mode 100644 index 236a6bef1..000000000 --- a/DsgTools/core/DSGToolsProcessingAlgs/Models/dsgToolsProcessingModel.py +++ /dev/null @@ -1,551 +0,0 @@ -# -*- coding: utf-8 -*- -""" -/*************************************************************************** - DsgTools - A QGIS plugin - Brazilian Army Cartographic Production Tools - ------------------- - begin : 2019-08-29 - git sha : $Format:%H$ - copyright : (C) 2019 by João P. Esperidião - Cartographic Engineer @ Brazilian Army - email : esperidiao.joao@eb.mil.br - ***************************************************************************/ -/*************************************************************************** - * * - * This program is free software; you can redistribute it and/or modify * - * it under the terms of the GNU General Public License as published by * - * the Free Software Foundation; either version 2 of the License, or * - * (at your option) any later version. * - * * - ***************************************************************************/ -""" - -import os, json -from time import time, sleep - -from qgis.core import ( - QgsTask, - QgsProject, - QgsMapLayer, - QgsLayerTreeLayer, - QgsProcessingFeedback, - QgsProcessingModelAlgorithm, - QgsVectorLayer, - QgsProcessingUtils, - QgsProcessingContext, -) -from qgis.PyQt.QtCore import pyqtSignal, QCoreApplication -from qgis.utils import iface -from processing.tools import dataobjects -import processing - - -class DsgToolsProcessingModel(QgsTask): - """ - Handles models and materializes QgsProcessingModels from a DSGTools default - model parameter set. - """ - - # supported input model formats - MODEL_TYPES = ["xml", "file", "model"] - # xml: XML string # - # file: path to a local file # - # model: qgis resgistered model # - modelFinished = pyqtSignal(QgsTask) - - # Appending status flags to the existing ones - n = max( - [ - QgsTask.Queued, - QgsTask.OnHold, - QgsTask.Running, - QgsTask.Complete, - QgsTask.Terminated, - ] - ) - ( - WarningFlags, - HaltedOnFlags, - HaltedOnPossibleFalsePositiveFlags, - FlagsIgnored, - ) = range(n + 1, n + 5) - del n - - def __init__(self, parameters, name, taskName=None, flags=None, feedback=None): - """ - Class constructor. - :param parameters: (dict) map of attributes for a model. - :param name: (str) name to identify current model. - :param taskName: (str) name to be exposed on progress bar. - :param flags: (list) a list of QgsTask flags to be set to current model. - :param feedback: (QgsProcessingFeedback) task progress tracking QGIS - object. - """ - super(DsgToolsProcessingModel, self).__init__( - # "", QgsTask.CanCancel if flags is None else flags - taskName - or QCoreApplication.translate( - "DsgToolsProcessingModel", - "DSGTools Quality Assurance Model: {0}".format(name), - ), - QgsTask.CanCancel if flags is None else flags, - ) - self._name = name - self._param = {} if self.validateParameters(parameters) else parameters - # self.setTitle(taskName or self.displayName()) - self.feedback = feedback or QgsProcessingFeedback() - self.feedback.canceled.connect(self.cancel) - self.output = { - "result": dict(), - "status": False, - "executionTime": 0.0, - "errorMessage": self.tr("Thread not started yet."), - "finishStatus": "initial", - } - - def setTitle(self, title): - """ - Defines task's title (e.g. text shown as the task's name on QGIS task - manager). - """ - self.setDescription(title) - - def validateParameters(self, parameters): - """ - Validates a set of parameters for a model composing a Workflow. - :param parameters: (dict) map of attributes for a model. - :return: (str) invalidation reason. - """ - if "displayName" not in parameters or not parameters["displayName"]: - # this is not a mandatory item, but it's a required tag - parameters["displayName"] = "" - if "flags" not in parameters or not parameters["flags"]: - # this is a mandatory item, but it's has a default to be set if missing - parameters["flags"] = { - "onFlagsRaised": "halt", - "enableLocalFlags": False, - "modelCanHaveFalsePositiveFlags": False, - "loadOutput": False, - } - if "flagLayerNames" not in parameters["flags"]: - parameters["flags"]["flagLayerNames"] = [] - if "pauseAfterExecution" not in parameters: - parameters["pauseAfterExecution"] = False - if "source" not in parameters or not parameters["source"]: - return self.tr("Model source is not defined.") - if ( - "type" not in parameters["source"] - or parameters["source"]["type"] not in self.MODEL_TYPES - ): - return self.tr("Input model type is not supported (or missing).") - if "data" not in parameters["source"] or not parameters["source"]["data"]: - return self.tr("Input model source was not identified.") - return "" - - @staticmethod - def modelFromFile(filepath): - """ - Initiates a model from a filepath. - :param filepath: (str) filepath for target file. - :return: (QgsProcessingModelAlgorithm) model as a processing algorithm. - """ - alg = QgsProcessingModelAlgorithm() - alg.fromFile(filepath) - alg.initAlgorithm() - return alg - - @staticmethod - def modelFromXml(xml): - """ - Creates a processing model object from XML text. - :param xml: (str) XML file contents. - :return: (QgsProcessingModelAlgorithm) model as a processing algorithm. - """ - temp = os.path.join( - os.path.dirname(__file__), "temp_model_{0}.model3".format(hash(time())) - ) - with open(temp, "w+", encoding="utf-8") as xmlFile: - xmlFile.write(xml) - alg = DsgToolsProcessingModel.modelFromFile(temp) - os.remove(temp) - return alg - - def description(self): - """ - Retrieves description text as set on model's definition. - """ - model = self.model() - return model.shortDescription() - - def metadata(self): - """ - A map to Worflow's metadata. - :return: (dict) metadata. - """ - if "metadata" not in self._param: - return {"author": "", "version": "", "lastModified": "", "originalName": ""} - return self._param["metadata"] - - def author(self): - """ - Retrieves the model's author name, if available. - :return: (str) author name. - """ - meta = self.metadata() - return meta["author"] if "author" in meta else "" - - def version(self): - """ - Retrieves the model's version, if available. - :return: (str) model's version. - """ - meta = self.metadata() - return meta["version"] if "version" in meta else "" - - def lastModified(self): - """ - Retrieves the model's last modification timestamp, if available. - :return: (str) last modification timestamp. - """ - meta = self.metadata() - return meta["lastModified"] if "lastModified" in meta else "" - - def originalName(self): - """ - When a model is imported from a file-based model, one might want to - store the original model's name. - :return: (str) original model's name. - """ - meta = self.metadata() - return meta["originalName"] if "originalName" in meta else "" - - def metadataText(self): - """ - Retrieves Workflow's metadata string. - :return: (str) Workflow's metadata string. - """ - if "metadata" not in self._param: - return "" - return self.tr("Model {name} v{version} ({lastModified}) by {author}.").format( - name=self.displayName(), **self.metadata() - ) - - def isValid(self): - """ - Checks whether current model is a valid instance of DSGTools processing - model. - :return: (bool) validity status. - """ - return self._param != dict() - - def data(self): - """ - Current model's source data. If it's from a file, it's its filepath, - if from an XML, its contents. - :return: (str) model's source data. - """ - return self._param["source"]["data"] if self.isValid() else "" - - def source(self): - """ - Current model's input mode (XML text, file...). - :return: (str) model's input source. - """ - return self._param["source"]["type"] if self.isValid() else "" - - def displayName(self): - """ - Model's friendly name, if available. - :return: (str) model's friendly name. - """ - if not self.isValid(): - return "" - return ( - self._param["displayName"] - if "displayName" in self._param - else self.tr("DSGTools Validation Model ({0})").format(self.name()) - ) - - def name(self): - """ - Name for model identification. - :return: (str) model's name. - """ - return self._name - - def model(self): - """ - Gets the processing model nested into parameters. - :return: (QgsProcessingModelAlgorithm) model as a processing algorithm. - """ - if not self.isValid(): - return QgsProcessingModelAlgorithm() - method = { - "xml": DsgToolsProcessingModel.modelFromXml, - "file": DsgToolsProcessingModel.modelFromFile, - } - return ( - method[self.source()](self.data()) - if self.source() in method - else QgsProcessingModelAlgorithm() - ) - - def flags(self): - """ - Models execution flag when running on Workflow behaviour. - :return: (dict) flag map. - """ - flagMap = dict() - if not self._param: - return {} - for flag, value in self._param["flags"].items(): - flagMap[flag] = value - return flagMap - - def onFlagsRaised(self): - """ - Model behavior when running on a Workflow if flags are raised. - :return: (str) model behaviour on Workflow. - """ - return self.flags()["onFlagsRaised"] if self.flags() else "halt" - - def modelCanHaveFalsePositiveFlags(self): - return ( - self.flags().get("modelCanHaveFalsePositiveFlags", False) - if self.flags() - else False - ) - - def pauseAfterExecution(self): - return self._param.get("pauseAfterExecution", False) - - def loadOutput(self): - """ - Model behavior when running on a Workflow if flags are raised. - :return: (str) model behaviour on Workflow. - """ - return self.flags()["loadOutput"] if self.flags() else False - - def flagLayerNames(self): - """ - Model behaviour when flags are raised. Tells which layers should be checked as flags. - :return: (list) list of layer names - """ - return self.flags()["flagLayerNames"] if self.flags() else [] - - def enableLocalFlags(self): - """ - Indicates whether model should store its to a local DSGTools default - database. - :return: (bool) whether flags would be written on a local database. - """ - return self.flags()["enableLocalFlags"] if self.flags() else False - - def childAlgorithms(self, model=None): - """ - A list of all algorithms' names nested into the model. - :return: (list-of-str) list of all algorithms. - """ - return [ - alg.algorithm().displayName() - for alg in (model or self.model()).childAlgorithms().values() - ] - - def modelParameters(self, model=None): - """ - A list of parameters needed to be filled for the model to run. - :param model: (QgsProcessingModelAlgorithm) model to have its parameters - checked. - :return: (list-of-str) - """ - # IMPORTANT: this method seems to be causing QGIS to crash when called - # from command line. Error is "corrupted double-linked list / Aborted (core dumped)" - # seems to be a QGIS mishandling, but should be lloked into deeper. - # It works just fine running on plugin's thread - e.g. does not crash - # whilst running an algorithm or using it internally - # It seems the culprit is the parameterDefinitions method, from - # QgsProcessingModelAlgorithm - if not self._param: - return [] - model = model or self.model() - return [param.name() for param in model.parameterDefinitions()] - - def addLayerToGroup( - self, layer, groupname, subgroupname=None, clearGroupBeforeAdding=False - ): - """ - Adds a layer to a group into layer panel. - :param layer: (QgsMapLayer) layer to be added to canvas. - :param groupname: (str) name for group to nest the layer. - :param subgroupname: (str) name for the subgroup to be added. - """ - subGroup = self.createGroups(groupname, subgroupname) - if clearGroupBeforeAdding: - self.clearGroup(subGroup) - layer = ( - layer - if isinstance(layer, QgsMapLayer) - else QgsProcessingUtils.mapLayerFromString(layer) - ) - QgsProject.instance().addMapLayer(layer, addToLegend=False) - subGroup.addLayer(layer) - - def createGroups(self, groupname, subgroupname): - root = QgsProject.instance().layerTreeRoot() - qaGroup = self.createGroup(groupname, root) - subGroup = self.createGroup(subgroupname, qaGroup) - return subGroup - - def createGroup(self, groupName, rootNode): - groupNode = rootNode.findGroup(groupName) - return groupNode if groupNode else rootNode.addGroup(groupName) - - def prepareGroup(self, model): - subGroup = self.createGroups( - "DSGTools_QA_Toolbox", self.model().model.displayName() - ) - self.clearGroup(subGroup) - - def clearGroup(self, group): - for lyrGroup in group.findLayers(): - lyr = lyrGroup.layer() - if isinstance(lyr, QgsVectorLayer): - lyr.rollBack() - group.removeAllChildren() - - def runModel(self, feedback=None): - """ - Executes current model. - :return: (dict) map to model's outputs. - :param feedback: (QgsProcessingFeedback) task progress tracking QGIS - object. - """ - # this tool understands every parameter to be filled as an output LAYER - # it also sets all output to a MEMORY LAYER. - model = self.model() - if self.isCanceled(): - return {} - context = dataobjects.createContext(feedback=feedback) - out = processing.run( - model, - {param: "memory:" for param in self.modelParameters(model)}, - feedback=feedback, - context=context, - ) - # not sure exactly when, but on 3.16 LTR output from model runs include - # new items on it. these new items break our implementation =) - # hence the popitems - out.pop("CHILD_INPUTS", None) - out.pop("CHILD_RESULTS", None) - return out - - def enableFeatureCount(self, lyr): - root = QgsProject.instance().layerTreeRoot() - lyrNode = root.findLayer(lyr.id()) - lyrNode.setCustomProperty("showFeatureCount", True) - - def export(self, filepath): - """ - Dumps model parameters as a JSON file. - :param filepath: (str) path to JSON file. - :return: (bool) operation success. - """ - with open(filepath, "w", encoding="utf-8") as fp: - fp.write(json.dumps(self._param, sort_keys=True, indent=4)) - return os.path.exists(filepath) - - def asDict(self): - """ - Dumps model parameters as a dict. Returns a copy of current parameters - and modifications on the output does not modify this object. - :return: (dict) DSGTools processing model definitions. - """ - return dict(self._param) - - def run(self): - """ - Method reimplemented in order to run models in thread as a QgsTask. - :return: (bool) task success status. - """ - start = time() - try: - if not self.feedback.isCanceled() or not self.isCanceled(): - self.output = {"result": dict(), "status": True, "errorMessage": ""} - for paramName, vl in self.runModel(self.feedback).items(): - baseName = paramName.rsplit(":", 1)[-1] - name = baseName - idx = 1 - while name in self.output["result"]: - name = "{0} ({1})".format(baseName, idx) - idx += 1 - # print(vl) - vl.setName(name) - # print("PASSED") - self.output["result"][name] = vl - except Exception as e: - self.output = { - "result": {}, - "status": False, - "errorMessage": self.tr("Model has failed:\n'{error}'").format( - error=str(e) - ), - } - self.output["executionTime"] = time() - start - return self.output["status"] - - def hasFlags(self): - """ - Iterates over the results and finds if there are flags. - """ - for key, lyr in self.output["result"].items(): - if ( - key in self._param["flags"]["flagLayerNames"] - and isinstance(lyr, QgsMapLayer) - and lyr.featureCount() > 0 - ): - return True - return False - - def finished(self, result): - """ - Reimplemented from parent QgsTask. Method works a postprocessing one, - always called right after run is finished (read the docs on QgsTask). - :param result: (bool) run returned valued. - """ - if self.isCanceled(): - return - self.loadOutputs() - if result and self.onFlagsRaised() == "halt" and self.hasFlags(): - self.cancel() - self.feedback.cancel() - self.output["finishStatus"] = "halt" - elif not result: - self.output["finishStatus"] = "failed" - msg = self.output.get("errorMessage", "") - self.feedback.pushWarning(msg) - else: - self.output["finishStatus"] = "finished" - self.modelFinished.emit(self) - - def loadOutputs(self): - loadOutput = self.loadOutput() - flagLayerNames = self.flagLayerNames() - context = QgsProcessingContext() - iface.mapCanvas().freeze(True) - for name, vl in self.output["result"].items(): - if vl is None: - continue - if vl.name() not in flagLayerNames and not loadOutput: - continue - if isinstance(vl, str): - vl = QgsProcessingUtils.mapLayerFromString(vl, context) - self.output["result"][name] = vl - if not isinstance(vl, QgsMapLayer) or not vl.isValid(): - continue - vl.setName(name.split(":", 2)[-1]) - if vl.name() in flagLayerNames and vl.featureCount() == 0: - continue - cloneVl = vl.clone() - self.addLayerToGroup(cloneVl, "DSGTools_QA_Toolbox", self.displayName()) - self.enableFeatureCount(cloneVl) - iface.mapCanvas().freeze(False) diff --git a/DsgTools/core/DSGToolsProcessingAlgs/Models/qualityAssuranceWorkflow.py b/DsgTools/core/DSGToolsProcessingAlgs/Models/qualityAssuranceWorkflow.py deleted file mode 100644 index 874caaa8c..000000000 --- a/DsgTools/core/DSGToolsProcessingAlgs/Models/qualityAssuranceWorkflow.py +++ /dev/null @@ -1,458 +0,0 @@ -# -*- coding: utf-8 -*- -""" -/*************************************************************************** - DsgTools - A QGIS plugin - Brazilian Army Cartographic Production Tools - ------------------- - begin : 2019-08-28 - git sha : $Format:%H$ - copyright : (C) 2019 by João P. Esperidião - Cartographic Engineer @ Brazilian Army - email : esperidiao.joao@eb.mil.br - ***************************************************************************/ -/*************************************************************************** - * * - * This program is free software; you can redistribute it and/or modify * - * it under the terms of the GNU General Public License as published by * - * the Free Software Foundation; either version 2 of the License, or * - * (at your option) any later version. * - * * - ***************************************************************************/ -""" - -from copy import deepcopy -import os, json -from time import sleep, time -from functools import partial - -from qgis.core import ( - QgsApplication, - QgsProcessingFeedback, - QgsProcessingMultiStepFeedback, -) -from qgis.PyQt.QtCore import QObject, pyqtSignal - -from DsgTools.core.DSGToolsProcessingAlgs.Models.dsgToolsProcessingModel import ( - DsgToolsProcessingModel, -) - - -class QualityAssuranceWorkflow(QObject): - """ - Works as a multi-model runner. Understands all models' parameters as an - output vector layer. - """ - - workflowFinished = pyqtSignal() - workflowPaused = pyqtSignal(DsgToolsProcessingModel) - haltedOnFlags = pyqtSignal(DsgToolsProcessingModel) - haltedOnPossibleFalsePositiveFlags = pyqtSignal(DsgToolsProcessingModel) - modelStarted = pyqtSignal(DsgToolsProcessingModel) - modelFinished = pyqtSignal(DsgToolsProcessingModel) - modelFinishedWithFlags = pyqtSignal(DsgToolsProcessingModel) - modelFailed = pyqtSignal(DsgToolsProcessingModel) - - def __init__(self, parameters, feedback=None): - """ - Class constructor. Materializes an workflow set of parameters. - :param parameters: (dict) map of workflow attributes. - :param feedback: (QgsProcessingFeedback) task progress tracking QGIS - object. - """ - super(QualityAssuranceWorkflow, self).__init__() - msg = self.validateParameters(parameters) - if msg: - raise Exception( - self.tr("Invalid workflow parameter:\n{msg}").format(msg=msg) - ) - self._param = parameters - self._modelOrderMap = dict() - self.output = self._param.get("output", {}) - self.feedback = feedback or QgsProcessingFeedback() - self._executionOrder = { - idx: model for idx, model in enumerate(self.validModels().values()) - } - - def validateParameters(self, parameters): - """ - Validates a set of parameters for a valid Workflow. - :param parameters: (dict) map of workflow attributes to be validated. - :return: (str) invalidation reason. - """ - if "displayName" not in parameters or not parameters["displayName"]: - # this is not a mandatory item, but it defaults to a value - parameters["displayName"] = self.tr("DSGTools Validation Workflow") - if "models" not in parameters or not parameters["models"]: - return self.tr("Workflow seems to have no models associated with it.") - for modelName, modelParam in parameters["models"].items(): - model = DsgToolsProcessingModel(modelParam, modelName) - if not model.isValid(): - return self.tr("Model {model} is invalid: '{reason}'.").format( - model=modelName, reason=model.validateParameters(modelParam) - ) - # if "flagLayer" not in parameters or not parameters["flagLayer"]: - # self.tr("No flag layer was provided.") - # if "historyLayer" not in parameters or not parameters["historyLayer"]: - # self.tr("No history layer was provided.") - return "" - - def metadata(self): - """ - A map to Workflow's metadata. - :return: (dict) metadata. - """ - return self._param["metadata"] if "metadata" in self._param else dict() - - def author(self): - """ - Retrieves Workflow's author, if available. - :return: (str) Workflow's author. - """ - meta = self.metadata() - return meta["author"] if "author" in meta else "" - - def version(self): - """ - Retrieves Workflow's version, if available. - :return: (str) Workflow's version. - """ - meta = self.metadata() - return meta["version"] if "version" in meta else "" - - def lastModified(self): - """ - Retrieves Workflow's last modification "timestamp", if available. - :return: (str) Workflow's last modification time and date. - """ - meta = self.metadata() - return meta["lastModified"] if "lastModified" in meta else "" - - def metadataText(self): - """ - Retrieves Workflow's metadata string. - :return: (str) Workflow's metadata string. - """ - if not self.metadata(): - return "" - return self.tr( - "Workflow {name} v{version} ({lastModified}) by {author}." - ).format(name=self.displayName(), **self.metadata()) - - def displayName(self): - """ - Friendly name for the workflow. - :return: (str) display name. - """ - return self._param["displayName"] if "displayName" in self._param else "" - - def name(self): - """ - Proxy method for displayName. - :return: (str) display name. - """ - return self.displayName() - - def models(self): - """ - Model parameters defined to run in this workflow. - :return: (dict) models maps to valid and invalid models. - """ - models = {"valid": dict(), "invalid": dict()} - self._multiStepFeedback = QgsProcessingMultiStepFeedback( - len(self._param["models"]), self.feedback - ) - self._multiStepFeedback.setCurrentStep(0) - for modelName, modelParam in self._param["models"].items(): - model = DsgToolsProcessingModel( - modelParam, modelName, feedback=self._multiStepFeedback - ) - if not model.isValid(): - models["invalid"][modelName] = model.validateParameters(modelParam) - else: - models["valid"][modelName] = model - return models - - def validModels(self): - """ - Returns all valid models from workflow parameters. - :return: (dict) models maps to valid and invalid models. - """ - models = dict() - self._multiStepFeedback = QgsProcessingMultiStepFeedback( - len(self._param["models"]), self.feedback - ) - self._multiStepFeedback.setCurrentStep(0) - for idx, (modelName, modelParam) in enumerate(self._param["models"].items()): - model = DsgToolsProcessingModel( - modelParam, modelName, feedback=self._multiStepFeedback - ) - if model.isValid(): - models[modelName] = model - self._modelOrderMap[modelName] = idx - return models - - def invalidModels(self): - """ - Returns all valid models from workflow parameters. - :return: (dict) models maps invalid models to their invalidation reason. - """ - models = dict() - self._multiStepFeedback = QgsProcessingMultiStepFeedback( - len(self._param["models"]), self.feedback - ) - self._multiStepFeedback.setCurrentStep(0) - for modelName, modelParam in self._param["models"].items(): - model = DsgToolsProcessingModel( - modelParam, modelName, feedback=self._multiStepFeedback - ) - if not model.isValid(): - models[modelName] = model.validateParameters(modelParam) - return models - - def hasInvalidModel(self): - """ - Checks if any of the nested models is invalid. - :return: (bool) if there are invalid models. - """ - models = dict() - for modelName, modelParam in self._param["models"].items(): - model = DsgToolsProcessingModel(modelParam, modelName) - if not model.isValid(): - return True - return False - - def export(self, filepath): - """ - Dumps workflow's parameters as a JSON file. - :param filepath: (str) path to JSON file. - :return: (bool) operation success. - """ - with open(filepath, "w", encoding="utf-8") as fp: - fp.write(json.dumps(self._param, indent=4)) - return os.path.exists(filepath) - - def asDict(self, withOutputDict=False): - """ - Dumps model parameters as a JSON file. - :param filepath: (str) path to JSON file. - :return: (dict) DSGTools processing model definitions. - """ - outputDict = dict(self._param) - if not withOutputDict: - return outputDict - outputCopy = dict(self.output) - for key, value in outputCopy.items(): - if "result" in value: - outputCopy[key].pop("result") - outputDict.update({"output": outputCopy}) - return outputDict - - def finished(self): - """ - Executes all post-processing actions. - """ - # Add default post-processing actions here! - self.workflowFinished.emit() - - def runOnMainThread(self): - """ - If, for some reason, Workflow should not be run from secondary threads, - this method provides a 'static' execution alternative. - :return: (dict) a map to each model's output. - """ - self.output = dict() - for model in self.validModels().values(): - start = time() - mName = model.name() - self.output[mName] = dict() - try: - self.output[mName]["result"] = { - k.split(":", 2)[-1]: v - for k, v in model.runModel(model.feedback).items() - } - self.output[mName]["status"] = True - except Exception as e: - self.output[mName]["result"] = None - self.output[mName]["status"] = False - self.output[mName]["executionTime"] = time() - start - self.finished() - return self.output - - def setupModelTask(self, model): - """ - Sets model to run on QGIS task manager. - """ - QgsApplication.taskManager().addTask(model) - - def hold(self): - """ - Puts current active tasks/models on hold. - """ - if not hasattr(self, "_executionOrder"): - return - for m in self._executionOrder.values(): - if m.status() == m.Running: - m.hold() - - def unhold(self): - """ - Puts current paused tasks/models back to active status. - """ - if not hasattr(self, "_executionOrder"): - return - for m in self._executionOrder.values(): - if m.status() == m.OnHold: - m.unhold() - - def currentModel(self): - """ - Retrieves the model currently running, if any. - :return: (DsgToolsProcessingModel) current active model. - """ - if not hasattr(self, "_executionOrder"): - return None - for m in self._executionOrder.values(): - if m.status() == m.Running: - return m - return None - - def raiseFlagWarning(self, model): - """ - Advises connected objects that flags were raised even though workflow - :param model: (DsgToolsProcessingModel) model to have its flags checked. - """ - if model.hasFlags(): - self.modelFinishedWithFlags.emit(model) - else: - self.modelFinished.emit(model) - - def raiseFlagError(self, model): - """ - It stops the workflow execution if flags are identified. - :param model: (DsgToolsProcessingModel) model to have its flags checked. - """ - if model.hasFlags(): - self.feedback.cancel() - self.haltedOnFlags.emit(model) - else: - self.modelFinished.emit(model) - return self.feedback.isCanceled() - - def handleFlags(self, model): - """ - Handles Workflow behaviour for a model's flag output. - :param model: (DsgToolsProcessingModel) model to have its output handled. - """ - onFlagsMethod = { - "warn": partial(self.raiseFlagWarning, model), - "halt": partial(self.raiseFlagError, model), - "ignore": partial(self.modelFinished.emit, model), - }[model.onFlagsRaised()]() - - def run(self, firstModelName=None, cooldown=None): - """ - Executes all models in secondary threads. - :param firstModelName: (str) first model's name to be executed. - :param cooldown: (float) time to wait till next model is started. - """ - self._executionOrder = { - idx: model for idx, model in enumerate(self.validModels().values()) - } - modelCount = len(self._executionOrder) - if self.hasInvalidModel() or modelCount == 0: - return None - - def modelCompleted(model, step): - self.output[model.name()] = model.output - self._multiStepFeedback.setCurrentStep(step) - self.handleFlags(model) - if model.pauseAfterExecution(): - QgsApplication.taskManager().cancelAll() - nextModel = self.getNextModel(model) - self.workflowPaused.emit(nextModel) - - if firstModelName is not None: - for idx, model in self._executionOrder.items(): - if model.name() == firstModelName: - initialIdx = idx - break - else: - # name was not found - return None - else: - initialIdx = 0 - self.output = dict() - for idx, currentModel in self._executionOrder.items(): - if idx < initialIdx: - continue - # all models MUST pass through this postprocessing method - currentModel.taskCompleted.connect( - partial(modelCompleted, currentModel, idx + 1) - ) - currentModel.begun.connect(partial(self.modelStarted.emit, currentModel)) - currentModel.taskTerminated.connect( - partial(self.modelFailed.emit, currentModel) - ) - if idx != modelCount - 1: - self._executionOrder[idx + 1].addSubTask( - currentModel, subTaskDependency=currentModel.ParentDependsOnSubTask - ) - else: - # last model indicates workflow finish - currentModel.taskCompleted.connect(self.finished) - # last model will trigger every dependent model till the first added to - # the task manager - self.setupModelTask(currentModel) - - def lastModelName(self, returnIdx=False): - """ - Gets the last model prepared to execute but has either failed or not - run. - :return: (str) first model's name not to run. - """ - if not hasattr(self, "_executionOrder"): - return None - modelCount = len(self._executionOrder) - for idx, model in self._executionOrder.items(): - modelName = self._executionOrder[idx].displayName() - if ( - modelName not in self.output - or self.output[modelName].get("finishStatus", "initial") != "finished" - ): - return modelName if not returnIdx else idx, modelName - else: - return None if not returnIdx else None, None - - def getOutputStatusDict(self): - return self.output - - def setOutputStatusDict(self, statusDict): - self.output = statusDict - - def modelCount(self): - return len(self._executionOrder) - - def getNextModel(self, currentModel): - currentModelIdx = self.getModelIndexByName(currentModel.name()) - if currentModelIdx + 1 > self.modelCount(): - return None - return self._executionOrder[currentModelIdx + 1] - - def getModelIndexByName(self, modelName): - for idx, model in self._executionOrder.items(): - if model.name() == modelName: - return idx - return None - - def getPreviousModelName(self, idx): - if idx - 1 < 0: - return None - return self._executionOrder[idx - 1].name() - - def getNextModelName(self, idx): - if idx + 1 > self.modelCount(): - return None - return self._executionOrder[idx + 1].name() - - def currentFlagsAreFalsePositive(self): - pass diff --git a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py index b24a9c944..503e78107 100644 --- a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py +++ b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py @@ -51,9 +51,6 @@ from DsgTools.gui.ProductionTools.Toolboxes.QualityAssuranceToolBox.workflowSetupDialog import ( WorkflowSetupDialog, ) -from DsgTools.core.DSGToolsProcessingAlgs.Models.qualityAssuranceWorkflow import ( - QualityAssuranceWorkflow, -) from DsgTools.core.DSGToolsWorkflow.workflowItem import DSGToolsWorkflowItem, ExecutionStatus from DsgTools.core.DSGToolsWorkflow.workflow import ( DSGToolsWorkflow, From 9356e074c55a6159ff49e48f55b9a8db9f0da9a9 Mon Sep 17 00:00:00 2001 From: phborba Date: Tue, 28 May 2024 17:59:20 -0300 Subject: [PATCH 16/23] =?UTF-8?q?corrige=20t=C3=A9rmino=20sem=20erros=20do?= =?UTF-8?q?=20workflow?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DsgTools/core/DSGToolsWorkflow/workflow.py | 4 ++++ DsgTools/core/DSGToolsWorkflow/workflowItem.py | 5 +++++ .../qualityAssuranceDockWidget.py | 13 ++++++++++++- 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/DsgTools/core/DSGToolsWorkflow/workflow.py b/DsgTools/core/DSGToolsWorkflow/workflow.py index 385b86eba..1815dded2 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflow.py +++ b/DsgTools/core/DSGToolsWorkflow/workflow.py @@ -80,6 +80,7 @@ class DSGToolsWorkflow(QObject): currentWorkflowItemStatusChanged = pyqtSignal(int, DSGToolsWorkflowItem) workflowHasBeenReset = pyqtSignal() workflowPaused = pyqtSignal() + currentWorkflowExecutionFinished = pyqtSignal() currentTaskChanged = pyqtSignal(int, QgsTask) def __post_init__(self) -> None: @@ -239,6 +240,9 @@ def run(self, resumeFromStart: bool = True) -> None: self.setCurrentWorkflowItem(0) self.workflowHasBeenReset.emit() currentWorkflowItem = self.getCurrentWorkflowItem() + if currentWorkflowItem is None: + self.currentWorkflowExecutionFinished.emit() + return if currentWorkflowItem.getStatus() in [ExecutionStatus.IGNORE_FLAGS]: self.currentStepIndex = self.getNextWorkflowStep() if self.currentStepIndex is None: diff --git a/DsgTools/core/DSGToolsWorkflow/workflowItem.py b/DsgTools/core/DSGToolsWorkflow/workflowItem.py index aac6a4e2b..b13ad1570 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflowItem.py +++ b/DsgTools/core/DSGToolsWorkflow/workflowItem.py @@ -251,6 +251,11 @@ def getFlagNames(self) -> List[str]: List[str]: List of flag layer names. """ return self.flags.flagLayerNames + + def getAllOutputNamesFromModel(self) -> List[str]: + return [ + outputDef.name().split(":")[-1] for outputDef in self.model.outputDefinitions() + ] def flagsCanHaveFalsePositiveResults(self) -> bool: return self.flags.modelCanHaveFalsePositiveFlags diff --git a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py index 503e78107..12b9736ac 100644 --- a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py +++ b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py @@ -438,6 +438,16 @@ def currentWorkflow(self): """ name = self.currentWorkflowName() return self.workflows.get(name, None) + + def currentWorkflowFinishedExecutionMessage(self): + currentWorkflow = self.currentWorkflow() + if currentWorkflow is None: + return + self.iface.messageBar().pushMessage( + self.tr("DSGTools Q&A Tool Box"), + self.tr(f"Workflow {currentWorkflow.displayName} execution has finished."), + Qgis.Info, duration=3 + ) def setRowColor(self, row, backgroundColor, foregroundColor): """ @@ -573,6 +583,7 @@ def setWorkflow(self, workflow): workflowItem.feedback.progressChanged.connect(partial(self.intWrapper, row)) self.tableWidget.setCellWidget(row, 2, pb) workflow.currentWorkflowItemStatusChanged.connect(self.setModelStatus) + workflow.currentWorkflowExecutionFinished.connect(self.currentWorkflowFinishedExecutionMessage) # workflow.currentTaskChanged.connect(self.setupProgressBar) def intWrapper(self, idx, v): @@ -726,7 +737,7 @@ def addWorkflowItem(self, workflow: DSGToolsWorkflow) -> None: def importWorkflowFromJsonPayload(self, data: List[Dict]) -> None: for workflow_dict in data: try: - workflow = QualityAssuranceWorkflow(workflow_dict) + workflow = dsgtools_workflow_from_dict(workflow_dict) except Exception as e: self.iface.messageBar().pushMessage( self.tr("DSGTools Q&A Tool Box"), From c75bf7f41cfacd27322e274780992c7e5cad8082 Mon Sep 17 00:00:00 2001 From: phborba Date: Wed, 29 May 2024 11:53:42 -0300 Subject: [PATCH 17/23] =?UTF-8?q?muda=20interface=20de=20ger=C3=AAncia=20d?= =?UTF-8?q?e=20modelo=20dentro=20da=20cria=C3=A7=C3=A3o=20do=20workflow.?= =?UTF-8?q?=20corrige=20bugs=20de=20loop=20infinito=20ao=20parar=20antes?= =?UTF-8?q?=20de=20executar=20o=20pr=C3=B3ximo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- DsgTools/core/DSGToolsWorkflow/workflow.py | 4 + .../core/DSGToolsWorkflow/workflowItem.py | 12 +- .../orderedTableWidget.py | 10 +- .../importExportFileWidget.py | 128 ++++++++++++++++++ .../importExportFileWidget.ui | 88 ++++++++++++ .../qualityAssuranceDockWidget.py | 6 +- .../workflowSetupDialog.py | 58 +++----- 7 files changed, 260 insertions(+), 46 deletions(-) create mode 100644 DsgTools/gui/CustomWidgets/SelectionWidgets/importExportFileWidget.py create mode 100644 DsgTools/gui/CustomWidgets/SelectionWidgets/importExportFileWidget.ui diff --git a/DsgTools/core/DSGToolsWorkflow/workflow.py b/DsgTools/core/DSGToolsWorkflow/workflow.py index 1815dded2..d03d608b4 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflow.py +++ b/DsgTools/core/DSGToolsWorkflow/workflow.py @@ -219,6 +219,9 @@ def postProcessWorkflowItem(self, workflowItem: DSGToolsWorkflowItem) -> None: self.multiStepFeedback.setProgress(100) return if workflowItem.pauseAfterExecution: + if workflowItem.getStatus() != ExecutionStatus.FINISHED: + return + self.currentStepIndex = self.getNextWorkflowStep() currentWorkflowItem = self.getCurrentWorkflowItem() currentWorkflowItem.pauseBeforeRunning() self.currentWorkflowItemStatusChanged.emit( @@ -249,6 +252,7 @@ def run(self, resumeFromStart: bool = True) -> None: self.multiStepFeedback.setProgress(100) return currentWorkflowItem = self.getCurrentWorkflowItem() + currentWorkflowItem.clearFlagsBeforeRunning() currentTask: QgsTask = self.prepareTask() currentWorkflowItem.changeCurrentStatus( status=ExecutionStatus.RUNNING, diff --git a/DsgTools/core/DSGToolsWorkflow/workflowItem.py b/DsgTools/core/DSGToolsWorkflow/workflowItem.py index b13ad1570..2efde4bda 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflowItem.py +++ b/DsgTools/core/DSGToolsWorkflow/workflowItem.py @@ -499,7 +499,7 @@ def loadOutputs(self, feedback): if vl.name() in flagLayerNames and vl.featureCount() == 0: continue cloneVl = vl.clone() - self.executionOutput.result[name] = cloneVl + self.executionOutput.result[cloneVl.name()] = cloneVl self.addLayerToGroup(cloneVl, self.displayName, clearGroupBeforeAdding=True) self.enableFeatureCount(cloneVl) iface.mapCanvas().freeze(False) @@ -522,6 +522,16 @@ def addLayerToGroup(self, layer, subgroupname, clearGroupBeforeAdding=False): ) QgsProject.instance().addMapLayer(layer, addToLegend=False) subGroup.addLayer(layer) + + def clearFlagsBeforeRunning(self): + lyrKeysToPop = [] + for lyrName, vl in self.executionOutput.result.items(): + if lyrName not in self.flags.flagLayerNames: + continue + QgsProject.instance().removeMapLayer(vl.id()) + lyrKeysToPop.append(lyrName) + for key in lyrKeysToPop: + self.executionOutput.result.pop(key) def createGroups(self, subgroupname): """Create groups in the layer panel. diff --git a/DsgTools/gui/CustomWidgets/OrderedPropertyWidgets/orderedTableWidget.py b/DsgTools/gui/CustomWidgets/OrderedPropertyWidgets/orderedTableWidget.py index c2af5cd98..b243437e8 100644 --- a/DsgTools/gui/CustomWidgets/OrderedPropertyWidgets/orderedTableWidget.py +++ b/DsgTools/gui/CustomWidgets/OrderedPropertyWidgets/orderedTableWidget.py @@ -228,7 +228,10 @@ def setValue(self, row, column, value): raise Exception( self.tr("Setter method must be defined for widget type.") ) - getattr(widget, setter)(value) + if isinstance(value, dict): + getattr(widget, setter)(**value) + else: + getattr(widget, setter)(value) def rowCount(self): """ @@ -340,7 +343,10 @@ def addRow(self, contents, row=None): else: widget = properties["widget"]() if value is not None: - getattr(widget, properties["setter"])(value) + if isinstance(value, dict): + getattr(widget, properties["setter"])(**value) + else: + getattr(widget, properties["setter"])(value) if isinstance(widget, QDoubleSpinBox): widget.setDecimals(16) self.tableWidget.setCellWidget(row, col, widget) diff --git a/DsgTools/gui/CustomWidgets/SelectionWidgets/importExportFileWidget.py b/DsgTools/gui/CustomWidgets/SelectionWidgets/importExportFileWidget.py new file mode 100644 index 000000000..7ffa86ffa --- /dev/null +++ b/DsgTools/gui/CustomWidgets/SelectionWidgets/importExportFileWidget.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- +""" +/*************************************************************************** + DsgTools + A QGIS plugin + Brazilian Army Cartographic Production Tools + ------------------- + begin : 2016-08-01 + git sha : $Format:%H$ + copyright : (C) 2016 by Philipe Borba - Cartographic Engineer @ Brazilian Army + email : borba.philipe@eb.mil.br + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +""" +import json +import os +from pathlib import Path + +from qgis.core import QgsMessageLog + +# Qt imports +from qgis.PyQt import QtWidgets, uic +from qgis.PyQt.QtCore import pyqtSlot, pyqtSignal, QSettings, QDir +from qgis.PyQt.QtSql import QSqlQuery +from qgis.PyQt.QtWidgets import QFileDialog + + +FORM_CLASS, _ = uic.loadUiType( + os.path.join(os.path.dirname(__file__), "importExportFileWidget.ui") +) + + +class ImportExportFileWidget(QtWidgets.QWidget, FORM_CLASS): + fileSelected = pyqtSignal() + fileExported = pyqtSignal(str) + + def __init__(self, parent=None): + """Constructor.""" + super(self.__class__, self).__init__(parent) + self.setupUi(self) + self.lineEdit.setReadOnly(True) + self.exportFilePushButton.setEnabled(False) + self.caption = "" + self.filter = "" + self.fileName = None + self.fileContent = None + + @pyqtSlot(bool) + def on_selectFilePushButton_clicked(self): + """ + Selects the correct way to choose files according to the type + """ + fd = QFileDialog() + fd.setDirectory(QDir.homePath()) + selectedFile = fd.getOpenFileName(caption=self.caption, filter=self.filter) + if isinstance(selectedFile, tuple): + self.fileName = Path(selectedFile[0]).name + selectedFile = ( + selectedFile[0] if isinstance(selectedFile, tuple) else selectedFile + ) + self.lineEdit.setText(self.fileName) + with open(self.selectedFilePath, "r") as f: + self.fileContent = f.readlines() + self.exportFilePushButton.setEnabled(True) + self.fileSelected.emit() + + @pyqtSlot(bool) + def on_exportFilePushButton_clicked(self): + if self.fileContent is None: + raise Exception("Invalid file content.") + fd = QFileDialog() + fd.setDirectory(QDir.homePath()) + filename = fd.getSaveFileName(caption=self.caption, filter=self.filter) + filename = ( + filename[0] + if Path(filename[0]).suffix in map(lambda x: x.strip(), self.filter.replace("(","").replace(")","").split("*")[1::]) + else f"{filename[0]}.{self.filter}" + ) + with open(filename, "w") as f: + f.writelines(self.fileContent) + self.fileExported.emit(filename[0]) + + def resetAll(self): + """ + Resets all + """ + self.lineEdit.clear() + + def setTitle(self, text): + """ + Sets the label title + """ + self.label.setText(text) + + def setCaption(self, caption): + """ + Sets the caption + """ + self.caption = caption + + def setFilter(self, filter): + """ + Sets the file filter + """ + self.filter = filter + + def setType(self, type): + """ + Sets selection type (e.g multi, single, dir) + """ + self.type = type + + def setFile(self, fileName, fileContent): + self.fileName = fileName + self.fileContent = fileContent + self.lineEdit.setText(self.fileName) + self.exportFilePushButton.setEnabled(self.fileName is not None and self.fileContent is not None) + + def getFile(self): + return (self.fileName, self.fileContent) \ No newline at end of file diff --git a/DsgTools/gui/CustomWidgets/SelectionWidgets/importExportFileWidget.ui b/DsgTools/gui/CustomWidgets/SelectionWidgets/importExportFileWidget.ui new file mode 100644 index 000000000..0d5bacb03 --- /dev/null +++ b/DsgTools/gui/CustomWidgets/SelectionWidgets/importExportFileWidget.ui @@ -0,0 +1,88 @@ + + + SelectFileWidget + + + + 0 + 0 + 201 + 26 + + + + + 0 + 0 + + + + + 0 + 26 + + + + Form + + + + 0 + + + 0 + + + 0 + + + 0 + + + + + Import file + + + + + + + :/plugins/DsgTools/icons/import.png:/plugins/DsgTools/icons/import.png + + + + + + + + 0 + 0 + + + + + + + + true + + + Export file + + + + + + + :/plugins/DsgTools/icons/export.png:/plugins/DsgTools/icons/export.png + + + + + + + + + + diff --git a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py index 12b9736ac..f2ae32bbc 100644 --- a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py +++ b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py @@ -386,7 +386,7 @@ def editCurrentWorkflow(self): os.path.dirname(__file__), "temp_workflow_{0}.workflow".format(hash(time())) ) with open(temp, "w+", encoding="utf-8") as f: - json.dump(workflow.asDict(), f) + json.dump(workflow.as_dict(), f) dlg.importWorkflow(temp) os.remove(temp) if dlg.exec_() != 1: @@ -448,6 +448,8 @@ def currentWorkflowFinishedExecutionMessage(self): self.tr(f"Workflow {currentWorkflow.displayName} execution has finished."), Qgis.Info, duration=3 ) + self.progressBar.setValue(100) + self.resumePushButton.setEnabled(False) def setRowColor(self, row, backgroundColor, foregroundColor): """ @@ -679,6 +681,8 @@ def runWorkflow(self): # outside thread execution setup and should all be handled from # within this method - at runtime sender = self.sender() + if sender.objectName() == "runPushButton": + self.resumePushButton.setEnabled(True) resumeFromStart = sender is None or sender.objectName() == "runPushButton" if resumeFromStart: workflow.resetWorkflowItems() diff --git a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py index 235cc162c..7cf672288 100644 --- a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py +++ b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py @@ -23,10 +23,12 @@ from functools import partial import os, json +from pathlib import Path from time import time from datetime import datetime from DsgTools.core.DSGToolsWorkflow.workflowItem import DSGToolsWorkflowItem +from DsgTools.gui.CustomWidgets.SelectionWidgets.importExportFileWidget import ImportExportFileWidget from qgis.PyQt import uic from qgis.core import Qgis from qgis.gui import QgsMessageBar @@ -103,8 +105,8 @@ def __init__(self, parent=None): "header": self.tr("Model source"), "type": "widget", "widget": self.modelWidget, - "setter": "setText", - "getter": "text", + "setter": "setFile", + "getter": "getFile", }, self.ON_FLAGS_HEADER: { "header": self.tr("On flags"), @@ -289,19 +291,14 @@ def modelWidget(self, filepath=None): :parma filepath: (str) path to a model. :return: (SelectFileWidget) DSGTools custom file selection widget. """ - widget = SelectFileWidget() - widget.label.hide() - widget.selectFilePushButton.setText("...") + widget = ImportExportFileWidget() widget.selectFilePushButton.setMaximumWidth(32) widget.lineEdit.setPlaceholderText(self.tr("Select a model...")) widget.lineEdit.setFrame(False) widget.setCaption(self.tr("Select a QGIS Processing model file")) widget.setFilter(self.tr("Select a QGIS Processing model (*.model *.model3)")) # defining setter and getter methods for composed widgets into OTW - widget.setText = widget.lineEdit.setText - widget.text = widget.lineEdit.text - if filepath is not None: - widget.setText(filepath) + widget.fileExported.connect(lambda x: self.pushMessage(x)) return widget def onFlagsWidget(self, option=None): @@ -434,7 +431,7 @@ def readRow(self, row): :return: (dict) parameters map. """ contents = self.orderedTableWidget.row(row) - filepath = contents[self.MODEL_SOURCE_HEADER].strip() + fileName, xml = contents[self.MODEL_SOURCE_HEADER] onFlagsIdx = contents[self.ON_FLAGS_HEADER] name = contents[self.MODEL_NAME_HEADER].strip() loadOutput = contents[self.LOAD_OUT_HEADER] @@ -443,11 +440,6 @@ def readRow(self, row): ] flagLayerNames = contents[self.FLAG_KEYS_HEADER].strip().split(",") pauseAfterExecution = contents[self.PAUSE_AFTER_EXECUTION] - if not os.path.exists(filepath): - xml = "" - else: - with open(filepath, "r", encoding="utf-8") as f: - xml = f.read() return { "displayName": name, "flags": { @@ -462,9 +454,7 @@ def readRow(self, row): "data": xml, }, "metadata": { - "originalName": os.path.relpath( - os.path.realpath(filepath), os.path.realpath(self.__qgisModelPath__) - ), + "originalName": fileName, }, } @@ -477,33 +467,12 @@ def setModelToRow(self, row: int, workflowItem: DSGToolsWorkflowItem): """ # all model files handled by this tool are read/written on QGIS model dir data = workflowItem.source.data - if workflowItem.source.type == "file" and os.path.exists(data): - with open(data, "r", encoding="utf-8") as f: - xml = f.read() - originalName = os.path.basename(data) - elif workflowItem.source.type == "xml": - xml = data - meta = workflowItem.metadata - originalName = ( - meta.originalName - if os.path.exists(meta.originalName) - else "temp_{0}.model3".format(hash(time())) - ) - else: - return False - path = os.path.join(self.__qgisModelPath__, originalName) - msg = self.tr( - "Model '{0}' is already imported would you like to overwrite it?" - ).format(path) - if os.path.exists(path) and self.confirmAction(msg, addPromptToAll=True): - os.remove(path) - if not os.path.exists(path): - with open(path, "w") as f: - f.write(xml) + meta = workflowItem.metadata + originalName = Path(meta.originalName).name self.orderedTableWidget.addRow( contents={ self.MODEL_NAME_HEADER: workflowItem.displayName, - self.MODEL_SOURCE_HEADER: path, + self.MODEL_SOURCE_HEADER: {"fileName":originalName, "fileContent":data}, self.ON_FLAGS_HEADER: { "halt": self.ON_FLAGS_HALT, "warn": self.ON_FLAGS_WARN, @@ -601,6 +570,11 @@ def exportWorkflow(self, filepath: str) -> bool: """ workflow = dsgtools_workflow_from_dict(self.workflowParameterMap()) return workflow.export(filepath=filepath) + + def pushMessage(self, message): + self.messageBar.pushMessage( + self.tr("Info"), message, level=Qgis.Warning, duration=5 + ) @pyqtSlot(bool, name="on_exportPushButton_clicked") def export(self): From 89639d3abeb954a31b25e79689036a7078245fca Mon Sep 17 00:00:00 2001 From: phborba Date: Wed, 29 May 2024 12:40:17 -0300 Subject: [PATCH 18/23] limpa camadas --- DsgTools/core/DSGToolsWorkflow/workflow.py | 17 ++++++++++++++++- DsgTools/core/DSGToolsWorkflow/workflowItem.py | 13 ++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/DsgTools/core/DSGToolsWorkflow/workflow.py b/DsgTools/core/DSGToolsWorkflow/workflow.py index d03d608b4..fff31075b 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflow.py +++ b/DsgTools/core/DSGToolsWorkflow/workflow.py @@ -32,6 +32,7 @@ QgsProcessingFeedback, QgsProcessingMultiStepFeedback, QgsTask, + QgsProject, ) from qgis.PyQt.QtCore import pyqtSignal, QObject @@ -242,8 +243,10 @@ def run(self, resumeFromStart: bool = True) -> None: self.resetWorkflowItems() self.setCurrentWorkflowItem(0) self.workflowHasBeenReset.emit() + self.removeEmptyGroups() currentWorkflowItem = self.getCurrentWorkflowItem() if currentWorkflowItem is None: + self.feedback.setProgress(100) self.currentWorkflowExecutionFinished.emit() return if currentWorkflowItem.getStatus() in [ExecutionStatus.IGNORE_FLAGS]: @@ -252,7 +255,7 @@ def run(self, resumeFromStart: bool = True) -> None: self.multiStepFeedback.setProgress(100) return currentWorkflowItem = self.getCurrentWorkflowItem() - currentWorkflowItem.clearFlagsBeforeRunning() + currentWorkflowItem.clearOutputs(onlyFlags=True) currentTask: QgsTask = self.prepareTask() currentWorkflowItem.changeCurrentStatus( status=ExecutionStatus.RUNNING, @@ -265,6 +268,18 @@ def run(self, resumeFromStart: bool = True) -> None: self.currentTaskChanged.emit(self.currentStepIndex, currentTask) QgsApplication.taskManager().addTask(currentTask) + def clearAllLayersBeforeRunning(self): + for workflowItem in self.workflowItemList: + workflowItem.clearOutputs() + + def removeEmptyGroups(self): + rootNode = QgsProject.instance().layerTreeRoot() + parentGroupName = "DSGTools_QA_Toolbox" + parentGroupNode = rootNode.findGroup(parentGroupName) + if parentGroupNode is None: + return + parentGroupNode.removeChildrenGroupWithoutLayers() + def setIgnoreFlagsStatusOnCurrentStep(self): """Set the status to ignore flags on the current workflow step.""" currentWorkflowItem = self.getCurrentWorkflowItem() diff --git a/DsgTools/core/DSGToolsWorkflow/workflowItem.py b/DsgTools/core/DSGToolsWorkflow/workflowItem.py index 2efde4bda..469bb3431 100644 --- a/DsgTools/core/DSGToolsWorkflow/workflowItem.py +++ b/DsgTools/core/DSGToolsWorkflow/workflowItem.py @@ -208,6 +208,8 @@ def __post_init__(self): def resetItem(self): """Reset the workflow item.""" + if hasattr(self, "executionOutput"): + self.clearOutputs() self.executionOutput = ModelExecutionOutput() def as_dict(self) -> Dict[str, str]: @@ -523,15 +525,20 @@ def addLayerToGroup(self, layer, subgroupname, clearGroupBeforeAdding=False): QgsProject.instance().addMapLayer(layer, addToLegend=False) subGroup.addLayer(layer) - def clearFlagsBeforeRunning(self): + def clearOutputs(self, onlyFlags=False): lyrKeysToPop = [] + iface.mapCanvas().freeze(True) for lyrName, vl in self.executionOutput.result.items(): - if lyrName not in self.flags.flagLayerNames: + if onlyFlags and lyrName not in self.flags.flagLayerNames: continue - QgsProject.instance().removeMapLayer(vl.id()) + try: + QgsProject.instance().removeMapLayer(vl.id()) + except: + pass lyrKeysToPop.append(lyrName) for key in lyrKeysToPop: self.executionOutput.result.pop(key) + iface.mapCanvas().freeze(False) def createGroups(self, subgroupname): """Create groups in the layer panel. From f4542dee9e3a938393d13f098574c4bea8576845 Mon Sep 17 00:00:00 2001 From: phborba Date: Fri, 31 May 2024 14:21:49 -0300 Subject: [PATCH 19/23] =?UTF-8?q?adiciona=20sele=C3=A7=C3=A3o=20de=20outpu?= =?UTF-8?q?ts=20para=20camada=20de=20flags=20e=20coloca=20dispositivo=20de?= =?UTF-8?q?=20acompanhar=20os=20processos=20executados=20(rolar=20a=20barr?= =?UTF-8?q?a=20vertical=20da=20caixa=20de=20ferramentas=20de=20controle=20?= =?UTF-8?q?de=20qualidade)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../customCheckableComboBox.py | 37 +++++ .../importExportFileWidget.py | 16 +- .../qualityAssuranceDockWidget.py | 4 + .../workflowSetupDialog.py | 140 ++++++++---------- .../workflowSetupDialog.ui | 4 +- 5 files changed, 118 insertions(+), 83 deletions(-) create mode 100644 DsgTools/gui/CustomWidgets/SelectionWidgets/customCheckableComboBox.py diff --git a/DsgTools/gui/CustomWidgets/SelectionWidgets/customCheckableComboBox.py b/DsgTools/gui/CustomWidgets/SelectionWidgets/customCheckableComboBox.py new file mode 100644 index 000000000..2f56e5356 --- /dev/null +++ b/DsgTools/gui/CustomWidgets/SelectionWidgets/customCheckableComboBox.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +""" +/*************************************************************************** + DsgTools + A QGIS plugin + Brazilian Army Cartographic Production Tools + ------------------- + begin : 2024-05-31 + git sha : $Format:%H$ + copyright : (C) 2024 by Philipe Borba - Cartographic Engineer @ Brazilian Army + email : borba.philipe@eb.mil.br + ***************************************************************************/ + +/*************************************************************************** + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * * + ***************************************************************************/ +""" + +from typing import List, Optional, Union +from qgis.gui import QgsCheckableComboBox + +class CustomCheckableComboBox(QgsCheckableComboBox): + def __init__(self): + super(CustomCheckableComboBox, self).__init__() + + def setData(self, items: Union[List[str], str], checkedItems: Optional[Union[List[str], str]] = None): + itemList = items.split(',') if isinstance(items, str) else items + self.addItems(itemList) + if checkedItems is None: + return + checkedItemList = checkedItems.split(',') if isinstance(checkedItems, str) else checkedItems + self.setCheckedItems(checkedItemList) diff --git a/DsgTools/gui/CustomWidgets/SelectionWidgets/importExportFileWidget.py b/DsgTools/gui/CustomWidgets/SelectionWidgets/importExportFileWidget.py index 7ffa86ffa..e5b49aaf3 100644 --- a/DsgTools/gui/CustomWidgets/SelectionWidgets/importExportFileWidget.py +++ b/DsgTools/gui/CustomWidgets/SelectionWidgets/importExportFileWidget.py @@ -39,7 +39,7 @@ class ImportExportFileWidget(QtWidgets.QWidget, FORM_CLASS): - fileSelected = pyqtSignal() + fileSelected = pyqtSignal(str, str) fileExported = pyqtSignal(str) def __init__(self, parent=None): @@ -67,10 +67,13 @@ def on_selectFilePushButton_clicked(self): selectedFile[0] if isinstance(selectedFile, tuple) else selectedFile ) self.lineEdit.setText(self.fileName) - with open(self.selectedFilePath, "r") as f: - self.fileContent = f.readlines() + if selectedFile == '': + self.fileContent = None + return + with open(selectedFile, "r") as f: + self.fileContent = f.read() self.exportFilePushButton.setEnabled(True) - self.fileSelected.emit() + self.fileSelected.emit(self.fileName, self.fileContent) @pyqtSlot(bool) def on_exportFilePushButton_clicked(self): @@ -85,7 +88,7 @@ def on_exportFilePushButton_clicked(self): else f"{filename[0]}.{self.filter}" ) with open(filename, "w") as f: - f.writelines(self.fileContent) + f.write(self.fileContent) self.fileExported.emit(filename[0]) def resetAll(self): @@ -123,6 +126,7 @@ def setFile(self, fileName, fileContent): self.fileContent = fileContent self.lineEdit.setText(self.fileName) self.exportFilePushButton.setEnabled(self.fileName is not None and self.fileContent is not None) + self.fileSelected.emit(self.fileName, self.fileContent) def getFile(self): - return (self.fileName, self.fileContent) \ No newline at end of file + return {"fileName": self.fileName, "fileContent":self.fileContent} \ No newline at end of file diff --git a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py index f2ae32bbc..8b7fde30b 100644 --- a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py +++ b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/qualityAssuranceDockWidget.py @@ -483,6 +483,10 @@ def setModelStatus(self, row, workflowItem): self.setRowStatus(row, code) self.tableWidget.cellWidget(row, 1).setText(status) self.setGuiState(code == ExecutionStatus.RUNNING) + if code == ExecutionStatus.RUNNING: + pageStep = self.tableWidget.verticalScrollBar().pageStep() + if row >= pageStep: + self.tableWidget.verticalScrollBar().setValue(row) if code in [ExecutionStatus.FAILED, ExecutionStatus.FINISHED_WITH_FLAGS, ExecutionStatus.IGNORE_FLAGS]: # advise user a model status has changed only if it came from a # signal call diff --git a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py index 7cf672288..7d61b30e6 100644 --- a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py +++ b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py @@ -27,12 +27,13 @@ from time import time from datetime import datetime -from DsgTools.core.DSGToolsWorkflow.workflowItem import DSGToolsWorkflowItem +from DsgTools.core.DSGToolsWorkflow.workflowItem import DSGToolsWorkflowItem, ModelSource +from DsgTools.gui.CustomWidgets.SelectionWidgets.customCheckableComboBox import CustomCheckableComboBox from DsgTools.gui.CustomWidgets.SelectionWidgets.importExportFileWidget import ImportExportFileWidget -from qgis.PyQt import uic +from qgis.PyQt import uic, QtCore, QtWidgets from qgis.core import Qgis -from qgis.gui import QgsMessageBar -from qgis.PyQt.QtCore import QSize, QCoreApplication, pyqtSlot +from qgis.gui import QgsMessageBar, QgsCheckableComboBox +from qgis.PyQt.QtCore import QSize, QCoreApplication, pyqtSlot, Qt from qgis.PyQt.QtWidgets import ( QDialog, QComboBox, @@ -74,11 +75,11 @@ class WorkflowSetupDialog(QDialog, FORM_CLASS): ( MODEL_NAME_HEADER, MODEL_SOURCE_HEADER, - ON_FLAGS_HEADER, - LOAD_OUT_HEADER, FLAG_KEYS_HEADER, + ON_FLAGS_HEADER, FLAG_CAN_BE_FALSE_POSITIVE_HEADER, - PAUSE_AFTER_EXECUTION, + PAUSE_AFTER_EXECUTION_HEADER, + LOAD_OUTPUTS_THAT_ARE_NOT_FLAG_HEADER, ) = range(7) def __init__(self, parent=None): @@ -108,6 +109,13 @@ def __init__(self, parent=None): "setter": "setFile", "getter": "getFile", }, + self.FLAG_KEYS_HEADER: { + "header": self.tr("Flag keys"), + "type": "widget", + "widget": self.loadFlagLayers, + "setter": "setData", + "getter": "checkedItems", + }, self.ON_FLAGS_HEADER: { "header": self.tr("On flags"), "type": "widget", @@ -115,20 +123,6 @@ def __init__(self, parent=None): "setter": "setCurrentIndex", "getter": "currentIndex", }, - self.LOAD_OUT_HEADER: { - "header": self.tr("Load output"), - "type": "widget", - "widget": self.loadOutputWidget, - "setter": "setChecked", - "getter": "isChecked", - }, - self.FLAG_KEYS_HEADER: { - "header": self.tr("Flag keys"), - "type": "widget", - "widget": self.loadFlagLayers, - "setter": "setText", - "getter": "text", - }, self.FLAG_CAN_BE_FALSE_POSITIVE_HEADER: { "header": self.tr("Flags can be false positive"), "type": "widget", @@ -136,29 +130,34 @@ def __init__(self, parent=None): "setter": "setChecked", "getter": "isChecked", }, - self.PAUSE_AFTER_EXECUTION: { + self.PAUSE_AFTER_EXECUTION_HEADER: { "header": self.tr("Pause after execution"), "type": "widget", "widget": self.pauseAfterExecutionWidget, "setter": "setChecked", "getter": "isChecked", }, + self.LOAD_OUTPUTS_THAT_ARE_NOT_FLAG_HEADER: { + "header": self.tr("Load output layers that are not flags"), + "type": "widget", + "widget": self.loadOutputWidget, + "setter": "setChecked", + "getter": "isChecked", + }, } ) self.orderedTableWidget.setHeaderDoubleClickBehaviour("replicate") + self.orderedTableWidget.horizontalHeader().setStretchLastSection(True) + self.orderedTableWidget.tableWidget.setHorizontalScrollBarPolicy(QtCore.Qt.ScrollBarAsNeeded) + self.orderedTableWidget.tableWidget.horizontalHeader().setDefaultSectionSize(150) + self.orderedTableWidget.tableWidget.horizontalHeader().setMinimumSectionSize(100) + self.orderedTableWidget.tableWidget.horizontalHeader().resizeSection(self.MODEL_NAME_HEADER, 200) + self.orderedTableWidget.tableWidget.horizontalHeader().resizeSection(self.MODEL_SOURCE_HEADER, 250) + self.orderedTableWidget.tableWidget.horizontalHeader().resizeSection(self.FLAG_KEYS_HEADER, 200) + self.orderedTableWidget.tableWidget.horizontalHeader().resizeSection(self.ON_FLAGS_HEADER, 100) self.promptToAll = None self.orderedTableWidget.rowAdded.connect(self.postAddRowStandard) - def _checkFalsePositiveAvailability(self, row): - comboBox = self.orderedTableWidget.itemAt(row, 2) - checkBox = self.orderedTableWidget.itemAt(row, 5) - flagType = comboBox.currentIndex() - if flagType != self.ON_FLAGS_HALT: - checkBox.setChecked(False) - checkBox.setEnabled(False) - return - checkBox.setEnabled(True) - def postAddRowStandard(self, row): """ Sets up widgets to work as expected right after they are added to GUI. @@ -167,41 +166,23 @@ def postAddRowStandard(self, row): # in standard GUI, the layer selectors are QgsMapLayerComboBox, and its # layer changed signal should be connected to the filter expression # widget setup - comboBox = self.orderedTableWidget.itemAt(row, 2) - comboBox.currentIndexChanged.connect( - partial(self._checkFalsePositiveAvailability, row) - ) - self.resizeTable() - - def resizeTable(self): - """ - Adjusts table columns sizes. - """ - dSize = ( - self.orderedTableWidget.geometry().width() - - self.orderedTableWidget.horizontalHeader().geometry().width() + importExportFileWidget = self.orderedTableWidget.itemAt(row, self.MODEL_SOURCE_HEADER) + importExportFileWidget.fileSelected.connect( + partial(self._getModelOutputs, row) ) - onFlagsColSize = self.orderedTableWidget.sectionSize(2) - loadOutColSize = self.orderedTableWidget.sectionSize(3) - flagsOutColSize = self.orderedTableWidget.sectionSize(4) - falsePositiveColSize = self.orderedTableWidget.sectionSize(5) - pauseAfterExecutionColSize = self.orderedTableWidget.sectionSize(6) - missingBarSize = ( - self.geometry().size().width() - - dSize - - onFlagsColSize - - falsePositiveColSize - - loadOutColSize - - flagsOutColSize - - pauseAfterExecutionColSize - ) - # the "-11" is empiric: it makes it fit header to table - self.orderedTableWidget.tableWidget.horizontalHeader().resizeSection( - 0, int(0.4 * missingBarSize) - 11 - ) - self.orderedTableWidget.tableWidget.horizontalHeader().resizeSection( - 1, missingBarSize - int(0.4 * missingBarSize) - 11 + + def _getModelOutputs(self, row, fileName, fileContent): + currentModelSource = ModelSource( + type="xml", + data=fileContent ) + currentModel = currentModelSource.modelFromXml() + outputs = [ + outputDef.name().split(":")[-1] for outputDef in currentModel.outputDefinitions() + ] + currentCheckableCombobBox = self.orderedTableWidget.itemAt(row, self.FLAG_KEYS_HEADER) + currentCheckableCombobBox.clear() + currentCheckableCombobBox.setData(outputs) def resizeEvent(self, e): """ @@ -216,7 +197,6 @@ def resizeEvent(self, e): 40, # this felt nicer than the original height (30) ) ) - self.resizeTable() def confirmAction(self, msg, showCancel=True, addPromptToAll=False): """ @@ -283,6 +263,7 @@ def modelNameWidget(self, name=None): if name is not None: le.setText(name) le.setFrame(False) + le.textChanged.connect(lambda x: le.setToolTip(x)) return le def modelWidget(self, filepath=None): @@ -357,7 +338,13 @@ def loadOutputWidget(self, option=None): return cb def loadFlagLayers(self): - return QLineEdit() + sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.MinimumExpanding, QtWidgets.QSizePolicy.Minimum) + sizePolicy.setHorizontalStretch(0) + sizePolicy.setVerticalStretch(0) + widget = CustomCheckableComboBox() + widget.setSizePolicy(sizePolicy) + widget.setMinimumSize(QtCore.QSize(100, 25)) + return widget def now(self): """ @@ -434,12 +421,12 @@ def readRow(self, row): fileName, xml = contents[self.MODEL_SOURCE_HEADER] onFlagsIdx = contents[self.ON_FLAGS_HEADER] name = contents[self.MODEL_NAME_HEADER].strip() - loadOutput = contents[self.LOAD_OUT_HEADER] + loadOutput = contents[self.LOAD_OUTPUTS_THAT_ARE_NOT_FLAG_HEADER] modelCanHaveFalsePositiveFlags = contents[ self.FLAG_CAN_BE_FALSE_POSITIVE_HEADER ] - flagLayerNames = contents[self.FLAG_KEYS_HEADER].strip().split(",") - pauseAfterExecution = contents[self.PAUSE_AFTER_EXECUTION] + flagLayerNames = contents[self.FLAG_KEYS_HEADER] + pauseAfterExecution = contents[self.PAUSE_AFTER_EXECUTION_HEADER] return { "displayName": name, "flags": { @@ -479,11 +466,14 @@ def setModelToRow(self, row: int, workflowItem: DSGToolsWorkflowItem): "ignore": self.ON_FLAGS_IGNORE, }[workflowItem.flags.onFlagsRaised], self.FLAG_CAN_BE_FALSE_POSITIVE_HEADER: workflowItem.flags.modelCanHaveFalsePositiveFlags, - self.LOAD_OUT_HEADER: workflowItem.flags.loadOutput, - self.FLAG_KEYS_HEADER: ",".join( - map(lambda x: str(x).strip(), workflowItem.flags.flagLayerNames) - ), - self.PAUSE_AFTER_EXECUTION: workflowItem.pauseAfterExecution, + self.LOAD_OUTPUTS_THAT_ARE_NOT_FLAG_HEADER: workflowItem.flags.loadOutput, + self.FLAG_KEYS_HEADER: { + "items": workflowItem.getAllOutputNamesFromModel(), + "checkedItems": ",".join( + map(lambda x: str(x).strip(), workflowItem.flags.flagLayerNames) + ) + }, + self.PAUSE_AFTER_EXECUTION_HEADER: workflowItem.pauseAfterExecution, } ) return True diff --git a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.ui b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.ui index 7b0fa368b..b572dd958 100644 --- a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.ui +++ b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.ui @@ -6,8 +6,8 @@ 0 0 - 1161 - 668 + 1400 + 550 From a131adeea701dce5a01de3908933e873e5a2c23c Mon Sep 17 00:00:00 2001 From: phborba Date: Fri, 31 May 2024 15:05:18 -0300 Subject: [PATCH 20/23] fix --- .../Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py index 7d61b30e6..54a95efb9 100644 --- a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py +++ b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py @@ -563,7 +563,7 @@ def exportWorkflow(self, filepath: str) -> bool: def pushMessage(self, message): self.messageBar.pushMessage( - self.tr("Info"), message, level=Qgis.Warning, duration=5 + self.tr("Info"), message, level=Qgis.Info, duration=5 ) @pyqtSlot(bool, name="on_exportPushButton_clicked") From 24cf5a6b844f9458777d52eab17eecb445a7ff8c Mon Sep 17 00:00:00 2001 From: phborba Date: Fri, 31 May 2024 15:17:20 -0300 Subject: [PATCH 21/23] export fix --- .../CustomWidgets/SelectionWidgets/importExportFileWidget.py | 4 +++- .../Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/DsgTools/gui/CustomWidgets/SelectionWidgets/importExportFileWidget.py b/DsgTools/gui/CustomWidgets/SelectionWidgets/importExportFileWidget.py index e5b49aaf3..277fe3757 100644 --- a/DsgTools/gui/CustomWidgets/SelectionWidgets/importExportFileWidget.py +++ b/DsgTools/gui/CustomWidgets/SelectionWidgets/importExportFileWidget.py @@ -87,9 +87,11 @@ def on_exportFilePushButton_clicked(self): if Path(filename[0]).suffix in map(lambda x: x.strip(), self.filter.replace("(","").replace(")","").split("*")[1::]) else f"{filename[0]}.{self.filter}" ) + if ".model3" not in filename or ".model" not in filename: + filename = filename + ".model3" with open(filename, "w") as f: f.write(self.fileContent) - self.fileExported.emit(filename[0]) + self.fileExported.emit(filename) def resetAll(self): """ diff --git a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py index 54a95efb9..4137befa0 100644 --- a/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py +++ b/DsgTools/gui/ProductionTools/Toolboxes/QualityAssuranceToolBox/workflowSetupDialog.py @@ -277,9 +277,9 @@ def modelWidget(self, filepath=None): widget.lineEdit.setPlaceholderText(self.tr("Select a model...")) widget.lineEdit.setFrame(False) widget.setCaption(self.tr("Select a QGIS Processing model file")) - widget.setFilter(self.tr("Select a QGIS Processing model (*.model *.model3)")) + widget.setFilter(self.tr("Select a QGIS Processing model (*.model3 *.model)")) # defining setter and getter methods for composed widgets into OTW - widget.fileExported.connect(lambda x: self.pushMessage(x)) + widget.fileExported.connect(lambda x: self.pushMessage(self.tr(f"Model source exported to file {x}"))) return widget def onFlagsWidget(self, option=None): From d0b36c1251f33e40c90b8f6dee97af83aedf13ae Mon Sep 17 00:00:00 2001 From: phborba Date: Fri, 31 May 2024 15:23:34 -0300 Subject: [PATCH 22/23] version fix --- CHANGELOG.md | 16 ++++++++++++++-- DsgTools/metadata.txt | 2 +- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 796b8c543..7ec11fa92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # CHANGELOG -## 4.13.37 - dev +## 4.13.49 - dev Novas Funcionalidades: @@ -8,6 +8,11 @@ Novas Funcionalidades: - Novo processo de identificar mudança de atributo em linhas (portado do ferramentas experimentais); - Novo processo BatchRunAlgorithmWithGeographicBoundsConstraint: roda em lote, como o BatchRunAlgorithm no modelo, porém tem uma camada de entrada para o limite geográfico (o limite geográfico pode filtrar as flags de saída ou ser input, como no caso do clean com alterações somente dentro da moldura); - Novo processo de identificar linhas não cobertas por outras linhas (identifica travessias hidroviárias que não se conectam com vias deslocamento ou com a moldura); +- Adiciona suporte para a EDGV 3.0 Topo; +- Novo processo de reclassificar pixel adjacente em raster para o vizinho mais próximo (útil para generalizar raster de vegetação classificada); +- Novo processo de reclassificar conjunto de pixels adjacentes em raster para o vizinho mais próximo (variação do algoritmo de reclassificar pixel adjacente em raster para o vizinho mais próximo que considera área do agrupamento de pixels); +- Novo processo de reclassificar conjunto de pixels adjacenter por meio de janela deslizante; +- Novo processo de selecionar conjuntos de linhas fechadas pequenos; Melhorias: @@ -29,6 +34,10 @@ Melhorias: - Adiciona camada de saída no algoritmo de verificação de ortografia (SpellChecker) para indicar se há erros; - A Caixa de Ferramentas de Controle de Qualidade agora só limpa as camadas que são flags dos processos, mantendo as entradas; - Adicionada uma verificação na execução da Caixa de Ferramentas de Controle de Qualidade para evitar que o usuário comece o processo novamente sem querer; +- Adiciona a opção de permitir linhas fechadas no processamento de unir linhas (utilizado para fechar curvas de nível em processamentos específicos de estilos na edição); +- Altera o valor default da ferramenta de revisão para Pan to Next; +- Adiciona a melhoria no menu de reclassificação para puxar os campos idênticos da camada de origem; +- Workflow refatorado para corrigir constantes crashes durante a utilização; Correção de bug: @@ -52,6 +61,9 @@ Correção de bug: - Corrige bug de estado guardado no projeto no workflow; - Corrige crashes no dsgtools nos processings (versão 4.13.35, para refrência em caso de problemas); - Corrige bug com camada vazia em models do workflow ao executar o processo de identificar linhas não cobertas por outras linhas (IdentifyUncoveredStartAndEndPointsAlgorithm); +- Corrige bug em Update Runway Altitude; +- Corrige bug em Identify Intertwined Lines para tratar caso de Geometry Collection (linhas que se cruzam e se sobrepoem); +- Corrige bug nos Batch Run, valor padrão removido para compatibilidade com models nas versões mais atuais do QGIS (a partir da 3.30), não afeta versões mais antigas do QGIS; ## 4.12.0 - 2023-12-13 @@ -378,4 +390,4 @@ Correção de bugs: - O problema onde a Ferramenta de Aquisição com Ângulos Retos e a Ferramenta de Aquisição à Mão Livre não atribuíam os valores padrões nos formulários da feição foi corrigido - Correção nos processings de geração de MI: remover MI que não existem -Changelog completo: +Changelog completo: \ No newline at end of file diff --git a/DsgTools/metadata.txt b/DsgTools/metadata.txt index bc30a9b83..e2cd82328 100644 --- a/DsgTools/metadata.txt +++ b/DsgTools/metadata.txt @@ -10,7 +10,7 @@ name=DSG Tools qgisMinimumVersion=3.22 description=Brazilian Army Cartographic Production Tools -version=4.13.37 +version=4.13.49 author=Brazilian Army Geographic Service email=dsgtools@dsg.eb.mil.br about= From 9c8ccdb2c25d28b7f1ce5df166b8b1c9efc4a168 Mon Sep 17 00:00:00 2001 From: phborba Date: Fri, 31 May 2024 15:27:21 -0300 Subject: [PATCH 23/23] fix --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ec11fa92..8f34f9cd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,8 @@ Melhorias: - Altera o valor default da ferramenta de revisão para Pan to Next; - Adiciona a melhoria no menu de reclassificação para puxar os campos idênticos da camada de origem; - Workflow refatorado para corrigir constantes crashes durante a utilização; +- Alterada a forma de importar e exportar os modelos de dentro do workflow; +- Alterada a interface de gerência de flags na construção do workflow (combo box selecionando as saídas); Correção de bug: