diff --git a/spine_items/data_connection/data_connection.py b/spine_items/data_connection/data_connection.py index 11d6e3d0..1b9ffaf8 100644 --- a/spine_items/data_connection/data_connection.py +++ b/spine_items/data_connection/data_connection.py @@ -8,11 +8,7 @@ # Public License for more details. You should have received a copy of the GNU Lesser General Public License along with # this program. If not, see . ###################################################################################################################### - -""" -Module for data connection class. - -""" +"""Module for data connection class.""" import os import shutil @@ -21,6 +17,7 @@ from PySide6.QtGui import QStandardItem, QStandardItemModel, QBrush from PySide6.QtWidgets import QFileDialog, QGraphicsItem, QFileIconProvider, QInputDialog, QMessageBox from spine_engine.utils.serialization import deserialize_path, serialize_path +from spinedb_api.helpers import remove_credentials_from_url from spinetoolbox.project_item.project_item import ProjectItem from spinetoolbox.widgets.custom_qwidgets import ToolBarWidget from spinetoolbox.helpers import open_url, same_path @@ -30,24 +27,29 @@ from .executable_item import ExecutableItem from .item_info import ItemInfo from .output_resources import scan_for_resources +from .utils import restore_database_references from ..database_validation import DatabaseConnectionValidator from ..widgets import UrlSelectorDialog -from ..utils import split_url_credentials +from ..utils import convert_to_sqlalchemy_url, convert_url_to_safe_string + +class _Role: + """Extra reference model data roles.""" -_DATA_FILE_PATH_ROLE = Qt.ItemDataRole.UserRole + 1 -_MISSING_ROLE = Qt.ItemDataRole.UserRole + 2 + DATA_FILE_PATH = Qt.ItemDataRole.UserRole + 1 + MISSING = Qt.ItemDataRole.UserRole + 2 + FILE_REFERENCE = Qt.ItemDataRole.UserRole + 3 + DB_URL_REFERENCE = Qt.ItemDataRole.UserRole + 4 _MISSING_ITEM_FOREGROUND = QBrush(Qt.red) class DataConnection(ProjectItem): - def __init__( - self, name, description, x, y, toolbox, project, file_references=None, db_references=None, db_credentials=None - ): - """Data Connection class. + """Data connection project item.""" + def __init__(self, name, description, x, y, toolbox, project, file_references=None, db_references=None): + """ Args: name (str): Object name description (str): Object description @@ -56,23 +58,18 @@ def __init__( toolbox (ToolboxUI): QMainWindow instance project (SpineToolboxProject): the project this item belongs to file_references (list, optional): a list of file paths - db_references (list, optional): a list of db urls - db_credentials (dict, optional): mapping urls from db_references to tuple (username, password) + db_references (list of dict, optional): a list of db urls """ super().__init__(name, description, x, y, project) if file_references is None: file_references = list() if db_references is None: db_references = list() - if db_credentials is None: - db_credentials = dict() self._toolbox = toolbox self.reference_model = QStandardItemModel() # References self.data_model = QStandardItemModel() # Paths of project internal files. These are found in DC data directory self.file_system_watcher = None self.file_references = list(file_references) - self.db_references = list(db_references) - self.db_credentials = db_credentials self._file_ref_root = QStandardItem("File paths") self._file_ref_root.setFlags(self._file_ref_root.flags() & ~Qt.ItemIsEditable) self._db_ref_root = QStandardItem("URLs") @@ -81,7 +78,7 @@ def __init__( self.any_refs_selected = False self.any_data_selected = False self.current_is_file_ref = False - self.populate_reference_list() + self.populate_reference_list(db_references) self.populate_data_list() self._database_validator = DatabaseConnectionValidator() @@ -89,6 +86,7 @@ def set_up(self): super().set_up() self.file_system_watcher = CustomFileSystemWatcher(self) self.file_system_watcher.add_persistent_file_paths(ref for ref in self.file_references if os.path.exists(ref)) + self._watch_sqlite_file(*self.db_reference_iter()) self.file_system_watcher.add_persistent_dir_path(self.data_dir) self.file_system_watcher.file_removed.connect(self._handle_file_removed) self.file_system_watcher.file_renamed.connect(self._handle_file_renamed) @@ -108,6 +106,18 @@ def item_category(): def executable_class(self): return ExecutableItem + def db_reference_iter(self): + """Iterates over database references. + + Yields: + dict: database URL + """ + for row in range(self._db_ref_root.rowCount()): + yield self._db_ref_root.child(row).data(_Role.DB_URL_REFERENCE) + + def has_db_references(self): + return self._db_ref_root.rowCount() != 0 + @Slot(QItemSelection, QItemSelection) def _update_selection_state(self, _selected, _deselected): self._do_update_selection_state() @@ -126,7 +136,6 @@ def make_signal_handler_dict(self): """Returns a dictionary of all shared signals and their handlers. This is to enable simpler connecting and disconnecting.""" s = super().make_signal_handler_dict() - # pylint: disable=unnecessary-lambda s[self._properties_ui.toolButton_minus.clicked] = self.remove_references s[self._properties_ui.toolButton_add.clicked] = self.copy_to_project s[self._properties_ui.treeView_dc_references.doubleClicked] = self.open_reference @@ -213,22 +222,39 @@ def _add_file_references(self, paths): @Slot(bool) def show_add_db_reference_dialog(self, _=False): - """Opens a dialog where user can select an url to be added as reference for this Data Connection.""" - selector = UrlSelectorDialog(self._toolbox.qsettings(), self._toolbox, self._toolbox) - selector.exec() - full_url = selector.url - if not full_url: # Cancel button clicked + """Opens a dialog where user can select a url to be added as reference for this Data Connection.""" + selector = UrlSelectorDialog(self._toolbox.qsettings(), False, self._toolbox, self._toolbox) + result = selector.exec() + if result == UrlSelectorDialog.DialogCode.Rejected: return - url, credentials = split_url_credentials(full_url) - if url in self.db_references: + url = selector.url_dict() + if self._has_db_reference(url): self._logger.msg_warning.emit(f"Reference to database {url} already exists") return + sa_url = convert_to_sqlalchemy_url(url, self.name, self._logger) self._database_validator.validate_url( - selector.dialect, full_url, self._log_database_reference_error, success_slot=None + url["dialect"], sa_url, self._log_database_reference_error, success_slot=None ) - self.db_credentials[url] = credentials self._toolbox.undo_stack.push(AddDCReferencesCommand(self, [], [url])) + def _has_db_reference(self, url): + """Checks if given database URL exists already. + + Ignores usernames and passwords. + + Args: + url (dict): URL to check + + Returns: + bool: True if db reference exists, False otherwise + """ + significant_keys = ("dialect", "host", "port", "database") + for row in range(self._db_ref_root.rowCount()): + existing_url = self._db_ref_root.child(row).data(_Role.DB_URL_REFERENCE) + if all(url[key] == existing_url[key] for key in significant_keys): + return True + return False + @Slot(str) def _log_database_reference_error(self, error): """Logs final database validation error messages. @@ -239,11 +265,17 @@ def _log_database_reference_error(self, error): self._logger.msg_error.emit(f"{self.name}: invalid database URL: {error}") def do_add_references(self, file_refs, db_refs): + """Adds file and databases references to DC and starts watching the files. + + Args: + file_refs (list of str): file reference paths + db_refs (list of dict): database reference URLs + """ file_refs = [os.path.abspath(ref) for ref in file_refs] self.file_references += file_refs self.file_system_watcher.add_persistent_file_paths(ref for ref in file_refs if os.path.exists(ref)) self._append_file_references_to_model(*file_refs) - self.db_references += db_refs + self._watch_sqlite_file(*db_refs) self._append_db_references_to_model(*db_refs) self._check_notifications() self._resources_to_successors_changed() @@ -266,7 +298,7 @@ def remove_references(self, _=False): if parent == file_ref_root_index: file_references.append(index.data(Qt.ItemDataRole.DisplayRole)) elif parent == db_ref_root_index: - db_references.append(index.data(Qt.ItemDataRole.DisplayRole)) + db_references.append(index.data(_Role.DB_URL_REFERENCE)) self._toolbox.undo_stack.push(RemoveDCReferencesCommand(self, file_references, db_references)) self._logger.msg.emit("Selected references removed") @@ -274,10 +306,11 @@ def do_remove_references(self, file_refs, db_refs): """Removes given paths from references. Args: - file_refs (list): List of removed file paths. - db_refs (list): List of removed urls. + file_refs (list of str): List of removed file paths. + db_refs (list of dict): List of removed urls. """ self.file_system_watcher.remove_persistent_file_paths(file_refs) + self._unwatch_sqlite_file(*db_refs) refs_removed = self._remove_file_references(*file_refs) refs_removed |= self._remove_db_references(*db_refs) if refs_removed: @@ -295,17 +328,16 @@ def _remove_file_references(self, *refs): def _remove_db_references(self, *refs): result = False + matches = {convert_url_to_safe_string(url) for url in refs} for k in reversed(range(self._db_ref_root.rowCount())): - if self._db_ref_root.child(k).text() in refs: - url = self.db_references.pop(k) - self.db_credentials.pop(url, None) + if self._db_ref_root.child(k).text() in matches: self._db_ref_root.removeRow(k) result = True return result def _remove_data_file(self, path): for k in reversed(range(self.data_model.rowCount())): - data_filepath = self.data_model.item(k).data(_DATA_FILE_PATH_ROLE) + data_filepath = self.data_model.item(k).data(_Role.DATA_FILE_PATH) if same_path(data_filepath, path): self.data_model.removeRow(k) return True @@ -314,9 +346,9 @@ def _remove_data_file(self, path): def _rename_data_file(self, old_path, new_path): for k in range(self.data_model.rowCount()): item = self.data_model.item(k) - if same_path(item.data(_DATA_FILE_PATH_ROLE), old_path): + if same_path(item.data(_Role.DATA_FILE_PATH), old_path): item.setText(os.path.basename(new_path)) - item.setData(new_path, _DATA_FILE_PATH_ROLE) + item.setData(new_path, _Role.DATA_FILE_PATH) return True return False @@ -370,6 +402,14 @@ def _try_to_mark_file_reference_missing(self, path): if same_path(item.text(), path): self._mark_as_missing(item) return True + for row in range(self._db_ref_root.rowCount()): + item = self._db_ref_root.child(row) + url = item.data(_Role.DB_URL_REFERENCE) + if url["dialect"] != "sqlite": + continue + if same_path(url["database"], path): + self._mark_as_missing(item) + return True return False @Slot(str, str) @@ -399,12 +439,12 @@ def replace_new_path(paths): file_refs = list(self.file_references) data_files = [os.path.join(self.data_dir, f) for f in self.data_files()] new_resources = scan_for_resources( - self, file_refs + data_files, self.db_references, self.db_credentials, self._project.project_dir + self, file_refs + data_files, list(self.db_reference_iter()), self._project.project_dir ) if not replace_new_path(file_refs): replace_new_path(data_files) old_resources = scan_for_resources( - self, file_refs + data_files, self.db_references, self.db_credentials, self._project.project_dir + self, file_refs + data_files, list(self.db_reference_iter()), self._project.project_dir ) self._resources_to_successors_replaced(old_resources, new_resources) @@ -430,9 +470,7 @@ def refresh(retry_count): if not same_path(path, item.text()): continue fixed_references.append(path) - item.clearData() - item.setFlags(~Qt.ItemIsEditable) - item.setData(path, Qt.ItemDataRole.DisplayRole) + self._mark_as_found(item) if not fixed_references: return self.file_system_watcher.add_persistent_file_paths(ref for ref in fixed_references) @@ -477,22 +515,53 @@ def do_copy_to_project(self, paths): def refresh_references(self): """Checks if missing file references have somehow come back to life.""" selected_indexes = self._properties_ui.treeView_dc_references.selectedIndexes() - fixed_references = [] if not selected_indexes: return for index in selected_indexes: - if self.reference_model.itemFromIndex(index.parent()) is not self._file_ref_root: - continue item = self.reference_model.itemFromIndex(index) - path = item.data(Qt.ItemDataRole.DisplayRole) - if item.data(_MISSING_ROLE) and os.path.exists(path): - fixed_references.append(path) - item.clearData() - item.setFlags(~Qt.ItemIsEditable) - item.setData(path, Qt.ItemDataRole.DisplayRole) - if not fixed_references: + if self.reference_model.itemFromIndex(index.parent()) is self._db_ref_root: + self.refresh_db_references(item) + else: + self.refresh_file_references(item) + + def refresh_file_references(self, item): + file_path = item.data(Qt.ItemDataRole.DisplayRole) + if item.data(_Role.MISSING) and os.path.exists(file_path): + self._mark_as_found(item) + else: + return + self.file_system_watcher.add_persistent_file_path(file_path) + self._check_notifications() + self._resources_to_successors_changed() + + def refresh_db_references(self, item): + """Checks if the db reference is valid""" + url = item.data(_Role.DB_URL_REFERENCE) + self._database_validator.validate_url( + url["dialect"], + convert_to_sqlalchemy_url(url), + self._log_database_reference_error, + success_slot=self._revive_db_reference, + ) + + @Slot(object) + def _revive_db_reference(self, url): + """Colors the db reference back to black. + + Args: + url (URL): SqlAlchemy URL + """ + url_text = remove_credentials_from_url(str(url)) + for row in range(self._db_ref_root.rowCount()): + item = self._db_ref_root.child(row) + if url_text == item.text(): + self._mark_as_found(item) + url_dict = item.data(_Role.DB_URL_REFERENCE) + if url_dict["dialect"] == "sqlite": + self.file_system_watcher.add_persistent_file_path(url_dict["database"]) + break + else: return - self.file_system_watcher.add_persistent_file_paths(ref for ref in fixed_references) self._check_notifications() self._resources_to_successors_changed() @@ -521,7 +590,7 @@ def open_data_file(self, index): if not index.isValid(): logging.error("Index not valid") return - data_file = index.data(_DATA_FILE_PATH_ROLE) + data_file = index.data(_Role.DATA_FILE_PATH) url = "file:///" + data_file # noinspection PyTypeChecker, PyCallByClass, PyArgumentList res = open_url(url) @@ -593,8 +662,12 @@ def delete_files_from_project(self, file_names): except OSError: self._logger.msg_error.emit(f"Removing file {path_to_remove} failed.\nCheck permissions.") - def populate_reference_list(self): - """List file references in QTreeView.""" + def populate_reference_list(self, db_references): + """List references in QTreeView. + + Args: + db_references (list of dict): database URLs + """ self.reference_model.clear() self.reference_model.setHorizontalHeaderItem(0, QStandardItem("References")) # Add header self._file_ref_root.removeRows(0, self._file_ref_root.rowCount()) @@ -602,7 +675,7 @@ def populate_reference_list(self): self._db_ref_root.removeRows(0, self._db_ref_root.rowCount()) self.reference_model.appendRow(self._db_ref_root) self._append_file_references_to_model(*self.file_references) - self._append_db_references_to_model(*self.db_references) + self._append_db_references_to_model(*db_references) def _append_file_references_to_model(self, *paths): non_existent_paths = [] @@ -627,15 +700,61 @@ def _mark_as_missing(item): item (QStandardItem): item to modify """ item.setData("The file is missing.", Qt.ItemDataRole.ToolTipRole) - item.setData(_MISSING_ITEM_FOREGROUND, Qt.ForegroundRole) - item.setData(True, _MISSING_ROLE) + item.setData(_MISSING_ITEM_FOREGROUND, Qt.ItemDataRole.ForegroundRole) + item.setData(True, _Role.MISSING) + + @staticmethod + def _mark_as_found(item): + """Modifies given model item to appear as existing reference. + + Args: + item (QStandardItem): item to modify + """ + item.setData(None, Qt.ItemDataRole.ToolTipRole) + item.setData(None, Qt.ItemDataRole.ForegroundRole) + item.setData(False, _Role.MISSING) def _append_db_references_to_model(self, *urls): + """Appends given database URLs to the model. + + Args: + *urls: dict-style URLs to add + """ for url in urls: - item = QStandardItem(url) + item = QStandardItem(convert_url_to_safe_string(url)) + item.setData(url, _Role.DB_URL_REFERENCE) item.setFlags(~Qt.ItemIsEditable) self._db_ref_root.appendRow(item) + def _watch_sqlite_file(self, *urls): + """Adds sqlite files to file system watcher's watched paths. + + Args: + *urls: dict-style URLs to watch + """ + for url in urls: + if url["dialect"] == "sqlite": + path = url["database"] + if os.path.exists(path): + self.file_system_watcher.add_persistent_file_path(path) + + def _unwatch_sqlite_file(self, *urls): + """Removes sqlite files from file system watcher's watched paths. + + Args: + *urls: dict-style URLs to watch + + Returns: + list of str: list of removed paths + """ + paths = [] + for url in urls: + if url["dialect"] == "sqlite": + path = url["database"] + if os.path.exists(path): + paths.append(path) + return self.file_system_watcher.remove_persistent_file_paths(paths) + def populate_data_list(self): """List project internal data (files) in QTreeView.""" self.data_model.clear() @@ -648,21 +767,21 @@ def _append_data_files_to_model(self, *paths): item.setFlags(~Qt.ItemIsEditable) icon = QFileIconProvider().icon(QFileInfo(path)) item.setData(icon, Qt.ItemDataRole.DecorationRole) - item.setData(path, _DATA_FILE_PATH_ROLE) + item.setData(path, _Role.DATA_FILE_PATH) self.data_model.appendRow(item) def resources_for_direct_successors(self): """see base class""" data_files = [os.path.join(self.data_dir, f) for f in self.data_files()] resources = scan_for_resources( - self, self.file_references + data_files, self.db_references, self.db_credentials, self._project.project_dir + self, self.file_references + data_files, list(self.db_reference_iter()), self._project.project_dir ) return resources def _check_notifications(self): """Sets or clears the exclamation mark icon.""" self.clear_notifications() - if not self.file_references and not self.db_references and not self.data_files(): + if not self.file_references and not self.has_db_references() and not self.data_files(): self.add_notification( "This Data Connection does not have any references or data. " "Add some in the Data Connection Properties panel." @@ -675,8 +794,19 @@ def item_dict(self): """Returns a dictionary corresponding to this item.""" d = super().item_dict() d["file_references"] = [serialize_path(ref, self._project.project_dir) for ref in self.file_references] - d["db_references"] = self.db_references - d["db_credentials"] = self.db_credentials + db_references = [] + db_credentials = {} + for url in self.db_reference_iter(): + serialized_url = dict(url) + username = serialized_url.pop("username") + password = serialized_url.pop("password") + if username: + db_credentials[convert_url_to_safe_string(serialized_url)] = username, password + if serialized_url["dialect"] == "sqlite": + serialized_url["database"] = serialize_path(serialized_url["database"], self._project.project_dir) + db_references.append(serialized_url) + d["db_references"] = db_references + d["db_credentials"] = db_credentials return d @staticmethod @@ -690,9 +820,10 @@ def from_dict(name, item_dict, toolbox, project): # FIXME: Do we want to convert references to file_references via upgrade? file_references = item_dict.get("file_references", list()) or item_dict.get("references", list()) file_references = [deserialize_path(r, project.project_dir) for r in file_references] - db_references = item_dict.get("db_references", list()) - db_credentials = item_dict.get("db_credentials", dict()) - return DataConnection(name, description, x, y, toolbox, project, file_references, db_references, db_credentials) + db_references = restore_database_references( + item_dict.get("db_references", []), item_dict.get("db_credentials", {}), project.project_dir + ) + return DataConnection(name, description, x, y, toolbox, project, file_references, db_references) def rename(self, new_name, rename_data_dir_message): """See base class.""" diff --git a/spine_items/data_connection/executable_item.py b/spine_items/data_connection/executable_item.py index 1acc50b5..c46beeb2 100644 --- a/spine_items/data_connection/executable_item.py +++ b/spine_items/data_connection/executable_item.py @@ -18,12 +18,13 @@ from spine_engine.utils.serialization import deserialize_path from .item_info import ItemInfo from .output_resources import scan_for_resources +from .utils import restore_database_references class ExecutableItem(ExecutableItemBase): """The executable parts of Data Connection.""" - def __init__(self, name, file_references, db_references, db_credentials, project_dir, logger): + def __init__(self, name, file_references, db_references, project_dir, logger): """ Args: name (str): item's name @@ -40,7 +41,6 @@ def __init__(self, name, file_references, db_references, db_credentials, project data_files.append(entry.path) self._file_paths = file_references + data_files self._urls = db_references - self._url_credentials = db_credentials @staticmethod def item_type(): @@ -49,13 +49,14 @@ def item_type(): def _output_resources_forward(self): """See base class.""" - return scan_for_resources(self, self._file_paths, self._urls, self._url_credentials, self._project_dir) + return scan_for_resources(self, self._file_paths, self._urls, self._project_dir) @classmethod def from_dict(cls, item_dict, name, project_dir, app_settings, specifications, logger): """See base class.""" file_references = item_dict["file_references"] file_references = [deserialize_path(r, project_dir) for r in file_references] - db_references = item_dict.get("db_references", []) - db_credentials = item_dict.get("db_credentials", {}) - return cls(name, file_references, db_references, db_credentials, project_dir, logger) + db_references = restore_database_references( + item_dict.get("db_references", []), item_dict.get("db_credentials", {}), project_dir + ) + return cls(name, file_references, db_references, project_dir, logger) diff --git a/spine_items/data_connection/output_resources.py b/spine_items/data_connection/output_resources.py index 9f4dab0c..d7b66a00 100644 --- a/spine_items/data_connection/output_resources.py +++ b/spine_items/data_connection/output_resources.py @@ -15,18 +15,18 @@ from pathlib import Path from spine_engine.project_item.project_item_resource import file_resource, transient_file_resource, url_resource from spine_engine.utils.serialization import path_in_dir -from ..utils import unsplit_url_credentials +from spinedb_api.helpers import remove_credentials_from_url +from ..utils import convert_to_sqlalchemy_url, unsplit_url_credentials -def scan_for_resources(provider, file_paths, urls, url_credentials, project_dir): +def scan_for_resources(provider, file_paths, urls, project_dir): """ Creates file and URL resources based on DC's references and data. Args: provider (ProjectItem or ExecutableItem): resource provider item file_paths (list of str): file paths - urls (list of str): urls - url_credentials (dict): mapping url from urls to tuple (username, password) + urls (list of dict): urls project_dir (str): absolute path to project directory Returns: @@ -55,8 +55,10 @@ def scan_for_resources(provider, file_paths, urls, url_credentials, project_dir) continue resources.append(resource) for url in urls: - credentials = url_credentials.get(url) - full_url = unsplit_url_credentials(url, credentials) if credentials is not None else url - resource = url_resource(provider.name, full_url, f"<{provider.name}>" + url) + str_url = str(convert_to_sqlalchemy_url(url)) + schema = url.get("schema") + resource = url_resource( + provider.name, str_url, f"<{provider.name}>" + remove_credentials_from_url(str_url), schema=schema + ) resources.append(resource) return resources diff --git a/spine_items/data_connection/utils.py b/spine_items/data_connection/utils.py new file mode 100644 index 00000000..1e8b5b4c --- /dev/null +++ b/spine_items/data_connection/utils.py @@ -0,0 +1,63 @@ +###################################################################################################################### +# Copyright (C) 2017-2022 Spine project consortium +# This file is part of Spine Items. +# Spine Items is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General +# Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) +# any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General +# Public License for more details. You should have received a copy of the GNU Lesser General Public License along with +# this program. If not, see . +###################################################################################################################### +"""This module contains utilities for Data Connection.""" +import sys +import urllib.parse +from spine_engine.utils.serialization import deserialize_path +from spine_items.utils import convert_url_to_safe_string + + +def restore_database_references(references_list, credentials_dict, project_dir): + """Restores data from serialized database references. + + Args: + references_list (list of dict): serialized database references + credentials_dict (dict): mapping from safe URL to (username, password) tuple + project_dir (str): path to project directory + + Returns: + list of dict: deserialized database references + """ + db_references = [] + for reference_dict in references_list: + if isinstance(reference_dict, str): + # legacy db reference + url = urllib.parse.urlparse(reference_dict) + dialect = _dialect_from_scheme(url.scheme) + database = url.path[1:] + db_reference = { + "dialect": dialect, + "host": url.hostname, + "port": url.port, + "database": database, + } + else: + db_reference = dict(reference_dict) + if db_reference["dialect"] == "sqlite": + db_reference["database"] = deserialize_path(db_reference["database"], project_dir) + + db_reference["username"], db_reference["password"] = credentials_dict.get( + convert_url_to_safe_string(db_reference), (None, None) + ) + db_references.append(db_reference) + return db_references + + +def _dialect_from_scheme(scheme): + """Parses dialect from URL scheme. + + Args: + scheme (str): URL scheme + + Returns: + str: dialect name + """ + return scheme.split("+")[0] diff --git a/spine_items/data_store/data_store.py b/spine_items/data_store/data_store.py index 6ea8760e..536ae6b7 100644 --- a/spine_items/data_store/data_store.py +++ b/spine_items/data_store/data_store.py @@ -97,7 +97,7 @@ def set_up(self): def parse_url(self, url): """Return a complete url dictionary from the given dict or string""" - base_url = dict(dialect="", username="", password="", host="", port="", database="") + base_url = dict(dialect="", username="", password="", host="", port="", database="", schema="") if isinstance(url, dict): if url.get("dialect") == "sqlite" and "database" in url and url["database"] is not None: # Convert relative database path back to absolute @@ -202,7 +202,7 @@ def do_update_url(self, **kwargs): old_url = convert_to_sqlalchemy_url(self._url, self.name) new_dialect = kwargs.get("dialect") if new_dialect == "sqlite": - kwargs.update({"username": "", "password": "", "host": "", "port": ""}) + kwargs.update({"username": "", "password": "", "host": "", "port": "", "schema": ""}) self._url.update(kwargs) new_url = convert_to_sqlalchemy_url(self._url, self.name) self.load_url_into_selections(self._url) diff --git a/spine_items/data_store/widgets/data_store_properties_widget.py b/spine_items/data_store/widgets/data_store_properties_widget.py index 9c59eca2..9fa22c4c 100644 --- a/spine_items/data_store/widgets/data_store_properties_widget.py +++ b/spine_items/data_store/widgets/data_store_properties_widget.py @@ -33,7 +33,9 @@ def __init__(self, toolbox): self._active_item = None self.ui = Ui_Form() self.ui.setupUi(self) - self.ui.url_selector_widget.setup(list(SUPPORTED_DIALECTS.keys()), self._select_sqlite_file, self._toolbox) + self.ui.url_selector_widget.setup( + list(SUPPORTED_DIALECTS.keys()), self._select_sqlite_file, True, self._toolbox + ) def set_item(self, data_store): """Sets the active project item for the properties widget. diff --git a/spine_items/exporter/widgets/export_list_item.py b/spine_items/exporter/widgets/export_list_item.py index 1b1ee325..8873ae65 100644 --- a/spine_items/exporter/widgets/export_list_item.py +++ b/spine_items/exporter/widgets/export_list_item.py @@ -121,7 +121,7 @@ def _emit_out_label_changed(self): @Slot(bool) def _show_url_dialog(self, _=False): """Opens the URL selector dialog.""" - dialog = UrlSelectorDialog(self._app_settings, self._logger, self) + dialog = UrlSelectorDialog(self._app_settings, True, self._logger, self) if self._out_url is not None: dialog.set_url_dict(self._out_url) dialog.exec_() diff --git a/spine_items/importer/connection_manager.py b/spine_items/importer/connection_manager.py index 2128c82b..12477bbe 100644 --- a/spine_items/importer/connection_manager.py +++ b/spine_items/importer/connection_manager.py @@ -8,11 +8,7 @@ # Public License for more details. You should have received a copy of the GNU Lesser General Public License along with # this program. If not, see . ###################################################################################################################### - -""" -Contains ConnectionManager class. - -""" +""" Contains ConnectionManager class. """ from PySide6.QtCore import QObject, Qt, QThread, Signal, Slot from PySide6.QtWidgets import QFileDialog @@ -60,11 +56,10 @@ def __init__(self, connection, connection_settings, parent): super().__init__(parent) self._thread = None self._worker = None - self._source = None self._current_table = None self._table_options = {} self._table_types = {} - self._defaul_table_column_type = {} + self._default_table_column_type = {} self._table_row_types = {} self._connection = connection self._connection_settings = connection_settings @@ -92,20 +87,12 @@ def table_types(self): @property def table_default_column_type(self): - return self._defaul_table_column_type + return self._default_table_column_type @property def table_row_types(self): return self._table_row_types - @property - def source(self): - return self._source - - @source.setter - def source(self, source): - self._source = source - @property def source_type(self): return self._connection.__name__ @@ -143,28 +130,19 @@ def request_default_mapping(self): if self.is_connected: self.default_mapping_requested.emit() - def connection_ui(self): - """ - Launches a modal ui that prompts the user to select source. - - ex: fileselect if source is a file. - """ - ext = self._connection.FILE_EXTENSIONS - source, action = QFileDialog.getOpenFileName(None, "", ext) - if not source or not action: - return False - self._source = source - return True - - def init_connection(self): + def init_connection(self, source, **source_extras): """Creates a Worker and a new thread to read source data. If there is an existing thread close that one. + + Args: + source (str): source file name or URL + **source_extras: source specific additional connection settings """ # close existing thread self.close_connection() # create new thread and worker self._thread = QThread() - self._worker = ConnectionWorker(self._source, self._connection, self._connection_settings) + self._worker = ConnectionWorker(self._connection, self._connection_settings) self._worker.moveToThread(self._thread) # connect worker signals self._worker.connectionReady.connect(self._handle_connection_ready) @@ -181,7 +159,7 @@ def init_connection(self): self.connection_closed.connect(self._worker.disconnect, type=Qt.ConnectionType.BlockingQueuedConnection) # when thread is started, connect worker to source - self._thread.started.connect(self._worker.init_connection) + self._thread.started.connect(lambda: self._worker.init_connection(source, dict(source_extras))) self._thread.start() @Slot() @@ -260,7 +238,7 @@ def update_table_default_column_type(self, column_type): Args: column_type (dict): mapping from table name to column type name """ - self._defaul_table_column_type.update(column_type) + self._default_table_column_type.update(column_type) def clear_table_default_column_type(self, table_name): """Clears default column type. @@ -268,7 +246,7 @@ def clear_table_default_column_type(self, table_name): Args: table_name (str): table name """ - self._defaul_table_column_type.pop(table_name, None) + self._default_table_column_type.pop(table_name, None) def set_table_row_types(self, types): """Sets connection manager types for current connector @@ -293,12 +271,7 @@ def close_connection(self): class ConnectionWorker(QObject): - """A class for delegating SourceConnection operations to another QThread. - - Args: - source (str): path of the source file - connection (class): A class derived from `SourceConnection` for connecting to the source file - """ + """A class for delegating SourceConnection operations to another QThread.""" connectionFailed = Signal(str) """Signal with error message if connection fails""" @@ -313,19 +286,26 @@ class ConnectionWorker(QObject): defaultMappingReady = Signal(dict) """Signal when default mapping is ready""" - def __init__(self, source, connection, connection_settings, parent=None): + def __init__(self, connection, connection_settings, parent=None): + """ + Args: + connection (class): A class derived from `SourceConnection` for connecting to the source file + connection_settings (dict): settings passed to the connection constructor + parent (QObject): parent object + """ super().__init__(parent) - self._source = source self._connection = connection(connection_settings) - @Slot() - def init_connection(self): - """ - Connect to data source + def init_connection(self, source, source_extras): + """Connect to data source. + + Args: + source (str): source file path or URL + source_extras (dict): source specific additional connection settings """ - if self._source: + if source: try: - self._connection.connect_to_source(self._source) + self._connection.connect_to_source(source, **source_extras) self.connectionReady.emit() except Exception as error: self.connectionFailed.emit(f"Could not connect to source: {error}") diff --git a/spine_items/importer/do_work.py b/spine_items/importer/do_work.py index 91acff03..031de72d 100644 --- a/spine_items/importer/do_work.py +++ b/spine_items/importer/do_work.py @@ -8,21 +8,22 @@ # Public License for more details. You should have received a copy of the GNU Lesser General Public License along with # this program. If not, see . ###################################################################################################################### - -""" -Importer's execute kernel (do_work), as target for a multiprocess.Process - -""" +""" Importer's execute kernel (do_work), as target for a multiprocess.Process """ import os -from spinedb_api import InvalidMapping + +from spine_engine.project_item.project_item_resource import get_source, get_source_extras +from spinedb_api import clear_filter_configs, InvalidMapping +from spinedb_api.helpers import remove_credentials_from_url from spinedb_api.spine_db_client import SpineDBClient from spinedb_api.parameter_value import to_database from spinedb_api.import_mapping.type_conversion import value_to_convert_spec from spine_engine.utils.helpers import create_log_file_timestamp -def do_work(process, mapping, cancel_on_error, on_conflict, logs_dir, sources, connector, to_server_urls, lock, logger): +def do_work( + process, mapping, cancel_on_error, on_conflict, logs_dir, source_resources, connector, to_server_urls, lock, logger +): all_data = [] all_errors = [] table_mappings = { @@ -47,13 +48,19 @@ def do_work(process, mapping, cancel_on_error, on_conflict, logs_dir, sources, c for tn, cols in mapping.get("table_row_types", {}).items() } to_clients = [SpineDBClient.from_server_url(server_url) for server_url in to_server_urls] - for src in sources: - file_anchor = f"{os.path.basename(src)}" - logger.msg.emit("Importing " + file_anchor) + for resource in source_resources: + src = get_source(resource) + if resource.hasfilepath: + source_anchor = f"{os.path.basename(src)}" + else: + safe_url = remove_credentials_from_url(src) + source_anchor = f"

{safe_url}

" + logger.msg.emit("Importing " + source_anchor) + extras = get_source_extras(resource) try: - connector.connect_to_source(src) + connector.connect_to_source(src, **extras) except Exception as error: # pylint: disable=broad-except - logger.msg_error.emit(f"Failed to connect to {file_anchor}: {error}") + logger.msg_error.emit(f"Failed to connect to {source_anchor}: {error}") return (False,) for name, mappings in table_mappings.items(): logger.msg.emit(f"Processing table {name}") @@ -84,6 +91,7 @@ def do_work(process, mapping, cancel_on_error, on_conflict, logs_dir, sources, c ) all_data.append(data) all_errors.extend(errors) + connector.disconnect() if all_data: for client in to_clients: lock.acquire() @@ -139,8 +147,9 @@ def _import_data_to_url(cancel_on_error, on_conflict, logs_dir, all_data, client logger.msg_warning.emit("Ignoring errors. Set Cancel import on error to bail out instead.") if all_import_count > 0: client.call_method("commit_session", "Import data by Spine Toolbox Importer") + clean_url = clear_filter_configs(remove_credentials_from_url(client.get_db_url())) logger.msg_success.emit( - f"Inserted {all_import_count} data with {len(all_import_errors)} errors into {client.get_db_url()}" + f"Inserted {all_import_count} data with {len(all_import_errors)} errors into {clean_url}" ) else: logger.msg_warning.emit("No new data imported") diff --git a/spine_items/importer/executable_item.py b/spine_items/importer/executable_item.py index 1da216ca..14a4ad2c 100644 --- a/spine_items/importer/executable_item.py +++ b/spine_items/importer/executable_item.py @@ -24,7 +24,7 @@ from spinedb_api.spine_io.importers.datapackage_reader import DataPackageConnector from spinedb_api.spine_io.importers.sqlalchemy_connector import SqlAlchemyConnector from spine_engine.project_item.executable_item_base import ExecutableItemBase -from spine_engine.project_item.project_item_resource import get_labelled_sources +from spine_engine.project_item.project_item_resource import get_labelled_source_resources from spine_engine.utils.returning_process import ReturningProcess from spine_engine.spine_engine import ItemExecutionFinishState from ..db_writer_executable_item_base import DBWriterExecutableItemBase @@ -72,12 +72,12 @@ def execute(self, forward_resources, backward_resources, lock): if not self._mapping: self._logger.msg_warning.emit(f"{self.name}: No mappings configured. Skipping.") return ItemExecutionFinishState.SKIPPED - labelled_sources = get_labelled_sources(forward_resources) - sources = list() + labelled_resources = get_labelled_source_resources(forward_resources) + selected_resources = [] for label in self._selected_files: - sources += labelled_sources.get(label, []) + selected_resources += labelled_resources.get(label, []) to_resources = [r for r in backward_resources if r.type_ == "database"] - if not sources or not to_resources: + if not selected_resources or not to_resources: return ItemExecutionFinishState.SUCCESS source_type = self._mapping["source_type"] if source_type == "GdxConnector": @@ -101,7 +101,7 @@ def execute(self, forward_resources, backward_resources, lock): self._cancel_on_error, self._on_conflict, self._logs_dir, - sources, + selected_resources, connector, to_server_urls, lock, diff --git a/spine_items/importer/importer.py b/spine_items/importer/importer.py index b005eed8..9ac734d1 100644 --- a/spine_items/importer/importer.py +++ b/spine_items/importer/importer.py @@ -220,10 +220,12 @@ def open_import_editor(self, index): index (QModelIndex): resource list index """ source = None + source_extras = None if index.isValid(): resource = self._file_model.resource(index) if resource.type_ == "url": source = resource.url + source_extras = resource.metadata else: if not resource.hasfilepath: self._logger.msg_error.emit("File does not exist yet.") @@ -232,7 +234,9 @@ def open_import_editor(self, index): self._logger.msg_error.emit(f"Cannot find file '{source}'.") else: source = resource.path - self._toolbox.show_specification_form(self.item_type(), self.specification(), self, source=source) + self._toolbox.show_specification_form( + self.item_type(), self.specification(), self, source=source, source_extras=source_extras + ) def select_connector_type(self, index): """Opens dialog to select connector type for the given index.""" diff --git a/spine_items/importer/importer_factory.py b/spine_items/importer/importer_factory.py index 69fec74a..c50ac2c4 100644 --- a/spine_items/importer/importer_factory.py +++ b/spine_items/importer/importer_factory.py @@ -61,4 +61,5 @@ def make_specification_menu(parent, index): def make_specification_editor(toolbox, specification=None, item=None, **kwargs): """See base class.""" source = kwargs.get("source") - return ImportEditorWindow(toolbox, specification, item, source) + source_extras = kwargs.get("source_extras") + return ImportEditorWindow(toolbox, specification, item, source, source_extras) diff --git a/spine_items/importer/widgets/import_editor_window.py b/spine_items/importer/widgets/import_editor_window.py index b12309b8..d6254698 100644 --- a/spine_items/importer/widgets/import_editor_window.py +++ b/spine_items/importer/widgets/import_editor_window.py @@ -63,7 +63,7 @@ class _FileLessConnector(SourceConnection): FILE_EXTENSIONS = "" OPTIONS = {} - def connect_to_source(self, _source): + def connect_to_source(self, source, **extras): pass def disconnect(self): @@ -75,16 +75,18 @@ def get_tables(self): def get_data_iterator(self, table, options, max_rows=-1): return iter([]), () - def __init__(self, toolbox, specification, item=None, source=None): + def __init__(self, toolbox, specification, item=None, source=None, source_extras=None): """ Args: toolbox (QMainWindow): ToolboxUI class - specification (ImporterSpecification) + specification (ImporterSpecification, optional): Importer specification + item (Importer, optional): Linked Importer item source (str, optional): Importee file path or URL + source_extras (dict, optional): Additional source settings such as database schema """ super().__init__(toolbox, specification, item) - self.takeCentralWidget().deleteLater() - self._filepath = source if source else self._FILE_LESS + self._source = source if source else self._FILE_LESS + self._source_extras = source_extras if source_extras is not None else {} self._mappings_model = MappingsModel(self._undo_stack, self) self._mappings_model.rowsInserted.connect(self._reselect_source_table) self._ui.source_list.setModel(self._mappings_model) @@ -113,7 +115,7 @@ def __init__(self, toolbox, specification, item=None, source=None): def showEvent(self, ev): """Select file path in the combobox, which calls the ``start_ui`` slot.""" super().showEvent(ev) - self._ui.comboBox_source_file.setCurrentText(self._filepath) + self._ui.comboBox_source_file.setCurrentText(self._source) def is_file_less(self): return self._ui.comboBox_source_file.currentText() == self._FILE_LESS @@ -130,7 +132,7 @@ def _save(self, exiting=None): @property def _duplicate_kwargs(self): - return dict(source=self._filepath) + return dict(source=self._source) def _make_ui(self): from ..ui.import_editor_window import Ui_MainWindow # pylint: disable=import-outside-toplevel @@ -193,7 +195,7 @@ def _show_open_file_dialog(self, _=False): self._ui.comboBox_source_file.setCurrentText(source) def _get_source_url(self): - selector = UrlSelectorDialog(self._toolbox.qsettigns(), self._toolbox, parent=self) + selector = UrlSelectorDialog(self._toolbox.qsettings(), False, self._toolbox, parent=self) selector.exec() return selector.url @@ -217,7 +219,15 @@ def _switch_connector(self, _=False): self._memoized_connector = None self.start_ui(self._FILE_LESS) - def _get_connector_from_mapping(self, filepath): + def _get_connector_from_mapping(self, source): + """Guesses connector for given source. + + Args: + source (str): importee file path or URL + + Returns: + type: connector class, or None if no suitable connector was found + """ if not self.specification: return None mapping = self.specification.mapping @@ -226,28 +236,28 @@ def _get_connector_from_mapping(self, filepath): return None connector = _CONNECTOR_NAME_TO_CLASS[source_type] file_extensions = connector.FILE_EXTENSIONS.split(";;") - if filepath != self._FILE_LESS and not any(fnmatch.fnmatch(filepath, ext) for ext in file_extensions): + if source != self._FILE_LESS and not any(fnmatch.fnmatch(source, ext) for ext in file_extensions): + if isinstance(connector, SqlAlchemyConnector) and self._is_url(source): + return connector return None return connector - def start_ui(self, filepath): + def start_ui(self, source): """ Args: - filepath (str): Importee path + source (str): Importee path/URL """ - connector = self._get_connector_from_mapping(filepath) + connector = self._get_connector_from_mapping(source) if connector is None: # Ask user - connector = self._get_connector(filepath) + connector = self._get_connector(source) if not connector: return if connector.__name__ == "SqlAlchemyConnector": - self._ui.dockWidget_source_files.setWindowTitle("Source URLs") self._ui.file_path_label.setText("URL") else: - self._ui.dockWidget_source_files.setWindowTitle("Source files") self._ui.file_path_label.setText("File path") - if filepath == self._FILE_LESS: + if source == self._FILE_LESS: self._FileLessConnector.__name__ = connector.__name__ self._FileLessConnector.OPTIONS = connector.OPTIONS connector = self._FileLessConnector @@ -259,7 +269,6 @@ def start_ui(self, filepath): if self._connection_manager: self._connection_manager.close_connection() self._connection_manager = ConnectionManager(connector, connector_settings, self) - self._connection_manager.source = filepath self._connection_manager.connection_failed.connect(self.connection_failed.emit) self._connection_manager.error.connect(self.show_error) for header in (self._ui.source_data_table.horizontalHeader(), self._ui.source_data_table.verticalHeader()): @@ -267,18 +276,18 @@ def start_ui(self, filepath): self._connection_manager.connection_ready.connect(self._handle_connection_ready) mapping = self.specification.mapping if self.specification else {} self._import_sources.set_connector(self._connection_manager, mapping) - self._connection_manager.init_connection() + self._connection_manager.init_connection(source, **self._source_extras) @Slot() def _handle_connection_ready(self): self._ui.export_mappings_action.setEnabled(True) self._ui.import_mappings_action.setEnabled(True) - def _get_connector(self, filepath): + def _get_connector(self, source): """Shows a QDialog to select a connector for the given source file. Args: - filepath (str): Path of the file acting as an importee + source (str): Path of the file acting as an importee Returns: Asynchronous data reader class for the given importee @@ -295,8 +304,12 @@ def _get_connector(self, filepath): row = None for k, conn in enumerate(connector_list): file_extensions = conn.FILE_EXTENSIONS.split(";;") - if any(fnmatch.fnmatch(filepath, ext) for ext in file_extensions): + if any(fnmatch.fnmatch(source, ext) for ext in file_extensions): row = k + break + else: + if self._is_url(source): + row = connector_names.index(SqlAlchemyConnector.DISPLAY_NAME) if row is not None: connector_list_wg.setCurrentRow(row) button_box = QDialogButtonBox(QDialogButtonBox.StandardButton.Ok | QDialogButtonBox.StandardButton.Cancel) @@ -371,6 +384,18 @@ def _reselect_source_table(self, parent, first, last): index = self._mappings_model.index(last, 0) self._ui.source_list.selectionModel().setCurrentIndex(index, QItemSelectionModel.ClearAndSelect) + @staticmethod + def _is_url(string): + """Tests if given string looks like a URL. + + Args: + string (str): string to test + + Returns: + bool: True if string looks like a URL, False otherwise + """ + return "://" in string + def tear_down(self): if not super().tear_down(): return False diff --git a/spine_items/resources_icons_rc.py b/spine_items/resources_icons_rc.py index 3f2f3ec1..34eaaf10 100644 --- a/spine_items/resources_icons_rc.py +++ b/spine_items/resources_icons_rc.py @@ -3380,69 +3380,69 @@ \x00\x00\x00\x00\x00\x02\x00\x00\x00\x17\x00\x00\x00\x02\ \x00\x00\x00\x00\x00\x00\x00\x00\ \x00\x00\x01<\x00\x00\x00\x00\x00\x01\x00\x008\xf8\ -\x00\x00\x01y\x13v\x91\x22\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x00|\x00\x00\x00\x00\x00\x01\x00\x00/b\ -\x00\x00\x01y\x13v\x91%\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x02\xb4\x00\x00\x00\x00\x00\x01\x00\x00\x9a\x89\ -\x00\x00\x01zg\xfb\x9d\x7f\ +\x00\x00\x01\x8aF\x12\x1c\xf1\ \x00\x00\x020\x00\x00\x00\x00\x00\x01\x00\x00}_\ -\x00\x00\x01y\x13v\x91$\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x01\x16\x00\x00\x00\x00\x00\x01\x00\x007\xc4\ -\x00\x00\x01y\x13v\x91$\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x00\xca\x00\x00\x00\x00\x00\x01\x00\x003u\ -\x00\x00\x01y\x13v\x91%\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x00\x5c\x00\x00\x00\x00\x00\x01\x00\x00-\x06\ -\x00\x00\x01y\x13v\x91*\ +\x00\x00\x01\x8aF\x12\x1c\xf2\ \x00\x00\x00&\x00\x02\x00\x00\x00\x09\x00\x00\x00\x19\ \x00\x00\x00\x00\x00\x00\x00\x00\ \x00\x00\x02l\x00\x00\x00\x00\x00\x01\x00\x00\x85\x9f\ -\x00\x00\x01y\x13v\x91#\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x00\x10\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\ -\x00\x00\x01y\x13v\x91)\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x01\x9c\x00\x00\x00\x00\x00\x01\x00\x00m\xff\ -\x00\x00\x01\x86T\xda\x92\x08\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x01b\x00\x00\x00\x00\x00\x01\x00\x00a\xaa\ -\x00\x00\x01y\x13v\x91*\ +\x00\x00\x01\x8aF\x12\x1c\xf2\ \x00\x00\x02\x94\x00\x00\x00\x00\x00\x01\x00\x00\x87s\ -\x00\x00\x01y\x13v\x91#\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x02T\x00\x00\x00\x00\x00\x01\x00\x00\x7f6\ -\x00\x00\x01y\x13v\x91)\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x01|\x00\x00\x00\x00\x00\x01\x00\x00c\xc3\ -\x00\x00\x01y\x13v\x91$\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x00\xfe\x00\x00\x00\x00\x00\x01\x00\x004\xe9\ -\x00\x00\x01\x80`9\xfa7\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x01\xc6\x00\x00\x00\x00\x00\x01\x00\x00p\x12\ -\x00\x00\x01y\x13v\x91\x22\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x00@\x00\x00\x00\x00\x00\x01\x00\x00\x06\xe5\ -\x00\x00\x01y\x13v\x91#\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x02\xca\x00\x00\x00\x00\x00\x01\x00\x00\x9dP\ -\x00\x00\x01y\x13v\x91*\ +\x00\x00\x01\x8aF\x12\x1c\xf1\ \x00\x00\x01\xdc\x00\x00\x00\x00\x00\x01\x00\x00q\xc5\ -\x00\x00\x01y\x13v\x91%\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x00\x92\x00\x00\x00\x00\x00\x01\x00\x000q\ -\x00\x00\x01y\x13v\x91%\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x02\x0c\x00\x00\x00\x00\x00\x01\x00\x00s[\ -\x00\x00\x01y\x13v\x91%\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x00\xac\x00\x00\x00\x00\x00\x01\x00\x001D\ -\x00\x00\x01\x80`9\xfa7\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x03\xe2\x00\x00\x00\x00\x00\x01\x00\x00\xc0\xab\ -\x00\x00\x01y\x13v\x91(\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x02\xe0\x00\x00\x00\x00\x00\x01\x00\x00\x9f]\ -\x00\x00\x01\x80`9\xfa?\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x03@\x00\x00\x00\x00\x00\x01\x00\x00\xa82\ -\x00\x00\x01y\x13v\x91(\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x03\x22\x00\x00\x00\x00\x00\x01\x00\x00\xa6Q\ -\x00\x00\x01y\x13v\x91'\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x03\x00\x00\x00\x00\x00\x00\x01\x00\x00\xa4>\ -\x00\x00\x01y\x13v\x91&\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x04\x14\x00\x00\x00\x00\x00\x01\x00\x00\xc2}\ -\x00\x00\x01y\x13v\x91&\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x03Z\x00\x00\x00\x00\x00\x01\x00\x00\xaa\xbc\ -\x00\x00\x01y\x13v\x91(\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x03\x8a\x00\x00\x00\x00\x00\x01\x00\x00\xado\ -\x00\x00\x01\x80`9\xfa:\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ \x00\x00\x03\xb6\x00\x00\x00\x00\x00\x01\x00\x00\xb7)\ -\x00\x00\x01\x80`9\xfa=\ +\x00\x00\x01\x8aF\x12\x1c\xe1\ " def qInitResources(): diff --git a/spine_items/ui/url_selector_widget.py b/spine_items/ui/url_selector_widget.py index f3777823..b8c38383 100644 --- a/spine_items/ui/url_selector_widget.py +++ b/spine_items/ui/url_selector_widget.py @@ -25,9 +25,9 @@ QFont, QFontDatabase, QGradient, QIcon, QImage, QKeySequence, QLinearGradient, QPainter, QPalette, QPixmap, QRadialGradient, QTransform) -from PySide6.QtWidgets import (QApplication, QComboBox, QFrame, QGridLayout, - QHBoxLayout, QLabel, QLineEdit, QSizePolicy, - QToolButton, QVBoxLayout, QWidget) +from PySide6.QtWidgets import (QApplication, QComboBox, QFormLayout, QHBoxLayout, + QLabel, QLineEdit, QSizePolicy, QToolButton, + QWidget) from spine_items.widgets import FileDropTargetLineEdit from spine_items import resources_icons_rc @@ -36,159 +36,124 @@ class Ui_Form(object): def setupUi(self, Form): if not Form.objectName(): Form.setObjectName(u"Form") - Form.resize(381, 221) - self.verticalLayout = QVBoxLayout(Form) - self.verticalLayout.setObjectName(u"verticalLayout") - self.frame = QFrame(Form) - self.frame.setObjectName(u"frame") - self.gridLayout = QGridLayout(self.frame) - self.gridLayout.setObjectName(u"gridLayout") - self.gridLayout.setContentsMargins(3, 3, 3, 3) - self.lineEdit_password = QLineEdit(self.frame) - self.lineEdit_password.setObjectName(u"lineEdit_password") - self.lineEdit_password.setEnabled(False) - sizePolicy = QSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) - sizePolicy.setHorizontalStretch(0) - sizePolicy.setVerticalStretch(0) - sizePolicy.setHeightForWidth(self.lineEdit_password.sizePolicy().hasHeightForWidth()) - self.lineEdit_password.setSizePolicy(sizePolicy) - self.lineEdit_password.setMinimumSize(QSize(0, 24)) - self.lineEdit_password.setMaximumSize(QSize(5000, 24)) - self.lineEdit_password.setEchoMode(QLineEdit.Password) - self.lineEdit_password.setClearButtonEnabled(True) - - self.gridLayout.addWidget(self.lineEdit_password, 3, 1, 1, 5) - - self.label_port = QLabel(self.frame) - self.label_port.setObjectName(u"label_port") - sizePolicy1 = QSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed) - sizePolicy1.setHorizontalStretch(1) - sizePolicy1.setVerticalStretch(0) - sizePolicy1.setHeightForWidth(self.label_port.sizePolicy().hasHeightForWidth()) - self.label_port.setSizePolicy(sizePolicy1) + Form.resize(386, 212) + self.formLayout = QFormLayout(Form) + self.formLayout.setObjectName(u"formLayout") + self.label_dialect = QLabel(Form) + self.label_dialect.setObjectName(u"label_dialect") - self.gridLayout.addWidget(self.label_port, 4, 4, 1, 1) + self.formLayout.setWidget(0, QFormLayout.LabelRole, self.label_dialect) - self.comboBox_dialect = QComboBox(self.frame) + self.comboBox_dialect = QComboBox(Form) self.comboBox_dialect.setObjectName(u"comboBox_dialect") - sizePolicy.setHeightForWidth(self.comboBox_dialect.sizePolicy().hasHeightForWidth()) - self.comboBox_dialect.setSizePolicy(sizePolicy) - self.comboBox_dialect.setMinimumSize(QSize(0, 24)) - self.comboBox_dialect.setMaximumSize(QSize(16777215, 24)) - self.gridLayout.addWidget(self.comboBox_dialect, 0, 1, 1, 5) + self.formLayout.setWidget(0, QFormLayout.FieldRole, self.comboBox_dialect) - self.label_dsn = QLabel(self.frame) + self.label_dsn = QLabel(Form) self.label_dsn.setObjectName(u"label_dsn") - sizePolicy2 = QSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed) - sizePolicy2.setHorizontalStretch(0) - sizePolicy2.setVerticalStretch(0) - sizePolicy2.setHeightForWidth(self.label_dsn.sizePolicy().hasHeightForWidth()) - self.label_dsn.setSizePolicy(sizePolicy2) - self.gridLayout.addWidget(self.label_dsn, 1, 0, 1, 1) + self.formLayout.setWidget(1, QFormLayout.LabelRole, self.label_dsn) - self.label_dialect = QLabel(self.frame) - self.label_dialect.setObjectName(u"label_dialect") - sizePolicy2.setHeightForWidth(self.label_dialect.sizePolicy().hasHeightForWidth()) - self.label_dialect.setSizePolicy(sizePolicy2) - self.label_dialect.setMaximumSize(QSize(16777215, 16777215)) + self.comboBox_dsn = QComboBox(Form) + self.comboBox_dsn.setObjectName(u"comboBox_dsn") + self.comboBox_dsn.setEnabled(False) - self.gridLayout.addWidget(self.label_dialect, 0, 0, 1, 1) + self.formLayout.setWidget(1, QFormLayout.FieldRole, self.comboBox_dsn) - self.lineEdit_port = QLineEdit(self.frame) - self.lineEdit_port.setObjectName(u"lineEdit_port") - self.lineEdit_port.setEnabled(False) - sizePolicy.setHeightForWidth(self.lineEdit_port.sizePolicy().hasHeightForWidth()) - self.lineEdit_port.setSizePolicy(sizePolicy) - self.lineEdit_port.setMinimumSize(QSize(0, 24)) - self.lineEdit_port.setMaximumSize(QSize(80, 24)) - self.lineEdit_port.setInputMethodHints(Qt.ImhNone) + self.label_username = QLabel(Form) + self.label_username.setObjectName(u"label_username") - self.gridLayout.addWidget(self.lineEdit_port, 4, 5, 1, 1) + self.formLayout.setWidget(2, QFormLayout.LabelRole, self.label_username) - self.label_username = QLabel(self.frame) - self.label_username.setObjectName(u"label_username") - sizePolicy2.setHeightForWidth(self.label_username.sizePolicy().hasHeightForWidth()) - self.label_username.setSizePolicy(sizePolicy2) + self.lineEdit_username = QLineEdit(Form) + self.lineEdit_username.setObjectName(u"lineEdit_username") + self.lineEdit_username.setEnabled(False) + self.lineEdit_username.setClearButtonEnabled(True) - self.gridLayout.addWidget(self.label_username, 2, 0, 1, 1) + self.formLayout.setWidget(2, QFormLayout.FieldRole, self.lineEdit_username) - self.label_database = QLabel(self.frame) - self.label_database.setObjectName(u"label_database") - sizePolicy2.setHeightForWidth(self.label_database.sizePolicy().hasHeightForWidth()) - self.label_database.setSizePolicy(sizePolicy2) + self.label_password = QLabel(Form) + self.label_password.setObjectName(u"label_password") - self.gridLayout.addWidget(self.label_database, 6, 0, 1, 1) + self.formLayout.setWidget(3, QFormLayout.LabelRole, self.label_password) - self.comboBox_dsn = QComboBox(self.frame) - self.comboBox_dsn.setObjectName(u"comboBox_dsn") - self.comboBox_dsn.setEnabled(False) - sizePolicy.setHeightForWidth(self.comboBox_dsn.sizePolicy().hasHeightForWidth()) - self.comboBox_dsn.setSizePolicy(sizePolicy) - self.comboBox_dsn.setMinimumSize(QSize(0, 24)) - self.comboBox_dsn.setMaximumSize(QSize(16777215, 24)) + self.lineEdit_password = QLineEdit(Form) + self.lineEdit_password.setObjectName(u"lineEdit_password") + self.lineEdit_password.setEnabled(False) + self.lineEdit_password.setEchoMode(QLineEdit.Password) + self.lineEdit_password.setClearButtonEnabled(True) - self.gridLayout.addWidget(self.comboBox_dsn, 1, 1, 1, 5) + self.formLayout.setWidget(3, QFormLayout.FieldRole, self.lineEdit_password) - self.lineEdit_host = QLineEdit(self.frame) + self.label_host = QLabel(Form) + self.label_host.setObjectName(u"label_host") + + self.formLayout.setWidget(4, QFormLayout.LabelRole, self.label_host) + + self.horizontalLayout = QHBoxLayout() + self.horizontalLayout.setObjectName(u"horizontalLayout") + self.lineEdit_host = QLineEdit(Form) self.lineEdit_host.setObjectName(u"lineEdit_host") self.lineEdit_host.setEnabled(False) - sizePolicy3 = QSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) - sizePolicy3.setHorizontalStretch(3) - sizePolicy3.setVerticalStretch(0) - sizePolicy3.setHeightForWidth(self.lineEdit_host.sizePolicy().hasHeightForWidth()) - self.lineEdit_host.setSizePolicy(sizePolicy3) - self.lineEdit_host.setMinimumSize(QSize(0, 24)) - self.lineEdit_host.setMaximumSize(QSize(5000, 24)) self.lineEdit_host.setClearButtonEnabled(True) - self.gridLayout.addWidget(self.lineEdit_host, 4, 1, 1, 3) + self.horizontalLayout.addWidget(self.lineEdit_host) - self.label_password = QLabel(self.frame) - self.label_password.setObjectName(u"label_password") - sizePolicy2.setHeightForWidth(self.label_password.sizePolicy().hasHeightForWidth()) - self.label_password.setSizePolicy(sizePolicy2) + self.label_port = QLabel(Form) + self.label_port.setObjectName(u"label_port") - self.gridLayout.addWidget(self.label_password, 3, 0, 1, 1) + self.horizontalLayout.addWidget(self.label_port) - self.lineEdit_username = QLineEdit(self.frame) - self.lineEdit_username.setObjectName(u"lineEdit_username") - self.lineEdit_username.setEnabled(False) - sizePolicy.setHeightForWidth(self.lineEdit_username.sizePolicy().hasHeightForWidth()) - self.lineEdit_username.setSizePolicy(sizePolicy) - self.lineEdit_username.setMinimumSize(QSize(0, 24)) - self.lineEdit_username.setMaximumSize(QSize(5000, 24)) - self.lineEdit_username.setClearButtonEnabled(True) + self.lineEdit_port = QLineEdit(Form) + self.lineEdit_port.setObjectName(u"lineEdit_port") + self.lineEdit_port.setEnabled(False) + sizePolicy = QSizePolicy(QSizePolicy.Preferred, QSizePolicy.Fixed) + sizePolicy.setHorizontalStretch(0) + sizePolicy.setVerticalStretch(0) + sizePolicy.setHeightForWidth(self.lineEdit_port.sizePolicy().hasHeightForWidth()) + self.lineEdit_port.setSizePolicy(sizePolicy) + self.lineEdit_port.setInputMethodHints(Qt.ImhNone) - self.gridLayout.addWidget(self.lineEdit_username, 2, 1, 1, 5) + self.horizontalLayout.addWidget(self.lineEdit_port) - self.label_host = QLabel(self.frame) - self.label_host.setObjectName(u"label_host") - sizePolicy2.setHeightForWidth(self.label_host.sizePolicy().hasHeightForWidth()) - self.label_host.setSizePolicy(sizePolicy2) - self.gridLayout.addWidget(self.label_host, 4, 0, 1, 1) + self.formLayout.setLayout(4, QFormLayout.FieldRole, self.horizontalLayout) + + self.schema_label = QLabel(Form) + self.schema_label.setObjectName(u"schema_label") + + self.formLayout.setWidget(5, QFormLayout.LabelRole, self.schema_label) + + self.schema_line_edit = QLineEdit(Form) + self.schema_line_edit.setObjectName(u"schema_line_edit") + self.schema_line_edit.setEnabled(False) + self.schema_line_edit.setClearButtonEnabled(True) + + self.formLayout.setWidget(5, QFormLayout.FieldRole, self.schema_line_edit) + + self.label_database = QLabel(Form) + self.label_database.setObjectName(u"label_database") + + self.formLayout.setWidget(6, QFormLayout.LabelRole, self.label_database) self.horizontalLayout_4 = QHBoxLayout() self.horizontalLayout_4.setObjectName(u"horizontalLayout_4") - self.lineEdit_database = FileDropTargetLineEdit(self.frame) + self.lineEdit_database = FileDropTargetLineEdit(Form) self.lineEdit_database.setObjectName(u"lineEdit_database") self.lineEdit_database.setEnabled(False) - sizePolicy.setHeightForWidth(self.lineEdit_database.sizePolicy().hasHeightForWidth()) - self.lineEdit_database.setSizePolicy(sizePolicy) - self.lineEdit_database.setMinimumSize(QSize(0, 24)) - self.lineEdit_database.setMaximumSize(QSize(16777215, 24)) self.lineEdit_database.setCursor(QCursor(Qt.IBeamCursor)) self.lineEdit_database.setClearButtonEnabled(True) self.horizontalLayout_4.addWidget(self.lineEdit_database) - self.toolButton_select_sqlite_file = QToolButton(self.frame) + self.toolButton_select_sqlite_file = QToolButton(Form) self.toolButton_select_sqlite_file.setObjectName(u"toolButton_select_sqlite_file") self.toolButton_select_sqlite_file.setEnabled(False) - sizePolicy.setHeightForWidth(self.toolButton_select_sqlite_file.sizePolicy().hasHeightForWidth()) - self.toolButton_select_sqlite_file.setSizePolicy(sizePolicy) + sizePolicy1 = QSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) + sizePolicy1.setHorizontalStretch(0) + sizePolicy1.setVerticalStretch(0) + sizePolicy1.setHeightForWidth(self.toolButton_select_sqlite_file.sizePolicy().hasHeightForWidth()) + self.toolButton_select_sqlite_file.setSizePolicy(sizePolicy1) self.toolButton_select_sqlite_file.setMinimumSize(QSize(22, 22)) self.toolButton_select_sqlite_file.setMaximumSize(QSize(22, 22)) icon = QIcon() @@ -198,17 +163,15 @@ def setupUi(self, Form): self.horizontalLayout_4.addWidget(self.toolButton_select_sqlite_file) - self.gridLayout.addLayout(self.horizontalLayout_4, 6, 1, 1, 5) - - - self.verticalLayout.addWidget(self.frame) + self.formLayout.setLayout(6, QFormLayout.FieldRole, self.horizontalLayout_4) QWidget.setTabOrder(self.comboBox_dialect, self.comboBox_dsn) QWidget.setTabOrder(self.comboBox_dsn, self.lineEdit_username) QWidget.setTabOrder(self.lineEdit_username, self.lineEdit_password) QWidget.setTabOrder(self.lineEdit_password, self.lineEdit_host) QWidget.setTabOrder(self.lineEdit_host, self.lineEdit_port) - QWidget.setTabOrder(self.lineEdit_port, self.lineEdit_database) + QWidget.setTabOrder(self.lineEdit_port, self.schema_line_edit) + QWidget.setTabOrder(self.schema_line_edit, self.lineEdit_database) QWidget.setTabOrder(self.lineEdit_database, self.toolButton_select_sqlite_file) self.retranslateUi(Form) @@ -218,18 +181,14 @@ def setupUi(self, Form): def retranslateUi(self, Form): Form.setWindowTitle(QCoreApplication.translate("Form", u"Form", None)) - self.lineEdit_password.setPlaceholderText("") - self.label_port.setText(QCoreApplication.translate("Form", u"Port:", None)) - self.label_dsn.setText(QCoreApplication.translate("Form", u"DSN:", None)) self.label_dialect.setText(QCoreApplication.translate("Form", u"Dialect:", None)) - self.lineEdit_port.setPlaceholderText("") + self.label_dsn.setText(QCoreApplication.translate("Form", u"DSN:", None)) self.label_username.setText(QCoreApplication.translate("Form", u"Username:", None)) - self.label_database.setText(QCoreApplication.translate("Form", u"Database:", None)) - self.lineEdit_host.setPlaceholderText("") self.label_password.setText(QCoreApplication.translate("Form", u"Password:", None)) - self.lineEdit_username.setPlaceholderText("") self.label_host.setText(QCoreApplication.translate("Form", u"Host:", None)) - self.lineEdit_database.setPlaceholderText("") + self.label_port.setText(QCoreApplication.translate("Form", u"Port:", None)) + self.schema_label.setText(QCoreApplication.translate("Form", u"Schema:", None)) + self.label_database.setText(QCoreApplication.translate("Form", u"Database:", None)) #if QT_CONFIG(tooltip) self.toolButton_select_sqlite_file.setToolTip(QCoreApplication.translate("Form", u"

Select SQLite file

", None)) #endif // QT_CONFIG(tooltip) diff --git a/spine_items/ui/url_selector_widget.ui b/spine_items/ui/url_selector_widget.ui index 190bda2f..d8e3375e 100644 --- a/spine_items/ui/url_selector_widget.ui +++ b/spine_items/ui/url_selector_widget.ui @@ -18,373 +18,192 @@ 0 0 - 381 - 221 + 386 + 212 Form - - - - - - 3 - - - 3 - - - 3 - - - 3 - - - - - false - - - - 0 - 0 - - - - - 0 - 24 - - - - - 5000 - 24 - - - - QLineEdit::Password - - - - - - true - - - - - - - - 1 - 0 - - - - Port: - - - - - - - - 0 - 0 - - - - - 0 - 24 - - - - - 16777215 - 24 - - - - - - - - - 0 - 0 - - - - DSN: - - - - - - - - 0 - 0 - - - - - 16777215 - 16777215 - - - - Dialect: - - - - - - - false - - - - 0 - 0 - - - - - 0 - 24 - - - - - 80 - 24 - - - - Qt::ImhNone - - - - - - - - - - - 0 - 0 - - - - Username: - - - - - - - - 0 - 0 - - - - Database: - - - - - - - false - - - - 0 - 0 - - - - - 0 - 24 - - - - - 16777215 - 24 - - - - - - - - false - - - - 3 - 0 - - - - - 0 - 24 - - - - - 5000 - 24 - - - - - - - true - - - - - - - - 0 - 0 - - - - Password: - - - - - - - false - - - - 0 - 0 - - - - - 0 - 24 - - - - - 5000 - 24 - - - - - - - true - - - - - - - - 0 - 0 - - - - Host: - - - - - - - - - false - - - - 0 - 0 - - - - - 0 - 24 - - - - - 16777215 - 24 - - - - IBeamCursor - - - - - - true - - - - - - - false - - - - 0 - 0 - - - - - 22 - 22 - - - - - 22 - 22 - - - - <html><head/><body><p>Select SQLite file</p></body></html> - - - - :/icons/folder-open-solid.svg:/icons/folder-open-solid.svg - - - - - - + + + + + Dialect: + + + + + + + + DSN: + + + + + + + false + + + + + + + Username: + + + + + + + false + + + true + + + + + + + Password: + + + + + + + false + + + QLineEdit::Password + + + true + + + + + + + Host: + + + + + + + + + false + + + true + + + + + + + Port: + + + + + + + false + + + + 0 + 0 + + + + Qt::ImhNone + + + + + + + + + Schema: + + + + + + + false + + + true + + + + + + + Database: + + + + + + + + + false + + + IBeamCursor + + + true + + + + + + + false + + + + 0 + 0 + + + + + 22 + 22 + + + + + 22 + 22 + + + + <html><head/><body><p>Select SQLite file</p></body></html> + + + + :/icons/folder-open-solid.svg:/icons/folder-open-solid.svg + + + + + @@ -401,6 +220,7 @@ lineEdit_password lineEdit_host lineEdit_port + schema_line_edit lineEdit_database toolButton_select_sqlite_file diff --git a/spine_items/utils.py b/spine_items/utils.py index 7f962d74..ce2c894d 100644 --- a/spine_items/utils.py +++ b/spine_items/utils.py @@ -14,12 +14,18 @@ """ import os.path +from contextlib import suppress from sqlalchemy import create_engine from sqlalchemy.engine.url import URL, make_url import spinedb_api from spinedb_api.filters.scenario_filter import scenario_name_from_dict from spine_engine.utils.queue_logger import SuppressedMessage +from spinedb_api.helpers import remove_credentials_from_url + + +class URLError(Exception): + """Exception for errors in URL dicts.""" def database_label(provider_name): @@ -43,43 +49,77 @@ def convert_to_sqlalchemy_url(urllib_url, item_name="", logger=None): logger.msg_error.emit(f"No URL specified for {selections}. Please specify one and try again") return None try: - url = {key: value for key, value in urllib_url.items() if value} + sa_url = _convert_url(urllib_url) + _validate_sa_url(sa_url, urllib_url["dialect"]) + return sa_url + except URLError as error: + logger.msg_error.emit(f"Unable to generate URL from {selections}: {error}") + return None + + +def _convert_url(url_dict): + """Converts URL dict to SqlAlchemy URL. + + Args: + url_dict (dict): URL dictionary + + Returns: + URL: SqlAlchemy URL + """ + try: + url = {key: value for key, value in url_dict.items() if value} dialect = url.pop("dialect") + with suppress(KeyError): + del url["schema"] if not dialect: - logger.msg_error.emit(f"Unable to generate URL from {selections}: invalid dialect {dialect}.") - return None + raise URLError(f"invalid dialect {dialect}.") if dialect == "sqlite": database = url.get("database", "") if database: url["database"] = os.path.abspath(database) - sa_url = URL("sqlite", **url) # pylint: disable=unexpected-keyword-arg - else: - db_api = spinedb_api.SUPPORTED_DIALECTS.get(dialect) - if db_api is None: - db_api = spinedb_api.helpers.UNSUPPORTED_DIALECTS[dialect] - drivername = f"{dialect}+{db_api}" - sa_url = URL(drivername, **url) # pylint: disable=unexpected-keyword-arg - except Exception as e: # pylint: disable=broad-except - # This is in case one of the keys has invalid format - logger.msg_error.emit(f"Unable to generate URL from {selections}: {e}") - return None + return URL("sqlite", **url) # pylint: disable=unexpected-keyword-arg + db_api = spinedb_api.SUPPORTED_DIALECTS.get(dialect) + if db_api is None: + db_api = spinedb_api.helpers.UNSUPPORTED_DIALECTS[dialect] + driver_name = f"{dialect}+{db_api}" + return URL(driver_name, **url) # pylint: disable=unexpected-keyword-arg + except Exception as error: + raise URLError(str(error)) from error + + +def _validate_sa_url(sa_url, dialect): + """Validates SqlAlchemy URL. + + Args: + sa_url (URL): SqlAlchemy URL to validate + dialect (str): dialect + + Raises: + URLError: raised if given URL is invalid + """ if not sa_url.database: - logger.msg_error.emit(f"Unable to generate URL from {selections}: database missing.") - return None + raise URLError(f"database missing") if dialect != "sqlite": if sa_url.host is None: - logger.msg_error.emit(f"Unable to generate URL from {selections}: missing host.") - return None + raise URLError(f"missing host") if sa_url.port is None: - logger.msg_error.emit(f"Unable to generate URL from {selections}: missing port.") - return None + raise URLError(f"missing port") if sa_url.username is None: - logger.msg_error.emit(f"Unable to generate URL from {selections}: missing username.") - return None + raise URLError(f"missing username") if sa_url.password is None: - logger.msg_error.emit(f"Unable to generate URL from {selections}: missing password.") - return None - return sa_url + raise URLError(f"missing password") + + +def convert_url_to_safe_string(url): + """Converts dict-style database URL to string without credentials. + + Args: + url (dict): URL to convert + + Returns: + str: URL as string + """ + return remove_credentials_from_url(str(_convert_url(url))) def check_database_url(sa_url): diff --git a/spine_items/widgets.py b/spine_items/widgets.py index 493d6c57..f7e73b2e 100644 --- a/spine_items/widgets.py +++ b/spine_items/widgets.py @@ -381,17 +381,21 @@ def __init__(self, parent=None): self._ui.lineEdit_username.textChanged.connect(lambda: self.url_changed.emit()) self._ui.lineEdit_password.textChanged.connect(lambda: self.url_changed.emit()) - def setup(self, dialects, select_sqlite_file_callback, logger): + def setup(self, dialects, select_sqlite_file_callback, hide_schema, logger): """Sets the widget up for usage. Args: dialects (Sequence of str): available SQL dialects select_sqlite_file_callback (Callable): function that returns a path to SQLite file or None + hide_schema (bool): True to hide the Schema field logger (LoggerInterface): logger """ self._get_sqlite_file_path = select_sqlite_file_callback self._logger = logger self._ui.comboBox_dialect.addItems(dialects) + if hide_schema: + self._ui.schema_line_edit.setVisible(False) + self._ui.schema_label.setVisible(False) def set_url(self, url): """Sets the URL for the widget. @@ -403,6 +407,7 @@ def set_url(self, url): host = url.get("host", "") port = url.get("port", "") database = url.get("database", "") + schema = url.get("schema", "") username = url.get("username", "") password = url.get("password", "") self.blockSignals(True) @@ -413,6 +418,7 @@ def set_url(self, url): _set_line_edit_text(self._ui.lineEdit_host, host) _set_line_edit_text(self._ui.lineEdit_port, port) _set_line_edit_text(self._ui.lineEdit_database, database) + _set_line_edit_text(self._ui.schema_line_edit, schema) _set_line_edit_text(self._ui.lineEdit_username, username) _set_line_edit_text(self._ui.lineEdit_password, password) self.blockSignals(False) @@ -428,6 +434,7 @@ def url_dict(self): "host": self._ui.lineEdit_host.text(), "port": self._ui.lineEdit_port.text(), "database": self._ui.lineEdit_database.text(), + "schema": self._ui.schema_line_edit.text(), "username": self._ui.lineEdit_username.text(), "password": self._ui.lineEdit_password.text(), } @@ -480,6 +487,7 @@ def enable_no_dialect(self): self._ui.lineEdit_database.setEnabled(False) self._ui.lineEdit_username.setEnabled(False) self._ui.lineEdit_password.setEnabled(False) + self._ui.schema_line_edit.setEnabled(False) def enable_mssql(self): """Adjusts controls to mssql connection specification.""" @@ -490,6 +498,7 @@ def enable_mssql(self): self._ui.lineEdit_database.setEnabled(False) self._ui.lineEdit_username.setEnabled(True) self._ui.lineEdit_password.setEnabled(True) + self._ui.schema_line_edit.setEnabled(True) self._ui.lineEdit_host.clear() self._ui.lineEdit_port.clear() self._ui.lineEdit_database.clear() @@ -504,6 +513,7 @@ def enable_sqlite(self): self._ui.lineEdit_database.setEnabled(True) self._ui.lineEdit_username.setEnabled(False) self._ui.lineEdit_password.setEnabled(False) + self._ui.schema_line_edit.setEnabled(False) self._ui.lineEdit_host.clear() self._ui.lineEdit_port.clear() self._ui.lineEdit_username.clear() @@ -519,15 +529,17 @@ def enable_common(self): self._ui.lineEdit_database.setEnabled(True) self._ui.lineEdit_username.setEnabled(True) self._ui.lineEdit_password.setEnabled(True) + self._ui.schema_line_edit.setEnabled(True) class UrlSelectorDialog(QDialog): msg_error = Signal(str) - def __init__(self, app_settings, logger, parent=None): + def __init__(self, app_settings, hide_schema, logger, parent=None): """ Args: app_settings (QSettings): Toolbox settings + hide_schema (bool): if True, hide the Schema field logger (LoggerInterface): logger parent (QWidget, optional): parent widget """ @@ -539,7 +551,7 @@ def __init__(self, app_settings, logger, parent=None): self._logger = logger self.ui = Ui_Dialog() self.ui.setupUi(self) - self.ui.url_selector_widget.setup(KNOWN_SQL_DIALECTS, self._browse_sqlite_file, self._logger) + self.ui.url_selector_widget.setup(KNOWN_SQL_DIALECTS, self._browse_sqlite_file, hide_schema, self._logger) self.ui.url_selector_widget.url_changed.connect(self._refresh_url) # Add status bar to form self.statusbar = QStatusBar(self) diff --git a/tests/data_connection/test_DataConnection.py b/tests/data_connection/test_DataConnection.py index 01add1c6..14b6bb66 100644 --- a/tests/data_connection/test_DataConnection.py +++ b/tests/data_connection/test_DataConnection.py @@ -21,7 +21,7 @@ from unittest import mock from unittest.mock import MagicMock, NonCallableMagicMock from PySide6.QtCore import QItemSelectionModel -from PySide6.QtWidgets import QApplication, QMessageBox +from PySide6.QtWidgets import QApplication, QDialog, QMessageBox from PySide6.QtGui import Qt from spinetoolbox.helpers import signal_waiter from spine_items.data_connection.data_connection import DataConnection @@ -94,7 +94,7 @@ def test_add_file_references(self): self.assertEqual(1, self._ref_model.rowCount(self._ref_model.index(0, 0))) self.assertEqual(str(a), self._ref_model.index(0, 0, self._ref_model.index(0, 0)).data()) self._data_connection.file_references = list() - self._data_connection.populate_reference_list() + self._data_connection.populate_reference_list([]) # Add two references (the other one is non-existing) # Note: non-existing files cannot be added with the toolbox but this tests a situation when # project.json file has references to files that do not exist anymore and user tries to add a @@ -118,25 +118,40 @@ def test_add_db_references(self): with mock.patch( "spine_items.data_connection.data_connection.UrlSelectorDialog.exec" ) as url_selector_exec, mock.patch( - "spine_items.data_connection.data_connection.UrlSelectorDialog.url", new_callable=mock.PropertyMock - ) as url_selector_url: + "spine_items.data_connection.data_connection.UrlSelectorDialog.url_dict" + ) as url_selector_url_dict: # Add nothing - url_selector_url.return_value = "" + url_selector_exec.return_value = QDialog.DialogCode.Rejected self._data_connection.show_add_db_reference_dialog() self.assertEqual(1, url_selector_exec.call_count) - self.assertEqual(0, len(self._data_connection.db_references)) + self.assertFalse(self._data_connection.has_db_references()) self.assertEqual(0, self._ref_model.rowCount(self._ref_model.index(1, 0))) # Add one url - url_selector_url.return_value = "mysql://randy:creamfraiche@host:3306/db" + url_selector_exec.return_value = QDialog.DialogCode.Accepted + url_selector_url_dict.return_value = { + "dialect": "mysql", + "host": "host", + "port": 3306, + "database": "db", + "username": "randy", + "password": "creamfraiche", + } self._data_connection.show_add_db_reference_dialog() self.assertEqual(2, url_selector_exec.call_count) - self.assertEqual(1, len(self._data_connection.db_references)) + self.assertEqual(1, len(list(self._data_connection.db_reference_iter()))) self.assertEqual(1, self._ref_model.rowCount(self._ref_model.index(1, 0))) # Add same url with different username and password (should not be added) - url_selector_url.return_value = "mysql://scott:tiger@host:3306/db" + url_selector_url_dict.return_value = { + "dialect": "mysql", + "host": "host", + "port": 3306, + "database": "db", + "username": "scott", + "password": "tiger", + } self._data_connection.show_add_db_reference_dialog() self.assertEqual(3, url_selector_exec.call_count) - self.assertEqual(1, len(self._data_connection.db_references)) + self.assertEqual(1, len(list(self._data_connection.db_reference_iter()))) self.assertEqual(1, self._ref_model.rowCount(self._ref_model.index(1, 0))) def test_remove_references(self): @@ -149,8 +164,8 @@ def test_remove_references(self): ) as mock_selected_indexes, mock.patch( "spine_items.data_connection.data_connection.UrlSelectorDialog.exec" ) as url_selector_exec, mock.patch( - "spine_items.data_connection.data_connection.UrlSelectorDialog.url", new_callable=mock.PropertyMock - ) as url_selector_url: + "spine_items.data_connection.data_connection.UrlSelectorDialog.url_dict" + ) as url_selector_url_dict: a = Path(temp_dir, "a.txt") a.touch() b = Path(temp_dir, "b.txt") @@ -167,12 +182,27 @@ def test_remove_references(self): self.assertEqual(2, len(self._data_connection.file_references)) self.assertEqual(2, self._ref_model.rowCount(self._ref_model.index(0, 0))) # Second add a couple of dbs as refs - url_selector_url.return_value = "mysql://scott:tiger@host:3306/db" + url_selector_exec.return_value = QDialog.DialogCode.Accepted + url_selector_url_dict.return_value = { + "dialect": "mysql", + "username": "scott", + "password": "tiger", + "host": "host", + "port": 3306, + "database": "db", + } self._data_connection.show_add_db_reference_dialog() - url_selector_url.return_value = "mysql://randy:creamfraiche@host:3307/db" + url_selector_url_dict.return_value = { + "dialect": "mysql", + "username": "randy", + "password": "creamfraiche", + "host": "host", + "port": 3307, + "database": "db", + } self._data_connection.show_add_db_reference_dialog() self.assertEqual(2, url_selector_exec.call_count) - self.assertEqual(2, len(self._data_connection.db_references)) + self.assertEqual(2, len(list(self._data_connection.db_reference_iter()))) self.assertEqual(2, self._ref_model.rowCount(self._ref_model.index(1, 0))) # Test with no indexes selected mock_selected_indexes.return_value = [] @@ -195,11 +225,25 @@ def test_remove_references(self): mock_selected_indexes.return_value = [db1_index] self._data_connection.remove_references() self.assertEqual(3, mock_selected_indexes.call_count) - self.assertEqual(1, len(self._data_connection.db_references)) + self.assertEqual(1, len(list(self._data_connection.db_reference_iter()))) self.assertEqual(1, self._ref_model.rowCount(self._ref_model.index(1, 0))) # Check that the remaining db is the one that's supposed to be there - self.assertEqual(["mysql://host:3307/db"], self._data_connection.db_references) - self.assertEqual("mysql://host:3307/db", self._ref_model.item(1).child(0).data(Qt.ItemDataRole.DisplayRole)) + self.assertEqual( + [ + { + "dialect": "mysql", + "host": "host", + "port": 3307, + "database": "db", + "username": "randy", + "password": "creamfraiche", + } + ], + list(self._data_connection.db_reference_iter()), + ) + self.assertEqual( + "mysql+pymysql://host:3307/db", self._ref_model.item(1).child(0).data(Qt.ItemDataRole.DisplayRole) + ) # Now remove the remaining file and db b_index = self._ref_model.index(0, 0, self._ref_model.index(0, 0)) db2_index = self._ref_model.index(0, 0, self._ref_model.index(1, 0)) @@ -208,7 +252,7 @@ def test_remove_references(self): self.assertEqual(4, mock_selected_indexes.call_count) self.assertEqual(0, len(self._data_connection.file_references)) self.assertEqual(0, self._ref_model.rowCount(self._ref_model.index(0, 0))) - self.assertEqual(0, len(self._data_connection.db_references)) + self.assertEqual(0, len(list(self._data_connection.db_reference_iter()))) self.assertEqual(0, self._ref_model.rowCount(self._ref_model.index(1, 0))) # Add a and b back and the two non-existing files as well # Select non-existing file c and remove it @@ -292,7 +336,7 @@ def test_remove_references_with_del_key(self): indexes = self._data_connection._properties_ui.treeView_dc_references.selectedIndexes() self._data_connection._properties_ui.treeView_dc_references.del_key_pressed.emit() self.assertEqual(0, len(self._data_connection.file_references)) - self.assertEqual(0, len(self._data_connection.db_references)) + self.assertEqual(0, len(list(self._data_connection.db_reference_iter()))) def test_renaming_file_marks_its_reference_as_missing(self): temp_dir = Path(self._temp_dir.name, "references") diff --git a/tests/data_connection/test_DataConnectionExecutable.py b/tests/data_connection/test_DataConnectionExecutable.py index d9cdb183..ab5238a8 100644 --- a/tests/data_connection/test_DataConnectionExecutable.py +++ b/tests/data_connection/test_DataConnectionExecutable.py @@ -53,7 +53,7 @@ def test_from_dict(self): self.assertEqual(2, len(item._file_paths)) def test_stop_execution(self): - executable = ExecutableItem("name", [], [], {}, self._temp_dir.name, mock.MagicMock()) + executable = ExecutableItem("name", [], [], self._temp_dir.name, mock.MagicMock()) with mock.patch( "spine_engine.project_item.executable_item_base.ExecutableItemBase.stop_execution" ) as mock_stop_execution: @@ -61,7 +61,7 @@ def test_stop_execution(self): mock_stop_execution.assert_called_once() def test_execute(self): - executable = ExecutableItem("name", [], [], {}, self._temp_dir.name, mock.MagicMock()) + executable = ExecutableItem("name", [], [], self._temp_dir.name, mock.MagicMock()) self.assertTrue(executable.execute([], [], Lock())) def test_output_resources_backward(self): @@ -69,7 +69,7 @@ def test_output_resources_backward(self): dc_data_dir.mkdir(parents=True) temp_file_path = pathlib.Path(dc_data_dir, "file.txt") temp_file_path.touch() - executable = ExecutableItem("name", ["file_reference"], [], {}, self._temp_dir.name, mock.MagicMock()) + executable = ExecutableItem("name", ["file_reference"], [], self._temp_dir.name, mock.MagicMock()) self.assertEqual(executable.output_resources(ExecutionDirection.BACKWARD), []) def test_output_resources_forward(self): @@ -79,7 +79,7 @@ def test_output_resources_forward(self): dc_data_dir.mkdir(parents=True) temp_file_path = pathlib.Path(dc_data_dir, "data_file") temp_file_path.touch() - executable = ExecutableItem("name", [str(file_reference)], [], {}, self._temp_dir.name, mock.MagicMock()) + executable = ExecutableItem("name", [str(file_reference)], [], self._temp_dir.name, mock.MagicMock()) output_resources = executable.output_resources(ExecutionDirection.FORWARD) self.assertEqual(len(output_resources), 2) resource = output_resources[0] diff --git a/tests/data_connection/test_output_resources.py b/tests/data_connection/test_output_resources.py index 61b703ee..b33525b9 100644 --- a/tests/data_connection/test_output_resources.py +++ b/tests/data_connection/test_output_resources.py @@ -18,29 +18,32 @@ class TestScanForResources(unittest.TestCase): - def test_url_reference_gets_added_as_url_resource(self): - data_connection = MagicMock() - data_connection.name = "test dc" - project_dir_path = PurePath(__file__).parent - data_connection.data_dir = str(project_dir_path / ".spinetoolbox" / "test_dc") - file_paths = [] - urls = ["gopher://long.gone.url"] - url_credentials = {} - resources = scan_for_resources(data_connection, file_paths, urls, url_credentials, str(project_dir_path)) - self.assertEqual(resources, [url_resource("test dc", "gopher://long.gone.url", "gopher://long.gone.url")]) - def test_credentials_do_not_show_in_url_resource_label(self): data_connection = MagicMock() data_connection.name = "test dc" project_dir_path = PurePath(__file__).parent data_connection.data_dir = str(project_dir_path / ".spinetoolbox" / "test_dc") file_paths = [] - urls = ["ssh://long.gone.url"] - url_credentials = {"ssh://long.gone.url": ("superman", "t0p s3cr3t")} - resources = scan_for_resources(data_connection, file_paths, urls, url_credentials, str(project_dir_path)) + urls = [ + { + "dialect": "postgresql", + "host": "long.gone.url", + "port": 5432, + "database": "warehouse", + "username": "superman", + "password": "t0p s3cr3t", + } + ] + resources = scan_for_resources(data_connection, file_paths, urls, str(project_dir_path)) self.assertEqual( resources, - [url_resource("test dc", "ssh://superman:t0p s3cr3t@long.gone.url", "ssh://long.gone.url")], + [ + url_resource( + "test dc", + "postgresql+psycopg2://superman:t0p s3cr3t@long.gone.url:5432/warehouse", + "postgresql+psycopg2://long.gone.url:5432/warehous", + ) + ], ) diff --git a/tests/importer/widgets/test_import_sources.py b/tests/importer/widgets/test_import_sources.py index 49dadb90..dcd332b1 100644 --- a/tests/importer/widgets/test_import_sources.py +++ b/tests/importer/widgets/test_import_sources.py @@ -52,7 +52,6 @@ def test_connector_fetches_empty_data(self): data = [] connection_settings = {"data": data} connector = ConnectionManager(_FixedTableReader, connection_settings, self._parent_widget) - connector.source = "no file test source" mapping = {"table_options": {"test data table": {"has_header": False}}} import_sources.set_connector(connector, mapping) self._mappings_model.append_new_table_with_mapping("test data table", None) @@ -60,7 +59,7 @@ def test_connector_fetches_empty_data(self): import_sources._change_selected_table(table_index, QModelIndex()) source_table_model = import_sources._source_data_model with signal_waiter(connector.connection_ready) as waiter: - connector.init_connection() + connector.init_connection("no file test source") waiter.wait() with signal_waiter(connector.data_ready) as waiter: source_table_model.fetchMore(QModelIndex()) @@ -92,7 +91,6 @@ def test_connector_fetches_data_only(self): data = [["data 1", "data 2"]] connection_settings = {"data": data} connector = ConnectionManager(_FixedTableReader, connection_settings, self._parent_widget) - connector.source = "no file test source" mapping = {"table_options": {"test data table": {"has_header": False}}} import_sources.set_connector(connector, mapping) self._mappings_model.append_new_table_with_mapping("test data table", None) @@ -100,7 +98,7 @@ def test_connector_fetches_data_only(self): import_sources._change_selected_table(table_index, QModelIndex()) source_table_model = import_sources._source_data_model with signal_waiter(connector.connection_ready) as waiter: - connector.init_connection() + connector.init_connection("no file test source") waiter.wait() with signal_waiter(connector.data_ready) as waiter: source_table_model.fetchMore(QModelIndex()) @@ -132,7 +130,6 @@ def test_connector_fetches_header_only(self): data = [["header 1", "header 2"]] connection_settings = {"data": data} connector = ConnectionManager(_FixedTableReader, connection_settings, self._parent_widget) - connector.source = "no file test source" mapping = {"table_options": {"test data table": {"has_header": True}}} import_sources.set_connector(connector, mapping) self._mappings_model.append_new_table_with_mapping("test data table", None) @@ -140,7 +137,7 @@ def test_connector_fetches_header_only(self): import_sources._change_selected_table(table_index, QModelIndex()) source_table_model = import_sources._source_data_model with signal_waiter(connector.connection_ready) as waiter: - connector.init_connection() + connector.init_connection("no file test source") waiter.wait() with signal_waiter(connector.data_ready) as waiter: source_table_model.fetchMore(QModelIndex()) @@ -172,7 +169,6 @@ def test_connector_fetches_data_and_header(self): data = [["header 1", "header 2"], ["data 1", "data 2"]] connection_settings = {"data": data} connector = ConnectionManager(_FixedTableReader, connection_settings, self._parent_widget) - connector.source = "no file test source" mapping = {"table_options": {"test data table": {"has_header": True}}} import_sources.set_connector(connector, mapping) self._mappings_model.append_new_table_with_mapping("test data table", None) @@ -180,7 +176,7 @@ def test_connector_fetches_data_and_header(self): import_sources._change_selected_table(table_index, QModelIndex()) source_table_model = import_sources._source_data_model with signal_waiter(connector.connection_ready) as waiter: - connector.init_connection() + connector.init_connection("no file test source") waiter.wait() with signal_waiter(connector.data_ready) as waiter: source_table_model.fetchMore(QModelIndex()) @@ -213,7 +209,7 @@ def __init__(self, settings): super().__init__(None) self._data = settings["data"] - def connect_to_source(self, source): + def connect_to_source(self, source, **extras): pass def disconnect(self):