From 61fe1f3a635e2a113ffd5ed498396427a3da37e5 Mon Sep 17 00:00:00 2001 From: Manu Date: Tue, 12 Mar 2024 15:09:07 +0000 Subject: [PATCH 1/6] Move slow code to async QThread workers (WIP) --- src/vorta/store/models.py | 4 +- src/vorta/store/utils.py | 14 + src/vorta/utils.py | 262 ++---------------- src/vorta/views/archive_tab.py | 67 +---- src/vorta/views/main_window.py | 9 +- src/vorta/views/repo_tab.py | 5 +- src/vorta/views/schedule_tab.py | 32 ++- src/vorta/views/source_tab.py | 24 +- src/vorta/views/utils.py | 36 +++ src/vorta/views/workers/__init__.py | 0 .../views/workers/archive_table_worker.py | 134 +++++++++ .../views/workers/file_path_info_worker.py | 122 ++++++++ src/vorta/views/workers/wifi_list_worker.py | 59 ++++ 13 files changed, 422 insertions(+), 346 deletions(-) create mode 100644 src/vorta/store/utils.py create mode 100644 src/vorta/views/workers/__init__.py create mode 100644 src/vorta/views/workers/archive_table_worker.py create mode 100644 src/vorta/views/workers/file_path_info_worker.py create mode 100644 src/vorta/views/workers/wifi_list_worker.py diff --git a/src/vorta/store/models.py b/src/vorta/store/models.py index 400299fb2..a5ab0fc0c 100644 --- a/src/vorta/store/models.py +++ b/src/vorta/store/models.py @@ -12,8 +12,8 @@ import peewee as pw from playhouse import signals -from vorta.utils import slugify -from vorta.views.utils import get_exclusion_presets +from vorta.store.utils import slugify +# from vorta.views.utils import get_exclusion_presets DB = pw.Proxy() logger = logging.getLogger(__name__) diff --git a/src/vorta/store/utils.py b/src/vorta/store/utils.py new file mode 100644 index 000000000..f810978f8 --- /dev/null +++ b/src/vorta/store/utils.py @@ -0,0 +1,14 @@ +import unicodedata +import re + +def slugify(value): + """ + Converts to lowercase, removes non-word characters (alphanumerics and + underscores) and converts spaces to hyphens. Also strips leading and + trailing whitespace. + + Copied from Django. + """ + value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii') + value = re.sub(r'[^\w\s-]', '', value).strip().lower() + return re.sub(r'[-\s]+', '-', value) diff --git a/src/vorta/utils.py b/src/vorta/utils.py index 033b486a5..f63ff966d 100644 --- a/src/vorta/utils.py +++ b/src/vorta/utils.py @@ -1,155 +1,33 @@ import argparse import errno -import fnmatch -import getpass import math +import getpass import os import re import socket import sys -import unicodedata -from datetime import datetime as dt +from datetime import datetime as dt, timedelta from functools import reduce from typing import Any, Callable, Iterable, List, Optional, Tuple, TypeVar -import psutil from PyQt6 import QtCore -from PyQt6.QtCore import QFileInfo, QThread, pyqtSignal -from PyQt6.QtWidgets import QApplication, QFileDialog, QSystemTrayIcon +from PyQt6.QtCore import QFileInfo, QThread, pyqtSignal, Qt +from PyQt6.QtWidgets import (QApplication, QFileDialog, QSystemTrayIcon, + QListWidgetItem, QTableWidgetItem) +from vorta.network_status.abc import NetworkStatusMonitor from vorta.borg._compatibility import BorgCompatibility from vorta.log import logger -from vorta.network_status.abc import NetworkStatusMonitor + # Used to store whether a user wanted to override the # default directory for the --development flag DEFAULT_DIR_FLAG = object() METRIC_UNITS = ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'] NONMETRIC_UNITS = ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi'] - -borg_compat = BorgCompatibility() _network_status_monitor = None - -class FilePathInfoAsync(QThread): - signal = pyqtSignal(str, str, str) - - def __init__(self, path, exclude_patterns_str): - self.path = path - QThread.__init__(self) - self.exiting = False - self.exclude_patterns = [] - for _line in (exclude_patterns_str or '').splitlines(): - line = _line.strip() - if line != '': - self.exclude_patterns.append(line) - - def run(self): - # logger.info("running thread to get path=%s...", self.path) - self.size, self.files_count = get_path_datasize(self.path, self.exclude_patterns) - self.signal.emit(self.path, str(self.size), str(self.files_count)) - - -def normalize_path(path): - """normalize paths for MacOS (but do nothing on other platforms)""" - # HFS+ converts paths to a canonical form, so users shouldn't be required to enter an exact match. - # Windows and Unix filesystems allow different forms, so users always have to enter an exact match. - return unicodedata.normalize('NFD', path) if sys.platform == 'darwin' else path - - -# prepare patterns as borg does -# see `FnmatchPattern._prepare` at -# https://github.com/borgbackup/borg/blob/master//src/borg/patterns.py -def prepare_pattern(pattern): - """Prepare and process fnmatch patterns as borg does""" - if pattern.endswith(os.path.sep): - # trailing sep indicates that the contents should be excluded - # but not the directory it self. - pattern = os.path.normpath(pattern).rstrip(os.path.sep) - pattern += os.path.sep + '*' + os.path.sep - else: - pattern = os.path.normpath(pattern) + os.path.sep + '*' - - pattern = pattern.lstrip(os.path.sep) # sep at beginning is removed - return re.compile(fnmatch.translate(pattern)) - - -def match(pattern: re.Pattern, path: str): - """Check whether a path matches the given pattern.""" - path = path.lstrip(os.path.sep) + os.path.sep - return pattern.match(path) is not None - - -def get_directory_size(dir_path, exclude_patterns): - '''Get number of files only and total size in bytes from a path. - Based off https://stackoverflow.com/a/17936789''' - exclude_patterns = [prepare_pattern(p) for p in exclude_patterns] - - data_size_filtered = 0 - seen = set() - seen_filtered = set() - - for dir_path, subdirectories, file_names in os.walk(dir_path, topdown=True): - is_excluded = False - for pattern in exclude_patterns: - if match(pattern, dir_path): - is_excluded = True - break - - if is_excluded: - subdirectories.clear() # so that os.walk won't walk them - continue - - for file_name in file_names: - file_path = os.path.join(dir_path, file_name) - - # Ignore symbolic links, since borg doesn't follow them - if os.path.islink(file_path): - continue - - is_excluded = False - for pattern in exclude_patterns: - if match(pattern, file_path): - is_excluded = True - break - - try: - stat = os.stat(file_path) - if stat.st_ino not in seen: # Visit each file only once - # this won't add the size of a hardlinked file - seen.add(stat.st_ino) - if not is_excluded: - data_size_filtered += stat.st_size - seen_filtered.add(stat.st_ino) - except (FileNotFoundError, PermissionError): - continue - - files_count_filtered = len(seen_filtered) - - return data_size_filtered, files_count_filtered - - -def get_network_status_monitor(): - global _network_status_monitor - if _network_status_monitor is None: - _network_status_monitor = NetworkStatusMonitor.get_network_status_monitor() - logger.info( - 'Using %s NetworkStatusMonitor implementation.', - _network_status_monitor.__class__.__name__, - ) - return _network_status_monitor - - -def get_path_datasize(path, exclude_patterns): - file_info = QFileInfo(path) - - if file_info.isDir(): - data_size, files_count = get_directory_size(file_info.absoluteFilePath(), exclude_patterns) - else: - data_size = file_info.size() - files_count = 1 - - return data_size, files_count +borg_compat = BorgCompatibility() def nested_dict(): @@ -220,22 +98,6 @@ def get_private_keys() -> List[str]: return available_private_keys -def sort_sizes(size_list): - """Sorts sizes with extensions. Assumes that size is already in largest unit possible""" - final_list = [] - for suffix in [" B", " KB", " MB", " GB", " TB", " PB", " EB", " ZB", " YB"]: - sub_list = [ - float(size[: -len(suffix)]) - for size in size_list - if size.endswith(suffix) and size[: -len(suffix)][-1].isnumeric() - ] - sub_list.sort() - final_list += [(str(size) + suffix) for size in sub_list] - # Skip additional loops - if len(final_list) == len(size_list): - break - return final_list - Number = TypeVar("Number", int, float) @@ -244,6 +106,16 @@ def clamp(n: Number, min_: Number, max_: Number) -> Number: """Restrict the number n inside a range""" return min(max_, max(n, min_)) +def get_network_status_monitor(): + global _network_status_monitor + if _network_status_monitor is None: + _network_status_monitor = NetworkStatusMonitor.get_network_status_monitor() + logger.info( + 'Using %s NetworkStatusMonitor implementation.', + _network_status_monitor.__class__.__name__, + ) + return _network_status_monitor + def find_best_unit_for_sizes(sizes: Iterable[int], metric: bool = True, precision: int = 1) -> int: """ @@ -303,37 +175,6 @@ def get_asset(path): return os.path.join(bundle_dir, path) -def get_sorted_wifis(profile): - """ - Get Wifi networks known to the OS (only current one on macOS) and - merge with networks from other profiles. Update last connected time. - """ - - from vorta.store.models import WifiSettingModel - - # Pull networks known to OS and all other backup profiles - system_wifis = get_network_status_monitor().get_known_wifis() - from_other_profiles = WifiSettingModel.select().where(WifiSettingModel.profile != profile.id).execute() - - for wifi in list(from_other_profiles) + system_wifis: - db_wifi, created = WifiSettingModel.get_or_create( - ssid=wifi.ssid, - profile=profile.id, - defaults={'last_connected': wifi.last_connected, 'allowed': True}, - ) - - # Update last connected time - if not created and db_wifi.last_connected != wifi.last_connected: - db_wifi.last_connected = wifi.last_connected - db_wifi.save() - - # Finally return list of networks and settings for that profile - return ( - WifiSettingModel.select() - .where(WifiSettingModel.profile == profile.id) - .order_by(-WifiSettingModel.last_connected) - ) - def parse_args(): parser = argparse.ArgumentParser(description='Vorta Backup GUI for Borg.') @@ -368,19 +209,6 @@ def parse_args(): return parser.parse_known_args()[0] -def slugify(value): - """ - Converts to lowercase, removes non-word characters (alphanumerics and - underscores) and converts spaces to hyphens. Also strips leading and - trailing whitespace. - - Copied from Django. - """ - value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii') - value = re.sub(r'[^\w\s-]', '', value).strip().lower() - return re.sub(r'[-\s]+', '-', value) - - def uses_dark_mode(): """ This function detects if we are running in dark mode (e.g. macOS dark mode). @@ -431,60 +259,6 @@ def format_archive_name(profile, archive_name_tpl): SHELL_PATTERN_ELEMENT = re.compile(r'([?\[\]*])') -def get_mount_points(repo_url): - mount_points = {} - repo_mounts = [] - for proc in psutil.process_iter(): - try: - name = proc.name() - if name == 'borg' or name.startswith('python'): - if 'mount' not in proc.cmdline(): - continue - - if borg_compat.check('V2'): - # command line syntax: - # `borg mount -r (-a )` - cmd = proc.cmdline() - if repo_url in cmd: - i = cmd.index(repo_url) - if len(cmd) > i + 1: - mount_point = cmd[i + 1] - - # Archive mount? - ao = '-a' in cmd - if ao or '--match-archives' in cmd: - i = cmd.index('-a' if ao else '--match-archives') - if len(cmd) >= i + 1 and not SHELL_PATTERN_ELEMENT.search(cmd[i + 1]): - mount_points[mount_point] = cmd[i + 1] - else: - repo_mounts.append(mount_point) - else: - for idx, parameter in enumerate(proc.cmdline()): - if parameter.startswith(repo_url): - # mount from this repo - - # The borg mount command specifies that the mount_point - # parameter comes after the archive name - if len(proc.cmdline()) > idx + 1: - mount_point = proc.cmdline()[idx + 1] - - # archive or full mount? - if parameter[len(repo_url) :].startswith('::'): - archive_name = parameter[len(repo_url) + 2 :] - mount_points[archive_name] = mount_point - break - else: - # repo mount point - repo_mounts.append(mount_point) - - except (psutil.ZombieProcess, psutil.AccessDenied, psutil.NoSuchProcess): - # Getting process details may fail (e.g. zombie process on macOS) - # or because the process is owned by another user. - # Also see https://github.com/giampaolo/psutil/issues/783 - continue - - return mount_points, repo_mounts - def is_system_tray_available(): app = QApplication.instance() diff --git a/src/vorta/views/archive_tab.py b/src/vorta/views/archive_tab.py index d2af5757b..6938265e3 100644 --- a/src/vorta/views/archive_tab.py +++ b/src/vorta/views/archive_tab.py @@ -1,6 +1,5 @@ import logging import sys -from datetime import timedelta from typing import Dict, Optional from PyQt6 import QtCore, uic @@ -36,17 +35,14 @@ from vorta.utils import ( borg_compat, choose_file_dialog, - find_best_unit_for_sizes, format_archive_name, - get_asset, - get_mount_points, - pretty_bytes, + get_asset ) from vorta.views import diff_result, extract_dialog from vorta.views.diff_result import DiffResultDialog, DiffTree from vorta.views.extract_dialog import ExtractDialog, ExtractTree -from vorta.views.source_tab import SizeItem from vorta.views.utils import get_colored_icon +from vorta.views.workers.archive_table_worker import PopulateArchiveTableAsync uifile = get_asset('UI/archivetab.ui') ArchiveTabUI, ArchiveTabBase = uic.loadUiType(uifile) @@ -73,6 +69,7 @@ def __init__(self, parent=None, app=None): super().__init__(parent) self.setupUi(parent) self.mount_points = {} # mapping of archive name to mount point + self.workers = [] self.repo_mount_point: Optional[str] = None # mount point of whole repo self.menu = None self.app = app @@ -241,10 +238,6 @@ def populate_from_profile(self): """Populate archive list and prune settings from profile.""" profile = self.profile() if profile.repo is not None: - # get mount points - self.mount_points, repo_mount_points = get_mount_points(profile.repo.url) - if repo_mount_points: - self.repo_mount_point = repo_mount_points[0] if profile.repo.name: repo_name = f"{profile.repo.name} ({profile.repo.url})" @@ -252,58 +245,10 @@ def populate_from_profile(self): repo_name = profile.repo.url self.toolBox.setItemText(0, self.tr('Archives for {}').format(repo_name)) - archives = [s for s in profile.repo.archives.select().order_by(ArchiveModel.time.desc())] + populateArchiveTableWorker = PopulateArchiveTableAsync(profile, self.mount_points, self.archiveTable) + self.workers.append(populateArchiveTableWorker) # preserve worker reference + populateArchiveTableWorker.start() - # if no archive's name can be found in self.mount_points, then hide the mount point column - if not any(a.name in self.mount_points for a in archives): - self.archiveTable.hideColumn(3) - else: - self.archiveTable.showColumn(3) - - sorting = self.archiveTable.isSortingEnabled() - self.archiveTable.setSortingEnabled(False) - best_unit = find_best_unit_for_sizes((a.size for a in archives), precision=SIZE_DECIMAL_DIGITS) - for row, archive in enumerate(archives): - self.archiveTable.insertRow(row) - - formatted_time = archive.time.strftime('%Y-%m-%d %H:%M') - self.archiveTable.setItem(row, 0, QTableWidgetItem(formatted_time)) - - # format units based on user settings for 'dynamic' or 'fixed' units - fixed_unit = best_unit if SettingsModel.get(key='enable_fixed_units').value else None - size = pretty_bytes(archive.size, fixed_unit=fixed_unit, precision=SIZE_DECIMAL_DIGITS) - self.archiveTable.setItem(row, 1, SizeItem(size)) - - if archive.duration is not None: - formatted_duration = str(timedelta(seconds=round(archive.duration))) - else: - formatted_duration = '' - - self.archiveTable.setItem(row, 2, QTableWidgetItem(formatted_duration)) - - mount_point = self.mount_points.get(archive.name) - if mount_point is not None: - item = QTableWidgetItem(mount_point) - self.archiveTable.setItem(row, 3, item) - - self.archiveTable.setItem(row, 4, QTableWidgetItem(archive.name)) - - if archive.trigger == 'scheduled': - item = QTableWidgetItem(get_colored_icon('clock-o'), '') - item.setToolTip(self.tr('Scheduled')) - self.archiveTable.setItem(row, 5, item) - elif archive.trigger == 'user': - item = QTableWidgetItem(get_colored_icon('user'), '') - item.setToolTip(self.tr('User initiated')) - item.setTextAlignment(Qt.AlignmentFlag.AlignRight) - self.archiveTable.setItem(row, 5, item) - - self.archiveTable.setRowCount(len(archives)) - self.archiveTable.setSortingEnabled(sorting) - item = self.archiveTable.item(0, 0) - self.archiveTable.scrollToItem(item) - - self.archiveTable.selectionModel().clearSelection() if self.remaining_refresh_archives == 0: self._toggle_all_buttons(enabled=True) else: diff --git a/src/vorta/views/main_window.py b/src/vorta/views/main_window.py index 78cf40f47..24b3350c2 100644 --- a/src/vorta/views/main_window.py +++ b/src/vorta/views/main_window.py @@ -180,10 +180,15 @@ def profile_selection_changed_action(self, index): if not backup_profile_id: return self.current_profile = BackupProfileModel.get(id=backup_profile_id) + logger.info('step 1') self.archiveTab.populate_from_profile() - self.repoTab.populate_from_profile() + logger.info('step 2') + self.repoTab.populate_from_profile() # 1s + logger.info('step 3') self.sourceTab.populate_from_profile() - self.scheduleTab.populate_from_profile() + logger.info('step 4') + self.scheduleTab.populate_from_profile() #1s + logger.info('step 5') SettingsModel.update({SettingsModel.str_value: self.current_profile.id}).where( SettingsModel.key == 'previous_profile_id' ).execute() diff --git a/src/vorta/views/repo_tab.py b/src/vorta/views/repo_tab.py index e9f38dbad..a74f620d5 100644 --- a/src/vorta/views/repo_tab.py +++ b/src/vorta/views/repo_tab.py @@ -173,8 +173,6 @@ def init_repo_stats(self): self.repoEncryption.setText(na) self.repoEncryption.setToolTip(no_repo_selected) - self.repo_changed.emit() - def init_ssh(self): keys = get_private_keys() self.sshComboBox.clear() @@ -264,6 +262,7 @@ def repo_select_action(self): profile.repo = self.repoSelector.currentData() profile.save() self.init_repo_stats() + self.repo_changed.emit() def process_new_repo(self, result): if result['returncode'] == 0: @@ -276,10 +275,12 @@ def process_new_repo(self, result): self.repoSelector.setCurrentIndex(self.repoSelector.count() - 1) self.repo_added.emit() self.init_repo_stats() + self.repo_changed.emit() def repo_unlink_action(self): profile = self.profile() self.init_repo_stats() + self.repo_changed.emit() msg = QMessageBox() msg.setStandardButtons(QMessageBox.StandardButton.Ok) diff --git a/src/vorta/views/schedule_tab.py b/src/vorta/views/schedule_tab.py index dc97cfa47..fd0f19bfa 100644 --- a/src/vorta/views/schedule_tab.py +++ b/src/vorta/views/schedule_tab.py @@ -12,7 +12,8 @@ from vorta.i18n import get_locale from vorta.scheduler import ScheduleStatusType from vorta.store.models import BackupProfileMixin, EventLogModel, WifiSettingModel -from vorta.utils import get_asset, get_sorted_wifis +from vorta.utils import get_asset +from vorta.views.workers.wifi_list_worker import PopulateWifiAsync from vorta.views.utils import get_colored_icon uifile = get_asset('UI/scheduletab.ui') @@ -33,6 +34,7 @@ def __init__(self, parent=None): self.setupUi(parent) self.app: application.VortaApp = QApplication.instance() self.toolBox.setCurrentIndex(0) + self.workers = [] self.schedulerRadioMapping = { 'off': self.scheduleOffRadio, @@ -171,7 +173,9 @@ def populate_from_profile(self): else: self.createCmdLineEdit.setEnabled(False) - self.populate_wifi() + populateWifiWorker = PopulateWifiAsync(profile, self.wifiListWidget) + self.workers.append(populateWifiWorker) # preserve reference + populateWifiWorker.start() self.populate_logs() self.draw_next_scheduled_backup() @@ -191,18 +195,18 @@ def draw_next_scheduled_backup(self): self.nextBackupDateTimeLabel.setText(text) self.nextBackupDateTimeLabel.repaint() - def populate_wifi(self): - self.wifiListWidget.clear() - for wifi in get_sorted_wifis(self.profile()): - item = QListWidgetItem() - item.setText(wifi.ssid) - item.setFlags(item.flags() | QtCore.Qt.ItemFlag.ItemIsUserCheckable) - if wifi.allowed: - item.setCheckState(QtCore.Qt.CheckState.Checked) - else: - item.setCheckState(QtCore.Qt.CheckState.Unchecked) - self.wifiListWidget.addItem(item) - self.wifiListWidget.itemChanged.connect(self.save_wifi_item) + # def populate_wifi(self): + # self.wifiListWidget.clear() + # for wifi in get_sorted_wifis(self.profile()): + # item = QListWidgetItem() + # item.setText(wifi.ssid) + # item.setFlags(item.flags() | QtCore.Qt.ItemFlag.ItemIsUserCheckable) + # if wifi.allowed: + # item.setCheckState(QtCore.Qt.CheckState.Checked) + # else: + # item.setCheckState(QtCore.Qt.CheckState.Unchecked) + # self.wifiListWidget.addItem(item) + # self.wifiListWidget.itemChanged.connect(self.save_wifi_item) def save_wifi_item(self, item): db_item = WifiSettingModel.get(ssid=item.text(), profile=self.profile().id) diff --git a/src/vorta/views/source_tab.py b/src/vorta/views/source_tab.py index ed63be2a7..e4a686f62 100644 --- a/src/vorta/views/source_tab.py +++ b/src/vorta/views/source_tab.py @@ -14,15 +14,14 @@ ) from vorta.store.models import BackupProfileMixin, SettingsModel, SourceFileModel +from vorta.views.workers.file_path_info_worker import FilePathInfoAsync from vorta.utils import ( - FilePathInfoAsync, choose_file_dialog, get_asset, - pretty_bytes, - sort_sizes, + pretty_bytes ) from vorta.views.exclude_dialog import ExcludeDialog -from vorta.views.utils import get_colored_icon +from vorta.views.utils import get_colored_icon, SizeItem uifile = get_asset('UI/sourcetab.ui') SourceUI, SourceBase = uic.loadUiType(uifile) @@ -36,23 +35,6 @@ class SourceColumn: FilesCount = 2 -class SizeItem(QTableWidgetItem): - def __init__(self, s): - super().__init__(s) - self.setTextAlignment(Qt.AlignmentFlag.AlignVCenter + Qt.AlignmentFlag.AlignRight) - - def __lt__(self, other): - if other.text() == '': - return False - elif self.text() == '': - return True - else: - return sort_sizes([self.text(), other.text()]) == [ - self.text(), - other.text(), - ] - - class FilesCount(QTableWidgetItem): def __lt__(self, other): # Verify that conversion is only performed on valid integers diff --git a/src/vorta/views/utils.py b/src/vorta/views/utils.py index 5a9697a73..9a3c24cb9 100644 --- a/src/vorta/views/utils.py +++ b/src/vorta/views/utils.py @@ -3,10 +3,46 @@ import sys from PyQt6.QtGui import QIcon, QImage, QPixmap +from PyQt6.QtCore import Qt +from PyQt6.QtWidgets import QTableWidgetItem from vorta.utils import get_asset, uses_dark_mode +class SizeItem(QTableWidgetItem): + def __init__(self, s): + super().__init__(s) + self.setTextAlignment(Qt.AlignmentFlag.AlignVCenter + Qt.AlignmentFlag.AlignRight) + + def __lt__(self, other): + if other.text() == '': + return False + elif self.text() == '': + return True + else: + return sort_sizes([self.text(), other.text()]) == [ + self.text(), + other.text(), + ] + + +def sort_sizes(size_list): + """Sorts sizes with extensions. Assumes that size is already in largest unit possible""" + final_list = [] + for suffix in [" B", " KB", " MB", " GB", " TB", " PB", " EB", " ZB", " YB"]: + sub_list = [ + float(size[: -len(suffix)]) + for size in size_list + if size.endswith(suffix) and size[: -len(suffix)][-1].isnumeric() + ] + sub_list.sort() + final_list += [(str(size) + suffix) for size in sub_list] + # Skip additional loops + if len(final_list) == len(size_list): + break + return final_list + + def get_colored_icon(icon_name, scaled_height=128, return_qpixmap=False): """ Return SVG icon in the correct color. diff --git a/src/vorta/views/workers/__init__.py b/src/vorta/views/workers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/vorta/views/workers/archive_table_worker.py b/src/vorta/views/workers/archive_table_worker.py new file mode 100644 index 000000000..a918f7cf6 --- /dev/null +++ b/src/vorta/views/workers/archive_table_worker.py @@ -0,0 +1,134 @@ + +from datetime import timedelta + +import psutil +from PyQt6.QtCore import QThread, Qt +from PyQt6.QtWidgets import QTableWidgetItem + +from vorta.store.models import ArchiveModel, SettingsModel +from vorta.views.utils import get_colored_icon, SizeItem +from vorta.utils import borg_compat, pretty_bytes, find_best_unit_for_sizes, SHELL_PATTERN_ELEMENT + +SIZE_DECIMAL_DIGITS = 1 + + +class PopulateArchiveTableAsync(QThread): + def __init__(self, profile, mount_points, archiveTable): + QThread.__init__(self) + self.profile = profile + self.mount_points = mount_points + self.archiveTable = archiveTable + + def run(self): + # get mount points + self.mount_points, repo_mount_points = get_mount_points(self.profile.repo.url) + if repo_mount_points: + self.repo_mount_point = repo_mount_points[0] + + archives = [s for s in self.profile.repo.archives.select().order_by(ArchiveModel.time.desc())] + + # if no archive's name can be found in self.mount_points, then hide the mount point column + if not any(a.name in self.mount_points for a in archives): + self.archiveTable.hideColumn(3) + else: + self.archiveTable.showColumn(3) + + sorting = self.archiveTable.isSortingEnabled() + self.archiveTable.setSortingEnabled(False) + best_unit = find_best_unit_for_sizes((a.size for a in archives), precision=SIZE_DECIMAL_DIGITS) + for row, archive in enumerate(archives): + self.archiveTable.insertRow(row) + + formatted_time = archive.time.strftime('%Y-%m-%d %H:%M') + self.archiveTable.setItem(row, 0, QTableWidgetItem(formatted_time)) + + # format units based on user settings for 'dynamic' or 'fixed' units + fixed_unit = best_unit if SettingsModel.get(key='enable_fixed_units').value else None + size = pretty_bytes(archive.size, fixed_unit=fixed_unit, precision=SIZE_DECIMAL_DIGITS) + self.archiveTable.setItem(row, 1, SizeItem(size)) + + if archive.duration is not None: + formatted_duration = str(timedelta(seconds=round(archive.duration))) + else: + formatted_duration = '' + + self.archiveTable.setItem(row, 2, QTableWidgetItem(formatted_duration)) + + mount_point = self.mount_points.get(archive.name) + if mount_point is not None: + item = QTableWidgetItem(mount_point) + self.archiveTable.setItem(row, 3, item) + + self.archiveTable.setItem(row, 4, QTableWidgetItem(archive.name)) + + if archive.trigger == 'scheduled': + item = QTableWidgetItem(get_colored_icon('clock-o'), '') + item.setToolTip(self.tr('Scheduled')) + self.archiveTable.setItem(row, 5, item) + elif archive.trigger == 'user': + item = QTableWidgetItem(get_colored_icon('user'), '') + item.setToolTip(self.tr('User initiated')) + item.setTextAlignment(Qt.AlignmentFlag.AlignRight) + self.archiveTable.setItem(row, 5, item) + + self.archiveTable.setRowCount(len(archives)) + self.archiveTable.setSortingEnabled(sorting) + item = self.archiveTable.item(0, 0) + self.archiveTable.scrollToItem(item) + + self.archiveTable.selectionModel().clearSelection() + + +def get_mount_points(repo_url): + mount_points = {} + repo_mounts = [] + for proc in psutil.process_iter(): + try: + name = proc.name() + if name == 'borg' or name.startswith('python'): + if 'mount' not in proc.cmdline(): + continue + + if borg_compat.check('V2'): + # command line syntax: + # `borg mount -r (-a )` + cmd = proc.cmdline() + if repo_url in cmd: + i = cmd.index(repo_url) + if len(cmd) > i + 1: + mount_point = cmd[i + 1] + + # Archive mount? + ao = '-a' in cmd + if ao or '--match-archives' in cmd: + i = cmd.index('-a' if ao else '--match-archives') + if len(cmd) >= i + 1 and not SHELL_PATTERN_ELEMENT.search(cmd[i + 1]): + mount_points[mount_point] = cmd[i + 1] + else: + repo_mounts.append(mount_point) + else: + for idx, parameter in enumerate(proc.cmdline()): + if parameter.startswith(repo_url): + # mount from this repo + + # The borg mount command specifies that the mount_point + # parameter comes after the archive name + if len(proc.cmdline()) > idx + 1: + mount_point = proc.cmdline()[idx + 1] + + # archive or full mount? + if parameter[len(repo_url) :].startswith('::'): + archive_name = parameter[len(repo_url) + 2 :] + mount_points[archive_name] = mount_point + break + else: + # repo mount point + repo_mounts.append(mount_point) + + except (psutil.ZombieProcess, psutil.AccessDenied, psutil.NoSuchProcess): + # Getting process details may fail (e.g. zombie process on macOS) + # or because the process is owned by another user. + # Also see https://github.com/giampaolo/psutil/issues/783 + continue + + return mount_points, repo_mounts diff --git a/src/vorta/views/workers/file_path_info_worker.py b/src/vorta/views/workers/file_path_info_worker.py new file mode 100644 index 000000000..30307a26a --- /dev/null +++ b/src/vorta/views/workers/file_path_info_worker.py @@ -0,0 +1,122 @@ + +from datetime import datetime as dt, timedelta +import unicodedata +import os +import sys +import re +import fnmatch +import psutil + +from PyQt6 import QtCore +from PyQt6.QtCore import QFileInfo, QThread, pyqtSignal, Qt +from PyQt6.QtWidgets import (QApplication, QFileDialog, QSystemTrayIcon, + QListWidgetItem, QTableWidgetItem) + + +class FilePathInfoAsync(QThread): + signal = pyqtSignal(str, str, str) + + def __init__(self, path, exclude_patterns_str): + self.path = path + QThread.__init__(self) + self.exiting = False + self.exclude_patterns = [] + for _line in (exclude_patterns_str or '').splitlines(): + line = _line.strip() + if line != '': + self.exclude_patterns.append(line) + + def run(self): + # logger.info("running thread to get path=%s...", self.path) + self.size, self.files_count = get_path_datasize(self.path, self.exclude_patterns) + self.signal.emit(self.path, str(self.size), str(self.files_count)) + + +def normalize_path(path): + """normalize paths for MacOS (but do nothing on other platforms)""" + # HFS+ converts paths to a canonical form, so users shouldn't be required to enter an exact match. + # Windows and Unix filesystems allow different forms, so users always have to enter an exact match. + return unicodedata.normalize('NFD', path) if sys.platform == 'darwin' else path + +def get_path_datasize(path, exclude_patterns): + file_info = QFileInfo(path) + + if file_info.isDir(): + data_size, files_count = get_directory_size(file_info.absoluteFilePath(), exclude_patterns) + else: + data_size = file_info.size() + files_count = 1 + + return data_size, files_count + + +# prepare patterns as borg does +# see `FnmatchPattern._prepare` at +# https://github.com/borgbackup/borg/blob/master//src/borg/patterns.py +def prepare_pattern(pattern): + """Prepare and process fnmatch patterns as borg does""" + if pattern.endswith(os.path.sep): + # trailing sep indicates that the contents should be excluded + # but not the directory it self. + pattern = os.path.normpath(pattern).rstrip(os.path.sep) + pattern += os.path.sep + '*' + os.path.sep + else: + pattern = os.path.normpath(pattern) + os.path.sep + '*' + + pattern = pattern.lstrip(os.path.sep) # sep at beginning is removed + return re.compile(fnmatch.translate(pattern)) + + +def match(pattern: re.Pattern, path: str): + """Check whether a path matches the given pattern.""" + path = path.lstrip(os.path.sep) + os.path.sep + return pattern.match(path) is not None + + +def get_directory_size(dir_path, exclude_patterns): + '''Get number of files only and total size in bytes from a path. + Based off https://stackoverflow.com/a/17936789''' + exclude_patterns = [prepare_pattern(p) for p in exclude_patterns] + + data_size_filtered = 0 + seen = set() + seen_filtered = set() + + for dir_path, subdirectories, file_names in os.walk(dir_path, topdown=True): + is_excluded = False + for pattern in exclude_patterns: + if match(pattern, dir_path): + is_excluded = True + break + + if is_excluded: + subdirectories.clear() # so that os.walk won't walk them + continue + + for file_name in file_names: + file_path = os.path.join(dir_path, file_name) + + # Ignore symbolic links, since borg doesn't follow them + if os.path.islink(file_path): + continue + + is_excluded = False + for pattern in exclude_patterns: + if match(pattern, file_path): + is_excluded = True + break + + try: + stat = os.stat(file_path) + if stat.st_ino not in seen: # Visit each file only once + # this won't add the size of a hardlinked file + seen.add(stat.st_ino) + if not is_excluded: + data_size_filtered += stat.st_size + seen_filtered.add(stat.st_ino) + except (FileNotFoundError, PermissionError): + continue + + files_count_filtered = len(seen_filtered) + + return data_size_filtered, files_count_filtered diff --git a/src/vorta/views/workers/wifi_list_worker.py b/src/vorta/views/workers/wifi_list_worker.py new file mode 100644 index 000000000..8c19f9427 --- /dev/null +++ b/src/vorta/views/workers/wifi_list_worker.py @@ -0,0 +1,59 @@ + +import logging +from PyQt6 import QtCore +from PyQt6.QtCore import QThread, pyqtSignal, Qt +from PyQt6.QtWidgets import QListWidgetItem + +from vorta.utils import get_network_status_monitor +from vorta.store.models import WifiSettingModel + +logger = logging.getLogger(__name__) + +class PopulateWifiAsync(QThread): + def __init__(self, profile, wifiListWidget): + QThread.__init__(self) + self.profile = profile + self.wifiListWidget = wifiListWidget + + def run(self): + self.wifiListWidget.clear() + for wifi in get_sorted_wifis(self.profile): + item = QListWidgetItem() + item.setText(wifi.ssid) + item.setFlags(item.flags() | QtCore.Qt.ItemFlag.ItemIsUserCheckable) + if wifi.allowed: + item.setCheckState(QtCore.Qt.CheckState.Checked) + else: + item.setCheckState(QtCore.Qt.CheckState.Unchecked) + self.wifiListWidget.addItem(item) + + +def get_sorted_wifis(profile): + """ + Get Wifi networks known to the OS (only current one on macOS) and + merge with networks from other profiles. Update last connected time. + """ + + + # Pull networks known to OS and all other backup profiles + system_wifis = get_network_status_monitor().get_known_wifis() + from_other_profiles = WifiSettingModel.select().where(WifiSettingModel.profile != profile.id).execute() + + for wifi in list(from_other_profiles) + system_wifis: + db_wifi, created = WifiSettingModel.get_or_create( + ssid=wifi.ssid, + profile=profile.id, + defaults={'last_connected': wifi.last_connected, 'allowed': True}, + ) + + # Update last connected time + if not created and db_wifi.last_connected != wifi.last_connected: + db_wifi.last_connected = wifi.last_connected + db_wifi.save() + + # Finally return list of networks and settings for that profile + return ( + WifiSettingModel.select() + .where(WifiSettingModel.profile == profile.id) + .order_by(-WifiSettingModel.last_connected) + ) From 504be709347104c0bb08e77b5fc9635acb9ded66 Mon Sep 17 00:00:00 2001 From: Manu Date: Sat, 16 Mar 2024 17:59:44 +0000 Subject: [PATCH 2/6] Avoid changing GUI in QThread --- src/vorta/views/archive_tab.py | 82 +++++++++-- src/vorta/views/main_window.py | 4 +- src/vorta/views/schedule_tab.py | 36 ++--- .../views/workers/archive_table_worker.py | 134 ------------------ .../views/workers/mount_points_worker.py | 73 ++++++++++ src/vorta/views/workers/wifi_list_worker.py | 79 +++++------ 6 files changed, 200 insertions(+), 208 deletions(-) delete mode 100644 src/vorta/views/workers/archive_table_worker.py create mode 100644 src/vorta/views/workers/mount_points_worker.py diff --git a/src/vorta/views/archive_tab.py b/src/vorta/views/archive_tab.py index 6938265e3..696e92dd4 100644 --- a/src/vorta/views/archive_tab.py +++ b/src/vorta/views/archive_tab.py @@ -1,6 +1,7 @@ import logging import sys from typing import Dict, Optional +from datetime import timedelta from PyQt6 import QtCore, uic from PyQt6.QtCore import QItemSelectionModel, QMimeData, QPoint, Qt, pyqtSlot @@ -36,13 +37,15 @@ borg_compat, choose_file_dialog, format_archive_name, - get_asset + get_asset, + pretty_bytes, + find_best_unit_for_sizes ) from vorta.views import diff_result, extract_dialog from vorta.views.diff_result import DiffResultDialog, DiffTree from vorta.views.extract_dialog import ExtractDialog, ExtractTree -from vorta.views.utils import get_colored_icon -from vorta.views.workers.archive_table_worker import PopulateArchiveTableAsync +from vorta.views.utils import get_colored_icon, SizeItem +from vorta.views.workers.mount_points_worker import MountPointsWorker uifile = get_asset('UI/archivetab.ui') ArchiveTabUI, ArchiveTabBase = uic.loadUiType(uifile) @@ -234,23 +237,85 @@ def _toggle_all_buttons(self, enabled=True): # Restore states self.on_selection_change() + def set_mount_points(self, mount_points, repo_mounts): + if len(repo_mounts) == 0: + return + + archives = [s for s in self.profile().repo.archives.select().order_by(ArchiveModel.time.desc())] + + # if no archive's name can be found in self.mount_points, then hide the mount point column + if not any(a.name in mount_points for a in archives): + return + else: + self.archiveTable.showColumn(3) + self.repo_mount_point = repo_mounts[0] + + for row, archive in enumerate(archives): + mount_point = self.mount_points.get(archive.name) + if mount_point is not None: + item = QTableWidgetItem(mount_point) + self.archiveTable.setItem(row, 3, item) + def populate_from_profile(self): """Populate archive list and prune settings from profile.""" + self.archiveTable.blockSignals(True) profile = self.profile() if profile.repo is not None: - if profile.repo.name: repo_name = f"{profile.repo.name} ({profile.repo.url})" else: repo_name = profile.repo.url self.toolBox.setItemText(0, self.tr('Archives for {}').format(repo_name)) - populateArchiveTableWorker = PopulateArchiveTableAsync(profile, self.mount_points, self.archiveTable) + populateArchiveTableWorker = MountPointsWorker(profile.repo.url) self.workers.append(populateArchiveTableWorker) # preserve worker reference + populateArchiveTableWorker.signal.connect(self.set_mount_points) populateArchiveTableWorker.start() + self.archiveTable.hideColumn(3) + + archives = [s for s in profile.repo.archives.select().order_by(ArchiveModel.time.desc())] + + sorting = self.archiveTable.isSortingEnabled() + self.archiveTable.setSortingEnabled(False) + best_unit = find_best_unit_for_sizes((a.size for a in archives), precision=SIZE_DECIMAL_DIGITS) + for row, archive in enumerate(archives): + self.archiveTable.insertRow(row) + + formatted_time = archive.time.strftime('%Y-%m-%d %H:%M') + self.archiveTable.setItem(row, 0, QTableWidgetItem(formatted_time)) - if self.remaining_refresh_archives == 0: - self._toggle_all_buttons(enabled=True) + # format units based on user settings for 'dynamic' or 'fixed' units + fixed_unit = best_unit if SettingsModel.get(key='enable_fixed_units').value else None + size = pretty_bytes(archive.size, fixed_unit=fixed_unit, precision=SIZE_DECIMAL_DIGITS) + self.archiveTable.setItem(row, 1, SizeItem(size)) + + if archive.duration is not None: + formatted_duration = str(timedelta(seconds=round(archive.duration))) + else: + formatted_duration = '' + + self.archiveTable.setItem(row, 2, QTableWidgetItem(formatted_duration)) + self.archiveTable.setItem(row, 4, QTableWidgetItem(archive.name)) + + if archive.trigger == 'scheduled': + item = QTableWidgetItem(get_colored_icon('clock-o'), '') + item.setToolTip(self.tr('Scheduled')) + self.archiveTable.setItem(row, 5, item) + elif archive.trigger == 'user': + item = QTableWidgetItem(get_colored_icon('user'), '') + item.setToolTip(self.tr('User initiated')) + item.setTextAlignment(Qt.AlignmentFlag.AlignRight) + self.archiveTable.setItem(row, 5, item) + + self.archiveTable.setRowCount(len(archives)) + self.archiveTable.setSortingEnabled(sorting) + item = self.archiveTable.item(0, 0) + self.archiveTable.scrollToItem(item) + + self.archiveTable.selectionModel().clearSelection() + + if self.remaining_refresh_archives == 0: + self._toggle_all_buttons(enabled=True) else: self.mount_points = {} self.archiveTable.setRowCount(0) @@ -261,13 +326,14 @@ def populate_from_profile(self): self.prunePrefixTemplate.setText(profile.prune_prefix) # Populate pruning options from database - profile = self.profile() for i in self.prune_intervals: getattr(self, f'prune_{i}').setValue(getattr(profile, f'prune_{i}')) getattr(self, f'prune_{i}').valueChanged.connect(self.save_prune_setting) self.prune_keep_within.setText(profile.prune_keep_within) self.prune_keep_within.editingFinished.connect(self.save_prune_setting) + self.archiveTable.blockSignals(False) + def on_selection_change(self, selected=None, deselected=None): """ React to a change of the selection of the archiveTableView. diff --git a/src/vorta/views/main_window.py b/src/vorta/views/main_window.py index 24b3350c2..ba8466fa8 100644 --- a/src/vorta/views/main_window.py +++ b/src/vorta/views/main_window.py @@ -183,11 +183,11 @@ def profile_selection_changed_action(self, index): logger.info('step 1') self.archiveTab.populate_from_profile() logger.info('step 2') - self.repoTab.populate_from_profile() # 1s + self.repoTab.populate_from_profile() logger.info('step 3') self.sourceTab.populate_from_profile() logger.info('step 4') - self.scheduleTab.populate_from_profile() #1s + self.scheduleTab.populate_from_profile() logger.info('step 5') SettingsModel.update({SettingsModel.str_value: self.current_profile.id}).where( SettingsModel.key == 'previous_profile_id' diff --git a/src/vorta/views/schedule_tab.py b/src/vorta/views/schedule_tab.py index fd0f19bfa..898dc47a6 100644 --- a/src/vorta/views/schedule_tab.py +++ b/src/vorta/views/schedule_tab.py @@ -13,7 +13,7 @@ from vorta.scheduler import ScheduleStatusType from vorta.store.models import BackupProfileMixin, EventLogModel, WifiSettingModel from vorta.utils import get_asset -from vorta.views.workers.wifi_list_worker import PopulateWifiAsync +from vorta.views.workers.wifi_list_worker import WifiListWorker from vorta.views.utils import get_colored_icon uifile = get_asset('UI/scheduletab.ui') @@ -173,12 +173,27 @@ def populate_from_profile(self): else: self.createCmdLineEdit.setEnabled(False) - populateWifiWorker = PopulateWifiAsync(profile, self.wifiListWidget) - self.workers.append(populateWifiWorker) # preserve reference - populateWifiWorker.start() + wifiListWorker = WifiListWorker(profile.id) + self.workers.append(wifiListWorker) # preserve reference + wifiListWorker.signal.connect(self.set_wifi_list) + wifiListWorker.start() + self.populate_logs() self.draw_next_scheduled_backup() + def set_wifi_list(self, wifi_list): + self.wifiListWidget.clear() + for wifi in wifi_list: + item = QListWidgetItem() + item.setText(wifi.ssid) + item.setFlags(item.flags() | QtCore.Qt.ItemFlag.ItemIsUserCheckable) + if wifi.allowed: + item.setCheckState(QtCore.Qt.CheckState.Checked) + else: + item.setCheckState(QtCore.Qt.CheckState.Unchecked) + self.wifiListWidget.addItem(item) + + def draw_next_scheduled_backup(self): status = self.app.scheduler.next_job_for_profile(self.profile().id) if status.type in ( @@ -195,19 +210,6 @@ def draw_next_scheduled_backup(self): self.nextBackupDateTimeLabel.setText(text) self.nextBackupDateTimeLabel.repaint() - # def populate_wifi(self): - # self.wifiListWidget.clear() - # for wifi in get_sorted_wifis(self.profile()): - # item = QListWidgetItem() - # item.setText(wifi.ssid) - # item.setFlags(item.flags() | QtCore.Qt.ItemFlag.ItemIsUserCheckable) - # if wifi.allowed: - # item.setCheckState(QtCore.Qt.CheckState.Checked) - # else: - # item.setCheckState(QtCore.Qt.CheckState.Unchecked) - # self.wifiListWidget.addItem(item) - # self.wifiListWidget.itemChanged.connect(self.save_wifi_item) - def save_wifi_item(self, item): db_item = WifiSettingModel.get(ssid=item.text(), profile=self.profile().id) db_item.allowed = item.checkState() == Qt.CheckState.Checked diff --git a/src/vorta/views/workers/archive_table_worker.py b/src/vorta/views/workers/archive_table_worker.py deleted file mode 100644 index a918f7cf6..000000000 --- a/src/vorta/views/workers/archive_table_worker.py +++ /dev/null @@ -1,134 +0,0 @@ - -from datetime import timedelta - -import psutil -from PyQt6.QtCore import QThread, Qt -from PyQt6.QtWidgets import QTableWidgetItem - -from vorta.store.models import ArchiveModel, SettingsModel -from vorta.views.utils import get_colored_icon, SizeItem -from vorta.utils import borg_compat, pretty_bytes, find_best_unit_for_sizes, SHELL_PATTERN_ELEMENT - -SIZE_DECIMAL_DIGITS = 1 - - -class PopulateArchiveTableAsync(QThread): - def __init__(self, profile, mount_points, archiveTable): - QThread.__init__(self) - self.profile = profile - self.mount_points = mount_points - self.archiveTable = archiveTable - - def run(self): - # get mount points - self.mount_points, repo_mount_points = get_mount_points(self.profile.repo.url) - if repo_mount_points: - self.repo_mount_point = repo_mount_points[0] - - archives = [s for s in self.profile.repo.archives.select().order_by(ArchiveModel.time.desc())] - - # if no archive's name can be found in self.mount_points, then hide the mount point column - if not any(a.name in self.mount_points for a in archives): - self.archiveTable.hideColumn(3) - else: - self.archiveTable.showColumn(3) - - sorting = self.archiveTable.isSortingEnabled() - self.archiveTable.setSortingEnabled(False) - best_unit = find_best_unit_for_sizes((a.size for a in archives), precision=SIZE_DECIMAL_DIGITS) - for row, archive in enumerate(archives): - self.archiveTable.insertRow(row) - - formatted_time = archive.time.strftime('%Y-%m-%d %H:%M') - self.archiveTable.setItem(row, 0, QTableWidgetItem(formatted_time)) - - # format units based on user settings for 'dynamic' or 'fixed' units - fixed_unit = best_unit if SettingsModel.get(key='enable_fixed_units').value else None - size = pretty_bytes(archive.size, fixed_unit=fixed_unit, precision=SIZE_DECIMAL_DIGITS) - self.archiveTable.setItem(row, 1, SizeItem(size)) - - if archive.duration is not None: - formatted_duration = str(timedelta(seconds=round(archive.duration))) - else: - formatted_duration = '' - - self.archiveTable.setItem(row, 2, QTableWidgetItem(formatted_duration)) - - mount_point = self.mount_points.get(archive.name) - if mount_point is not None: - item = QTableWidgetItem(mount_point) - self.archiveTable.setItem(row, 3, item) - - self.archiveTable.setItem(row, 4, QTableWidgetItem(archive.name)) - - if archive.trigger == 'scheduled': - item = QTableWidgetItem(get_colored_icon('clock-o'), '') - item.setToolTip(self.tr('Scheduled')) - self.archiveTable.setItem(row, 5, item) - elif archive.trigger == 'user': - item = QTableWidgetItem(get_colored_icon('user'), '') - item.setToolTip(self.tr('User initiated')) - item.setTextAlignment(Qt.AlignmentFlag.AlignRight) - self.archiveTable.setItem(row, 5, item) - - self.archiveTable.setRowCount(len(archives)) - self.archiveTable.setSortingEnabled(sorting) - item = self.archiveTable.item(0, 0) - self.archiveTable.scrollToItem(item) - - self.archiveTable.selectionModel().clearSelection() - - -def get_mount_points(repo_url): - mount_points = {} - repo_mounts = [] - for proc in psutil.process_iter(): - try: - name = proc.name() - if name == 'borg' or name.startswith('python'): - if 'mount' not in proc.cmdline(): - continue - - if borg_compat.check('V2'): - # command line syntax: - # `borg mount -r (-a )` - cmd = proc.cmdline() - if repo_url in cmd: - i = cmd.index(repo_url) - if len(cmd) > i + 1: - mount_point = cmd[i + 1] - - # Archive mount? - ao = '-a' in cmd - if ao or '--match-archives' in cmd: - i = cmd.index('-a' if ao else '--match-archives') - if len(cmd) >= i + 1 and not SHELL_PATTERN_ELEMENT.search(cmd[i + 1]): - mount_points[mount_point] = cmd[i + 1] - else: - repo_mounts.append(mount_point) - else: - for idx, parameter in enumerate(proc.cmdline()): - if parameter.startswith(repo_url): - # mount from this repo - - # The borg mount command specifies that the mount_point - # parameter comes after the archive name - if len(proc.cmdline()) > idx + 1: - mount_point = proc.cmdline()[idx + 1] - - # archive or full mount? - if parameter[len(repo_url) :].startswith('::'): - archive_name = parameter[len(repo_url) + 2 :] - mount_points[archive_name] = mount_point - break - else: - # repo mount point - repo_mounts.append(mount_point) - - except (psutil.ZombieProcess, psutil.AccessDenied, psutil.NoSuchProcess): - # Getting process details may fail (e.g. zombie process on macOS) - # or because the process is owned by another user. - # Also see https://github.com/giampaolo/psutil/issues/783 - continue - - return mount_points, repo_mounts diff --git a/src/vorta/views/workers/mount_points_worker.py b/src/vorta/views/workers/mount_points_worker.py new file mode 100644 index 000000000..9d88bc371 --- /dev/null +++ b/src/vorta/views/workers/mount_points_worker.py @@ -0,0 +1,73 @@ + + +import psutil +from PyQt6.QtCore import QThread, Qt, pyqtSignal +from PyQt6.QtWidgets import QTableWidgetItem + +from vorta.store.models import ArchiveModel, SettingsModel +from vorta.views.utils import get_colored_icon, SizeItem +from vorta.utils import borg_compat, pretty_bytes, find_best_unit_for_sizes, SHELL_PATTERN_ELEMENT + +SIZE_DECIMAL_DIGITS = 1 + + +class MountPointsWorker(QThread): + signal = pyqtSignal(dict, list) + + def __init__(self, repo_url): + QThread.__init__(self) + self.repo_url = repo_url + + def run(self): + mount_points = {} + repo_mounts = [] + for proc in psutil.process_iter(): + try: + name = proc.name() + if name == 'borg' or name.startswith('python'): + if 'mount' not in proc.cmdline(): + continue + + if borg_compat.check('V2'): + # command line syntax: + # `borg mount -r (-a )` + cmd = proc.cmdline() + if self.repo_url in cmd: + i = cmd.index(self.repo_url) + if len(cmd) > i + 1: + mount_point = cmd[i + 1] + + # Archive mount? + ao = '-a' in cmd + if ao or '--match-archives' in cmd: + i = cmd.index('-a' if ao else '--match-archives') + if len(cmd) >= i + 1 and not SHELL_PATTERN_ELEMENT.search(cmd[i + 1]): + mount_points[mount_point] = cmd[i + 1] + else: + repo_mounts.append(mount_point) + else: + for idx, parameter in enumerate(proc.cmdline()): + if parameter.startswith(self.repo_url): + # mount from this repo + + # The borg mount command specifies that the mount_point + # parameter comes after the archive name + if len(proc.cmdline()) > idx + 1: + mount_point = proc.cmdline()[idx + 1] + + # archive or full mount? + if parameter[len(self.repo_url) :].startswith('::'): + archive_name = parameter[len(self.repo_url) + 2 :] + mount_points[archive_name] = mount_point + break + else: + # repo mount point + repo_mounts.append(mount_point) + + except (psutil.ZombieProcess, psutil.AccessDenied, psutil.NoSuchProcess): + # Getting process details may fail (e.g. zombie process on macOS) + # or because the process is owned by another user. + # Also see https://github.com/giampaolo/psutil/issues/783 + continue + + self.signal.emit(mount_points, repo_mounts) diff --git a/src/vorta/views/workers/wifi_list_worker.py b/src/vorta/views/workers/wifi_list_worker.py index 8c19f9427..b94afbe3e 100644 --- a/src/vorta/views/workers/wifi_list_worker.py +++ b/src/vorta/views/workers/wifi_list_worker.py @@ -1,59 +1,44 @@ import logging -from PyQt6 import QtCore -from PyQt6.QtCore import QThread, pyqtSignal, Qt -from PyQt6.QtWidgets import QListWidgetItem +from PyQt6.QtCore import QThread, pyqtSignal from vorta.utils import get_network_status_monitor from vorta.store.models import WifiSettingModel logger = logging.getLogger(__name__) -class PopulateWifiAsync(QThread): - def __init__(self, profile, wifiListWidget): +class WifiListWorker(QThread): + signal = pyqtSignal(list) + + def __init__(self, profile_id): QThread.__init__(self) - self.profile = profile - self.wifiListWidget = wifiListWidget + self.profile_id = profile_id def run(self): - self.wifiListWidget.clear() - for wifi in get_sorted_wifis(self.profile): - item = QListWidgetItem() - item.setText(wifi.ssid) - item.setFlags(item.flags() | QtCore.Qt.ItemFlag.ItemIsUserCheckable) - if wifi.allowed: - item.setCheckState(QtCore.Qt.CheckState.Checked) - else: - item.setCheckState(QtCore.Qt.CheckState.Unchecked) - self.wifiListWidget.addItem(item) - - -def get_sorted_wifis(profile): - """ - Get Wifi networks known to the OS (only current one on macOS) and - merge with networks from other profiles. Update last connected time. - """ - - - # Pull networks known to OS and all other backup profiles - system_wifis = get_network_status_monitor().get_known_wifis() - from_other_profiles = WifiSettingModel.select().where(WifiSettingModel.profile != profile.id).execute() - - for wifi in list(from_other_profiles) + system_wifis: - db_wifi, created = WifiSettingModel.get_or_create( - ssid=wifi.ssid, - profile=profile.id, - defaults={'last_connected': wifi.last_connected, 'allowed': True}, + """ + Get Wifi networks known to the OS (only current one on macOS) and + merge with networks from other profiles. Update last connected time. + """ + + # Pull networks known to OS and all other backup profiles + system_wifis = get_network_status_monitor().get_known_wifis() + from_other_profiles = WifiSettingModel.select().where(WifiSettingModel.profile != self.profile_id).execute() + + for wifi in list(from_other_profiles) + system_wifis: + db_wifi, created = WifiSettingModel.get_or_create( + ssid=wifi.ssid, + profile=self.profile_id, + defaults={'last_connected': wifi.last_connected, 'allowed': True}, + ) + + # Update last connected time + if not created and db_wifi.last_connected != wifi.last_connected: + db_wifi.last_connected = wifi.last_connected + db_wifi.save() + + # Finally return list of networks and settings for that profile + self.signal.emit( + WifiSettingModel.select() + .where(WifiSettingModel.profile == self.profile_id) + .order_by(-WifiSettingModel.last_connected) ) - - # Update last connected time - if not created and db_wifi.last_connected != wifi.last_connected: - db_wifi.last_connected = wifi.last_connected - db_wifi.save() - - # Finally return list of networks and settings for that profile - return ( - WifiSettingModel.select() - .where(WifiSettingModel.profile == profile.id) - .order_by(-WifiSettingModel.last_connected) - ) From 7bbf372c129d306a61e9892fc9b47bb333583afc Mon Sep 17 00:00:00 2001 From: Manu Date: Sat, 16 Mar 2024 18:16:32 +0000 Subject: [PATCH 3/6] Lint --- requirements.d/Brewfile | 1 + src/vorta/store/models.py | 2 +- src/vorta/store/utils.py | 3 ++- src/vorta/utils.py | 19 +++++++++---------- src/vorta/views/archive_tab.py | 6 +++--- src/vorta/views/schedule_tab.py | 3 +-- src/vorta/views/source_tab.py | 10 +++------- src/vorta/views/utils.py | 2 +- .../views/workers/file_path_info_worker.py | 15 +++++---------- .../views/workers/mount_points_worker.py | 12 +++++------- src/vorta/views/workers/wifi_list_worker.py | 5 +++-- 11 files changed, 34 insertions(+), 44 deletions(-) diff --git a/requirements.d/Brewfile b/requirements.d/Brewfile index a3f4b08bc..6dbea82b0 100644 --- a/requirements.d/Brewfile +++ b/requirements.d/Brewfile @@ -4,6 +4,7 @@ brew 'create-dmg' brew 'qt' brew 'hub' +brew 'pre-commit' brew 'xmlstarlet' cask 'qt-creator' cask 'sparkle' diff --git a/src/vorta/store/models.py b/src/vorta/store/models.py index a5ab0fc0c..d809814b4 100644 --- a/src/vorta/store/models.py +++ b/src/vorta/store/models.py @@ -13,7 +13,7 @@ from playhouse import signals from vorta.store.utils import slugify -# from vorta.views.utils import get_exclusion_presets +from vorta.views.utils import get_exclusion_presets DB = pw.Proxy() logger = logging.getLogger(__name__) diff --git a/src/vorta/store/utils.py b/src/vorta/store/utils.py index f810978f8..fc013309f 100644 --- a/src/vorta/store/utils.py +++ b/src/vorta/store/utils.py @@ -1,5 +1,6 @@ -import unicodedata import re +import unicodedata + def slugify(value): """ diff --git a/src/vorta/utils.py b/src/vorta/utils.py index f63ff966d..9ea88c259 100644 --- a/src/vorta/utils.py +++ b/src/vorta/utils.py @@ -1,24 +1,25 @@ import argparse import errno -import math import getpass +import math import os import re import socket import sys -from datetime import datetime as dt, timedelta +from datetime import datetime as dt from functools import reduce from typing import Any, Callable, Iterable, List, Optional, Tuple, TypeVar from PyQt6 import QtCore -from PyQt6.QtCore import QFileInfo, QThread, pyqtSignal, Qt -from PyQt6.QtWidgets import (QApplication, QFileDialog, QSystemTrayIcon, - QListWidgetItem, QTableWidgetItem) +from PyQt6.QtWidgets import ( + QApplication, + QFileDialog, + QSystemTrayIcon, +) -from vorta.network_status.abc import NetworkStatusMonitor from vorta.borg._compatibility import BorgCompatibility from vorta.log import logger - +from vorta.network_status.abc import NetworkStatusMonitor # Used to store whether a user wanted to override the # default directory for the --development flag @@ -98,7 +99,6 @@ def get_private_keys() -> List[str]: return available_private_keys - Number = TypeVar("Number", int, float) @@ -106,6 +106,7 @@ def clamp(n: Number, min_: Number, max_: Number) -> Number: """Restrict the number n inside a range""" return min(max_, max(n, min_)) + def get_network_status_monitor(): global _network_status_monitor if _network_status_monitor is None: @@ -175,7 +176,6 @@ def get_asset(path): return os.path.join(bundle_dir, path) - def parse_args(): parser = argparse.ArgumentParser(description='Vorta Backup GUI for Borg.') parser.add_argument('--version', '-V', action='store_true', help="Show version and exit.") @@ -259,7 +259,6 @@ def format_archive_name(profile, archive_name_tpl): SHELL_PATTERN_ELEMENT = re.compile(r'([?\[\]*])') - def is_system_tray_available(): app = QApplication.instance() if app is None: diff --git a/src/vorta/views/archive_tab.py b/src/vorta/views/archive_tab.py index 696e92dd4..9408b3cba 100644 --- a/src/vorta/views/archive_tab.py +++ b/src/vorta/views/archive_tab.py @@ -1,7 +1,7 @@ import logging import sys -from typing import Dict, Optional from datetime import timedelta +from typing import Dict, Optional from PyQt6 import QtCore, uic from PyQt6.QtCore import QItemSelectionModel, QMimeData, QPoint, Qt, pyqtSlot @@ -36,15 +36,15 @@ from vorta.utils import ( borg_compat, choose_file_dialog, + find_best_unit_for_sizes, format_archive_name, get_asset, pretty_bytes, - find_best_unit_for_sizes ) from vorta.views import diff_result, extract_dialog from vorta.views.diff_result import DiffResultDialog, DiffTree from vorta.views.extract_dialog import ExtractDialog, ExtractTree -from vorta.views.utils import get_colored_icon, SizeItem +from vorta.views.utils import SizeItem, get_colored_icon from vorta.views.workers.mount_points_worker import MountPointsWorker uifile = get_asset('UI/archivetab.ui') diff --git a/src/vorta/views/schedule_tab.py b/src/vorta/views/schedule_tab.py index 898dc47a6..f5c3b60b5 100644 --- a/src/vorta/views/schedule_tab.py +++ b/src/vorta/views/schedule_tab.py @@ -13,8 +13,8 @@ from vorta.scheduler import ScheduleStatusType from vorta.store.models import BackupProfileMixin, EventLogModel, WifiSettingModel from vorta.utils import get_asset -from vorta.views.workers.wifi_list_worker import WifiListWorker from vorta.views.utils import get_colored_icon +from vorta.views.workers.wifi_list_worker import WifiListWorker uifile = get_asset('UI/scheduletab.ui') ScheduleUI, ScheduleBase = uic.loadUiType(uifile) @@ -193,7 +193,6 @@ def set_wifi_list(self, wifi_list): item.setCheckState(QtCore.Qt.CheckState.Unchecked) self.wifiListWidget.addItem(item) - def draw_next_scheduled_backup(self): status = self.app.scheduler.next_job_for_profile(self.profile().id) if status.type in ( diff --git a/src/vorta/views/source_tab.py b/src/vorta/views/source_tab.py index e4a686f62..83da6a404 100644 --- a/src/vorta/views/source_tab.py +++ b/src/vorta/views/source_tab.py @@ -14,14 +14,10 @@ ) from vorta.store.models import BackupProfileMixin, SettingsModel, SourceFileModel -from vorta.views.workers.file_path_info_worker import FilePathInfoAsync -from vorta.utils import ( - choose_file_dialog, - get_asset, - pretty_bytes -) +from vorta.utils import choose_file_dialog, get_asset, pretty_bytes from vorta.views.exclude_dialog import ExcludeDialog -from vorta.views.utils import get_colored_icon, SizeItem +from vorta.views.utils import SizeItem, get_colored_icon +from vorta.views.workers.file_path_info_worker import FilePathInfoAsync uifile = get_asset('UI/sourcetab.ui') SourceUI, SourceBase = uic.loadUiType(uifile) diff --git a/src/vorta/views/utils.py b/src/vorta/views/utils.py index 9a3c24cb9..a4be4f76a 100644 --- a/src/vorta/views/utils.py +++ b/src/vorta/views/utils.py @@ -2,8 +2,8 @@ import os import sys -from PyQt6.QtGui import QIcon, QImage, QPixmap from PyQt6.QtCore import Qt +from PyQt6.QtGui import QIcon, QImage, QPixmap from PyQt6.QtWidgets import QTableWidgetItem from vorta.utils import get_asset, uses_dark_mode diff --git a/src/vorta/views/workers/file_path_info_worker.py b/src/vorta/views/workers/file_path_info_worker.py index 30307a26a..a4950252a 100644 --- a/src/vorta/views/workers/file_path_info_worker.py +++ b/src/vorta/views/workers/file_path_info_worker.py @@ -1,16 +1,10 @@ - -from datetime import datetime as dt, timedelta -import unicodedata +import fnmatch import os -import sys import re -import fnmatch -import psutil +import sys +import unicodedata -from PyQt6 import QtCore -from PyQt6.QtCore import QFileInfo, QThread, pyqtSignal, Qt -from PyQt6.QtWidgets import (QApplication, QFileDialog, QSystemTrayIcon, - QListWidgetItem, QTableWidgetItem) +from PyQt6.QtCore import QFileInfo, QThread, pyqtSignal class FilePathInfoAsync(QThread): @@ -38,6 +32,7 @@ def normalize_path(path): # Windows and Unix filesystems allow different forms, so users always have to enter an exact match. return unicodedata.normalize('NFD', path) if sys.platform == 'darwin' else path + def get_path_datasize(path, exclude_patterns): file_info = QFileInfo(path) diff --git a/src/vorta/views/workers/mount_points_worker.py b/src/vorta/views/workers/mount_points_worker.py index 9d88bc371..9836ae8df 100644 --- a/src/vorta/views/workers/mount_points_worker.py +++ b/src/vorta/views/workers/mount_points_worker.py @@ -1,12 +1,10 @@ - - import psutil -from PyQt6.QtCore import QThread, Qt, pyqtSignal -from PyQt6.QtWidgets import QTableWidgetItem +from PyQt6.QtCore import QThread, pyqtSignal -from vorta.store.models import ArchiveModel, SettingsModel -from vorta.views.utils import get_colored_icon, SizeItem -from vorta.utils import borg_compat, pretty_bytes, find_best_unit_for_sizes, SHELL_PATTERN_ELEMENT +from vorta.utils import ( + SHELL_PATTERN_ELEMENT, + borg_compat, +) SIZE_DECIMAL_DIGITS = 1 diff --git a/src/vorta/views/workers/wifi_list_worker.py b/src/vorta/views/workers/wifi_list_worker.py index b94afbe3e..cfcb6b3bc 100644 --- a/src/vorta/views/workers/wifi_list_worker.py +++ b/src/vorta/views/workers/wifi_list_worker.py @@ -1,12 +1,13 @@ - import logging + from PyQt6.QtCore import QThread, pyqtSignal -from vorta.utils import get_network_status_monitor from vorta.store.models import WifiSettingModel +from vorta.utils import get_network_status_monitor logger = logging.getLogger(__name__) + class WifiListWorker(QThread): signal = pyqtSignal(list) From 6d6351a5dcc47a1b1d43761ccde4c59feb264343 Mon Sep 17 00:00:00 2001 From: Manu Date: Sat, 16 Mar 2024 18:30:13 +0000 Subject: [PATCH 4/6] Lint 2 --- setup.cfg | 2 +- src/vorta/views/workers/mount_points_worker.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/setup.cfg b/setup.cfg index e0cdd9319..745536cf6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -105,7 +105,7 @@ extension-pkg-whitelist=PyQt6 load-plugins= [pylint.messages control] -disable= W0503,W0511,C0301,R0903,R0201,W0212,C0114,C0115,C0116,C0103,E0611,E1120,C0415,R0914,R0912,R0915 +disable= W0511,C0301,R0903,W0212,C0114,C0115,C0116,C0103,E0611,E1120,C0415,R0914,R0912,R0915 [pylint.format] max-line-length=120 diff --git a/src/vorta/views/workers/mount_points_worker.py b/src/vorta/views/workers/mount_points_worker.py index 9836ae8df..2df4f6c92 100644 --- a/src/vorta/views/workers/mount_points_worker.py +++ b/src/vorta/views/workers/mount_points_worker.py @@ -1,15 +1,13 @@ import psutil from PyQt6.QtCore import QThread, pyqtSignal -from vorta.utils import ( - SHELL_PATTERN_ELEMENT, - borg_compat, -) +from vorta.utils import SHELL_PATTERN_ELEMENT, borg_compat SIZE_DECIMAL_DIGITS = 1 class MountPointsWorker(QThread): + signal = pyqtSignal(dict, list) def __init__(self, repo_url): From 8c024cfbcbcd409367e8093a2b5818aa6d1cd176 Mon Sep 17 00:00:00 2001 From: Manu Date: Mon, 18 Mar 2024 15:07:27 +0000 Subject: [PATCH 5/6] Start fixing tests --- src/vorta/views/archive_tab.py | 12 ++++---- src/vorta/views/main_window.py | 3 +- tests/unit/conftest.py | 6 ++++ tests/unit/test_utils.py | 52 +-------------------------------- tests/unit/test_workers.py | 53 ++++++++++++++++++++++++++++++++++ 5 files changed, 67 insertions(+), 59 deletions(-) create mode 100644 tests/unit/test_workers.py diff --git a/src/vorta/views/archive_tab.py b/src/vorta/views/archive_tab.py index 9408b3cba..5b43c58a0 100644 --- a/src/vorta/views/archive_tab.py +++ b/src/vorta/views/archive_tab.py @@ -273,7 +273,7 @@ def populate_from_profile(self): populateArchiveTableWorker.start() self.archiveTable.hideColumn(3) - archives = [s for s in profile.repo.archives.select().order_by(ArchiveModel.time.desc())] + archives = list(profile.repo.archives.select().order_by(ArchiveModel.time.desc())) sorting = self.archiveTable.isSortingEnabled() self.archiveTable.setSortingEnabled(False) @@ -307,12 +307,12 @@ def populate_from_profile(self): item.setTextAlignment(Qt.AlignmentFlag.AlignRight) self.archiveTable.setItem(row, 5, item) - self.archiveTable.setRowCount(len(archives)) - self.archiveTable.setSortingEnabled(sorting) - item = self.archiveTable.item(0, 0) - self.archiveTable.scrollToItem(item) + self.archiveTable.setRowCount(len(archives)) + self.archiveTable.setSortingEnabled(sorting) + item = self.archiveTable.item(0, 0) + self.archiveTable.scrollToItem(item) - self.archiveTable.selectionModel().clearSelection() + self.archiveTable.selectionModel().clearSelection() if self.remaining_refresh_archives == 0: self._toggle_all_buttons(enabled=True) diff --git a/src/vorta/views/main_window.py b/src/vorta/views/main_window.py index ba8466fa8..cf70fc9bd 100644 --- a/src/vorta/views/main_window.py +++ b/src/vorta/views/main_window.py @@ -263,8 +263,7 @@ def profile_imported_event(profile): self.tr('Profile {} imported.').format(profile.name), ) self.repoTab.populate_from_profile() - self.scheduleTab.populate_logs() - self.scheduleTab.populate_wifi() + self.scheduleTab.populate_from_profile() self.miscTab.populate() self.populate_profile_selector() diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index e622a2118..ad64fffc5 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -74,6 +74,12 @@ def init_db(qapp, qtbot, tmpdir_factory): qapp.backup_finished_event.disconnect() qapp.scheduler.schedule_changed.disconnect() qtbot.waitUntil(lambda: not qapp.jobs_manager.is_worker_running(), **pytest._wait_defaults) + for worker in qapp.main_window.archiveTab.workers: + worker.quit() + worker.exit() + for worker in qapp.main_window.scheduleTab.workers: + worker.quit() + worker.exit() mock_db.close() diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index ea529c089..3180a859e 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1,16 +1,13 @@ -import sys import uuid import pytest from vorta.keyring.abc import VortaKeyring from vorta.utils import ( find_best_unit_for_sizes, - get_path_datasize, is_system_tray_available, - normalize_path, pretty_bytes, - sort_sizes, ) +from vorta.views.utils import sort_sizes def test_keyring(): @@ -110,53 +107,6 @@ def test_pretty_bytes_nonfixed_units(size, metric, expected_output): assert output == expected_output -def test_normalize_path(): - """ - Test that path is normalized for macOS, but does nothing for other platforms. - """ - input_path = '/Users/username/caf\u00e9/file.txt' - expected_output = '/Users/username/café/file.txt' - - actual_output = normalize_path(input_path) - - if sys.platform == 'darwin': - assert actual_output == expected_output - else: - assert actual_output == input_path - - -def test_get_path_datasize(tmpdir): - """ - Test that get_path_datasize() works correctly when passed excluded patterns. - """ - # Create a temporary directory for testing - test_dir = tmpdir.mkdir("test_dir") - test_file = test_dir.join("test_file.txt") - test_file.write("Hello, World!") - - # Create a subdirectory with a file to exclude - excluded_dir = test_dir.mkdir("excluded_dir") - excluded_file = excluded_dir.join("excluded_file.txt") - excluded_file.write("Excluded file, should not be checked.") - - exclude_patterns = [f"{excluded_dir}"] - - # Test when the path is a directory - data_size, files_count = get_path_datasize(str(test_dir), exclude_patterns) - assert data_size == len("Hello, World!") - assert files_count == 1 - - # Test when the path is a file - data_size, files_count = get_path_datasize(str(test_file), exclude_patterns) - assert data_size == len("Hello, World!") - assert files_count == 1 - - # Test when the path is a directory with an excluded file - data_size, files_count = get_path_datasize(str(excluded_dir), exclude_patterns) - assert data_size == 0 - assert files_count == 0 - - def test_is_system_tray_available(mocker): """ Sanity check to ensure proper behavior diff --git a/tests/unit/test_workers.py b/tests/unit/test_workers.py new file mode 100644 index 000000000..121d5e26f --- /dev/null +++ b/tests/unit/test_workers.py @@ -0,0 +1,53 @@ +import sys + +from vorta.views.workers.file_path_info_worker import ( + get_path_datasize, + normalize_path, +) + + +def test_normalize_path(): + """ + Test that path is normalized for macOS, but does nothing for other platforms. + """ + input_path = '/Users/username/caf\u00e9/file.txt' + expected_output = '/Users/username/café/file.txt' + + actual_output = normalize_path(input_path) + + if sys.platform == 'darwin': + assert actual_output == expected_output + else: + assert actual_output == input_path + + +def test_get_path_datasize(tmpdir): + """ + Test that get_path_datasize() works correctly when passed excluded patterns. + """ + # Create a temporary directory for testing + test_dir = tmpdir.mkdir("test_dir") + test_file = test_dir.join("test_file.txt") + test_file.write("Hello, World!") + + # Create a subdirectory with a file to exclude + excluded_dir = test_dir.mkdir("excluded_dir") + excluded_file = excluded_dir.join("excluded_file.txt") + excluded_file.write("Excluded file, should not be checked.") + + exclude_patterns = [f"{excluded_dir}"] + + # Test when the path is a directory + data_size, files_count = get_path_datasize(str(test_dir), exclude_patterns) + assert data_size == len("Hello, World!") + assert files_count == 1 + + # Test when the path is a file + data_size, files_count = get_path_datasize(str(test_file), exclude_patterns) + assert data_size == len("Hello, World!") + assert files_count == 1 + + # Test when the path is a directory with an excluded file + data_size, files_count = get_path_datasize(str(excluded_dir), exclude_patterns) + assert data_size == 0 + assert files_count == 0 From 7d96a31274a2e04d42973da6302540cd0fb09da0 Mon Sep 17 00:00:00 2001 From: Manu Date: Mon, 18 Mar 2024 16:15:49 +0000 Subject: [PATCH 6/6] Wait for Qthreads to quit after tests --- src/vorta/views/main_window.py | 5 ----- tests/unit/conftest.py | 10 ++++------ 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/src/vorta/views/main_window.py b/src/vorta/views/main_window.py index cf70fc9bd..6a985be3a 100644 --- a/src/vorta/views/main_window.py +++ b/src/vorta/views/main_window.py @@ -180,15 +180,10 @@ def profile_selection_changed_action(self, index): if not backup_profile_id: return self.current_profile = BackupProfileModel.get(id=backup_profile_id) - logger.info('step 1') self.archiveTab.populate_from_profile() - logger.info('step 2') self.repoTab.populate_from_profile() - logger.info('step 3') self.sourceTab.populate_from_profile() - logger.info('step 4') self.scheduleTab.populate_from_profile() - logger.info('step 5') SettingsModel.update({SettingsModel.str_value: self.current_profile.id}).where( SettingsModel.key == 'previous_profile_id' ).execute() diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index ad64fffc5..39f2af378 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -72,14 +72,12 @@ def init_db(qapp, qtbot, tmpdir_factory): qapp.jobs_manager.cancel_all_jobs() qapp.backup_finished_event.disconnect() - qapp.scheduler.schedule_changed.disconnect() - qtbot.waitUntil(lambda: not qapp.jobs_manager.is_worker_running(), **pytest._wait_defaults) - for worker in qapp.main_window.archiveTab.workers: - worker.quit() - worker.exit() - for worker in qapp.main_window.scheduleTab.workers: + for worker in qapp.main_window.archiveTab.workers + qapp.main_window.scheduleTab.workers: worker.quit() + worker.wait() worker.exit() + qtbot.waitUntil(lambda: not qapp.jobs_manager.is_worker_running(), **pytest._wait_defaults) + qapp.scheduler.schedule_changed.disconnect() mock_db.close()