Skip to content

Commit

Permalink
Refactor BorgThread into 2 separate classes.
Browse files Browse the repository at this point in the history
  • Loading branch information
m3nu committed Nov 2, 2018
1 parent ae7f52d commit 20252cf
Show file tree
Hide file tree
Showing 11 changed files with 341 additions and 312 deletions.
6 changes: 3 additions & 3 deletions src/vorta/application.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from .tray_menu import TrayMenu
from .scheduler import VortaScheduler
from .models import BackupProfileModel, SnapshotModel, BackupProfileMixin
from .borg_runner import BorgThread
from .borg_create import BorgCreateThread
from .views.main_window import MainWindow


Expand Down Expand Up @@ -34,9 +34,9 @@ def __init__(self, args):
self.main_window.show()

def create_backup_action(self):
msg = BorgThread.prepare_create_cmd()
msg = BorgCreateThread.prepare()
if msg['ok']:
self.thread = BorgThread(msg['cmd'], msg['params'], parent=self)
self.thread = BorgCreateThread(msg['cmd'], msg['params'], parent=self)
self.thread.start()
else:
self.backup_log_event.emit(msg['message'])
Expand Down
2 changes: 1 addition & 1 deletion src/vorta/assets/UI/repotab.ui
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@
<item row="3" column="1">
<widget class="QLabel" name="label_5">
<property name="text">
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;SSH account with Borg installed server-side. Also try &lt;a href=&quot;https://www.borgbase.com?utm_source=vorta&quot;&gt;&lt;span style=&quot; text-decoration: underline; color:#0000ff;&quot;&gt;BorgBase&lt;/span&gt;&lt;/a&gt;. 100GB free during Beta.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;SSH account with Borg installed server-side. Or try &lt;a href=&quot;https://www.borgbase.com?utm_source=vorta&quot;&gt;&lt;span style=&quot; text-decoration: underline; color:#0000ff;&quot;&gt;BorgBase&lt;/span&gt;&lt;/a&gt;. 100GB free during Beta.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
<property name="openExternalLinks">
<bool>true</bool>
Expand Down
119 changes: 15 additions & 104 deletions src/vorta/borg_runner.py → src/vorta/borg_create.py
Original file line number Diff line number Diff line change
@@ -1,113 +1,16 @@
import json
import os
import sys
import shutil
import tempfile
import platform
from dateutil import parser
from datetime import datetime as dt
from PyQt5 import QtCore
from PyQt5.QtWidgets import QApplication
from subprocess import Popen, PIPE

from .models import SourceDirModel, BackupProfileModel, EventLogModel, WifiSettingModel, SnapshotModel, BackupProfileMixin
from .models import SourceDirModel, BackupProfileModel, WifiSettingModel, SnapshotModel, BackupProfileMixin
from .utils import get_current_wifi, keyring
from .borg_thread import BorgThread


class BorgThread(QtCore.QThread, BackupProfileMixin):
mutex = QtCore.QMutex()

def __init__(self, cmd, params, parent=None):
"""
Thread to run Borg operations in. It will connect to the main app instance and
emit events via it.
Functions, like `prepare_create_cmd` and `process_create_result` are structured
around Borg subcommands. They may move to their own class in the future.
:param cmd: Borg command line
:param params: To pass extra options that are later formatted centrally.
:param parent: Parent window. Needs `thread.wait()` if none. (scheduler)
"""
super().__init__(parent)
self.app = QApplication.instance()
self.app.backup_cancelled_event.connect(self.cancel)

# Find packaged borg binary. Prefer globally installed.
if not shutil.which('borg'):
meipass_borg = os.path.join(sys._MEIPASS, 'bin', 'borg')
if os.path.isfile(meipass_borg):
cmd[0] = meipass_borg
self.cmd = cmd

env = os.environ.copy()
env['BORG_HOSTNAME_IS_UNIQUE'] = '1'
if params.get('password') and params['password'] is not None:
env['BORG_PASSPHRASE'] = params['password']

env['BORG_RSH'] = 'ssh -oStrictHostKeyChecking=no'
if params.get('ssh_key') and params['ssh_key']:
env['BORG_RSH'] += f' -i ~/.ssh/{params["ssh_key"]}'

self.env = env
self.params = params
self.process = None

@classmethod
def is_running(cls):
if cls.mutex.tryLock():
cls.mutex.unlock()
return False
else:
return True

def run(self):
self.app.backup_started_event.emit()
self.app.backup_log_event.emit('Backup started.')
self.mutex.lock()
log_entry = EventLogModel(category='borg-run', subcommand=self.cmd[1])
log_entry.save()
self.process = Popen(self.cmd, stdout=PIPE, stderr=PIPE, bufsize=1, universal_newlines=True, env=self.env)
for line in iter(self.process.stderr.readline, ''):
try:
parsed = json.loads(line)
if parsed['type'] == 'log_message':
self.app.backup_log_event.emit(f'{parsed["levelname"]}: {parsed["message"]}')
elif parsed['type'] == 'file_status':
self.app.backup_log_event.emit(f'{parsed["path"]} ({parsed["status"]})')
except json.decoder.JSONDecodeError:
self.app.backup_log_event.emit(line.strip())

self.process.wait()
stdout = self.process.stdout.read()
result = {
'params': self.params,
'returncode': self.process.returncode,
'cmd': self.cmd
}
try:
result['data'] = json.loads(stdout)
except:
result['data'] = {}

log_entry.returncode = self.process.returncode
log_entry.save()

# If result function is available for subcommand, run it.
result_func = f'process_{self.cmd[1]}_result'
if hasattr(self, result_func):
getattr(self, result_func)(result)

self.app.backup_finished_event.emit(result)
self.mutex.unlock()

def cancel(self):
if self.isRunning():
self.mutex.unlock()
self.process.kill()
self.terminate()

def process_create_result(self, result):
class BorgCreateThread(BorgThread, BackupProfileMixin):
def process_result(self, result):
if result['returncode'] == 0:
new_snapshot, created = SnapshotModel.get_or_create(
snapshot_id=result['data']['archive']['id'],
Expand All @@ -129,8 +32,18 @@ def process_create_result(self, result):
repo.total_unique_chunks = stats['total_unique_chunks']
repo.save()

def log_event(self, msg):
self.app.backup_log_event.emit(msg)

def started_event(self):
self.app.backup_started_event.emit()
self.app.backup_log_event.emit('Backup started.')

def finished_event(self, result):
self.app.backup_finished_event.emit(result)

@classmethod
def prepare_create_cmd(cls):
def prepare(cls):
"""
`borg create` is called from different places and needs some preparation.
Centralize it here and return the required arguments to the caller.
Expand Down Expand Up @@ -198,5 +111,3 @@ def prepare_create_cmd(cls):
ret['params'] = params

return ret


119 changes: 119 additions & 0 deletions src/vorta/borg_thread.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
import json
import os
import sys
import shutil
import signal
from PyQt5 import QtCore
from PyQt5.QtWidgets import QApplication
from subprocess import Popen, PIPE

from .models import EventLogModel

mutex = QtCore.QMutex()


class BorgThread(QtCore.QThread):
"""
Base class to run `borg` command line jobs. If a command needs more pre- or past-processing
it should sublass `BorgThread`.
"""

updated = QtCore.pyqtSignal(str)
result = QtCore.pyqtSignal(dict)

def __init__(self, cmd, params, parent=None):
"""
Thread to run Borg operations in.
:param cmd: Borg command line
:param params: To pass extra options that are later formatted centrally.
:param parent: Parent window. Needs `thread.wait()` if none. (scheduler)
"""

super().__init__(parent)
self.app = QApplication.instance()
self.app.backup_cancelled_event.connect(self.cancel)

# Find packaged borg binary. Prefer globally installed.
if not shutil.which('borg'):
meipass_borg = os.path.join(sys._MEIPASS, 'bin', 'borg')
if os.path.isfile(meipass_borg):
cmd[0] = meipass_borg
self.cmd = cmd

env = os.environ.copy()
env['BORG_HOSTNAME_IS_UNIQUE'] = '1'
if params.get('password') and params['password'] is not None:
env['BORG_PASSPHRASE'] = params['password']

env['BORG_RSH'] = 'ssh -oStrictHostKeyChecking=no'
if params.get('ssh_key') and params['ssh_key']:
env['BORG_RSH'] += f' -i ~/.ssh/{params["ssh_key"]}'

self.env = env
self.params = params
self.process = None

@classmethod
def is_running(cls):
if mutex.tryLock():
mutex.unlock()
return False
else:
return True

def run(self):
self.started_event()
mutex.lock()
log_entry = EventLogModel(category='borg-run', subcommand=self.cmd[1])
log_entry.save()

self.process = Popen(self.cmd, stdout=PIPE, stderr=PIPE, bufsize=1,
universal_newlines=True, env=self.env, preexec_fn=os.setsid)

for line in iter(self.process.stderr.readline, ''):
try:
parsed = json.loads(line)
if parsed['type'] == 'log_message':
self.log_event(f'{parsed["levelname"]}: {parsed["message"]}')
elif parsed['type'] == 'file_status':
self.log_event(f'{parsed["path"]} ({parsed["status"]})')
except json.decoder.JSONDecodeError:
self.log_event(line.strip())

self.process.wait()
stdout = self.process.stdout.read()
result = {
'params': self.params,
'returncode': self.process.returncode,
'cmd': self.cmd
}
try:
result['data'] = json.loads(stdout)
except:
result['data'] = {}

log_entry.returncode = self.process.returncode
log_entry.save()

self.process_result(result)
self.finished_event(result)
mutex.unlock()

def cancel(self):
if self.isRunning():
mutex.unlock()
os.killpg(os.getpgid(self.process.pid), signal.SIGTERM)
self.terminate()

def process_result(self, result):
pass

def log_event(self, msg):
self.updated.emit(msg)

def started_event(self):
self.updated.emit('Task started')

def finished_event(self, result):
self.result.emit(result)
6 changes: 3 additions & 3 deletions src/vorta/scheduler.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from apscheduler.schedulers.qt import QtScheduler
from apscheduler.triggers import cron

from .borg_runner import BorgThread
from .borg_create import BorgCreateThread
from .models import BackupProfileMixin


Expand Down Expand Up @@ -35,8 +35,8 @@ def next_job(self):

@classmethod
def create_backup(cls):
msg = BorgThread.prepare_create_cmd()
msg = BorgCreateThread.prepare()
if msg['ok']:
thread = BorgThread(msg['cmd'], msg['params'])
thread = BorgCreateThread(msg['cmd'], msg['params'])
thread.start()
thread.wait()
2 changes: 1 addition & 1 deletion src/vorta/tray_menu.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from PyQt5.QtGui import QIcon

from .utils import get_asset
from .borg_runner import BorgThread
from .borg_thread import BorgThread


class TrayMenu(QSystemTrayIcon):
Expand Down
Loading

0 comments on commit 20252cf

Please sign in to comment.