diff --git a/.gitignore b/.gitignore index b2b6d48..c5b8e85 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ __pycache__/ build/ +dist/ diff --git a/asInvoker.manifest b/asInvoker.manifest new file mode 100644 index 0000000..4b9797e --- /dev/null +++ b/asInvoker.manifest @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/build.spec b/build.spec new file mode 100644 index 0000000..e1339c7 --- /dev/null +++ b/build.spec @@ -0,0 +1,62 @@ +# -*- mode: python ; coding: utf-8 -*- + +block_cipher = None + + +a = Analysis(['update.py'], + binaries=[], + datas=[], + hiddenimports=[], + hookspath=[], + runtime_hooks=[], + excludes=['matplotlib', 'numpy'], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False) +pyz = PYZ(a.pure, a.zipped_data, + cipher=block_cipher) +exe = EXE(pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + [], + name='FlashpointUpdater', + debug=False, + manifest='asInvoker.manifest', + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=True , icon='icon.ico') + +b = Analysis(['update_gui.py'], + binaries=[], + datas=[('icon.png', '.')], + hiddenimports=[], + hookspath=[], + runtime_hooks=[], + excludes=['matplotlib', 'numpy'], + win_no_prefer_redirects=False, + win_private_assemblies=False, + cipher=block_cipher, + noarchive=False) +pyz = PYZ(b.pure, b.zipped_data, + cipher=block_cipher) +exe = EXE(pyz, + b.scripts, + b.binaries, + b.zipfiles, + b.datas, + [], + name='FlashpointUpdaterQt', + debug=False, + manifest='asInvoker.manifest', + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=False , icon='icon.ico') diff --git a/config.json b/config.json index 4ac3aa3..efbe299 100644 --- a/config.json +++ b/config.json @@ -1,4 +1,4 @@ { - "index_endpoint": "https://unstable.life/fp-index", - "file_endpoint": "https://unstable.life/fp" + "index_endpoint": "https://unstable.life/fp-index/", + "file_endpoint": "https://unstable.life/fp/" } diff --git a/core.py b/core.py new file mode 100644 index 0000000..144eb76 --- /dev/null +++ b/core.py @@ -0,0 +1,145 @@ +from collections import namedtuple +from urllib.parse import urljoin +import threading +import requests +import datetime +import logging +import queue +import time +import lzma +import json +import math + +# Allows you to retrieve the arguments passed to a function and +# an arbitrary value by passing a 'store' or '_store' argument +# as its return value. Good for use with concurrent.futures. +def wrap_call(function, *args, **kwargs): + store = kwargs.pop('_store', kwargs.pop('store', None)) + Wrap = namedtuple('Wrap', ['result', 'store', 'args', 'kwargs']) + return Wrap(function(*args, **kwargs), store, args, kwargs) + +class IndexServer(object): + # Index metadata schema: + # { name.str: { path.str, lzma.bool, info.str }, ... } + def __init__(self, endpoint): + self.endpoint = endpoint + r = requests.get(urljoin(self.endpoint, 'meta.json')) + r.raise_for_status() + self.meta = r.json() + + def available_indexes(self): + return self.meta['indexes'].keys() + + def get_latest(self): + return self.meta['latest'] + + def get_anchor(self): + return self.meta.get('anchor', None) + + def autodetect_anchor(self, anchor_hash): + anchor = self.get_anchor() + autodetect = dict() + if anchor: + autodetect = anchor['autodetect'] + return autodetect.get(anchor_hash, None) + + def info(self, name): + return self.meta['indexes'][name]['info'] + + def fetch(self, name, reporter, block_size=2048): + index_meta = self.meta['indexes'][name] + r = requests.get(urljoin(self.endpoint, index_meta['path']), stream=True) + r.raise_for_status() + data = bytearray() + total_size = int(r.headers.get('content-length', 0)) + for chunk, report in reporter.task_it('Fetching index %s' % name, r.iter_content(block_size), length=math.ceil(total_size / block_size)): + report() + data += chunk + if index_meta['lzma']: + data = lzma.decompress(data) + return json.loads(data) + +class PoisonPill(object): + pass + +class Task(object): + def __init__(self, title, unit, length): + self.title = title + self.unit = unit + self.length = length + +class ProgressReporter(object): + def __init__(self, logger='reporter'): + self._start = None + self._stopped = False + self._task = None + self._report_event = threading.Event() + self._step_queue = queue.Queue() + self._task_queue = queue.Queue() + self.logger = logging.getLogger(logger) + + def stop(self): + self._stopped = True + self._step_queue.put(PoisonPill()) + self._task_queue.put(PoisonPill()) + + def is_stopped(self): + return self._stopped + + def task(self, title, unit=None, length=None): + if self._stopped: + raise ValueError('operation on stopped reporter') + self._task = Task(title, unit, length) + self._task_queue.put(self._task) + if not self._start: + self._start = time.time() + else: + self._step_queue.put(PoisonPill()) + + def task_it(self, title, iterator, unit=None, length=None): + if not length: + length = len(iterator) if iterator else None + self.task(title, unit=unit, length=length) + for item in iterator: + if self._stopped: + raise ValueError('operation on stopped reporter') + yield item, self.report + if not self._report_event.isSet(): + raise RuntimeError('report not called in previous iteration') + self._report_event.clear() + + def report(self, payload=None): + if not self._report_event.isSet(): + self._step_queue.put(payload) + self._report_event.set() + + def get_current_task(self): + return self._task + + def tasks(self): + while True: + payload = self._task_queue.get() + if isinstance(payload, PoisonPill): + break + self.logger.info('*** Task Start: %s ***' % payload.title) + yield payload + self.logger.info('*** Task End: %s ***' % payload.title) + + def steps(self): + while True: + payload = self._step_queue.get() + if isinstance(payload, PoisonPill): + break + #self.logger.info('Step > %s' % payload) + yield payload + + def step(self, payload=None): + if self._stopped: + raise ValueError('operation on stopped reporter') + self._step_queue.put(payload) + + def elapsed(self): + elapsed = 0 + if self._start: + elapsed = time.time() - self._start + return str(datetime.timedelta(seconds=elapsed)) diff --git a/icon.ico b/icon.ico new file mode 100644 index 0000000..0ee9513 Binary files /dev/null and b/icon.ico differ diff --git a/icon.png b/icon.png new file mode 100644 index 0000000..220f529 Binary files /dev/null and b/icon.png differ diff --git a/requirements.txt b/requirements.txt index 75aaba4..b24ad18 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ requests backoff tqdm +pyqt5 diff --git a/setup.py b/setup.py deleted file mode 100644 index 198398c..0000000 --- a/setup.py +++ /dev/null @@ -1,19 +0,0 @@ -import sys -import os -from cx_Freeze import setup, Executable - -# Dependencies are automatically detected, but it might need fine tuning. -build_exe_options = {"packages": ["os", "asyncio", "idna.idnadata"], "excludes": ["numpy", "matplotlib"], 'include_files': ['config.json']} - -PYTHON_INSTALL_DIR = os.path.dirname(os.path.dirname(os.__file__)) -if sys.platform == "win32": - build_exe_options['include_files'] += [ - os.path.join(PYTHON_INSTALL_DIR, 'DLLs', 'libcrypto-1_1.dll'), - os.path.join(PYTHON_INSTALL_DIR, 'DLLs', 'libssl-1_1.dll'), - ] - -setup( name = "flashpoint-updater", - version = "0.1", - description = "Updater for BlueMaxima's Flashpoint", - options = {"build_exe": build_exe_options}, - executables = [Executable("update.py"), Executable("index.py")]) diff --git a/update.py b/update.py index 3bd1f73..f677288 100644 --- a/update.py +++ b/update.py @@ -1,22 +1,25 @@ #!/usr/bin/env python3 -from index import win_path +from core import IndexServer, Task, ProgressReporter from tqdm import tqdm -from urllib.parse import quote +from urllib.parse import quote, urljoin from concurrent.futures import as_completed import concurrent.futures +import threading import urllib3 import datetime import requests import backoff import shutil +import index import stat import json import lzma import time +import core import sys import os -@backoff.on_exception(backoff.expo, (requests.exceptions.RequestException, urllib3.exceptions.ProtocolError)) +@backoff.on_exception(backoff.expo, (requests.exceptions.RequestException, urllib3.exceptions.ProtocolError, urllib3.exceptions.ReadTimeoutError), logger='reporter') def download_file(session, url, dest): with session.get(url, stream=True, timeout=10) as r: with open(dest, 'wb') as f: @@ -26,54 +29,40 @@ def download_file(session, url, dest): def chown_file(path): os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) -def fetch_index(version, endpoint): - r = requests.get('%s/%s.json.xz' % (endpoint, version)) - return json.loads(lzma.decompress(r.content)) - -if __name__ == '__main__': - - with open('config.json', 'r') as f: - config = json.load(f) - - if len(sys.argv) != 4: - print('Usage: update.py ') - sys.exit(0) - - flashpoint = win_path(sys.argv[1]) - if not os.path.isdir(flashpoint): - print('Error: Flashpoint path not found.') - sys.exit(0) - - endpoint = config['index_endpoint'] - try: - current, target = fetch_index(sys.argv[2], endpoint), fetch_index(sys.argv[3], endpoint) - except requests.exceptions.RequestException: - print('Could not retrieve indexes for the versions specified.') - sys.exit(0) - - start = time.time() +def perform_update(flashpoint, current, target, file_endpoint, reporter): tmp = os.path.join(flashpoint, '.tmp') - os.mkdir(tmp) + try: + os.mkdir(tmp) + except FileExistsError: + reporter.logger.info('Temp folder exists. We are resuming.') to_download = list() - print('Preparing contents...') - for hash in tqdm(target['files'], unit=' files', ascii=True): + for hash, report in reporter.task_it('Preparing contents...', target['files'], unit='hash'): + report(hash) + tmpPath = os.path.join(tmp, hash) if hash in current['files']: path = os.path.normpath(current['files'][hash][0]) - os.rename(os.path.join(flashpoint, path), os.path.join(tmp, hash)) - else: + if not os.path.isfile(tmpPath): + try: + os.rename(os.path.join(flashpoint, path), tmpPath) + except FileNotFoundError: + reporter.logger.warning('File from current index not found. Queued for download: %s (%s)' % (path, hash)) + to_download.append(hash) + elif not (os.path.isfile(tmpPath) and index.hash(tmpPath, 'sha1') == hash): to_download.append(hash) + else: + reporter.logger.info('File from target index already in temp folder. Skipped: %s' % hash) - print('Downloading new data...') session = requests.Session() with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: tasks = list() for hash in to_download: - url = '%s/%s' % (config['file_endpoint'], quote(target['files'][hash][0])) - tasks.append(executor.submit(download_file, session, url, os.path.join(tmp, hash))) - for future in tqdm(as_completed(tasks), total=len(tasks), unit=' files', ascii=True): - future.result() + path = target['files'][hash][0] + url = urljoin(file_endpoint, quote(path)) + tasks.append(executor.submit(core.wrap_call, download_file, session, url, os.path.join(tmp, hash), store=path)) + for future, report in reporter.task_it('Downloading new data...', as_completed(tasks), length=len(tasks), unit='file'): + report(os.path.basename(future.result().store)) - print('Removing obsolete files...') + reporter.task('Removing obsolete files...') for r, d, f in os.walk(flashpoint, topdown=False): if r == tmp: continue @@ -87,8 +76,8 @@ def fetch_index(version, endpoint): chown_file(path) os.rmdir(path) - print('Creating file structure...') - for hash in tqdm(target['files'], unit=' files', ascii=True): + for hash, report in reporter.task_it('Creating file structure...', target['files'], unit='hash'): + report(hash) paths = target['files'][hash] while paths: path = os.path.normpath(paths.pop(0)) @@ -106,4 +95,44 @@ def fetch_index(version, endpoint): os.makedirs(os.path.join(flashpoint, os.path.normpath(path))) os.rmdir(tmp) - print('Update completed in %s' % str(datetime.timedelta(seconds=time.time() - start))) + reporter.stop() + +if __name__ == '__main__': + + with open('config.json', 'r') as f: + config = json.load(f) + + if len(sys.argv) != 4: + print('Usage: update.py ') + sys.exit(0) + + flashpoint = index.win_path(sys.argv[1]) + if not os.path.isdir(flashpoint): + print('Error: Flashpoint path not found.') + sys.exit(0) + + try: + server = IndexServer(config['index_endpoint']) + except requests.exceptions.RequestException as e: + print('Could not retrieve index metadata: %s' % str(e)) + sys.exit(0) + + def worker(reporter, root_path, server, file_endpoint, current, target): + try: + current, target = server.fetch(current, reporter), server.fetch(target, reporter) + except KeyError as e: + print('Could not find index: %s' % e.args[0]) + sys.exit(0) + except requests.exceptions.RequestException as e: + print('Could not retrieve index: %s' % str(e)) + sys.exit(0) + perform_update(root_path, current, target, file_endpoint, reporter) + + reporter = ProgressReporter() + threading.Thread(target=worker, args=(reporter, flashpoint, server, config['file_endpoint'], sys.argv[2], sys.argv[3])).start() + for task in reporter.tasks(): + print(task.title) + for step in tqdm(reporter.steps(), total=task.length, unit=task.unit or 'it', ascii=True): + pass + + print('Update completed in %s' % reporter.elapsed()) diff --git a/update_gui.py b/update_gui.py new file mode 100644 index 0000000..483cd53 --- /dev/null +++ b/update_gui.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +from core import IndexServer, Task, ProgressReporter +from PyQt5.QtGui import QIcon +from PyQt5.QtCore import QThread, QSize, Qt, pyqtSignal +from PyQt5.QtWidgets import * +from datetime import datetime +import threading +import requests +import logging +import ctypes +import update +import index +import json +import sys +import os +import re + +class UpdateThread(QThread): + sig_exc = pyqtSignal(Exception) + + def __init__(self, reporter, root_path, server, file_endpoint, current, target): + QThread.__init__(self) + self.reporter = reporter + self.root_path = root_path + self.server = server + self.file_endpoint = file_endpoint + self.current = current + self.target = target + + def run(self): + try: + current = self.server.fetch(self.current, self.reporter) + target = self.server.fetch(self.target, self.reporter) + update.perform_update(self.root_path, current, target, self.file_endpoint, self.reporter) + except Exception as e: + if not self.reporter.is_stopped(): # Interrupted from outside (on exit) + self.reporter.logger.critical('Update failed', exc_info=True) + self.reporter.stop() + self.sig_exc.emit(e) + +class ReporterThread(QThread): + sig_task = pyqtSignal(str, str, int) + sig_step = pyqtSignal(object) + sig_done = pyqtSignal(str) + + def __init__(self, reporter): + QThread.__init__(self) + self.reporter = reporter + + def run(self): + for task in self.reporter.tasks(): + self.sig_task.emit(task.title, task.unit, task.length or 0) + for step in self.reporter.steps(): + self.sig_step.emit(step) + self.sig_done.emit(self.reporter.elapsed()) + +class Updater(QDialog): + def __init__(self, server, file_endpoint, parent=None): + super(Updater, self).__init__(parent) + self.server = server + self.file_endpoint = file_endpoint + self.reporter_thread = None + self.update_thread = None + self.step_unit = None + self.progress = 0 + + self.root_path = QLineEdit() + self.browse_button = QPushButton('Browse...') + self.browse_button.clicked.connect(self.on_browse_button) + self.progress_bar = QProgressBar() + self.progress_bar.setValue(0) + self.from_combo_box = QComboBox() + self.from_combo_box.addItems(self.server.available_indexes()) + self.from_combo_box.setEnabled(False) + self.autodetect_checkbox = QCheckBox('Autodetect') + self.autodetect_checkbox.setChecked(True) + self.autodetect_checkbox.toggled.connect(self.on_autodetect_checked) + self.to_combo_box = QComboBox() + self.to_combo_box.addItems(self.server.available_indexes()) + self.update_button = QPushButton('Go!') + self.update_button.clicked.connect(self.on_update_button) + self.status_label = QLabel('Idle.') + self.step_label = QLabel() + + # Update to the latest version by default + pos = self.to_combo_box.findText(self.server.get_latest()) + if pos != -1: + self.to_combo_box.setCurrentIndex(pos) + + self.win_taskbar = None + if os.name == 'nt': + from PyQt5.QtWinExtras import QWinTaskbarButton + self.win_taskbar = QWinTaskbarButton(self) + self.win_taskbar.progress().setVisible(True) + + bottom = QVBoxLayout() + bottom.addWidget(self.status_label) + bottom.addWidget(self.progress_bar) + bottom.addWidget(self.step_label) + + layout = QGridLayout() + layout.addWidget(QLabel('Root Path'), 0, 0) + layout.addWidget(self.root_path, 0, 1) + layout.addWidget(self.browse_button, 0, 2) + layout.addWidget(QLabel('From'), 1, 0) + layout.addWidget(self.from_combo_box, 1, 1) + layout.addWidget(self.autodetect_checkbox, 1, 2) + layout.addWidget(QLabel('To'), 2, 0) + layout.addWidget(self.to_combo_box, 2, 1) + layout.addWidget(self.update_button, 2, 2) + layout.addLayout(bottom, 3, 0, 2, 0) + + self.setLayout(layout) + self.setGeometry(100, 100, 350, 100) + self.setWindowFlag(Qt.WindowContextHelpButtonHint, False) + + def set_task(self, task, unit, length): + self.status_label.setText(task) + self.progress_bar.setRange(0, length) + self.progress_bar.setValue(0) + if self.win_taskbar: + self.win_taskbar.progress().setRange(0, length) + self.win_taskbar.progress().setValue(0) + self.step_unit = unit + if not self.step_unit: + self.step_label.setText('') + + def step(self, payload): + self.progress_bar.setValue(self.progress_bar.value() + 1) + if self.win_taskbar: + self.win_taskbar.progress().setValue(self.progress_bar.value()) + if self.step_unit: + self.step_label.setText('Current %s: %s' % (self.step_unit, payload)) + + def set_done(self, elapsed): + self.update_button.setEnabled(True) + self.status_label.setText('Finished in %s' % elapsed) + self.step_label.setText('') + + def update_failed(self, exception): + QMessageBox.critical(self, 'Update failed', str(exception)) + + def perform_autodetect(self): + path = str(self.root_path.text()) + if self.autodetect_checkbox.isChecked() and self.server.get_anchor(): + try: + hash = index.hash(os.path.join(path, self.server.get_anchor()['file']), 'sha1') + pos = self.from_combo_box.findText(self.server.autodetect_anchor(hash)) + if pos != -1: + self.from_combo_box.setCurrentIndex(pos) + except FileNotFoundError: + pass + + def on_autodetect_checked(self): + self.from_combo_box.setEnabled(not self.autodetect_checkbox.isChecked()) + self.perform_autodetect() + + def on_browse_button(self): + dialog = QFileDialog(self, 'Select root path...') + dialog.setFileMode(QFileDialog.DirectoryOnly) + if dialog.exec_() == QDialog.Accepted: + folder = dialog.selectedFiles()[0] + if self.server.get_anchor(): + for r, d, f in os.walk(folder, topdown=True): + if r.count(os.sep) - folder.count(os.sep) == 1: + del d[:] + if self.server.get_anchor()['file'] in f: + folder = os.path.normpath(r) + break + self.root_path.setText(folder) + self.perform_autodetect() + + def on_update_button(self): + root_path = index.win_path(str(self.root_path.text())) + if not os.path.isdir(root_path): + QMessageBox.critical(self, 'Cannot proceed', 'Please make sure that the root path exists.') + return + self.update_button.setEnabled(False) + current = str(self.from_combo_box.currentText()) + target = str(self.to_combo_box.currentText()) + logger.info('Starting update from %s to %s' % (current, target)) + reporter = ProgressReporter() + self.reporter_thread = ReporterThread(reporter) + self.reporter_thread.sig_task.connect(self.set_task) + self.reporter_thread.sig_step.connect(self.step) + self.reporter_thread.sig_done.connect(self.set_done) + self.reporter_thread.start() + self.update_thread = UpdateThread(reporter, root_path, self.server, self.file_endpoint, current, target) + self.update_thread.sig_exc.connect(self.update_failed) + self.update_thread.start() + + def showEvent(self, event): + self.setFixedSize(self.size()) # Make non-resizable + if self.win_taskbar: + self.win_taskbar.setWindow(updater.windowHandle()) + event.accept() + + def closeEvent(self, event): + if self.update_thread and self.update_thread.isRunning(): + self.update_thread.reporter.stop() + self.update_thread.wait() + self.reporter_thread.wait() + event.accept() + +os.makedirs('logs', exist_ok=True) +logger = logging.getLogger('reporter') +logger.setLevel(logging.INFO) +logFormatter = logging.Formatter('%(asctime)s [%(levelname)s] %(message)s', '%Y-%m-%d %H:%M:%S') +fileHandler = logging.FileHandler(datetime.now().strftime('logs/update_%Y-%m-%d_%H-%M-%S.log'), delay=True) +fileHandler.setFormatter(logFormatter) +logger.addHandler(fileHandler) + +if os.name == 'nt': + ctypes.windll.shell32.SetCurrentProcessExplicitAppUserModelID('flashpoint.updater') + +res_path = getattr(sys, '_MEIPASS', os.path.dirname(os.path.abspath(__file__))) + +app = QApplication([sys.argv]) +app.setApplicationName('Flashpoint Updater') +app.setWindowIcon(QIcon(os.path.join(res_path, 'icon.png'))) + +try: + with open('config.json', 'r') as f: + config = json.load(f) +except FileNotFoundError: + logger.critical('Could not find configuration file. Aborted.') + QMessageBox.critical(None, 'Initialization error', 'Config file not found!') + sys.exit(0) +except ValueError: + logger.critical('Could parse configuration file. Aborted.') + QMessageBox.critical(None, 'Initialization error', 'Config file cannot be parsed!') + sys.exit(0) + +try: + server = IndexServer(config['index_endpoint']) +except requests.exceptions.RequestException as e: + logger.critical('Could not retrieve index metadata: %s' % str(e)) + QMessageBox.critical(None, 'Initialization error', 'Could not retrieve index metadata. Please, check the log file for more details.') + sys.exit(0) + +updater = Updater(server, config['file_endpoint']) +updater.show() +app.exec_()