Initial commit

.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+build/

README.md

@@ -0,0 +1,32 @@
+# Flashpoint Updater
+
+The updater for BlueMaxima's Flashpoint.
+Currently a work in progress.
+
+## End-user Setup
+
+### Windows
+
+1. Download the latest release.
+2. Unpack anywhere.
+3. Run it from the command-line as such:
+`update.exe <flashpoint-path> <current-version> <target-version>`
+
+##### Example: `update.exe C:\Flashpoint 5.5 6.1`
+
+### Mac/Linux
+
+1. Install Python 3.
+2. Clone the repository.
+3. Run in the project root: `pip install -r requirements.txt`
+4. Use it like: `update.py /media/ext1/Flashpoint <flashpoint-path> <current-version> <target-version>` 
+
+## Server Setup
+
+The updater works by fetching differing files from two version indexes. These indexes contain SHA-1 hashes of all the files in the project mapped to an array of their paths.
+
+The updater script expects indexes to be available at the location specified by `index_endpoint` in `config.json`. Example: `https://unstable.life/fp-index/6.1.json.xz`
+
+Similarly, files will be fetched in the location specified by `file_endpoint`.
+
+To generate indexes, use `index.py`: `index.py /media/ext1/Flashpoint 6.2.json.xz`

config.json

@@ -0,0 +1,4 @@
+{
+  "index_endpoint": "https://unstable.life/fp-index",
+  "file_endpoint": "https://unstable.life/fp"
+}

index.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+from tqdm import tqdm
+import json
+import lzma
+import os
+import sys
+import hashlib
+import posixpath
+
+# Allows accessing files that exceed MAX_PATH in Windows
+# See: https://docs.microsoft.com/en-us/windows/desktop/fileio/naming-a-file#maximum-path-length-limitation
+def win_path(path):
+    if os.name == 'nt':
+        path = os.path.abspath(path)
+        prefix = '\\\\?\\'
+        if not path.startswith(prefix):
+            # Handle shared paths
+            if path.startswith('\\\\'):
+                prefix += 'UNC'
+                path = path[1::] # Remove leading slash
+            path = prefix + path
+    return path
+
+def hash(file, hashalg, bufsize=2**16):
+    hash = hashlib.new(hashalg)
+    with open(file, 'rb') as f:
+        buf = f.read(bufsize)
+        while len(buf) > 0:
+            hash.update(buf)
+            buf = f.read(bufsize)
+    return hash.hexdigest()
+
+def index(path, hashalg):
+    files = dict()
+    empty = list()
+    path = win_path(path)
+    with tqdm(unit=' files') as pbar:
+        for r, d, f in os.walk(path):
+            # Include empty folders
+            rel = os.path.relpath(r, path).replace(os.path.sep, '/')
+            if len(d) == 0 and len(f) == 0:
+                empty.append(rel)
+            else:
+                for x, f in ((x if rel == '.' else posixpath.join(rel, x), os.path.join(r, x)) for x in f):
+                    files.setdefault(hash(f, hashalg), list()).append(x)
+                    pbar.update(1)
+    return files, empty
+
+if __name__ == '__main__':
+
+    if len(sys.argv) != 3:
+        print('Usage: index.py <path> <out.json.xz>')
+        sys.exit(0)
+
+    files, empty = index(sys.argv[1], 'sha1')
+    print('Applying LZMA compression...')
+    with lzma.open(sys.argv[2], 'wt', encoding='utf-8', preset=9) as f:
+        json.dump({'files': files, 'empty': empty}, f, separators=(',', ':'), ensure_ascii=False)

requirements.txt

@@ -0,0 +1,3 @@
+requests
+backoff
+tqdm

setup.py

@@ -0,0 +1,19 @@
+import sys
+import os
+from cx_Freeze import setup, Executable
+
+# Dependencies are automatically detected, but it might need fine tuning.
+build_exe_options = {"packages": ["os", "asyncio", "idna.idnadata"], "excludes": ["numpy", "matplotlib"], 'include_files': ['config.json']}
+
+PYTHON_INSTALL_DIR = os.path.dirname(os.path.dirname(os.__file__))
+if sys.platform == "win32":
+    build_exe_options['include_files'] += [
+        os.path.join(PYTHON_INSTALL_DIR, 'DLLs', 'libcrypto-1_1.dll'),
+        os.path.join(PYTHON_INSTALL_DIR, 'DLLs', 'libssl-1_1.dll'),
+    ]
+
+setup(  name = "flashpoint-updater",
+        version = "0.1",
+        description = "Updater for BlueMaxima's Flashpoint",
+        options = {"build_exe": build_exe_options},
+        executables = [Executable("update.py"), Executable("index.py")])

update.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+from index import win_path
+from tqdm import tqdm
+from urllib.parse import quote
+from concurrent.futures import as_completed
+import concurrent.futures
+import urllib3
+import datetime
+import requests
+import backoff
+import shutil
+import stat
+import json
+import lzma
+import time
+import sys
+import os
+
+@backoff.on_exception(backoff.expo, (requests.exceptions.RequestException, urllib3.exceptions.ProtocolError))
+def download_file(session, url, dest):
+    with session.get(url, stream=True, timeout=10) as r:
+        with open(dest, 'wb') as f:
+            shutil.copyfileobj(r.raw, f)
+
+# Fix for "read-only" files on Windows
+def chown_file(path):
+    os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
+
+def fetch_index(version, endpoint):
+    r = requests.get('%s/%s.json.xz' % (endpoint, version))
+    return json.loads(lzma.decompress(r.content))
+
+if __name__ == '__main__':
+
+    with open('config.json', 'r') as f:
+        config = json.load(f)
+
+    if len(sys.argv) != 4:
+        print('Usage: update.py <flashpoint-path> <current-version> <target-version>')
+        sys.exit(0)
+
+    flashpoint = win_path(sys.argv[1])
+    if not os.path.isdir(flashpoint):
+        print('Error: Flashpoint path not found.')
+        sys.exit(0)
+
+    endpoint = config['index_endpoint']
+    try:
+        current, target = fetch_index(sys.argv[2], endpoint), fetch_index(sys.argv[3], endpoint)
+    except requests.exceptions.RequestException:
+        print('Could not retrieve indexes for the versions specified.')
+        sys.exit(0)
+
+    start = time.time()
+    tmp = os.path.join(flashpoint, '.tmp')
+    os.mkdir(tmp)
+    to_download = list()
+    print('Preparing contents...')
+    for hash in tqdm(target['files'], unit=' files', ascii=True):
+        if hash in current['files']:
+            path = os.path.normpath(current['files'][hash][0])
+            os.rename(os.path.join(flashpoint, path), os.path.join(tmp, hash))
+        else:
+            to_download.append(hash)
+
+    print('Downloading new data...')
+    session = requests.Session()
+    with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
+        tasks = list()
+        for hash in to_download:
+            url = '%s/%s' % (config['file_endpoint'], quote(target['files'][hash][0]))
+            tasks.append(executor.submit(download_file, session, url, os.path.join(tmp, hash)))
+        for future in tqdm(as_completed(tasks), total=len(tasks), unit=' files', ascii=True):
+            future.result()
+
+    print('Removing obsolete files...')
+    for r, d, f in os.walk(flashpoint, topdown=False):
+        if r == tmp:
+            continue
+        for x in f:
+            path = os.path.join(r, x)
+            chown_file(path)
+            os.remove(path)
+        for x in d:
+            path = os.path.join(r, x)
+            if path != tmp:
+                chown_file(path)
+                os.rmdir(path)
+
+    print('Creating file structure...')
+    for hash in tqdm(target['files'], unit=' files', ascii=True):
+        paths = target['files'][hash]
+        while paths:
+            path = os.path.normpath(paths.pop(0))
+            parent = os.path.dirname(path)
+            if parent:
+                os.makedirs(os.path.join(flashpoint, parent), exist_ok=True)
+            tmpfile = os.path.join(tmp, hash)
+            dest = os.path.join(flashpoint, path)
+            if paths:
+                shutil.copy(tmpfile, dest)
+            else: # No more paths, we can move instead
+                os.rename(tmpfile, dest)
+
+    for path in target['empty']:
+        os.makedirs(os.path.join(flashpoint, os.path.normpath(path)))
+
+    os.rmdir(tmp)
+    print('Update completed in %s' % str(datetime.timedelta(seconds=time.time() - start)))