From c546465e7c640067f9362e3e04e90a49c808972e Mon Sep 17 00:00:00 2001 From: K0lb3 Date: Mon, 8 Nov 2021 10:47:15 +0100 Subject: [PATCH] Initial commit --- .gitattributes | 2 + .gitignore | 103 +++++++++++++++++++++++++ AssetBatchConverter.py | 163 +++++++++++++++++++++++++++++++++++++++ README.md | 24 ++++++ download_assets.py | 169 +++++++++++++++++++++++++++++++++++++++++ reaxtract_assets.py | 13 ++++ version.txt | 1 + 7 files changed, 475 insertions(+) create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 AssetBatchConverter.py create mode 100644 README.md create mode 100644 download_assets.py create mode 100644 reaxtract_assets.py create mode 100644 version.txt diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..dfe0770 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Auto detect text files and perform LF normalization +* text=auto diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..302c2dc --- /dev/null +++ b/.gitignore @@ -0,0 +1,103 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +extracted/ +raw/ +*.apk +lib/ +update_master.py + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +assets/asset_list.txt diff --git a/AssetBatchConverter.py b/AssetBatchConverter.py new file mode 100644 index 0000000..df3a288 --- /dev/null +++ b/AssetBatchConverter.py @@ -0,0 +1,163 @@ +import os +import UnityPy +from collections import Counter +import json + +TYPES = [ + # Images + 'Sprite', + 'Texture2D', + # Text (filish) + 'TextAsset', + 'Shader', + 'MonoBehaviour', + 'Mesh' + # Font + 'Font', + # Audio + 'AudioClip', +] + +ROOT = os.path.dirname(os.path.realpath(__file__)) +DST = os.path.join(ROOT, "extracted") +IGNOR_DIR_COUNT=0 + +def extract_assets(src): + # load source + env = UnityPy.load(src) + + # iterate over assets + for asset in env.assets: + # assets without container / internal path will be ignored for now + if not asset.container: + continue + # filter objects and put Texture2Ds at the end of the list + objs = sorted( + ( + obj + for obj in asset.get_objects() + if obj.type.name in TYPES + ), + key=lambda x: 1 if x.type == "Texture2D" else 0 + ) + cobjs = sorted( + ( + (key, obj) + for key, obj in asset.container.items() + if obj.type.name in TYPES + ), + key=lambda x: 1 if x[1].type == "Texture2D" else 0 + ) + # check which mode we will have to use + num_cont = len(cobjs) + num_objs = len(objs) + + # check if container contains all important assets, if yes, just ignore the container + if num_objs <= num_cont * 2: + for asset_path, obj in cobjs: + fp = os.path.join(DST, *asset_path.split('/') + [IGNOR_DIR_COUNT:]) + export_obj(obj, fp) + + # otherwise use the container to generate a path for the normal objects + else: + extracted = [] + # find the most common path + occurence_count = Counter(os.path.splitext(asset_path)[ + 0] for asset_path in asset.container.keys()) + local_path = os.path.join( + DST, *occurence_count.most_common(1)[0][0].split('/')[IGNOR_DIR_COUNT:]) + + for obj in objs: + if obj.path_id not in extracted: + try: + extracted.extend(export_obj( + obj, local_path, append_name=True)) + except Exception as e: + print(e, obj.path_id) + + +def export_obj(obj, fp: str, append_name: bool = False) -> list: + if obj.type not in TYPES: + return [] + + data = obj.read() + if append_name: + fp = os.path.join(fp, data.name) + + fp, extension = os.path.splitext(fp) + os.makedirs(os.path.dirname(fp), exist_ok=True) + + # streamlineable types + export = None + if obj.type == 'TextAsset': + if not extension: + extension = '.txt' + export = data.script + + elif obj.type == "Font": + if data.m_FontData: + extension = ".ttf" + if data.m_FontData[0:4] == b"OTTO": + extension = ".otf" + export = data.m_FontData + else: + return [obj.path_id] + + elif obj.type == "Mesh": + extension = ".obf" + export = data.export().encode("utf8") + + elif obj.type == "Shader": + extension = ".txt" + export = data.export().encode("utf8") + + elif obj.type == "MonoBehaviour": + # The data structure of MonoBehaviours is custom + # and is stored as nodes + # If this structure doesn't exist, + # it might still help to at least save the binary data, + # which can then be inspected in detail. + if obj.serialized_type.nodes: + extension = ".json" + export = json.dumps( + obj.read_typetree(), + indent=4, + ensure_ascii=False + ).encode("utf8") + else: + extension = ".bin" + export = data.raw_data + + if export: + with open(f"{fp}{extension}", "wb") as f: + f.write(export) + + # non-streamlineable types + if obj.type == "Sprite": + data.image.save(f"{fp}.png") + + return [obj.path_id, data.m_RD.texture.path_id, getattr(data.m_RD.alphaTexture, 'path_id', None)] + + elif obj.type == "Texture2D": + if not os.path.exists(fp) and data.m_Width: + # textures can have size 0..... + data.image.save(f"{fp}.png") + + elif obj.type == "AudioClip": + samples = data.samples + if len(samples) == 0: + pass + elif len(samples) == 1: + with open(f"{fp}.wav", "wb") as f: + f.write(list(data.samples.values())[0]) + else: + os.makedirs(fp, exist_ok=True) + for name, clip_data in samples.items(): + with open(os.path.join(fp, f"{name}.wav"), "wb") as f: + f.write(clip_data) + return [obj.path_id] + + +if __name__ == '__main__': + main() diff --git a/README.md b/README.md new file mode 100644 index 0000000..4665f49 --- /dev/null +++ b/README.md @@ -0,0 +1,24 @@ +# Blue Archive asset downloader + +A small project that downloads all assets of the global version of Blue Archive and extracts them while it's at it. + +The script updates the assets and even its own parameters on its own, +so all you have to do is execute the download_assets.py script after every update to get the latest files. + +## Script Requirements + +``` +- Python 3.6+ +- UnityPy 1.7.21 +- requests + +```cmd +pip install UnityPy==1.7.21 +pip install requests +``` + + +## TODO + +- decryption of some files +- diff --git a/download_assets.py b/download_assets.py new file mode 100644 index 0000000..97b28e9 --- /dev/null +++ b/download_assets.py @@ -0,0 +1,169 @@ +import requests +import os +from AssetBatchConverter import extract_assets +import UnityPy + +ROOT = os.path.dirname(os.path.realpath(__file__)) +RAW = os.path.join(ROOT, "raw") +EXT = os.path.join(ROOT, "extracted") +VERSION = os.path.join(ROOT, "version.txt") + +import UnityPy + + +def main(): + path = ROOT + app_id = "com.nexon.bluearchive" + + print("Fetching version") + if os.path.exists(VERSION): + with open(VERSION, "rt") as f: + version = f.read() + else: + print("no local version found") + version = update_apk_version(app_id, path) + print(version) + + print("Fetch latest resource version") + try: + check = version_check(app_id, build_version=version) + except Exception as e: + print("error during resource version request") + print("updating apk settings") + version = update_apk_version(app_id, path) + check = version_check(app_id, build_version = version) + + print("Updating resources/assets") + update_resources(check["patch"]["resource_path"]) + + +def version_check(app_id: str, api_version: str="v1.1", build_version: str = "1.35.115378"): + req = requests.post( + f"https://api-pub.nexon.com/patch/{api_version}/version-check", + json={ + "market_game_id": app_id, + "language": "en", + "advertising_id": "00000000-0000-0000-0000-000000000000", + "market_code": "playstore", + "country": "US", + "sdk_version": "187", # doesn't seem to matter + "curr_build_version": build_version, + "curr_build_number": int(build_version.rsplit(".", 1)[1]), + "curr_patch_version": 0, + }, + headers={ + "Content-Type": "application/json; charset=utf-8", + "User-Agent": "Dalvik/2.1.0 (Linux; U; Android 5.1.1; SM-A908N Build/LMY49I)", + "Host": "api-pub.nexon.com", + "Connection": "Keep-Alive", + "Accept-Encoding": "gzip", + }, + ) + res = req.json() + + # latest_build_version = res["latest_build_version"] + # latest_build_number = res["latest_build_number"] + # if latest_build_version != build_version or latest_build_number != build_number: + # return version_check(api_version, latest_build_version, latest_build_number) + return res + + +def update_resources(resource_path, lang="en"): + # 1. get resource data + req = requests.get(resource_path) + res = req.json() + + # 2. update resources + resource_main_url = resource_path.rsplit("/", 1)[0] + for resource in res["resources"]: + # { + # "group": "group2", + # "resource_path": "JA/group2/e392fcd3de13100b67589ef873b1f6d4.bundle", + # "resource_size": 25206, + # "resource_hash": "42092be2cf4d14381107205e40ab08b1" + # }, + update_resource(resource_main_url, resource) + + +def update_resource(resource_main_url, resource): + url = f"{resource_main_url}/{resource['resource_path']}" + if url.endswith(".bundle"): + raw_path = os.path.join(RAW, *resource["resource_path"].split("/")) + else: + raw_path = os.path.join(EXT, *resource["resource_path"].split("/")) + os.makedirs(os.path.dirname(raw_path), exist_ok=True) + + if not ( + os.path.exists(raw_path) + and os.path.getsize(raw_path) == resource["resource_size"] + ): + print(raw_path) + data = requests.get(url).content + with open(raw_path, "wb") as f: + f.write(data) + + # Unity BundleFile + if url.endswith(".bundle"): + extract_assets(data) + # extract with UnityPy + + +def update_apk_version(apk_id, path): + print("downloading latest apk from QooApp") + apk_data = download_QooApp_apk(apk_id) + with open(os.path.join(path, "current.apk"), "wb") as f: + f.write(apk_data) + print("extracing app_version and api_version") + version = extract_apk_version(apk_data) + with open(VERSION, "wt") as f: + f.write(version) + + return version + + +def extract_apk_version(apk_data): + from zipfile import ZipFile + import io + import re + + with io.BytesIO(apk_data) as stream: + with ZipFile(stream) as zip: + # devs are dumb shit and keep moving the app version around + with zip.open("assets/bin/Data/globalgamemanagers", "r") as f: + env = UnityPy.load(f) + for obj in env.objects: + if obj.type.name == "PlayerSettings": + build_version = re.search( + b"\d+?\.\d+?\.\d+", obj.get_raw_data() + )[0].decode() + return build_version + # smali\com\nexon\pub\bar\q.smali has the v1.1 for Nexus + + +def download_QooApp_apk(apk): + from urllib.request import urlopen, Request + from urllib.parse import urlencode + + query = urlencode( + { + "supported_abis": "x86,armeabi-v7a,armeabi", + "sdk_version": "22", + } + ) + res = urlopen( + Request( + url=f"https://api.qoo-app.com/v6/apps/{apk}/download?{query}", + headers={ + "accept-encoding": "gzip", + "user-agent": "QooApp 8.1.7", + "device-id": "80e65e35094bedcc", + }, + method="GET", + ) + ) + data = urlopen(res.url).read() + return data + + +if __name__ == "__main__": + main() diff --git a/reaxtract_assets.py b/reaxtract_assets.py new file mode 100644 index 0000000..7bf4086 --- /dev/null +++ b/reaxtract_assets.py @@ -0,0 +1,13 @@ +from download_assets import ROOT, RAW, EXT +import AssetBatchConverter + +AssetBatchConverter.DST= EXT + + +import os +for root, dirs, files in os.walk(RAW): + for f in files: + if not f.endswith(".bundle"): + continue + fp = os.path.join(root, f) + AssetBatchConverter.extract_assets(fp) \ No newline at end of file diff --git a/version.txt b/version.txt new file mode 100644 index 0000000..2f40b12 --- /dev/null +++ b/version.txt @@ -0,0 +1 @@ +1.35.115378 \ No newline at end of file