From 77e9543c6d9edc219265cad89c5e6b5c98231d38 Mon Sep 17 00:00:00 2001 From: Rune Daugaard Harlyk Date: Mon, 8 Apr 2024 14:16:48 +0200 Subject: [PATCH 1/3] Simpifies build script --- scripts/build_interface.py | 247 +++++++++++++++++-------------------- 1 file changed, 111 insertions(+), 136 deletions(-) diff --git a/scripts/build_interface.py b/scripts/build_interface.py index ce7265a0..033132c9 100644 --- a/scripts/build_interface.py +++ b/scripts/build_interface.py @@ -7,15 +7,14 @@ # Copyright (C) 2018 - 2023 rjwats # Copyright (C) 2023 theelims # Copyright (C) 2023 Maxtrium B.V. [ code available under dual license ] +# Copyright (C) 2024 runeharlyk # # All Rights Reserved. This software may be modified and distributed under # the terms of the LGPL v3 license. See the LICENSE file for details. from pathlib import Path from shutil import copytree, rmtree, copyfileobj -from subprocess import check_output, Popen, PIPE, STDOUT, CalledProcessError -from os.path import exists -from typing import Final +from os.path import exists, getmtime import os import gzip import mimetypes @@ -24,161 +23,137 @@ Import("env") -# print("Current build environment:") -# print(env.ParseFlags(env["BUILD_FLAGS"]).get("CPPDEFINES")) +project_dir = env["PROJECT_DIR"] +buildFlags = env.ParseFlags(env["BUILD_FLAGS"]) -# print("Current CLI targets", COMMAND_LINE_TARGETS) -# print("Current Build targets", BUILD_TARGETS) +interface_dir = project_dir + "/interface" +output_file = project_dir + "/lib/framework/WWWData.h" +source_www_dir = interface_dir + "/src" +build_dir = interface_dir + "/build" +filesystem_dir = project_dir + "/data/www" -OUTPUTFILE: Final[str] = env["PROJECT_DIR"] + "/lib/framework/WWWData.h" -SOURCEWWWDIR: Final[str] = env["PROJECT_DIR"] + "/interface/src" -def OutputFileExits(): - return os.path.exists(OUTPUTFILE) +def find_latest_timestamp_for_app(): + return max( + (getmtime(f) for f in glob.glob(f"{source_www_dir}/**/*", recursive=True)) + ) -def findLastestTimeStampWWWInterface(): - list_of_files = glob.glob(SOURCEWWWDIR+'/**/*', recursive=True) - # print(list_of_files) - latest_file = max(list_of_files, key=os.path.getmtime) - # print(latest_file) - return os.path.getmtime(latest_file) +def should_regenerate_output_file(): + if not flag_exists("EMBED_WWW") or not exists(output_file): + return True + last_source_change = find_latest_timestamp_for_app() + last_build = getmtime(output_file) -def findTimestampOutputFile(): - return os.path.getmtime(OUTPUTFILE) + print( + f"Newest file: {datetime.fromtimestamp(last_source_change)}, output file: {datetime.fromtimestamp(last_build)}" + ) -def needtoRegenerateOutputFile(): - if not flagExists("EMBED_WWW"): - return True - else: - if (OutputFileExits()): - latestWWWInterface = findLastestTimeStampWWWInterface() - timestampOutputFile = findTimestampOutputFile() - - # print timestamp of newest file in interface directory and timestamp of outputfile nicely formatted as time - print(f'Newest interface file: {datetime.fromtimestamp(latestWWWInterface):%Y-%m-%d %H:%M:%S}, WWW Outputfile: {datetime.fromtimestamp(timestampOutputFile):%Y-%m-%d %H:%M:%S}') - # print(f'Newest interface file: {latestWWWInterface:.2f}, WWW Outputfile: {timestampOutputFile:.2f}') - - sourceEdited=( timestampOutputFile < latestWWWInterface ) - if (sourceEdited): - print("Svelte source files are updated. Need to regenerate.") - return True - else: - print("Current outputfile is O.K. No need to regenerate.") - return False - - else: - print("WWW outputfile does not exists. Need to regenerate.") - return True + return last_build < last_source_change -def gzipFile(file): + +def gzip_file(file): with open(file, 'rb') as f_in: with gzip.open(file + '.gz', 'wb') as f_out: copyfileobj(f_in, f_out) os.remove(file) -def flagExists(flag): - buildFlags = env.ParseFlags(env["BUILD_FLAGS"]) + +def flag_exists(flag): for define in buildFlags.get("CPPDEFINES"): if (define == flag or (isinstance(define, list) and define[0] == flag)): return True + return False -def buildProgMem(): - mimetypes.init() - progmem = open(OUTPUTFILE,"w") - - progmem.write("#include \n") - progmem.write("#include \n") - - - progmemCounter = 0 - - assetMap = {} - - # Iterate over all files in the build directory - for path in Path("build").rglob("*.*"): - asset_path = path.relative_to("build").as_posix() - print("Converting " + str(asset_path)) - - asset_var = 'ESP_SVELTEKIT_DATA_' + str(progmemCounter) - asset_mime = mimetypes.types_map['.' + asset_path.split('.')[-1]] - - progmem.write('// ' + str(asset_path) + '\n') - progmem.write('const uint8_t ' + asset_var + '[] = {\n ') - progmemCounter += 1 - - # Open path as binary file, compress and read into byte array - size = 0 - with open(path, "rb") as f: - zipBuffer = gzip.compress(f.read()) - for byte in zipBuffer: - if not (size % 16): - progmem.write('\n ') - - progmem.write(f"0x{byte:02X}" + ',') - size += 1 - - progmem.write('\n};\n\n') - assetMap[asset_path] = { "name": asset_var, "mime": asset_mime, "size": size } - - progmem.write('typedef std::function RouteRegistrationHandler;\n\n') - progmem.write('class WWWData {\n') - progmem.write(' public:\n') - progmem.write(' static void registerRoutes(RouteRegistrationHandler handler) {\n') - - for asset_path, asset in assetMap.items(): - progmem.write(' handler("/' + str(asset_path) + '", "' + asset['mime'] + '", ' + asset['name'] + ', ' + str(asset['size']) + ');\n') - - progmem.write(' }\n') - progmem.write('};\n') - - progmem.write('\n') - - -def buildWeb(): - os.chdir("interface") - print("Building interface with npm") - try: - env.Execute("npm install") - env.Execute("npm run build") - buildPath = Path("build") - wwwPath = Path("../data/www") - if not flagExists("EMBED_WWW"): - if wwwPath.exists() and wwwPath.is_dir(): - rmtree(wwwPath) - print("Copying and compress interface to data directory") - copytree(buildPath, wwwPath) - for currentpath, folders, files in os.walk(wwwPath): - for file in files: - gzipFile(os.path.join(currentpath, file)) - else: - print("Converting interface to PROGMEM") - buildProgMem() - - finally: - os.chdir("..") - if not flagExists("EMBED_WWW"): - print("Build LittleFS file system image and upload to ESP32") - env.Execute("pio run --target uploadfs") -print("running: build_interface.py") +def get_package_manager(): + if exists(os.path.join(interface_dir, "pnpm-lock.yaml")): + return "pnpm" + return "npm" -# Dump global construction environment (for debug purpose) -#print(env.Dump()) -# Dump project construction environment (for debug purpose) -#print(projenv.Dump()) +def build_webapp(): + package_manager = get_package_manager() + print(f"Building interface with {package_manager}") + os.chdir(interface_dir) + env.Execute(f"{package_manager} install") + env.Execute(f"{package_manager} run build") + os.chdir("..") -if (needtoRegenerateOutputFile()): - buildWeb() -#env.AddPreAction("${BUILD_DIR}/src/HTTPServer.o", buildWebInterface) +def embed_webapp(): + if flag_exists("EMBED_WWW"): + print("Converting interface to PROGMEM") + build_progmem() + return + add_app_to_filesystem() -# if ("upload" in BUILD_TARGETS): -# print(BUILD_TARGETS) -# if (needtoRegenerateOutputFile()): -# buildWeb() -# else: -# print("Skipping build interface step for target(s): " + ", ".join(BUILD_TARGETS)) +def build_progmem(): + mimetypes.init() + with open(output_file, "w") as progmem: + progmem.write("#include \n") + progmem.write("#include \n") + + assetMap = {} + + for idx, path in enumerate(Path(build_dir).rglob("*.*")): + asset_path = path.relative_to(build_dir).as_posix() + asset_mime = ( + mimetypes.guess_type(asset_path)[0] or "application/octet-stream" + ) + print(f"Converting {asset_path}") + + asset_var = f"ESP_SVELTEKIT_DATA_{idx}" + progmem.write(f"// {asset_path}\n") + progmem.write(f"const uint8_t {asset_var}[] = {{\n\t") + file_data = gzip.compress(path.read_bytes()) + + for i, byte in enumerate(file_data): + if i and not (i % 16): + progmem.write("\n\t") + progmem.write(f"0x{byte:02X},") + + progmem.write("\n};\n\n") + assetMap[asset_path] = { + "name": asset_var, + "mime": asset_mime, + "size": len(file_data), + } + + progmem.write( + "typedef std::function RouteRegistrationHandler;\n\n" + ) + progmem.write("class WWWData {\n") + progmem.write("\tpublic:\n") + progmem.write( + "\t\tstatic void registerRoutes(RouteRegistrationHandler handler) {\n" + ) + + for asset_path, asset in assetMap.items(): + progmem.write( + f'\t\t\thandler("/{asset_path}", "{asset["mime"]}", {asset["name"]}, {asset["size"]});\n' + ) + + progmem.write("\t\t}\n") + progmem.write("};\n\n") + + +def add_app_to_filesystem(): + build_path = Path(build_dir) + www_path = Path(filesystem_dir) + if www_path.exists() and www_path.is_dir(): + rmtree(www_path) + print("Copying and compress interface to data directory") + copytree(build_path, www_path) + for current_path, _, files in os.walk(www_path): + for file in files: + gzip_file(os.path.join(current_path, file)) + print("Build LittleFS file system image and upload to ESP32") + env.Execute("pio run --target uploadfs") + +print("running: build_interface.py") +if should_regenerate_output_file(): + build_webapp() + embed_webapp() From e9b6467bd37d003879a18a2d8d731a6fa768f257 Mon Sep 17 00:00:00 2001 From: Rune Harlyk Date: Mon, 8 Apr 2024 21:07:28 +0200 Subject: [PATCH 2/3] Adds yarn as package manager choice --- scripts/build_interface.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build_interface.py b/scripts/build_interface.py index 033132c9..7313be52 100644 --- a/scripts/build_interface.py +++ b/scripts/build_interface.py @@ -69,6 +69,8 @@ def flag_exists(flag): def get_package_manager(): if exists(os.path.join(interface_dir, "pnpm-lock.yaml")): return "pnpm" + if exists(os.path.join(interface_dir, "yarn.lock")): + return "yarn" return "npm" From cbc50712116b8a5fcb114172bb2397fa17839779 Mon Sep 17 00:00:00 2001 From: Rune Harlyk Date: Mon, 8 Apr 2024 21:08:30 +0200 Subject: [PATCH 3/3] Adds check for missing lock file --- scripts/build_interface.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/scripts/build_interface.py b/scripts/build_interface.py index 7313be52..af5ce3d4 100644 --- a/scripts/build_interface.py +++ b/scripts/build_interface.py @@ -71,16 +71,21 @@ def get_package_manager(): return "pnpm" if exists(os.path.join(interface_dir, "yarn.lock")): return "yarn" - return "npm" + if exists(os.path.join(interface_dir, "package-lock.json")): + return "npm" def build_webapp(): - package_manager = get_package_manager() - print(f"Building interface with {package_manager}") - os.chdir(interface_dir) - env.Execute(f"{package_manager} install") - env.Execute(f"{package_manager} run build") - os.chdir("..") + if package_manager := get_package_manager(): + print(f"Building interface with {package_manager}") + os.chdir(interface_dir) + env.Execute(f"{package_manager} install") + env.Execute(f"{package_manager} run build") + os.chdir("..") + else: + raise Exception( + "No lock-file found. Please install dependencies for interface (eg. npm install)" + ) def embed_webapp():