diff --git a/.github/workflows/build_QeexoML.yml b/.github/workflows/build_QeexoML.yml index 50308c8..21d11f2 100644 --- a/.github/workflows/build_QeexoML.yml +++ b/.github/workflows/build_QeexoML.yml @@ -39,7 +39,18 @@ jobs: su -l arm_mlops_docker -c "/home/arm_mlops_docker/cmsis-toolbox-linux-amd64/bin/cpackget add https://github.com/ARM-software/CMSIS_6/releases/download/dev%2Fv6.0.0-dev35/ARM.CMSIS.6.0.0-dev35+geb1d42a.pack --agree-embedded-license" su -l arm_mlops_docker -c "/home/arm_mlops_docker/cmsis-toolbox-linux-amd64/bin/cpackget add https://github.com/ARM-software/CMSIS-DFP/releases/download/dev%2Fv1.0.0-dev12/ARM.CMSIS_DFP.1.0.0-dev12+g285da21.pack --agree-embedded-license" - - name: Build Model for Cortex-M7 with AC6 (size) + - name: Build custom Qeexo project CMSIS pack + run: | + export PATH=/home/arm_mlops_docker/cmsis-toolbox-linux-amd64/bin/:$PATH + pip3 install requests pyyml semantic_version + cd /__w/AVH-MLOps/AVH-MLOps/QeexoAutoML/Pack && python3 gen_pack.py --path QxLib_PID12345 --version 1.00 --candidate_rev=run${{ github.run_id }} + + - name: Install custom Qeexo project CMSIS pack + run: | + su -l arm_mlops_docker -c "/home/arm_mlops_docker/cmsis-toolbox-linux-amd64/bin/cpackget add /__w/AVH-MLOps/AVH-MLOps/QeexoAutoML/Pack/QxLib_PID12345/build/Qeexo.QxLib_PID12345.1.0.0-run${{ github.run_id }}.pack --agree-embedded-license" + + + - name: Build Test Project for Cortex-M7 with AC6 (size) run: | su - arm_mlops_docker -c "/home/arm_mlops_docker/cmsis-toolbox-linux-amd64/bin/cbuild /__w/AVH-MLOps/AVH-MLOps/QeexoAutoML/MLOps.csolution.yml --packs --context +CM7 --toolchain AC6" diff --git a/QeexoAutoML/Include/QxAutoMLUser.h b/QeexoAutoML/Pack/Lib/QxAutoMLUser.h similarity index 100% rename from QeexoAutoML/Include/QxAutoMLUser.h rename to QeexoAutoML/Pack/Lib/QxAutoMLUser.h diff --git a/QeexoAutoML/Lib/libQxClassifyEngine.a b/QeexoAutoML/Pack/Lib/libQxClassifyEngine.a similarity index 100% rename from QeexoAutoML/Lib/libQxClassifyEngine.a rename to QeexoAutoML/Pack/Lib/libQxClassifyEngine.a diff --git a/QeexoAutoML/Pack/Qeexo.QxLib_PID12345.pdsc b/QeexoAutoML/Pack/Qeexo.QxLib_PID12345.pdsc new file mode 100644 index 0000000..76995a7 --- /dev/null +++ b/QeexoAutoML/Pack/Qeexo.QxLib_PID12345.pdsc @@ -0,0 +1,20 @@ + + + Qeexo + QxLib_PID12345 + 1.0.0 + Qeexo Library Pack Project ID 12345 + + + + + + + + + + + + diff --git a/QeexoAutoML/Pack/QxLib_PID12345/config.yml b/QeexoAutoML/Pack/QxLib_PID12345/config.yml new file mode 100644 index 0000000..ed3ff71 --- /dev/null +++ b/QeexoAutoML/Pack/QxLib_PID12345/config.yml @@ -0,0 +1,15 @@ +name: QxLib_PID12345 +vendor: Qeexo +version: "0.0" + +pdsc: ./contributions/merge/Qeexo.QxLib_PID12345.pdsc +add: ./contributions/add +build: build +out: ./ + +remote_source: #https://github.com/google/flatbuffers/archive/master.zip +local_source: ../Lib/ +src_dest: ./Lib/ + + + diff --git a/QeexoAutoML/Pack/QxLib_PID12345/contributions/add/LICENSE.txt b/QeexoAutoML/Pack/QxLib_PID12345/contributions/add/LICENSE.txt new file mode 100644 index 0000000..8da1b89 --- /dev/null +++ b/QeexoAutoML/Pack/QxLib_PID12345/contributions/add/LICENSE.txt @@ -0,0 +1 @@ + EULA of the library \ No newline at end of file diff --git a/QeexoAutoML/Pack/QxLib_PID12345/contributions/merge/Qeexo.QxLib_PID12345.pdsc.tpl b/QeexoAutoML/Pack/QxLib_PID12345/contributions/merge/Qeexo.QxLib_PID12345.pdsc.tpl new file mode 100644 index 0000000..a8dcd9f --- /dev/null +++ b/QeexoAutoML/Pack/QxLib_PID12345/contributions/merge/Qeexo.QxLib_PID12345.pdsc.tpl @@ -0,0 +1,22 @@ + + + + Qeexo + QxLib_PID12345 + + + Qeexo AutoML project ID 12345 %{RELEASE_VERSION}% + + + http://www.qeexo.com/ + Qeexo Library Pack Project ID 12345 + + + AutoML Library Project ID 12345 + + + + + + + diff --git a/QeexoAutoML/Pack/gen_pack.py b/QeexoAutoML/Pack/gen_pack.py new file mode 100644 index 0000000..5be451f --- /dev/null +++ b/QeexoAutoML/Pack/gen_pack.py @@ -0,0 +1,194 @@ +# Version: 0.5 +# Date: 2021-05-25 +# This python script generates a CMSIS Software Pack +# +# Requirements: +# python 3.6+ +# +# Usage: +# +# python3 gen_pack.py +# +# required: +# --path - Path cotaining the config.yml and all assets +# +# optional: +# --version - Specify a custom version number +# --date-tag - Tag the version with todays date +# config.yml - Pack build configuration +# +# +# + + +import os, platform +import requests +import zipfile +import shutil +import subprocess +import time +import datetime + +import yaml +import argparse +import re + +import distutils +from distutils import dir_util + +import semantic_version + +def sanitize_SemVer(unsanitized): + """Uses a whitelist to avoid generating bad SemVer.""" + return str(semantic_version.Version.coerce(unsanitized)) + +def main(unparsed_args, flags): + print (flags) + + with open( os.path.abspath(flags.path + "/config.yml"), "r") as ymlfile: + cfg = yaml.safe_load(ymlfile) + + if shutil.which("packchk") is None: + print("Error: packchk is not available on path") + exit() + + PACK_BUILD = cfg["build"] + pack_path = os.path.realpath(os.path.join(os.path.abspath(flags.path), PACK_BUILD)) + print(pack_path) + + CONTRIB_ADD = cfg["add"] + add_path = os.path.realpath(os.path.join(os.path.abspath(flags.path), CONTRIB_ADD)) + print(add_path) + + SOURCE_ADD = cfg["local_source"] + src_path = os.path.realpath(os.path.join(os.path.abspath(flags.path), SOURCE_ADD)) + src_dest = os.path.realpath(os.path.join(pack_path, cfg["src_dest"])) + print(src_path) + + OUTPATH = cfg["out"] + out_path = os.path.realpath(os.path.join(os.path.abspath(flags.path), OUTPATH)) + print(out_path) + + PDSCPATH = cfg["pdsc"] + pdsc_path = os.path.realpath(os.path.join(os.path.abspath(flags.path), PDSCPATH)) + print(pdsc_path) + + print (">>> Preparing build environment directories.") + if os.path.isdir(pack_path): + shutil.rmtree(pack_path) + os.mkdir(pack_path) + + print (">>> Determining version string.") + pack_version = cfg["version"] + print (cfg["version"]) + if flags.version is not None: + pack_version = flags.version + now = datetime.datetime.now() + calversion = datetime.datetime.today().strftime('%Y%m%d') + tmpl_pdsc_date = now.strftime('%Y-%m-%d') + if flags.date_tag: + pack_version = pack_version + "." + calversion + elif flags.candidate_rev: + pack_version = pack_version + "-" + flags.candidate_rev + else: + pack_version = pack_version + pack_version = sanitize_SemVer(pack_version) + print(">>> Version: ", pack_version) + + if cfg["local_source"] is not None: + print (">>> Merging local source:") + print (">>> ", src_path, "==>" ,src_dest) + os.mkdir(src_dest) + distutils.dir_util.copy_tree(src_path, src_dest, verbose=1) + + if cfg["remote_source"] is not None: + print (">>> Merging remote source:") + print (">>> ", cfg["remote_source"], "==>" ,src_dest) + upstream_tmp = requests.get(cfg["remote_source"]) + open("./download.zip", 'wb').write(upstream_tmp.content) + with zipfile.ZipFile("./download.zip", 'r') as zip_ref: + zip_ref.extractall(src_dest) + + if cfg["add"] is not None: + print (">>> Merging additions.") + print (">>> ", add_path, "==>" ,pack_path) + distutils.dir_util.copy_tree(add_path, pack_path) + + history_str = "" + # read file into string + if flags.history != "": + with open(flags.history, "r") as history_file: + history_str = history_file.read() + history_str = history_str.replace("tensorflow.tensorflow-lite-micro", cfg["vendor"]+"."+cfg["name"]) + print(history_str) + + if cfg["pdsc"] is not None: + print (">>> Merging pdsc.") + new_pdsc_path = os.path.join(pack_path , os.path.basename(pdsc_path)) + #load pdsc template from ../templates + with open(pdsc_path+".tpl", 'r') as input_template_file: + template_file_text = input_template_file.read() + pdsc_path = new_pdsc_path + template_file_text = re.sub(r'%{RELEASE_VERSION}%', pack_version, template_file_text) + template_file_text = re.sub(r'%{RELEASE_DATE}%', tmpl_pdsc_date, template_file_text) + template_file_text = re.sub(r'%{HISTORY}%', history_str, template_file_text) + with open(pdsc_path, 'w') as output_file: + output_file.write(template_file_text) + print(template_file_text) + + print (">>> Running PackCheck") + command = "packchk -n PackName.txt " + pdsc_path + print (">>>", command) + os.system(command) + + with open (os.getcwd() + "/PackName.txt", "r") as packversion_file: + packfile_name=packversion_file.readline() + + os.chdir(pack_path) + + print ("Creating zip package: " + packfile_name) + with zipfile.ZipFile(packfile_name,"w",zipfile.ZIP_DEFLATED,allowZip64=True) as zf: + for root, _, filenames in os.walk('./'): + for name in filenames: + name = os.path.join(root, name) + name = os.path.normpath(name) + zf.write(name, name) + + +def parse_args(): + """Converts the raw arguments into accessible flags.""" + parser = argparse.ArgumentParser(usage='Use "python %(prog)s --help" for more information', formatter_class=argparse.HelpFormatter) + parser.register('type', 'bool', lambda v: v.lower() == 'true') + parser.add_argument( + '--path', + type=str, + default='', + help='Path to pack assets.') + parser.add_argument( + '--version', + type=str, + default=None, + help='Set or override version with custom setting.') + parser.add_argument( + '--candidate_rev', + type=str, + default="", + help='Release candidate versioning, e.g. rc1, rc2, etc.') + parser.add_argument( + '--date_tag', + default=False, + action='store_true', + help='Tag versions with date.') + parser.add_argument( + '--history', + type=str, + default="", + help='Release history.') + + flags, unparsed_args = parser.parse_known_args() + + main(unparsed_args, flags) + +if __name__ == '__main__': + parse_args() + diff --git a/QeexoAutoML/Test/ML_Test.cproject.yml b/QeexoAutoML/Test/ML_Test.cproject.yml index 449ae25..c355d34 100644 --- a/QeexoAutoML/Test/ML_Test.cproject.yml +++ b/QeexoAutoML/Test/ML_Test.cproject.yml @@ -5,11 +5,13 @@ project: packs: - pack: ARM::CMSIS@>=6.0.0-dev0 - pack: ARM::CMSIS-View@1.0.0 + - pack: ARM::CMSIS-DSP@1.15.0 - pack: ARM::CMSIS-RTX@>=1.0.0-dev0 + - pack: Qeexo::QxLib_PID12345 + add-path: - - ../Include - ./sensor_data components: @@ -31,6 +33,7 @@ project: - ARM_MATH_AUTOVECTORIZE - component: ARM::CMSIS:RTOS2:Keil RTX5&Source - component: CMSIS-View:Event Recorder&Semihosting + - component: Qeexo::Machine Learning:Qeexo AutoML:Qeexo Custom Library PID12345 groups: - group: main @@ -44,7 +47,6 @@ project: - group: QeexoAutoML files: - file: ./src/QxAutoMLWork.c - - file: ../Lib/libQxClassifyEngine.a layers: - layer: $Target-Layer$