Cleanup scons mess (#246)

* call autogen once per project

* use jinja for test_runner generation

* use pathlib for pathing

* clean up build.scons

* use emitter to define test.scons targets

* change smoke to its own project type, update scons docs

* use single option for target, fix test output format

* clean up lint format

* fix new target

* update python modules for build

* remove py can msg definition

* remove py-can-msg-gen from build.scons

* print out failed tests

* merge with main, resolve issues

* update build system doc

* unchange can id

* fix sim

* msg for when flash fails

.github/workflows/main.yml

@@ -11,7 +11,7 @@ on:
 
 env:
   # Set defines for builds/tests
-  DEFINES: "LOG_LEVEL_VERBOSITY=LOG_LEVEL_WARN"
+  DEFINES: "LOG_LEVEL=LOG_LEVEL_WARN"
 
 jobs:
   build:

.gitignore

@@ -23,15 +23,6 @@ __pycache__
 *.pyc
 
 # Autogenerated code
-*_getters.h
-*_pack_msg.h
-*_rx_all.c
-*_rx_structs.h
-*_setters.h
-*_tx_all.c
-*_tx_structs.h
-*_unpack_msg.h
 *can_board_ids.h
-*can_codegen.h
 *system_can.dbc
 x86_flash

SConstruct

@@ -1,148 +1,117 @@
-import os
-import sys
+from scons.common import flash_run
 import subprocess
-from scons.common import parse_config
+
+
+def set_target(option, opt, value, parser):
+    if opt == '--project':
+        target = f'projects/{value}'
+    if opt == '--smoke':
+        target = f'smoke/{value}'
+    if opt == '--library':
+        target = f'libraries/{value}'
+    if opt == '--python' or opt == '--py':
+        target = f'py/{value}'
+    setattr(parser.values, option.dest, target)
+
 
 ###########################################################
 # Build arguments
 ###########################################################
-
 AddOption(
     '--platform',
     dest='platform',
-    type='string',
-    action='store',
-    default='arm'
+    type='choice',
+    choices=("arm", "x86"),
+    default='arm',
+    help="Specifies target platform. One of 'arm' or 'x86'. Defaults to 'arm' if not provided."
 )
 
 AddOption(
-    '--project',
-    dest='project',
-    type='string',
-    action='store',
-)
-
-AddOption(
-    '--library',
-    dest='library',
-    type='string',
-    action='store',
-)
-
-AddOption(
-    '--python',
-    dest='python',
-    type='string',
-    action='store'
-)
-AddOption(
-    '--py',
-    dest='python',
-    type='string',
-    action='store'
+    '--library', '--project', '--smoke', '--python', '--py',
+    type="string", dest="name",
+    action='callback', callback=set_target,
+    help="Specify the target.   e.g. --library=ms-common, --project=leds, --smoke=adc, --python=example",
 )
 
 AddOption(
     '--test',
     dest='testfile',
     type='string',
-    action='store'
+    action='store',
+    help="Additionally specify the name of test to run for 'test' command."
 )
 
 AddOption(
     '--define',
     dest='define',
     type='string',
-    action='store'
+    action='store',
+    help="Add CPP defines to a build.   e.g. --define='LOG_LEVEL=LOG_LEVEL_WARN'"
 )
 
 AddOption(
-    '--name',
-    dest='name',
+    '--task',
+    dest='task',
     type='string',
     action='store'
 )
 
-# Adding Memory Report Argument to Environment Flags
-# Note platform needs to be explicitly set to arm
-
 AddOption(
     '--mem-report',
     dest='mem-report',
-    type='string',
-    action='store',
+    action='store_true',
+    help="(arm) Reports the memory space after a build."
+)
+
+AddOption(
+    '--sanitizer',
+    dest='sanitizer',
+    type='choice',
+    choices=("asan", "tsan"),
+    default="none",
+    help="(x86) Specifies the sanitizer. One of 'asan' for Address sanitizer or 'tsan' for Thread sanitizer. Defaults to none."
 )
 
 PLATFORM = GetOption('platform')
-PROJECT = GetOption('project')
-LIBRARY = GetOption('library')
-PYTHON = GetOption('python')
-MEM_REPORT = GetOption('mem-report')
-NAME = GetOption('name')
+TARGET = GetOption('name')
 
 ###########################################################
 # Environment setup
 ###########################################################
 
 # Retrieve the construction environment from the appropriate platform script
-if PLATFORM == 'x86':
-    env = SConscript('platform/x86.py')
-elif PLATFORM == 'arm':
-    env = SConscript('platform/arm.py')
-
-
-# env.VariantDir('build', '.', duplicate=0)
-
-TYPE = None
-if PROJECT:
-    TYPE = 'project'
-elif LIBRARY:
-    TYPE = 'library'
-elif PYTHON:
-    TYPE = 'python'
-TARGET = PROJECT or LIBRARY or PYTHON
+env = SConscript(f'platform/{PLATFORM}.py')
 
 VARS = {
     "PLATFORM": PLATFORM,
-    "TYPE": TYPE,
     "TARGET": TARGET,
-    "NAME": NAME,
     "env": env,
 }
 
-env["VARS"] = VARS
+COMMAND = COMMAND_LINE_TARGETS[0] if COMMAND_LINE_TARGETS else ""
 
-# Add flags when compiling a test
-TEST_CFLAGS = ['-DMS_TEST=1']
-if 'test' in COMMAND_LINE_TARGETS: # are we running "scons test"?
-    env['CCFLAGS'] += TEST_CFLAGS
 # Parse asan / tsan and Adding Sanitizer Argument to Environment Flags
 # Note platform needs to be explicitly set to x86
 
-AddOption(
-    '--sanitizer',
-    dest='sanitizer',
-    type='string',
-    action='store',
-    default="none"
-)
 SANITIZER = GetOption('sanitizer')
 
 if SANITIZER == 'asan':
-    env['CCFLAGS']   += ["-fsanitize=address"]
-    env['CXXFLAGS']  += ["-fsanitize=address"]
+    env['CCFLAGS'] += ["-fsanitize=address"]
+    env['CXXFLAGS'] += ["-fsanitize=address"]
     env['LINKFLAGS'] += ["-fsanitize=address"]
 elif SANITIZER == 'tsan':
-    env['CCFLAGS']   += ["-fsanitize=thread"]
-    env['CXXFLAGS']  += ["-fsanitize=thread"]
+    env['CCFLAGS'] += ["-fsanitize=thread"]
+    env['CXXFLAGS'] += ["-fsanitize=thread"]
     env['LINKFLAGS'] += ["-fsanitize=thread"]
 
-env['CCCOMSTR']     = "Compiling  $TARGET"
-env['LINKCOMSTR']   = "Linking    $TARGET"
-env['ARCOMSTR']     = "Archiving  $TARGET"
-env['ASCOMSTR']     = "Assembling $TARGET"
+env['CCCOMSTR'] = "Compiling  $TARGET"
+env['ARCOMSTR'] = "Archiving  $TARGET"
+env['ASCOMSTR'] = "Assembling $TARGET"
+env['LINKCOMSTR'] = "Linking    $TARGET"
 env['RANLIBCOMSTR'] = "Indexing   $TARGET"
 
+env.Append(CPPDEFINES=[GetOption('define')])
+
 ###########################################################
 # Directory setup
 ###########################################################
@@ -151,131 +120,77 @@ BUILD_DIR = Dir('#/build').Dir(PLATFORM)
 BIN_DIR = BUILD_DIR.Dir('bin')
 OBJ_DIR = BUILD_DIR.Dir('obj')
 
-PROJ_DIR = Dir('#/projects')
-SMOKE_DIR = Dir('#/smoke')
-
-PLATFORM_DIR = Dir('#/platform')
-
 VariantDir(OBJ_DIR, '.', duplicate=0)
 
-COMMAND = COMMAND_LINE_TARGETS[0] if COMMAND_LINE_TARGETS else None
-
-###########################################################
-# Build
-###########################################################
-if COMMAND == None or COMMAND == "test":
-    SConscript('scons/build.scons', exports='VARS')
-
 ###########################################################
 # Testing
 ###########################################################
 if COMMAND == "test":
+    # Add flags when compiling a test
+    TEST_CFLAGS = ['-DMS_TEST=1']
+    env['CCFLAGS'] += TEST_CFLAGS
     SConscript('scons/test.scons', exports='VARS')
+    SConscript('scons/build.scons', exports='VARS')
 
 ###########################################################
 # Helper targets
 ###########################################################
-if COMMAND == "new_task":
+elif COMMAND == "new":
     SConscript('scons/new_target.scons', exports='VARS')
 
 ###########################################################
 # Clean
 ###########################################################
-# 'clean' is a dummy file that doesn't get created
-# This is required for phony targets for scons to be happy
-Command('#/clean', [], 'rm -rf build/*')
-# Alias('clean', clean)
+elif COMMAND == "clean":
+    AlwaysBuild(Command('#/clean', [], 'rm -rf build/*'))
 
 ###########################################################
 # Linting and Formatting
 ###########################################################
-if COMMAND == "lint" or COMMAND == "format":
+elif COMMAND == "lint" or COMMAND == "format":
     SConscript('scons/lint_format.scons', exports='VARS')
 
-# ELFs are used for gdb and x86
-def proj_elf(proj_name, is_smoke=False):
-    return BIN_DIR.Dir(SMOKE_DIR.name if is_smoke else PROJ_DIR.name).File(proj_name)
-
-# .bin is used for flashing to MCU
-def proj_bin(proj_name, is_smoke=False):
-    return proj_elf(proj_name, is_smoke).File(proj_name + '.bin')
+###########################################################
+# Build
+###########################################################
+else:  # command not recognised, default to build
+    SConscript('scons/build.scons', exports='VARS')
 
 ###########################################################
 # Helper targets for x86
 ###########################################################
-if PLATFORM == 'x86' and TYPE == 'project':
+if PLATFORM == 'x86' and TARGET:
+    project_elf = BIN_DIR.File(TARGET)
     # os.exec the x86 project ELF file to simulate it
-    def sim_run(target, source, env):
-        path = proj_elf(TARGET, env.get("smoke")).path
-        print('Simulating', path)
-        os.execv(path, [path])
 
-    sim = Command('sim.txt', [], sim_run)
-    Depends(sim, proj_elf(TARGET))
-    Alias('sim', sim)
+    def sim_run(target, source, env):
+        print('Simulating', project_elf)
+        subprocess.run([project_elf.path])
 
-    sim_smoke = Command('sim_smoke.txt', [], sim_run, smoke=True)
-    Depends(sim_smoke, proj_elf(TARGET, True))
-    Alias('sim_smoke', sim_smoke)
+    AlwaysBuild(Command('#/sim', project_elf, sim_run))
 
     # open gdb with the elf file
     def gdb_run(target, source, env):
-        path = proj_elf(TARGET, env.get("smoke")).path
-        os.execv('/usr/bin/gdb', ['/usr/bin/gdb', path])
-
-    gdb = Command('gdb.txt', [], gdb_run)
-    Depends(gdb, proj_elf(TARGET))
-    Alias('gdb', gdb)
+        subprocess.run(['/usr/bin/gdb', project_elf.path])
 
-    gdb_smoke = Command('gdb_smoke.txt', [], gdb_run, smoke=True)
-    Depends(gdb_smoke, proj_elf(TARGET, True))
-    Alias('gdb_smoke', gdb_smoke)
+    AlwaysBuild(Command('#/gdb', project_elf, gdb_run))
 
 ###########################################################
 # Helper targets for arm
 ###########################################################
-if PLATFORM == 'arm' and TYPE == 'project':
+if PLATFORM == 'arm' and TARGET:
+    project_bin = BIN_DIR.File(TARGET + '.bin')
     # display memory info for the project
-    if MEM_REPORT:
-        get_mem_report = Action("python3 scons/mem_report.py " + "build/arm/bin/projects/{}".format(TARGET))
-        env.AddPostAction(proj_bin(TARGET, False), get_mem_report)
-
+    if GetOption('mem-report'):
+        AlwaysBuild(Command("#/mem-report", project_bin,
+                            f"python3 scons/mem_report.py build/arm/bin/{TARGET}"))
+        Default("#/mem-report")
+
     # flash the MCU using openocd
-    def flash_run(target, source, env):
-        import serial
-        output = subprocess.check_output(["ls", "/dev/serial/by-id/"])
-        device_path = f"/dev/serial/by-id/{str(output, 'ASCII').strip()}"
-        serialData = serial.Serial(device_path,115200)
-
-        OPENOCD = 'openocd'
-        OPENOCD_SCRIPT_DIR = '/usr/share/openocd/scripts/'
-        PROBE = 'cmsis-dap'
-        OPENOCD_CFG = [
-            OPENOCD,
-            '-s {}'.format(OPENOCD_SCRIPT_DIR),
-            '-f interface/{}.cfg'.format(PROBE),
-            '-f target/stm32f1x.cfg',
-            '-f {}/stm32f1-openocd.cfg'.format(PLATFORM_DIR),
-            '-c "stm32f1x.cpu configure -rtos FreeRTOS"',
-            '-c "stm_flash {}"'.format(proj_bin(TARGET, env.get("smoke"))),
-            '-c shutdown'
-        ]
-        cmd = 'sudo {}'.format(' '.join(OPENOCD_CFG))
-        subprocess.run(cmd, shell=True)
-
+    def flash_run_target(target, source, env):
+        serialData = flash_run(project_bin)
         while True:
             line: str = serialData.readline().decode("utf-8")
             print(line, end='')
-            if line.startswith('OK'):
-                break
-            if line.startswith('FAIL'):
-                fails += 1
-                break
-
-    flash = Command('flash.txt', [], flash_run)
-    Depends(flash, proj_bin(TARGET))
-    Alias('flash', flash)
-
-    flash_smoke = Command('flash_smoke.txt', [], flash_run, smoke=True)
-    Depends(flash_smoke, proj_bin(TARGET, True))
-    Alias('flash_smoke', flash_smoke)
+
+    AlwaysBuild(Command('#/flash', project_bin, flash_run_target))