Add support for custom build runners

doc/user/build_runners.rst

@@ -0,0 +1,45 @@
+Build runners
+=============
+
+Edalize will by default create a Makefile which it subsequently executes to offload the process of keeping track of what needs to be rebuilt when source files change. In Edalize we call make a build runner since it runs the build process.
+
+In some cases it can be necessary to augment or completely replace the makefile generation, e.g. to initialize some environment before launching or to integrate with a custom build system.
+
+It is therefore possible to replace the default build runner with a custom tool by creating a new module under the `edalize.build_runner` namespace. In this module Edalize expects to find a class with the same name as the module but capitalized. This class should have the following three functions defined.
+
+**__init__(self, flow_options)** Constructor that also receives the flow_options defined.
+
+**get_build_command(self)** Returns a tuple where the first element is the command to launch (e.g. `make` if we are executing a Makefile) and the second element is a list of options to send to the command (e.g. `["-j4", "-d"]` for a `make` process).
+
+**write(self, commands: EdaCommands, work_root: Path)** Write any required files needed for building. For the `make` build runner, this creates the actual Makefile.
+
+Below is an example of a build runner that extends the `make` build runner to copy the build tree to a server over ssh and the execute it from there.
+
+
+Build runner examplee::
+
+  from typing import List
+  from pathlib import Path
+  
+  from edalize.build_runners.make import Make
+  from edalize.utils import EdaCommands
+  
+  class Sshmake(Make):
+  
+      def __init__(self, flow_options):
+          super().__init__(flow_options)
+          self.build_host = "5446.54.40.138"
+  
+      def get_build_command(self):
+          return ("sh", ["launchme.sh"])
+  
+      def write(self, commands: EdaCommands, work_root: Path):
+          # Write Makefile
+          super().write(commands, work_root)
+  
+          # Write launcher script that copies files to build host and runs it
+          outfile = work_root / Path("launchme.sh")
+          with open(outfile, "w") as f:
+              f.write("#Auto generated by Edalize\n\n")
+              f.write(f"scp -r {work_root} {self.build_host}\n")
+              f.write(f"ssh {self.build_host} make " + ' '.join(self.build_options)+ "\n")

doc/user/index.rst

@@ -11,3 +11,5 @@ The Edalize flow is divided into three stages called `configure`, `build` and `r
 .. include:: configure.rst
 .. include:: build.rst
 .. include:: run.rst
+
+.. include:: build_runners.rst

edalize/build_runners/make.py

@@ -0,0 +1,48 @@
+from typing import List
+from pathlib import Path
+
+from edalize.utils import EdaCommands
+
+
+class Make(object):
+    def __init__(self, flow_options):
+        self.build_options = flow_options.get("flow_make_options", [])
+
+    def get_build_command(self):
+        return ("make", self.build_options)
+
+    def write(self, commands: EdaCommands, work_root: Path):
+        outfile = work_root / Path("Makefile")
+        with open(outfile, "w") as f:
+            f.write("#Auto generated by Edalize\n\n")
+            for v in commands.variables:
+                f.write(v + "\n")
+            if commands.variables:
+                f.write("\n\n")
+            if not commands.default_target:
+                raise RuntimeError("Internal Edalize error. Missing default target")
+
+            f.write(f"all: {commands.default_target}\n")
+
+            for c in commands.commands:
+                f.write(f"\n{' '.join(c.targets)}:")
+                for d in c.depends:
+                    f.write(" " + d)
+                if c.order_only_deps:
+                    f.write(" |")
+                    for d in c.order_only_deps:
+                        f.write(" " + d)
+
+                f.write("\n")
+
+                env_prefix = ""
+                if c.variables:
+                    env_prefix += "env "
+                    for key, value in c.variables.items():
+                        env_prefix += f"{key}={value} "
+
+                for command in c.commands:
+                    if command:
+                        f.write(
+                            f"\t$(EDALIZE_LAUNCHER) {env_prefix}{' '.join([str(x) for x in command])}\n"
+                        )

edalize/flows/edaflow.py

@@ -129,6 +129,10 @@ def get_nodes(self):
 class Edaflow(object):
 
     FLOW_OPTIONS = {
+        "build_runner": {
+            "type": "str",
+            "desc": "Tool to execute the build graph (Defaults to make)",
+        },
         "frontends": {
             "type": "str",
             "desc": "Tools to run before main flow",
@@ -307,6 +311,15 @@ def __init__(self, edam, work_root, verbose=False):
         self.set_run_command()
         self.add_scripts("run", "post_run")
 
+        # Initialize build runner
+        _br = self.flow_options.get("build_runner", "make")
+        try:
+            self.build_runner = getattr(
+                import_module(f"edalize.build_runners.{_br}"), _br.capitalize()
+            )(self.flow_options)
+        except ModuleNotFoundError:
+            raise RuntimeError(f"Could not find build runner '{_br}'")
+
     def set_run_command(self):
         self.commands.add([], ["run"], ["pre_run"])
 
@@ -317,7 +330,7 @@ def configure(self):
             node.inst.configure()
 
         # Write out execution file
-        self.commands.write(os.path.join(self.work_root, "Makefile"))
+        self.build_runner.write(self.commands, self.work_root)
 
     def _run_tool(self, cmd, args=[], cwd=None, quiet=False, env={}):
         logger.debug("Running " + cmd)
@@ -353,9 +366,8 @@ def _run_tool(self, cmd, args=[], cwd=None, quiet=False, env={}):
         return cp.returncode, cp.stdout, cp.stderr
 
     def build(self):
-        # FIXME: Get run command (e.g. make, ninja, cloud thingie..) from self.commands
-        make_options = self.flow_options.get("flow_make_options", [])
-        self._run_tool("make", args=make_options, cwd=self.work_root)
+        (cmd, args) = self.build_runner.get_build_command()
+        self._run_tool(cmd, args=args, cwd=self.work_root)
 
     # Most flows won't have a run phase
     def run(self, args=None):

pyproject.toml

@@ -39,4 +39,4 @@ script-files = ["scripts/el_docker"]
 write_to = "edalize/version.py"
 
 [tool.setuptools.packages.find]
-include = ["edalize", "edalize.tools", "edalize.flows"]
+include = ["edalize", "edalize.tools", "edalize.flows", "edalize.build_runners"]