Add --template param to solution creation (#194)

* Use env key instead of value when adding modules * Add CLI entrypoint * Call iotedgedev addmodule when create solution * Add default routes when adding modules * Update template * Update test * Revert changes to monitor timeout * Add temp sensor route when adding module * Add copy_template function * Add default ToIoTHub route when creating solution * Update system module images to GA version * Fix incorrect string escape * Replace image placeholder with real image URL when building images * Enable Node.js module creation * Refind the logic to parse image placeholder * Won't add tempsensor route when adding modules * Minor refinement * Add nested_set utility method * Add default route from temp sensor when solution creation * Rename var_dict to replacement * Use name in env as default when new solution * WIP support for BYPASS_MODULES * WIP support for BYPASS_MODULES * WIP support for BYPASS_MODULES * Update utility methods * Update image_tag_map key type to tuple * Update Docker SDK version and remove iotedgeruntime from requirements.txt * Disable outdated test cases temporarily * Add unit test for deploymentmanifest.py * Add unit test for utility.py * Fix error on Py27 * Compare lists order-insensitively * Fix PyTest failure on Python 3
Azure · Jul 19, 2018 · 797ae78 · 797ae78
1 parent 57f87c2
commit 797ae78
Show file tree

Hide file tree

Showing 36 changed files with 660 additions and 424 deletions.
diff --git a/.env.tmp b/.env.tmp
@@ -39,7 +39,7 @@ RUNTIME_CONFIG_DIR="."
 RUNTIME_HOST_NAME="."
     # "." - Auto detect
 
-RUNTIME_TAG="1.0-preview"
+RUNTIME_TAG="1.0"
 
 RUNTIME_VERBOSITY="INFO"
     # "DEBUG", "INFO", "ERROR", "WARNING"
@@ -51,9 +51,9 @@ RUNTIME_LOG_LEVEL="info"
 # MODULES
 #
 
-ACTIVE_MODULES="*" 
-    # "*" - to build all modules
-    # "filtermodule, module1" - Comma delimited list of modules to build
+BYPASS_MODULES=""
+    # "" - to build all modules
+    # "filtermodule, module1" - Comma delimited list of modules to bypass when building
 
 ACTIVE_DOCKER_PLATFORMS="amd64"
     # "*" - to build all docker files

diff --git a/iotedgedev/cli.py b/iotedgedev/cli.py
@@ -61,14 +61,25 @@ def main(set_config, az_cli=None):
               required=False,
               help="Creates a new Azure IoT Edge Solution. Use `--create .` to create in current folder. Use `--create TEXT` to create in a subfolder.")
 @click.argument("name", required=False)
-def solution(create, name):
+@click.option('--module',
+              required=False,
+              default=envvars.get_envvar("DEFAULT_MODULE_NAME", default="filtermodule"),
+              show_default=True,
+              help="Specify the name of the default IoT Edge module.")
+@click.option("--template",
+              default="csharp",
+              show_default=True,
+              required=False,
+              type=click.Choice(["csharp", "nodejs", "python", "csharpfunction"]),
+              help="Specify the template used to create the default IoT Edge module.")
+def solution(create, name, module, template):
 
     utility = Utility(envvars, output)
     sol = Solution(output, utility)
     if name:
-        sol.create(name)
+        sol.create(name, module, template)
     elif create:
-        sol.create(create)
+        sol.create(create, module, template)
 
 
 @click.command(context_settings=CONTEXT_SETTINGS, help="Creates Solution and Azure Resources")
@@ -107,7 +118,7 @@ def e2e(ctx):
                 required=True)
 @click.option("--template",
               required=True,
-              type=click.Choice(["csharp", "python", "csharpfunction"]),
+              type=click.Choice(["csharp", "nodejs", "python", "csharpfunction"]),
               help="Specify the template used to create the new IoT Edge module.")
 @click.pass_context
 def addmodule(ctx, name, template):
@@ -444,7 +455,7 @@ def azure(setup,
 @click.option("--template",
               default="csharp",
               required=False,
-              type=click.Choice(["csharp", "python", "csharpfunction"]),
+              type=click.Choice(["csharp", "nodejs", "python", "csharpfunction"]),
               help="Specify the template used to create the new IoT Edge module.")
 @click.option('--build',
               default=False,

diff --git a/iotedgedev/deploymentmanifest.py b/iotedgedev/deploymentmanifest.py
@@ -10,12 +10,13 @@
 
 
 class DeploymentManifest:
-    def __init__(self, envvars, output, path, is_template):
+    def __init__(self, envvars, output, utility, path, is_template):
+        self.utility = utility
         self.output = output
         try:
             self.path = path
             self.is_template = is_template
-            self.json = json.load(open(path))
+            self.json = json.loads(self.utility.get_file_contents(path, expand_env=True))
         except FileNotFoundError:
             self.output.error('Deployment manifest template file "{0}" not found'.format(path))
             if is_template:
@@ -27,7 +28,7 @@ def __init__(self, envvars, output, path, is_template):
                         envvars.save_envvar("DEPLOYMENT_CONFIG_TEMPLATE_FILE", path)
             else:
                 self.output.error('Deployment manifest file "{0}" not found'.format(path))
-                sys.exit()
+                sys.exit(1)
 
     def add_module_template(self, module_name):
         """Add a module template to the deployment manifest with amd64 as the default platform"""
@@ -37,12 +38,36 @@ def add_module_template(self, module_name):
             "status": "running",
             "restartPolicy": "always",
             "settings": {
-              "image": \"{MODULES.""" + module_name + """.amd64}\",
+              "image": \"${MODULES.""" + module_name + """.amd64}\",
               "createOptions": ""
             }
         }"""
 
-        self.json["moduleContent"]["$edgeAgent"]["properties.desired"]["modules"][module_name] = json.loads(new_module)
+        self.utility.nested_set(self.json, ["moduleContent", "$edgeAgent", "properties.desired", "modules", module_name], json.loads(new_module))
+
+        self.add_default_route(module_name)
+
+    def add_default_route(self, module_name):
+        """Add a default route to send messages to IoT Hub"""
+        new_route_name = "{0}ToIoTHub".format(module_name)
+        new_route = "FROM /messages/modules/{0}/outputs/* INTO $upstream".format(module_name)
+
+        self.utility.nested_set(self.json, ["moduleContent", "$edgeHub", "properties.desired", "routes", new_route_name], new_route)
+
+    def get_modules_to_process(self):
+        """Get modules to process from deployment manifest template"""
+        user_modules = self.json.get("moduleContent", {}).get("$edgeAgent", {}).get("properties.desired", {}).get("modules", {})
+        modules_to_process = []
+        for _, module_info in user_modules.items():
+            image = module_info.get("settings", {}).get("image", "")
+            # If the image is placeholder, e.g., ${MODULES.NodeModule.amd64}, parse module folder and platform from the placeholder
+            if image.startswith("${") and image.endswith("}") and len(image.split(".")) > 2:
+                first_dot = image.index(".")
+                second_dot = image.index(".", first_dot + 1)
+                module_dir = image[first_dot+1:second_dot]
+                module_platform = image[second_dot+1:image.index("}")]
+                modules_to_process.append((module_dir, module_platform))
+        return modules_to_process
 
     def save(self):
         """Dump the JSON to the disk"""

diff --git a/iotedgedev/dockercls.py b/iotedgedev/dockercls.py
@@ -23,6 +23,9 @@ def __init__(self, envvars, utility, output):
             self.docker_client = docker.from_env()
             self.docker_api = docker.APIClient()
 
+    def get_os_type(self):
+        return self.docker_client.info()["OSType"].lower()
+
     def init_registry(self):
 
         self.output.header("INITIALIZING CONTAINER REGISTRY")

diff --git a/iotedgedev/envvars.py b/iotedgedev/envvars.py
@@ -109,7 +109,7 @@ def load(self, force=False):
                 self.RUNTIME_CONFIG_DIR = self.get_envvar("RUNTIME_CONFIG_DIR", default=".")
                 if self.RUNTIME_CONFIG_DIR == ".":
                     self.set_envvar("RUNTIME_CONFIG_DIR", self.get_runtime_config_dir())
-                self.ACTIVE_MODULES = self.get_envvar("ACTIVE_MODULES")
+                self.BYPASS_MODULES = self.get_envvar("BYPASS_MODULES")
                 self.ACTIVE_DOCKER_PLATFORMS = self.get_envvar("ACTIVE_DOCKER_PLATFORMS", altkeys=["ACTIVE_DOCKER_ARCH"])
                 self.CONTAINER_REGISTRY_SERVER = self.get_envvar("CONTAINER_REGISTRY_SERVER")
                 self.CONTAINER_REGISTRY_USERNAME = self.get_envvar("CONTAINER_REGISTRY_USERNAME")

diff --git a/iotedgedev/module.py b/iotedgedev/module.py
@@ -1,6 +1,5 @@
-
-import os
 import json
+import os
 import sys
 
 
@@ -17,8 +16,7 @@ def __init__(self, output, utility, module_json_file):
     def load_module_json(self):
         if os.path.exists(self.module_json_file):
             try:
-                self.file_json_content = json.loads(
-                    self.utility.get_file_contents(self.module_json_file))
+                self.file_json_content = json.loads(self.utility.get_file_contents(self.module_json_file, expand_env=True))
 
                 self.module_language = self.file_json_content.get(
                     "language").lower()
@@ -37,15 +35,21 @@ def language(self):
 
     @property
     def platforms(self):
-        return self.file_json_content.get("image").get("tag").get("platforms")
+        return self.file_json_content.get("image", {}).get("tag", {}).get("platforms", "")
 
     @property
     def tag_version(self):
-        tag = self.file_json_content.get("image").get("tag").get("version")
-        if tag == "":
-            tag = "0.0.0"
+        tag = self.file_json_content.get("image", {}).get("tag", {}).get("version", "0.0.0")
 
         return tag
 
-    def get_platform_by_key(self, platform):
-        return self.file_json_content.get("image").get("tag").get("platforms").get(platform)
+    @property
+    def repository(self):
+        return self.file_json_content.get("image", {}).get("repository", "")
+
+    @property
+    def build_options(self):
+        return self.file_json_content.get("image", {}).get("buildOptions", [])
+
+    def get_dockerfile_by_platform(self, platform):
+        return self.file_json_content.get("image", {}).get("tag", {}).get("platforms", {}).get(platform, "")
diff --git a/iotedgedev/modules.py b/iotedgedev/modules.py
@@ -1,17 +1,16 @@
 import os
 import re
+import sys
 
 from .deploymentmanifest import DeploymentManifest
 from .dotnet import DotNet
 from .module import Module
-from .modulesprocessorfactory import ModulesProcessorFactory
 
 
 class Modules:
     def __init__(self, envvars, utility, output, dock):
         self.envvars = envvars
         self.utility = utility
-        self.utility.set_config()
         self.output = output
         self.dock = dock
         self.dock.init_registry()
@@ -30,15 +29,15 @@ def add(self, name, template):
             self.output.error("Module \"{0}\" already exists under {1}".format(name, os.path.abspath(self.envvars.MODULES_PATH)))
             return
 
-        deployment_manifest = DeploymentManifest(self.envvars, self.output, self.envvars.DEPLOYMENT_CONFIG_TEMPLATE_FILE, True)
+        deployment_manifest = DeploymentManifest(self.envvars, self.output, self.utility, self.envvars.DEPLOYMENT_CONFIG_TEMPLATE_FILE, True)
 
-        repo = "{0}/{1}".format(self.envvars.CONTAINER_REGISTRY_SERVER, name.lower())
+        repo = "{0}/{1}".format("${CONTAINER_REGISTRY_SERVER}", name.lower())
         if template == "csharp":
             dotnet = DotNet(self.envvars, self.output, self.utility)
             dotnet.install_module_template()
             dotnet.create_custom_module(name, repo, cwd)
         elif template == "nodejs":
-            self.utility.check_dependency("yo azure-iot-edge-module --help".split(), "To add new Node.js modules, the Yeoman tool and Azure IoT Edge Node.js module generator")
+            self.utility.check_dependency("yo azure-iot-edge-module --help".split(), "To add new Node.js modules, the Yeoman tool and Azure IoT Edge Node.js module generator", shell=True)
             cmd = "yo azure-iot-edge-module -n {0} -r {1}".format(name, repo)
             self.output.header(cmd)
             self.utility.exe_proc(cmd.split(), shell=True, cwd=cwd)
@@ -66,72 +65,97 @@ def push(self, no_build=False):
         self.build_push(no_build=no_build)
 
     def build_push(self, no_build=False, no_push=False):
-
         self.output.header("BUILDING MODULES", suppress=no_build)
 
-        # Get all the modules to build as specified in config.
-        modules_to_process = self.utility.get_active_modules()
+        bypass_modules = self.utility.get_bypass_modules()
+        active_platform = self.utility.get_active_docker_platform()
+
+        # map (module name, platform) tuple to tag.
+        # sample: (('filtermodule', 'amd64'), 'localhost:5000/filtermodule:0.0.1-amd64')
+        image_tag_map = {}
+        # map image tag to (module name, dockerfile) tuple
+        # sample: ('localhost:5000/filtermodule:0.0.1-amd64', ('filtermodule', '/test_solution/modules/filtermodule/Dockerfile.amd64'))
+        tag_dockerfile_map = {}
+        # map image tag to build options
+        # sample: ('localhost:5000/filtermodule:0.0.1-amd64', ["--add-host=github.com:192.30.255.112"])
+        tag_build_options_map = {}
+        # image tags to build
+        # sample: 'localhost:5000/filtermodule:0.0.1-amd64'
+        tags_to_build = set()
 
         for module in os.listdir(self.envvars.MODULES_PATH):
-
-            if len(modules_to_process) == 0 or modules_to_process[0] == "*" or module in modules_to_process:
-
+            if module not in bypass_modules:
                 module_dir = os.path.join(self.envvars.MODULES_PATH, module)
-
-                self.output.info("BUILDING MODULE: {0}".format(module_dir), suppress=no_build)
-
                 module_json = Module(self.output, self.utility, os.path.join(module_dir, "module.json"))
-                mod_proc = ModulesProcessorFactory(self.envvars, self.utility, self.output, module_dir).get(module_json.language)
-
-                # build module
+                for platform in module_json.platforms:
+                    # get the Dockerfile from module.json
+                    dockerfile = os.path.abspath(os.path.join(module_dir, module_json.get_dockerfile_by_platform(platform)))
+                    container_tag = "" if self.envvars.CONTAINER_TAG == "" else "-" + self.envvars.CONTAINER_TAG
+                    tag = "{0}:{1}{2}-{3}".format(module_json.repository, module_json.tag_version, container_tag, platform).lower()
+                    image_tag_map[(module, platform)] = tag
+                    tag_dockerfile_map[tag] = (module, dockerfile)
+                    tag_build_options_map[tag] = module_json.build_options
+                    if len(active_platform) > 0 and (active_platform[0] == "*" or platform in active_platform):
+                        tags_to_build.add(tag)
+
+        deployment_manifest = DeploymentManifest(self.envvars, self.output, self.utility, self.envvars.DEPLOYMENT_CONFIG_TEMPLATE_FILE, True)
+        modules_to_process = deployment_manifest.get_modules_to_process()
+
+        replacements = {}
+        for module, platform in modules_to_process:
+            if module not in bypass_modules:
+                key = (module, platform)
+                if key in image_tag_map:
+                    tag = image_tag_map.get(key)
+                    tags_to_build.add(tag)
+                    replacements["${{MODULES.{0}.{1}}}".format(module, platform)] = tag
+
+        for tag in tags_to_build:
+            if tag in tag_dockerfile_map:
+                module = tag_dockerfile_map.get(tag)[0]
+                dockerfile = tag_dockerfile_map.get(tag)[1]
+                self.output.info("BUILDING MODULE: {0}".format(module), suppress=no_build)
+                self.output.info("PROCESSING DOCKERFILE: {0}".format(dockerfile), suppress=no_build)
+                self.output.info("BUILDING DOCKER IMAGE: {0}".format(tag), suppress=no_build)
+
+                # BUILD DOCKER IMAGE
                 if not no_build:
-                    if not mod_proc.build():
-                        continue
-
-                docker_arch_process = [docker_arch.strip() for docker_arch in self.envvars.ACTIVE_DOCKER_PLATFORMS.split(",") if docker_arch]
-
-                for arch in module_json.platforms:
-                    if len(docker_arch_process) == 0 or docker_arch_process[0] == "*" or arch in docker_arch_process:
-
-                        # get the docker file from module.json
-                        docker_file = module_json.get_platform_by_key(arch)
-
-                        self.output.info("PROCESSING DOCKER FILE: " + docker_file, suppress=no_build)
-
-                        docker_file_name = os.path.basename(docker_file)
-                        container_tag = "" if self.envvars.CONTAINER_TAG == "" else "-" + self.envvars.CONTAINER_TAG
-                        tag_name = module_json.tag_version + container_tag
-
-                        # publish module
-                        if not no_build:
-                            self.output.info("PUBLISHING MODULE: " + module_dir)
-                            mod_proc.publish()
-
-                        image_destination_name = "{0}/{1}:{2}-{3}".format(self.envvars.CONTAINER_REGISTRY_SERVER, module, tag_name, arch).lower()
-
-                        self.output.info("BUILDING DOCKER IMAGE: " + image_destination_name, suppress=no_build)
-
-                        # cd to the module folder to build the docker image
-                        project_dir = os.getcwd()
-                        os.chdir(os.path.join(project_dir, module_dir))
-
-                        # BUILD DOCKER IMAGE
-
-                        if not no_build:
-                            build_result = self.dock.docker_client.images.build(tag=image_destination_name, path=".", dockerfile=docker_file_name, buildargs={"EXE_DIR": mod_proc.exe_dir})
-
-                            self.output.info("DOCKER IMAGE DETAILS: {0}".format(build_result))
-
-                        # CD BACK UP
-                        os.chdir(project_dir)
-
-                        if not no_push:
-                            # PUSH TO CONTAINER REGISTRY
-                            self.output.info("PUSHING DOCKER IMAGE TO: " + image_destination_name)
-
-                            for line in self.dock.docker_client.images.push(repository=image_destination_name, stream=True, auth_config={
-                                                                            "username": self.envvars.CONTAINER_REGISTRY_USERNAME, "password": self.envvars.CONTAINER_REGISTRY_PASSWORD}):
-                                self.output.procout(self.utility.decode(line).replace("\\u003e", ">"))
-
-                self.output.footer("BUILD COMPLETE", suppress=no_build)
-                self.output.footer("PUSH COMPLETE", suppress=no_push)
+                    # TODO: apply build options
+                    build_options = self.filter_build_options(tag_build_options_map.get(tag, None))
+
+                    context_path = os.path.abspath(os.path.join(self.envvars.MODULES_PATH, module))
+                    dockerfile_relative = os.path.relpath(dockerfile, context_path)
+                    # a hack to workaround Python Docker SDK's bug with Linux container mode on Windows
+                    if self.dock.get_os_type() == "linux" and sys.platform == "win32":
+                        dockerfile = dockerfile.replace("\\", "/")
+                        dockerfile_relative = dockerfile_relative.replace("\\", "/")
+
+                    build_result = self.dock.docker_client.images.build(tag=tag, path=context_path, dockerfile=dockerfile_relative)
+
+                    self.output.info("DOCKER IMAGE DETAILS: {0}".format(build_result))
+
+                if not no_push:
+                    # PUSH TO CONTAINER REGISTRY
+                    self.output.info("PUSHING DOCKER IMAGE: " + tag)
+
+                    for line in self.dock.docker_client.images.push(repository=tag, stream=True, auth_config={
+                            "username": self.envvars.CONTAINER_REGISTRY_USERNAME, "password": self.envvars.CONTAINER_REGISTRY_PASSWORD}):
+                        self.output.procout(self.utility.decode(line).replace("\\u003e", ">"))
+            self.output.footer("BUILD COMPLETE", suppress=no_build)
+            self.output.footer("PUSH COMPLETE", suppress=no_push)
+        self.utility.set_config(force=True, replacements=replacements)
+
+    @staticmethod
+    def filter_build_options(build_options):
+        """Remove build options which will be ignored"""
+        if build_options is None:
+            return None
+
+        filtered_build_options = []
+        for build_option in build_options:
+            build_option = build_option.strip()
+            parsed_option = re.compile(r"\s+").split(build_option)
+            if parsed_option and ["--rm", "--tag", "-t", "--file", "-f"].index(parsed_option[0]) < 0:
+                filtered_build_options.append(build_option)
+
+        return filtered_build_options