diff --git a/.env.tmp b/.env.tmp index 4ab41880..f0a04e46 100644 --- a/.env.tmp +++ b/.env.tmp @@ -39,7 +39,7 @@ RUNTIME_CONFIG_DIR="." RUNTIME_HOST_NAME="." # "." - Auto detect -RUNTIME_TAG="1.0-preview" +RUNTIME_TAG="1.0" RUNTIME_VERBOSITY="INFO" # "DEBUG", "INFO", "ERROR", "WARNING" @@ -51,9 +51,9 @@ RUNTIME_LOG_LEVEL="info" # MODULES # -ACTIVE_MODULES="*" - # "*" - to build all modules - # "filtermodule, module1" - Comma delimited list of modules to build +BYPASS_MODULES="" + # "" - to build all modules + # "filtermodule, module1" - Comma delimited list of modules to bypass when building ACTIVE_DOCKER_PLATFORMS="amd64" # "*" - to build all docker files diff --git a/iotedgedev/cli.py b/iotedgedev/cli.py index 29e4d863..aa44fbf8 100644 --- a/iotedgedev/cli.py +++ b/iotedgedev/cli.py @@ -61,14 +61,25 @@ def main(set_config, az_cli=None): required=False, help="Creates a new Azure IoT Edge Solution. Use `--create .` to create in current folder. Use `--create TEXT` to create in a subfolder.") @click.argument("name", required=False) -def solution(create, name): +@click.option('--module', + required=False, + default=envvars.get_envvar("DEFAULT_MODULE_NAME", default="filtermodule"), + show_default=True, + help="Specify the name of the default IoT Edge module.") +@click.option("--template", + default="csharp", + show_default=True, + required=False, + type=click.Choice(["csharp", "nodejs", "python", "csharpfunction"]), + help="Specify the template used to create the default IoT Edge module.") +def solution(create, name, module, template): utility = Utility(envvars, output) sol = Solution(output, utility) if name: - sol.create(name) + sol.create(name, module, template) elif create: - sol.create(create) + sol.create(create, module, template) @click.command(context_settings=CONTEXT_SETTINGS, help="Creates Solution and Azure Resources") @@ -107,7 +118,7 @@ def e2e(ctx): required=True) @click.option("--template", required=True, - type=click.Choice(["csharp", "python", "csharpfunction"]), + type=click.Choice(["csharp", "nodejs", "python", "csharpfunction"]), help="Specify the template used to create the new IoT Edge module.") @click.pass_context def addmodule(ctx, name, template): @@ -444,7 +455,7 @@ def azure(setup, @click.option("--template", default="csharp", required=False, - type=click.Choice(["csharp", "python", "csharpfunction"]), + type=click.Choice(["csharp", "nodejs", "python", "csharpfunction"]), help="Specify the template used to create the new IoT Edge module.") @click.option('--build', default=False, diff --git a/iotedgedev/deploymentmanifest.py b/iotedgedev/deploymentmanifest.py index cd2a7537..9d5028fc 100644 --- a/iotedgedev/deploymentmanifest.py +++ b/iotedgedev/deploymentmanifest.py @@ -10,12 +10,13 @@ class DeploymentManifest: - def __init__(self, envvars, output, path, is_template): + def __init__(self, envvars, output, utility, path, is_template): + self.utility = utility self.output = output try: self.path = path self.is_template = is_template - self.json = json.load(open(path)) + self.json = json.loads(self.utility.get_file_contents(path, expand_env=True)) except FileNotFoundError: self.output.error('Deployment manifest template file "{0}" not found'.format(path)) if is_template: @@ -27,7 +28,7 @@ def __init__(self, envvars, output, path, is_template): envvars.save_envvar("DEPLOYMENT_CONFIG_TEMPLATE_FILE", path) else: self.output.error('Deployment manifest file "{0}" not found'.format(path)) - sys.exit() + sys.exit(1) def add_module_template(self, module_name): """Add a module template to the deployment manifest with amd64 as the default platform""" @@ -37,12 +38,36 @@ def add_module_template(self, module_name): "status": "running", "restartPolicy": "always", "settings": { - "image": \"{MODULES.""" + module_name + """.amd64}\", + "image": \"${MODULES.""" + module_name + """.amd64}\", "createOptions": "" } }""" - self.json["moduleContent"]["$edgeAgent"]["properties.desired"]["modules"][module_name] = json.loads(new_module) + self.utility.nested_set(self.json, ["moduleContent", "$edgeAgent", "properties.desired", "modules", module_name], json.loads(new_module)) + + self.add_default_route(module_name) + + def add_default_route(self, module_name): + """Add a default route to send messages to IoT Hub""" + new_route_name = "{0}ToIoTHub".format(module_name) + new_route = "FROM /messages/modules/{0}/outputs/* INTO $upstream".format(module_name) + + self.utility.nested_set(self.json, ["moduleContent", "$edgeHub", "properties.desired", "routes", new_route_name], new_route) + + def get_modules_to_process(self): + """Get modules to process from deployment manifest template""" + user_modules = self.json.get("moduleContent", {}).get("$edgeAgent", {}).get("properties.desired", {}).get("modules", {}) + modules_to_process = [] + for _, module_info in user_modules.items(): + image = module_info.get("settings", {}).get("image", "") + # If the image is placeholder, e.g., ${MODULES.NodeModule.amd64}, parse module folder and platform from the placeholder + if image.startswith("${") and image.endswith("}") and len(image.split(".")) > 2: + first_dot = image.index(".") + second_dot = image.index(".", first_dot + 1) + module_dir = image[first_dot+1:second_dot] + module_platform = image[second_dot+1:image.index("}")] + modules_to_process.append((module_dir, module_platform)) + return modules_to_process def save(self): """Dump the JSON to the disk""" diff --git a/iotedgedev/dockercls.py b/iotedgedev/dockercls.py index 7cd59a52..147859d2 100644 --- a/iotedgedev/dockercls.py +++ b/iotedgedev/dockercls.py @@ -23,6 +23,9 @@ def __init__(self, envvars, utility, output): self.docker_client = docker.from_env() self.docker_api = docker.APIClient() + def get_os_type(self): + return self.docker_client.info()["OSType"].lower() + def init_registry(self): self.output.header("INITIALIZING CONTAINER REGISTRY") diff --git a/iotedgedev/envvars.py b/iotedgedev/envvars.py index 60b41826..e56f7221 100644 --- a/iotedgedev/envvars.py +++ b/iotedgedev/envvars.py @@ -109,7 +109,7 @@ def load(self, force=False): self.RUNTIME_CONFIG_DIR = self.get_envvar("RUNTIME_CONFIG_DIR", default=".") if self.RUNTIME_CONFIG_DIR == ".": self.set_envvar("RUNTIME_CONFIG_DIR", self.get_runtime_config_dir()) - self.ACTIVE_MODULES = self.get_envvar("ACTIVE_MODULES") + self.BYPASS_MODULES = self.get_envvar("BYPASS_MODULES") self.ACTIVE_DOCKER_PLATFORMS = self.get_envvar("ACTIVE_DOCKER_PLATFORMS", altkeys=["ACTIVE_DOCKER_ARCH"]) self.CONTAINER_REGISTRY_SERVER = self.get_envvar("CONTAINER_REGISTRY_SERVER") self.CONTAINER_REGISTRY_USERNAME = self.get_envvar("CONTAINER_REGISTRY_USERNAME") diff --git a/iotedgedev/module.py b/iotedgedev/module.py index d348c017..f1ec31ad 100644 --- a/iotedgedev/module.py +++ b/iotedgedev/module.py @@ -1,6 +1,5 @@ - -import os import json +import os import sys @@ -17,8 +16,7 @@ def __init__(self, output, utility, module_json_file): def load_module_json(self): if os.path.exists(self.module_json_file): try: - self.file_json_content = json.loads( - self.utility.get_file_contents(self.module_json_file)) + self.file_json_content = json.loads(self.utility.get_file_contents(self.module_json_file, expand_env=True)) self.module_language = self.file_json_content.get( "language").lower() @@ -37,15 +35,21 @@ def language(self): @property def platforms(self): - return self.file_json_content.get("image").get("tag").get("platforms") + return self.file_json_content.get("image", {}).get("tag", {}).get("platforms", "") @property def tag_version(self): - tag = self.file_json_content.get("image").get("tag").get("version") - if tag == "": - tag = "0.0.0" + tag = self.file_json_content.get("image", {}).get("tag", {}).get("version", "0.0.0") return tag - def get_platform_by_key(self, platform): - return self.file_json_content.get("image").get("tag").get("platforms").get(platform) + @property + def repository(self): + return self.file_json_content.get("image", {}).get("repository", "") + + @property + def build_options(self): + return self.file_json_content.get("image", {}).get("buildOptions", []) + + def get_dockerfile_by_platform(self, platform): + return self.file_json_content.get("image", {}).get("tag", {}).get("platforms", {}).get(platform, "") diff --git a/iotedgedev/modules.py b/iotedgedev/modules.py index a2a15d97..d36367e5 100644 --- a/iotedgedev/modules.py +++ b/iotedgedev/modules.py @@ -1,17 +1,16 @@ import os import re +import sys from .deploymentmanifest import DeploymentManifest from .dotnet import DotNet from .module import Module -from .modulesprocessorfactory import ModulesProcessorFactory class Modules: def __init__(self, envvars, utility, output, dock): self.envvars = envvars self.utility = utility - self.utility.set_config() self.output = output self.dock = dock self.dock.init_registry() @@ -30,15 +29,15 @@ def add(self, name, template): self.output.error("Module \"{0}\" already exists under {1}".format(name, os.path.abspath(self.envvars.MODULES_PATH))) return - deployment_manifest = DeploymentManifest(self.envvars, self.output, self.envvars.DEPLOYMENT_CONFIG_TEMPLATE_FILE, True) + deployment_manifest = DeploymentManifest(self.envvars, self.output, self.utility, self.envvars.DEPLOYMENT_CONFIG_TEMPLATE_FILE, True) - repo = "{0}/{1}".format(self.envvars.CONTAINER_REGISTRY_SERVER, name.lower()) + repo = "{0}/{1}".format("${CONTAINER_REGISTRY_SERVER}", name.lower()) if template == "csharp": dotnet = DotNet(self.envvars, self.output, self.utility) dotnet.install_module_template() dotnet.create_custom_module(name, repo, cwd) elif template == "nodejs": - self.utility.check_dependency("yo azure-iot-edge-module --help".split(), "To add new Node.js modules, the Yeoman tool and Azure IoT Edge Node.js module generator") + self.utility.check_dependency("yo azure-iot-edge-module --help".split(), "To add new Node.js modules, the Yeoman tool and Azure IoT Edge Node.js module generator", shell=True) cmd = "yo azure-iot-edge-module -n {0} -r {1}".format(name, repo) self.output.header(cmd) self.utility.exe_proc(cmd.split(), shell=True, cwd=cwd) @@ -66,72 +65,97 @@ def push(self, no_build=False): self.build_push(no_build=no_build) def build_push(self, no_build=False, no_push=False): - self.output.header("BUILDING MODULES", suppress=no_build) - # Get all the modules to build as specified in config. - modules_to_process = self.utility.get_active_modules() + bypass_modules = self.utility.get_bypass_modules() + active_platform = self.utility.get_active_docker_platform() + + # map (module name, platform) tuple to tag. + # sample: (('filtermodule', 'amd64'), 'localhost:5000/filtermodule:0.0.1-amd64') + image_tag_map = {} + # map image tag to (module name, dockerfile) tuple + # sample: ('localhost:5000/filtermodule:0.0.1-amd64', ('filtermodule', '/test_solution/modules/filtermodule/Dockerfile.amd64')) + tag_dockerfile_map = {} + # map image tag to build options + # sample: ('localhost:5000/filtermodule:0.0.1-amd64', ["--add-host=github.com:192.30.255.112"]) + tag_build_options_map = {} + # image tags to build + # sample: 'localhost:5000/filtermodule:0.0.1-amd64' + tags_to_build = set() for module in os.listdir(self.envvars.MODULES_PATH): - - if len(modules_to_process) == 0 or modules_to_process[0] == "*" or module in modules_to_process: - + if module not in bypass_modules: module_dir = os.path.join(self.envvars.MODULES_PATH, module) - - self.output.info("BUILDING MODULE: {0}".format(module_dir), suppress=no_build) - module_json = Module(self.output, self.utility, os.path.join(module_dir, "module.json")) - mod_proc = ModulesProcessorFactory(self.envvars, self.utility, self.output, module_dir).get(module_json.language) - - # build module + for platform in module_json.platforms: + # get the Dockerfile from module.json + dockerfile = os.path.abspath(os.path.join(module_dir, module_json.get_dockerfile_by_platform(platform))) + container_tag = "" if self.envvars.CONTAINER_TAG == "" else "-" + self.envvars.CONTAINER_TAG + tag = "{0}:{1}{2}-{3}".format(module_json.repository, module_json.tag_version, container_tag, platform).lower() + image_tag_map[(module, platform)] = tag + tag_dockerfile_map[tag] = (module, dockerfile) + tag_build_options_map[tag] = module_json.build_options + if len(active_platform) > 0 and (active_platform[0] == "*" or platform in active_platform): + tags_to_build.add(tag) + + deployment_manifest = DeploymentManifest(self.envvars, self.output, self.utility, self.envvars.DEPLOYMENT_CONFIG_TEMPLATE_FILE, True) + modules_to_process = deployment_manifest.get_modules_to_process() + + replacements = {} + for module, platform in modules_to_process: + if module not in bypass_modules: + key = (module, platform) + if key in image_tag_map: + tag = image_tag_map.get(key) + tags_to_build.add(tag) + replacements["${{MODULES.{0}.{1}}}".format(module, platform)] = tag + + for tag in tags_to_build: + if tag in tag_dockerfile_map: + module = tag_dockerfile_map.get(tag)[0] + dockerfile = tag_dockerfile_map.get(tag)[1] + self.output.info("BUILDING MODULE: {0}".format(module), suppress=no_build) + self.output.info("PROCESSING DOCKERFILE: {0}".format(dockerfile), suppress=no_build) + self.output.info("BUILDING DOCKER IMAGE: {0}".format(tag), suppress=no_build) + + # BUILD DOCKER IMAGE if not no_build: - if not mod_proc.build(): - continue - - docker_arch_process = [docker_arch.strip() for docker_arch in self.envvars.ACTIVE_DOCKER_PLATFORMS.split(",") if docker_arch] - - for arch in module_json.platforms: - if len(docker_arch_process) == 0 or docker_arch_process[0] == "*" or arch in docker_arch_process: - - # get the docker file from module.json - docker_file = module_json.get_platform_by_key(arch) - - self.output.info("PROCESSING DOCKER FILE: " + docker_file, suppress=no_build) - - docker_file_name = os.path.basename(docker_file) - container_tag = "" if self.envvars.CONTAINER_TAG == "" else "-" + self.envvars.CONTAINER_TAG - tag_name = module_json.tag_version + container_tag - - # publish module - if not no_build: - self.output.info("PUBLISHING MODULE: " + module_dir) - mod_proc.publish() - - image_destination_name = "{0}/{1}:{2}-{3}".format(self.envvars.CONTAINER_REGISTRY_SERVER, module, tag_name, arch).lower() - - self.output.info("BUILDING DOCKER IMAGE: " + image_destination_name, suppress=no_build) - - # cd to the module folder to build the docker image - project_dir = os.getcwd() - os.chdir(os.path.join(project_dir, module_dir)) - - # BUILD DOCKER IMAGE - - if not no_build: - build_result = self.dock.docker_client.images.build(tag=image_destination_name, path=".", dockerfile=docker_file_name, buildargs={"EXE_DIR": mod_proc.exe_dir}) - - self.output.info("DOCKER IMAGE DETAILS: {0}".format(build_result)) - - # CD BACK UP - os.chdir(project_dir) - - if not no_push: - # PUSH TO CONTAINER REGISTRY - self.output.info("PUSHING DOCKER IMAGE TO: " + image_destination_name) - - for line in self.dock.docker_client.images.push(repository=image_destination_name, stream=True, auth_config={ - "username": self.envvars.CONTAINER_REGISTRY_USERNAME, "password": self.envvars.CONTAINER_REGISTRY_PASSWORD}): - self.output.procout(self.utility.decode(line).replace("\\u003e", ">")) - - self.output.footer("BUILD COMPLETE", suppress=no_build) - self.output.footer("PUSH COMPLETE", suppress=no_push) + # TODO: apply build options + build_options = self.filter_build_options(tag_build_options_map.get(tag, None)) + + context_path = os.path.abspath(os.path.join(self.envvars.MODULES_PATH, module)) + dockerfile_relative = os.path.relpath(dockerfile, context_path) + # a hack to workaround Python Docker SDK's bug with Linux container mode on Windows + if self.dock.get_os_type() == "linux" and sys.platform == "win32": + dockerfile = dockerfile.replace("\\", "/") + dockerfile_relative = dockerfile_relative.replace("\\", "/") + + build_result = self.dock.docker_client.images.build(tag=tag, path=context_path, dockerfile=dockerfile_relative) + + self.output.info("DOCKER IMAGE DETAILS: {0}".format(build_result)) + + if not no_push: + # PUSH TO CONTAINER REGISTRY + self.output.info("PUSHING DOCKER IMAGE: " + tag) + + for line in self.dock.docker_client.images.push(repository=tag, stream=True, auth_config={ + "username": self.envvars.CONTAINER_REGISTRY_USERNAME, "password": self.envvars.CONTAINER_REGISTRY_PASSWORD}): + self.output.procout(self.utility.decode(line).replace("\\u003e", ">")) + self.output.footer("BUILD COMPLETE", suppress=no_build) + self.output.footer("PUSH COMPLETE", suppress=no_push) + self.utility.set_config(force=True, replacements=replacements) + + @staticmethod + def filter_build_options(build_options): + """Remove build options which will be ignored""" + if build_options is None: + return None + + filtered_build_options = [] + for build_option in build_options: + build_option = build_option.strip() + parsed_option = re.compile(r"\s+").split(build_option) + if parsed_option and ["--rm", "--tag", "-t", "--file", "-f"].index(parsed_option[0]) < 0: + filtered_build_options.append(build_option) + + return filtered_build_options diff --git a/iotedgedev/runtime.py b/iotedgedev/runtime.py index e45b834d..3bc7f3e8 100644 --- a/iotedgedev/runtime.py +++ b/iotedgedev/runtime.py @@ -32,4 +32,3 @@ def restart(self): self.dock.remove_modules() self.setup() self.start() - diff --git a/iotedgedev/solution.py b/iotedgedev/solution.py index 43bba9a4..3e945b63 100644 --- a/iotedgedev/solution.py +++ b/iotedgedev/solution.py @@ -1,12 +1,13 @@ import os import zipfile + class Solution: def __init__(self, output, utility): self.output = output self.utility = utility - def create(self, name): + def create(self, name, module, template): if name == ".": dir_path = os.getcwd() else: @@ -25,13 +26,17 @@ def create(self, name): self.output.error("Error while trying to load template.zip") self.output.error(str(ex)) - if name == ".": - name = "" - zipf = zipfile.ZipFile(template_zip) zipf.extractall(name) + self.utility.copy_template(os.path.join(dir_path, "deployment.template.json"), None, {"%MODULE%": module}, False) + os.rename(os.path.join(name, ".env.tmp"), os.path.join(name, ".env")) + + mod_cmd = "iotedgedev addmodule {0} --template {1}".format(module, template) + self.output.header(mod_cmd) + self.utility.call_proc(mod_cmd.split(), cwd=name) + self.output.footer("Azure IoT Edge Solution Created") - if name != "": + if name != ".": self.output.info("Execute 'cd {0}' to navigate to your new solution.".format(name)) diff --git a/iotedgedev/template/.env.tmp b/iotedgedev/template/.env.tmp index 4ab41880..e8a40207 100644 --- a/iotedgedev/template/.env.tmp +++ b/iotedgedev/template/.env.tmp @@ -39,7 +39,7 @@ RUNTIME_CONFIG_DIR="." RUNTIME_HOST_NAME="." # "." - Auto detect -RUNTIME_TAG="1.0-preview" +RUNTIME_TAG="1.0" RUNTIME_VERBOSITY="INFO" # "DEBUG", "INFO", "ERROR", "WARNING" @@ -51,13 +51,14 @@ RUNTIME_LOG_LEVEL="info" # MODULES # -ACTIVE_MODULES="*" - # "*" - to build all modules - # "filtermodule, module1" - Comma delimited list of modules to build +BYPASS_MODULES="" + # "" - to build all modules + # "filtermodule, module1" - Comma delimited list of modules to bypass when building -ACTIVE_DOCKER_PLATFORMS="amd64" - # "*" - to build all docker files - # "amd64,amd64.debug" - Comma delimted list of docker files to build +ACTIVE_DOCKER_PLATFORMS="" + # "" - to only build Dockerfiles specified in DEPLOYMENT_CONFIG_TEMPLATE_FILE + # "*" - to build all Dockerfiles + # "amd64,amd64.debug" - Comma delimited list of Dockerfiles to build CONTAINER_TAG="" diff --git a/iotedgedev/template/deployment.template.json b/iotedgedev/template/deployment.template.json index 2946e7ad..32bcbf17 100644 --- a/iotedgedev/template/deployment.template.json +++ b/iotedgedev/template/deployment.template.json @@ -14,7 +14,7 @@ "edgeAgent": { "type": "docker", "settings": { - "image": "microsoft/azureiotedge-agent:${RUNTIME_TAG}", + "image": "mcr.microsoft.com/azureiotedge-agent:${RUNTIME_TAG}", "createOptions": "" } }, @@ -23,8 +23,8 @@ "status": "running", "restartPolicy": "always", "settings": { - "image": "microsoft/azureiotedge-hub:${RUNTIME_TAG}", - "createOptions": "" + "image": "mcr.microsoft.com/azureiotedge-hub:${RUNTIME_TAG}", + "createOptions": "{\"HostConfig\":{\"PortBindings\":{\"8883/tcp\":[{\"HostPort\":\"8883\"}],\"443/tcp\":[{\"HostPort\":\"443\"}]}}}" } } }, @@ -35,17 +35,7 @@ "status": "running", "restartPolicy": "always", "settings": { - "image": "microsoft/azureiotedge-simulated-temperature-sensor:${RUNTIME_TAG}", - "createOptions": "" - } - }, - "filtermodule": { - "version": "1.0", - "type": "docker", - "status": "running", - "restartPolicy": "always", - "settings": { - "image": "${CONTAINER_REGISTRY_SERVER}/filtermodule:0.0.1-amd64", + "image": "mcr.microsoft.com/azureiotedge-simulated-temperature-sensor:${RUNTIME_TAG}", "createOptions": "" } } @@ -56,19 +46,12 @@ "properties.desired": { "schemaVersion": "1.0", "routes": { - "sensorToFilter": "FROM /messages/modules/temp-sensor-module/outputs/temperatureOutput INTO BrokeredEndpoint(\"/modules/filtermodule/inputs/input1\")", - "filterToIoTHub": "FROM /messages/modules/filtermodule/outputs/output1 INTO $upstream" + "sensorTo%MODULE%": "FROM /messages/modules/tempSensor/outputs/temperatureOutput INTO BrokeredEndpoint(\"/modules/%MODULE%/inputs/input1\")" }, "storeAndForwardConfiguration": { "timeToLiveSecs": 7200 } } - }, - "filtermodule": { - "properties.desired": { - "schemaVersion": "1.0", - "TemperatureThreshold": 21 - } } } } \ No newline at end of file diff --git a/iotedgedev/template/modules/filtermodule/.gitignore b/iotedgedev/template/modules/filtermodule/.gitignore deleted file mode 100644 index 39a824ac..00000000 --- a/iotedgedev/template/modules/filtermodule/.gitignore +++ /dev/null @@ -1,31 +0,0 @@ -# .NET Core -project.lock.json -project.fragment.lock.json -artifacts/ -**/Properties/launchSettings.json - -*_i.c -*_p.c -*_i.h -*.ilk -*.meta -*.obj -*.pch -*.pdb -*.pgc -*.pgd -*.rsp -*.sbr -*.tlb -*.tli -*.tlh -*.tmp -*.tmp_proj -*.log -*.vspscc -*.vssscc -.builds -*.pidb -*.svclog -*.scc -.vs \ No newline at end of file diff --git a/iotedgedev/template/modules/filtermodule/Dockerfile b/iotedgedev/template/modules/filtermodule/Dockerfile deleted file mode 100644 index 3df26fa2..00000000 --- a/iotedgedev/template/modules/filtermodule/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -FROM microsoft/dotnet:2.0-sdk AS build-env -WORKDIR /app - -COPY *.csproj ./ -RUN dotnet restore - -COPY . ./ -RUN dotnet publish -c Release -o out - -FROM microsoft/dotnet:2.0-runtime -WORKDIR /app -COPY --from=build-env /app/out ./ -ENTRYPOINT ["dotnet", "filtermodule.dll"] \ No newline at end of file diff --git a/iotedgedev/template/modules/filtermodule/Dockerfile.amd64.debug b/iotedgedev/template/modules/filtermodule/Dockerfile.amd64.debug deleted file mode 100644 index d003a0d6..00000000 --- a/iotedgedev/template/modules/filtermodule/Dockerfile.amd64.debug +++ /dev/null @@ -1,18 +0,0 @@ -FROM microsoft/dotnet:2.0-sdk AS build-env -WORKDIR /app - -COPY *.csproj ./ -RUN dotnet restore - -COPY . ./ -RUN dotnet publish -c Debug -o out - -FROM microsoft/dotnet:2.0-runtime-stretch -WORKDIR /app -COPY --from=build-env /app/out ./ - -RUN apt-get update -RUN apt-get install -y unzip procps -RUN curl -sSL https://aka.ms/getvsdbgsh | bash /dev/stdin -v latest -l ~/vsdbg - -ENTRYPOINT ["dotnet", "filtermodule.dll"] \ No newline at end of file diff --git a/iotedgedev/template/modules/filtermodule/Dockerfile.arm32v7 b/iotedgedev/template/modules/filtermodule/Dockerfile.arm32v7 deleted file mode 100644 index d5968179..00000000 --- a/iotedgedev/template/modules/filtermodule/Dockerfile.arm32v7 +++ /dev/null @@ -1,5 +0,0 @@ -FROM microsoft/dotnet:2.0.0-runtime-stretch-arm32v7 -ARG EXE_DIR=. -WORKDIR /app -COPY $EXE_DIR/ ./ -CMD ["dotnet", "filtermodule.dll"] \ No newline at end of file diff --git a/iotedgedev/template/modules/filtermodule/Program.cs b/iotedgedev/template/modules/filtermodule/Program.cs deleted file mode 100644 index b5f565b3..00000000 --- a/iotedgedev/template/modules/filtermodule/Program.cs +++ /dev/null @@ -1,128 +0,0 @@ -namespace filtermodule -{ - using System; - using System.IO; - using System.Runtime.InteropServices; - using System.Runtime.Loader; - using System.Security.Cryptography.X509Certificates; - using System.Text; - using System.Threading; - using System.Threading.Tasks; - using Microsoft.Azure.Devices.Client; - using Microsoft.Azure.Devices.Client.Transport.Mqtt; - - class Program - { - static int counter; - - static void Main(string[] args) - { - // The Edge runtime gives us the connection string we need -- it is injected as an environment variable - string connectionString = Environment.GetEnvironmentVariable("EdgeHubConnectionString"); - - // Cert verification is not yet fully functional when using Windows OS for the container - bool bypassCertVerification = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); - if (!bypassCertVerification) InstallCert(); - Init(connectionString, bypassCertVerification).Wait(); - - // Wait until the app unloads or is cancelled - var cts = new CancellationTokenSource(); - AssemblyLoadContext.Default.Unloading += (ctx) => cts.Cancel(); - Console.CancelKeyPress += (sender, cpe) => cts.Cancel(); - WhenCancelled(cts.Token).Wait(); - } - - /// - /// Handles cleanup operations when app is cancelled or unloads - /// - public static Task WhenCancelled(CancellationToken cancellationToken) - { - var tcs = new TaskCompletionSource(); - cancellationToken.Register(s => ((TaskCompletionSource)s).SetResult(true), tcs); - return tcs.Task; - } - - /// - /// Add certificate in local cert store for use by client for secure connection to IoT Edge runtime - /// - static void InstallCert() - { - string certPath = Environment.GetEnvironmentVariable("EdgeModuleCACertificateFile"); - if (string.IsNullOrWhiteSpace(certPath)) - { - // We cannot proceed further without a proper cert file - Console.WriteLine($"Missing path to certificate collection file: {certPath}"); - throw new InvalidOperationException("Missing path to certificate file."); - } - else if (!File.Exists(certPath)) - { - // We cannot proceed further without a proper cert file - Console.WriteLine($"Missing path to certificate collection file: {certPath}"); - throw new InvalidOperationException("Missing certificate file."); - } - X509Store store = new X509Store(StoreName.Root, StoreLocation.CurrentUser); - store.Open(OpenFlags.ReadWrite); - store.Add(new X509Certificate2(X509Certificate2.CreateFromCertFile(certPath))); - Console.WriteLine("Added Cert: " + certPath); - store.Close(); - } - - - /// - /// Initializes the DeviceClient and sets up the callback to receive - /// messages containing temperature information - /// - static async Task Init(string connectionString, bool bypassCertVerification = false) - { - Console.WriteLine("Connection String {0}", connectionString); - - MqttTransportSettings mqttSetting = new MqttTransportSettings(TransportType.Mqtt_Tcp_Only); - // During dev you might want to bypass the cert verification. It is highly recommended to verify certs systematically in production - if (bypassCertVerification) - { - mqttSetting.RemoteCertificateValidationCallback = (sender, certificate, chain, sslPolicyErrors) => true; - } - ITransportSettings[] settings = { mqttSetting }; - - // Open a connection to the Edge runtime - DeviceClient ioTHubModuleClient = DeviceClient.CreateFromConnectionString(connectionString, settings); - await ioTHubModuleClient.OpenAsync(); - Console.WriteLine("IoT Hub module client initialized."); - - // Register callback to be called when a message is received by the module - await ioTHubModuleClient.SetInputMessageHandlerAsync("input1", PipeMessage, ioTHubModuleClient); - } - - /// - /// This method is called whenever the module is sent a message from the EdgeHub. - /// It just pipe the messages without any change. - /// It prints all the incoming messages. - /// - static async Task PipeMessage(Message message, object userContext) - { - int counterValue = Interlocked.Increment(ref counter); - - var deviceClient = userContext as DeviceClient; - if (deviceClient == null) - { - throw new InvalidOperationException("UserContext doesn't contain " + "expected values"); - } - - byte[] messageBytes = message.GetBytes(); - string messageString = Encoding.UTF8.GetString(messageBytes); - Console.WriteLine($"Received message: {counterValue}, Body: [{messageString}]"); - - if (!string.IsNullOrEmpty(messageString)) - { - var pipeMessage = new Message(messageBytes); - foreach (var prop in message.Properties) - { - pipeMessage.Properties.Add(prop.Key, prop.Value); - } - await deviceClient.SendEventAsync("output1", pipeMessage); - Console.WriteLine("Received message sent"); - } - return MessageResponse.Completed; - } - } -} diff --git a/iotedgedev/template/modules/filtermodule/filtermodule.csproj b/iotedgedev/template/modules/filtermodule/filtermodule.csproj deleted file mode 100644 index ecab4e50..00000000 --- a/iotedgedev/template/modules/filtermodule/filtermodule.csproj +++ /dev/null @@ -1,22 +0,0 @@ - - - Exe - netcoreapp2.0 - - - - True - - - - - - - - - - - - - - \ No newline at end of file diff --git a/iotedgedev/template/modules/filtermodule/module.json b/iotedgedev/template/modules/filtermodule/module.json deleted file mode 100644 index 3d1b2f65..00000000 --- a/iotedgedev/template/modules/filtermodule/module.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "$schema-version": "0.0.1", - "description": "", - "image": { - "repository": "/", - "tag": { - "version": "0.0.1", - "platforms": { - "amd64": "./Dockerfile", - "amd64.debug": "./Dockerfile.amd64.debug", - "arm32v7": "./Dockerfile.arm32v7", - "windows-amd64": "./Dockerfile" - } - } - }, - "language": "csharp" -} \ No newline at end of file diff --git a/iotedgedev/template/runtime.template.json b/iotedgedev/template/runtime.template.json index 38ae19a5..6f306242 100644 --- a/iotedgedev/template/runtime.template.json +++ b/iotedgedev/template/runtime.template.json @@ -1,7 +1,7 @@ { "deployment": { "docker": { - "edgeRuntimeImage": "microsoft/azureiotedge-agent:${RUNTIME_TAG}", + "edgeRuntimeImage": "mcr.microsoft.com/azureiotedge-agent:${RUNTIME_TAG}", "loggingOptions": { "log-driver": "json-file", "log-opts": { diff --git a/iotedgedev/template/template.zip b/iotedgedev/template/template.zip index 459f1e9b..545faec7 100644 Binary files a/iotedgedev/template/template.zip and b/iotedgedev/template/template.zip differ diff --git a/iotedgedev/utility.py b/iotedgedev/utility.py index bca6be5c..13fc09b8 100644 --- a/iotedgedev/utility.py +++ b/iotedgedev/utility.py @@ -1,17 +1,19 @@ -from base64 import b64encode, b64decode import fnmatch -from hashlib import sha256 -from hmac import HMAC import json import os import subprocess import sys +from base64 import b64decode, b64encode +from hashlib import sha256 +from hmac import HMAC from time import time + +from .moduletype import ModuleType + if sys.version_info.major >= 3: from urllib.parse import quote, urlencode else: from urllib import quote, urlencode -from .moduletype import ModuleType class Utility: @@ -78,9 +80,13 @@ def get_iot_hub_sas_token(self, uri, key, policy_name, expiry=3600): return "SharedAccessSignature " + urlencode(rawtoken) - def get_file_contents(self, file): + def get_file_contents(self, file, expand_env=False): with open(file, "r") as file: - return file.read() + content = file.read() + if expand_env: + return os.path.expandvars(content) + else: + return content def decode(self, val): return val.decode("utf-8").strip() @@ -90,9 +96,12 @@ def get_config_files(self): return [os.path.join(os.getcwd(), f) for f in os.listdir(os.getcwd()) if f.endswith("template.json")] - def get_active_modules(self): + def get_bypass_modules(self): return [module.strip() - for module in self.envvars.ACTIVE_MODULES.split(",") if module] + for module in self.envvars.BYPASS_MODULES.split(",") if module] + + def get_active_docker_platform(self): + return [platform.strip() for platform in self.envvars.ACTIVE_DOCKER_PLATFORMS.split(",") if platform] def get_modules_in_config(self, moduleType): modules_config = json.load(open(self.envvars.DEPLOYMENT_CONFIG_FILE_PATH)) @@ -112,7 +121,7 @@ def get_modules_in_config(self, moduleType): return_modules.update(user_modules) return return_modules - def set_config(self, force=False): + def set_config(self, force=False, replacements=None): if not self.config_set or force: self.output.header("PROCESSING CONFIG FILES") @@ -137,12 +146,34 @@ def set_config(self, force=False): self.output.info("Expanding '{0}' to '{1}'".format( os.path.basename(config_file), build_config_file)) - config_file_expanded = os.path.expandvars( - self.get_file_contents(config_file)) - - with open(build_config_file, "w") as config_file_build: - config_file_build.write(config_file_expanded) + self.copy_template(config_file, build_config_file, replacements, True) self.output.line() self.config_set = True + + def copy_template(self, src, dest=None, replacements=None, expand_env=True): + """Read file at src, replace the keys in replacements with their values, optionally expand environment variables, and save to dest""" + if dest is None: + dest = src + + content = self.get_file_contents(src) + + if replacements: + for key, value in replacements.items(): + content = content.replace(key, value) + + if expand_env: + content = os.path.expandvars(content) + + with open(dest, "w") as dest_file: + dest_file.write(content) + + def nested_set(self, dic, keys, value): + current = dic + for key in keys[:-1]: + if key not in current: + current[key] = {} + current = current.get(key) + + current[keys[-1]] = value diff --git a/requirements.txt b/requirements.txt index f500aea7..2211c91c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,7 @@ -docker==3.0.0 +docker>=3.4 python-dotenv requests fstrings -azure-iot-edge-runtime-ctl==1.0.0rc22 azure-cli-iot azure-cli-profile azure-cli-extension diff --git a/setup.cfg b/setup.cfg index 8d4e9549..d0a4c042 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,3 +19,5 @@ exclude = docs [aliases] +[pytest] +norecursedirs=tests/utility diff --git a/setup.py b/setup.py index 6128ec05..698fcac8 100644 --- a/setup.py +++ b/setup.py @@ -31,11 +31,10 @@ def run(self): requirements = [ 'Click>=6.0', - 'docker==3.0.0', + 'docker>=3.4', 'python-dotenv', 'requests', 'fstrings', - 'azure-iot-edge-runtime-ctl==1.0.0rc22', 'azure-cli-iot', 'azure-cli-profile', 'azure-cli-extension', diff --git a/tests/assets/deployment.template_1.json b/tests/assets/deployment.template_1.json new file mode 100644 index 00000000..fbad7000 --- /dev/null +++ b/tests/assets/deployment.template_1.json @@ -0,0 +1,79 @@ +{ + "moduleContent": { + "$edgeAgent": { + "properties.desired": { + "schemaVersion": "1.0", + "runtime": { + "type": "docker", + "settings": { + "minDockerVersion": "v1.25", + "loggingOptions": "" + } + }, + "systemModules": { + "edgeAgent": { + "type": "docker", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-agent:1.0", + "createOptions": "" + } + }, + "edgeHub": { + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-hub:1.0", + "createOptions": "{\"HostConfig\":{\"PortBindings\":{\"8883/tcp\":[{\"HostPort\":\"8883\"}],\"443/tcp\":[{\"HostPort\":\"443\"}]}}}" + } + } + }, + "modules": { + "temp-sensor-module": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-simulated-temperature-sensor:1.0", + "createOptions": "" + } + }, + "csharpmodule": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "${MODULES.csharpmodule.amd64}", + "createOptions": "" + } + }, + "csharpfunction": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "${MODULES.csharpfunction.amd64.debug}", + "createOptions": "" + } + } + } + } + }, + "$edgeHub": { + "properties.desired": { + "schemaVersion": "1.0", + "routes": { + "sensorTocsharpmodule": "FROM /messages/modules/tempSensor/outputs/temperatureOutput INTO BrokeredEndpoint(\"/modules/csharpmodule/inputs/input1\")", + "csharpmoduleToIoTHub": "FROM /messages/modules/csharpmodule/outputs/* INTO $upstream", + "csharpfunctionToIoTHub": "FROM /messages/modules/csharpfunction/outputs/* INTO $upstream" + }, + "storeAndForwardConfiguration": { + "timeToLiveSecs": 7200 + } + } + } + } +} \ No newline at end of file diff --git a/tests/assets/deployment.template_2.json b/tests/assets/deployment.template_2.json new file mode 100644 index 00000000..a837d4e9 --- /dev/null +++ b/tests/assets/deployment.template_2.json @@ -0,0 +1,79 @@ +{ + "moduleContent": { + "$edgeAgent": { + "properties.desired": { + "schemaVersion": "1.0", + "runtime": { + "type": "docker", + "settings": { + "minDockerVersion": "v1.25", + "loggingOptions": "" + } + }, + "systemModules": { + "edgeAgent": { + "type": "docker", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-agent:1.0", + "createOptions": "" + } + }, + "edgeHub": { + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-hub:1.0", + "createOptions": "{\"HostConfig\":{\"PortBindings\":{\"8883/tcp\":[{\"HostPort\":\"8883\"}],\"443/tcp\":[{\"HostPort\":\"443\"}]}}}" + } + } + }, + "modules": { + "temp-sensor-module": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-simulated-temperature-sensor:1.0", + "createOptions": "" + } + }, + "csharpmodule": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "localhost:5000/csharpmodule:0.0.1-amd64", + "createOptions": "" + } + }, + "csharpfunction": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "localhost:5000/csharpfunction:0.0.1-amd64.debug", + "createOptions": "" + } + } + } + } + }, + "$edgeHub": { + "properties.desired": { + "schemaVersion": "1.0", + "routes": { + "sensorTocsharpmodule": "FROM /messages/modules/tempSensor/outputs/temperatureOutput INTO BrokeredEndpoint(\"/modules/csharpmodule/inputs/input1\")", + "csharpmoduleToIoTHub": "FROM /messages/modules/csharpmodule/outputs/* INTO $upstream", + "csharpfunctionToIoTHub": "FROM /messages/modules/csharpfunction/outputs/* INTO $upstream" + }, + "storeAndForwardConfiguration": { + "timeToLiveSecs": 7200 + } + } + } + } +} \ No newline at end of file diff --git a/tests/assets/deployment.template_3.json b/tests/assets/deployment.template_3.json new file mode 100644 index 00000000..d654b844 --- /dev/null +++ b/tests/assets/deployment.template_3.json @@ -0,0 +1,90 @@ +{ + "moduleContent": { + "$edgeAgent": { + "properties.desired": { + "schemaVersion": "1.0", + "runtime": { + "type": "docker", + "settings": { + "minDockerVersion": "v1.25", + "loggingOptions": "" + } + }, + "systemModules": { + "edgeAgent": { + "type": "docker", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-agent:1.0", + "createOptions": "" + } + }, + "edgeHub": { + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-hub:1.0", + "createOptions": "{\"HostConfig\":{\"PortBindings\":{\"8883/tcp\":[{\"HostPort\":\"8883\"}],\"443/tcp\":[{\"HostPort\":\"443\"}]}}}" + } + } + }, + "modules": { + "temp-sensor-module": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "mcr.microsoft.com/azureiotedge-simulated-temperature-sensor:1.0", + "createOptions": "" + } + }, + "csharpmodule": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "${MODULES.csharpmodule.amd64}", + "createOptions": "" + } + }, + "csharpfunction": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "${MODULES.csharpfunction.amd64.debug}", + "createOptions": "" + } + }, + "csharpmodule2": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "${MODULES.csharpmodule2.amd64}", + "createOptions": "" + } + } + } + } + }, + "$edgeHub": { + "properties.desired": { + "schemaVersion": "1.0", + "routes": { + "sensorTocsharpmodule": "FROM /messages/modules/tempSensor/outputs/temperatureOutput INTO BrokeredEndpoint(\"/modules/csharpmodule/inputs/input1\")", + "csharpmoduleToIoTHub": "FROM /messages/modules/csharpmodule/outputs/* INTO $upstream", + "csharpfunctionToIoTHub": "FROM /messages/modules/csharpfunction/outputs/* INTO $upstream", + "csharpmodule2ToIoTHub": "FROM /messages/modules/csharpmodule2/outputs/* INTO $upstream" + }, + "storeAndForwardConfiguration": { + "timeToLiveSecs": 7200 + } + } + } + } +} \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..8f17ebeb --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,4 @@ +import os +import sys + +sys.path.append(os.path.join(os.path.dirname(__file__), "utility")) diff --git a/tests/node_solution/.env.tmp b/tests/node_solution/.env.tmp index a559f85d..4eeccabd 100644 --- a/tests/node_solution/.env.tmp +++ b/tests/node_solution/.env.tmp @@ -39,7 +39,7 @@ RUNTIME_CONFIG_DIR="." RUNTIME_HOST_NAME="." # "." - Auto detect -RUNTIME_TAG="1.0-preview" +RUNTIME_TAG="1.0" RUNTIME_VERBOSITY="INFO" # "DEBUG", "INFO", "ERROR", "WARNING" diff --git a/tests/test_azurecli.py b/tests/test_azurecli.py index d5f2e06d..0fe44c7a 100644 --- a/tests/test_azurecli.py +++ b/tests/test_azurecli.py @@ -1,29 +1,33 @@ import pytest + from iotedgedev.azurecli import get_query_argument_for_id_and_name pytestmark = pytest.mark.unit + def get_terms(query): - # These tests are all asserting that the query contains two terms enclosed in - # [?], separated by || - # They don't care about the order. Tests will fail if the square brackets and || + # These tests are all asserting that the query contains two terms enclosed in + # [?], separated by || + # They don't care about the order. Tests will fail if the square brackets and || # contract is violated, but we'd want them to fail in that case. return query[2:len(query)-1].split(" || ") + def test_lowercase_token_should_be_lowercase_for_name_and_id(): token = "abc123" query = get_query_argument_for_id_and_name(token) terms = get_terms(query) assert len(terms) == 2 - assert "starts_with(@.id,'abc123')" in terms + assert "starts_with(@.id,'abc123')" in terms assert "contains(@.name,'abc123')" in terms + def test_mixedcase_token_should_be_lowercase_for_id_but_unmodified_for_name(): token = "AbC123" query = get_query_argument_for_id_and_name(token) terms = get_terms(query) assert len(terms) == 2 - assert "starts_with(@.id,'abc123')" in terms - assert "contains(@.name,'AbC123')" in terms \ No newline at end of file + assert "starts_with(@.id,'abc123')" in terms + assert "contains(@.name,'AbC123')" in terms diff --git a/tests/test_connectionstring.py b/tests/test_connectionstring.py index 509daa0d..19a2ba66 100644 --- a/tests/test_connectionstring.py +++ b/tests/test_connectionstring.py @@ -1,7 +1,8 @@ -import os import pytest -from dotenv import load_dotenv -from iotedgedev.connectionstring import ConnectionString, IoTHubConnectionString, DeviceConnectionString + +from iotedgedev.connectionstring import (ConnectionString, + DeviceConnectionString, + IoTHubConnectionString) pytestmark = pytest.mark.unit @@ -13,24 +14,29 @@ invalid_iothub_connectionstring = "HostName=testhub.azure-devices.net;SharedAccessKey=moregibberish" invalid_device_connectionstring = "HostName=testhub.azure-devices.net;DeviceId=;SharedAccessKey=othergibberish" + def test_empty_connectionstring(): connectionstring = ConnectionString(emptystring) assert not connectionstring.data + def test_empty_iothub_connectionstring(): connectionstring = IoTHubConnectionString(emptystring) assert not connectionstring.data + def test_empty_device_connectionstring(): connectionstring = DeviceConnectionString(emptystring) assert not connectionstring.data + def test_valid_connectionstring(): connectionstring = ConnectionString(valid_connectionstring) assert connectionstring.HostName == "testhub.azure-devices.net" assert connectionstring.HubName == "testhub" assert connectionstring.SharedAccessKey == "gibberish" + def test_valid_iothub_connectionstring(): connectionstring = IoTHubConnectionString(valid_iothub_connectionstring) assert connectionstring.HostName == "testhub.azure-devices.net" @@ -38,6 +44,7 @@ def test_valid_iothub_connectionstring(): assert connectionstring.SharedAccessKeyName == "iothubowner" assert connectionstring.SharedAccessKey == "moregibberish" + def test_valid_devicehub_connectionstring(): connectionstring = DeviceConnectionString(valid_device_connectionstring) assert connectionstring.HostName == "testhub.azure-devices.net" @@ -45,17 +52,20 @@ def test_valid_devicehub_connectionstring(): assert connectionstring.DeviceId == "testdevice" assert connectionstring.SharedAccessKey == "othergibberish" + def test_invalid_connectionstring(): connectionstring = ConnectionString(invalid_connectionstring) assert connectionstring.HubName != "testhub" + def test_invalid_iothub_connectionstring(): with pytest.raises(KeyError): IoTHubConnectionString(invalid_iothub_connectionstring) + def test_invalid_devicehub_connectionstring(): connectionstring = DeviceConnectionString(invalid_device_connectionstring) assert connectionstring.HostName == "testhub.azure-devices.net" assert connectionstring.HubName == "testhub" assert not connectionstring.DeviceId - assert connectionstring.SharedAccessKey == "othergibberish" \ No newline at end of file + assert connectionstring.SharedAccessKey == "othergibberish" diff --git a/tests/test_deploymentmanifest.py b/tests/test_deploymentmanifest.py new file mode 100644 index 00000000..3a2921a1 --- /dev/null +++ b/tests/test_deploymentmanifest.py @@ -0,0 +1,50 @@ +import json +import os + +import pytest + +from iotedgedev.deploymentmanifest import DeploymentManifest +from iotedgedev.envvars import EnvVars +from iotedgedev.output import Output +from iotedgedev.utility import Utility +from utility import assert_list_equal + +pytestmark = pytest.mark.unit + +tests_dir = os.path.join(os.getcwd(), "tests") +test_assets_dir = os.path.join(tests_dir, "assets") +test_file_1 = os.path.join(test_assets_dir, "deployment.template_1.json") +test_file_2 = os.path.join(test_assets_dir, "deployment.template_2.json") +test_file_3 = os.path.join(test_assets_dir, "deployment.template_3.json") + + +@pytest.fixture +def deployment_manifest(): + output = Output() + envvars = EnvVars(output) + envvars.load() + utility = Utility(envvars, output) + + def _deployment_manifest(path): + return DeploymentManifest(envvars, output, utility, path, True) + + return _deployment_manifest + + +def test_get_modules_to_process(deployment_manifest): + deployment_manifest = deployment_manifest(test_file_1) + modules_to_process = deployment_manifest.get_modules_to_process() + assert_list_equal(modules_to_process, [("csharpmodule", "amd64"), ("csharpfunction", "amd64.debug")]) + + +def test_get_modules_to_process_empty(deployment_manifest): + deployment_manifest = deployment_manifest(test_file_2) + modules_to_process = deployment_manifest.get_modules_to_process() + assert_list_equal(modules_to_process, []) + + +def test_add_module_template(deployment_manifest): + deployment_manifest = deployment_manifest(test_file_1) + deployment_manifest.add_module_template("csharpmodule2") + with open(test_file_3, "r") as expected: + assert deployment_manifest.json == json.load(expected) diff --git a/tests/test_envvars.py b/tests/test_envvars.py index daf0bc3d..340810a4 100644 --- a/tests/test_envvars.py +++ b/tests/test_envvars.py @@ -5,24 +5,28 @@ pytestmark = pytest.mark.unit + def test_valid_get_envvar(): output = Output() envvars = EnvVars(output) loglevel = envvars.get_envvar("RUNTIME_LOG_LEVEL") assert loglevel == "info" or "debug" + def test_invalid_get_envvar(): output = Output() envvars = EnvVars(output) testerval = envvars.get_envvar("TESTER") assert not testerval - + + def test_valid_load(): output = Output() envvars = EnvVars(output) envvars.load() assert envvars.RUNTIME_LOG_LEVEL == "info" or "debug" + def test_valid_verify_envvar_has_val(): output = Output() envvars = EnvVars(output) @@ -30,16 +34,19 @@ def test_valid_verify_envvar_has_val(): result = envvars.verify_envvar_has_val("RUNTIME_LOG_LEVEL", envvars.RUNTIME_LOG_LEVEL) assert not result + def test_valid_get_envvar_key_if_val(): output = Output() envvars = EnvVars(output) assert envvars.get_envvar_key_if_val("RUNTIME_LOG_LEVEL") + def test_invalid_get_envvar_key_if_val(): output = Output() envvars = EnvVars(output) assert not envvars.get_envvar_key_if_val("TESTER") + def test_set_envvar(): output = Output() envvars = EnvVars(output) @@ -48,4 +55,3 @@ def test_set_envvar(): setlevel = envvars.get_envvar("RUNTIME_LOG_LEVEL") assert setlevel == "debug" envvars.set_envvar("RUNTIME_LOG_LEVEL", loglevel) - \ No newline at end of file diff --git a/tests/test_iotedgedev.py b/tests/test_iotedgedev.py index 9b50ee73..0c1438ca 100644 --- a/tests/test_iotedgedev.py +++ b/tests/test_iotedgedev.py @@ -1,9 +1,13 @@ import json import os import shutil -import pytest +import pytest from click.testing import CliRunner +from dotenv import load_dotenv + +from iotedgedev.connectionstring import (DeviceConnectionString, + IoTHubConnectionString) pytestmark = pytest.mark.e2e @@ -106,7 +110,7 @@ def test_module_add(): runner = CliRunner() add_module_and_verify(cli.main, runner, "csharp") - # add_module_and_verify(cli.main, runner, "nodejs") + add_module_and_verify(cli.main, runner, "nodejs") add_module_and_verify(cli.main, runner, "python") add_module_and_verify(cli.main, runner, "csharpfunction") @@ -161,49 +165,49 @@ def test_deploy_modules(request): assert 'DEPLOYMENT COMPLETE' in result.output -@pytest.fixture -def test_start_runtime(request): +# @pytest.fixture +# def test_start_runtime(request): - os.chdir(test_solution_dir) +# os.chdir(test_solution_dir) - cli = __import__("iotedgedev.cli", fromlist=['main']) - runner = CliRunner() - result = runner.invoke(cli.main, ['start']) - print(result.output) +# cli = __import__("iotedgedev.cli", fromlist=['main']) +# runner = CliRunner() +# result = runner.invoke(cli.main, ['start']) +# print(result.output) - assert 'Runtime started' in result.output +# assert 'Runtime started' in result.output -@pytest.fixture -def test_monitor(request, capfd): +# @pytest.fixture +# def test_monitor(request, capfd): - os.chdir(test_solution_dir) +# os.chdir(test_solution_dir) - cli = __import__("iotedgedev.cli", fromlist=['main']) - runner = CliRunner() - result = runner.invoke(cli.main, ['monitor', '--timeout', '60000']) - out, err = capfd.readouterr() - print(out) - print(err) - print(result.output) +# cli = __import__("iotedgedev.cli", fromlist=['main']) +# runner = CliRunner() +# result = runner.invoke(cli.main, ['monitor', '--timeout', '60000']) +# out, err = capfd.readouterr() +# print(out) +# print(err) +# print(result.output) - assert 'timeCreated' in out +# assert 'timeCreated' in out -@pytest.fixture -def test_stop(request): +# @pytest.fixture +# def test_stop(request): - os.chdir(test_solution_dir) +# os.chdir(test_solution_dir) - cli = __import__("iotedgedev.cli", fromlist=['main']) - runner = CliRunner() - result = runner.invoke(cli.main, ['stop']) - print(result.output) +# cli = __import__("iotedgedev.cli", fromlist=['main']) +# runner = CliRunner() +# result = runner.invoke(cli.main, ['stop']) +# print(result.output) - assert 'Runtime stopped' in result.output +# assert 'Runtime stopped' in result.output -def test_e2e(test_push_modules, test_deploy_modules, test_start_runtime, test_monitor, test_stop): +def test_e2e(test_push_modules, test_deploy_modules): print('Testing E2E') @@ -219,9 +223,10 @@ def clean(): return -def test_node(setup_node_solution, test_push_modules, test_deploy_modules, test_start_runtime, test_monitor, test_stop): +def test_node(setup_node_solution, test_push_modules, test_deploy_modules): print('Testing Node Solution') + def test_valid_env_iothub_connectionstring(): load_dotenv(".env") env_iothub_connectionstring = os.getenv("IOTHUB_CONNECTION_STRING") @@ -231,6 +236,7 @@ def test_valid_env_iothub_connectionstring(): assert connectionstring.SharedAccessKey assert connectionstring.SharedAccessKeyName + def test_valid_env_device_connectionstring(): load_dotenv(".env") env_device_connectionstring = os.getenv("DEVICE_CONNECTION_STRING") @@ -240,6 +246,7 @@ def test_valid_env_device_connectionstring(): assert connectionstring.SharedAccessKey assert connectionstring.DeviceId + ''' def test_load_no_dotenv(): diff --git a/tests/test_utility.py b/tests/test_utility.py new file mode 100644 index 00000000..67862563 --- /dev/null +++ b/tests/test_utility.py @@ -0,0 +1,44 @@ +import os + +import pytest + +from iotedgedev.envvars import EnvVars +from iotedgedev.output import Output +from iotedgedev.utility import Utility +from utility import assert_json_file_equal + +pytestmark = pytest.mark.unit + +tests_dir = os.path.join(os.getcwd(), "tests") +test_assets_dir = os.path.join(tests_dir, "assets") +test_file_1 = os.path.join(test_assets_dir, "deployment.template_1.json") +test_file_2 = os.path.join(test_assets_dir, "deployment.template_2.json") + + +@pytest.fixture +def utility(): + output = Output() + envvars = EnvVars(output) + envvars.load() + return Utility(envvars, output) + + +def test_copy_template(utility, tmpdir): + replacements = { + "${MODULES.csharpmodule.amd64}": "localhost:5000/csharpmodule:0.0.1-amd64", + "${MODULES.csharpfunction.amd64.debug}": "localhost:5000/csharpfunction:0.0.1-amd64.debug" + } + dest = tmpdir.join("deployment_template_1.dest.json").strpath + utility.copy_template(test_file_1, dest, replacements=replacements, expand_env=False) + assert_json_file_equal(test_file_2, dest) + + +def test_copy_template_expand_env(utility, tmpdir): + replacements = { + "${MODULES.csharpmodule.amd64}": "${CONTAINER_REGISTRY_SERVER}/csharpmodule:0.0.1-amd64", + "${MODULES.csharpfunction.amd64.debug}": "${CONTAINER_REGISTRY_SERVER}/csharpfunction:0.0.1-amd64.debug" + } + os.environ["CONTAINER_REGISTRY_SERVER"] = "localhost:5000" + dest = tmpdir.join("deployment_template_2.dest.json").strpath + utility.copy_template(test_file_1, dest, replacements=replacements, expand_env=True) + assert_json_file_equal(test_file_2, dest) diff --git a/tests/utility/utility.py b/tests/utility/utility.py new file mode 100644 index 00000000..36999b5d --- /dev/null +++ b/tests/utility/utility.py @@ -0,0 +1,11 @@ +import json + + +def assert_list_equal(list1, list2): + return len(list1) == len(list2) and sorted(list1) == sorted(list2) + + +def assert_json_file_equal(file1, file2): + with open(file1, "r") as f1: + with open(file2, "r") as f2: + assert json.load(f1) == json.load(f2)