From da0d642b09029fe94835eed438b9df8192b88421 Mon Sep 17 00:00:00 2001 From: Ray Fang Date: Fri, 17 Aug 2018 22:20:45 +0800 Subject: [PATCH] Add telemetry (#258) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add iotedgehubdev class * Build solution before start iotedgehubdev * Update templates to fix route error * Fail fast if device connection string is not set before setup * Allow start iotedgehubdev in verbose mode * Add modulecred command * Add gateway_host option to setup command * Add start command * Remove references to runtime * Remove runtime.template.json from template.zip * Only start local registry when pushing images so config/deployment.json only generated after build * Check if the solution is built before starting simulator * Instruct users to setup for starting * Add tests * Enlarge monitor timeout when testing simulator * Fix a issue with spaces in paths * Prevent deploy command overwriting configs * Promote setup, start and stop commands to root * Update unit of monitor timeout to seconds * Add telemetry modules * Update config folder to home dir * Add with_telemetry decorator to commands * Add tests * Defer exception handling to with_telemetry decorator * Fix test failure * Fix unfinished conflict resolve * Added support for windows containers via nanoserver (#158) * added initial support for windows containers * initial support for nanoserver * corrected az cli path * only az not working * all dependencies correctly working * added required pip packages * removed unneeded file * removed external dependencies * completed support for nanoserver * fixed all issued mentioned in PR comments * installing both python2 and python3 on nanoserver * set path to point to python3, since python2 is not working yet * added missing parenthesis * set python3 as default python * mapped folder set automatically * improved build script * added setuptools install * set to use latest tag * added support for powershell script * automatically map folder * added sock mapping * removed pip from requirements since it may generate errors * ignored .backup files * fixed spacings * install modules locally * fixed bug the prevented arguments to be handled correctly * fixed bug the prevented arguments to be handled correctly * added build script alpha version * comments cleanup * build will now also push docker images * Bump version: 0.86.0 → 0.87.0 * version set back to 0.81.0 * fixed some bugs, added parameter for pipy login * modularized build script * fixed syntax * removed renaming of python3 executables * switch to docker folder if needed * exit on error * correctly handled directory switching * switched to ubuntu 18.04 for python 3.6 default support * using double slash to make script work * improved docker image build performances * file system cleanup * added platform parameter * Container docker compose support (#243) Adding docker compose to container * Support for multiple registries (#193) * envvar parsing of multiple container registries * rename value, support for pushing modules based on module.json * string comparison code clean up * modify envvars with better values & refactor dockercls and .env.tmp * modified variable names, minor fixes, added envvar testing specific to container registry * add tests for additional cr, comments to explain code, fix merge conflict * add additional testing for mutliple registries, fix logic around given/expected env vars * fix env load in tests * Tell travis to use DOTENV_FILE * Mod process timeout for new monitor-events (#253) * Merge upstream changes * Rename config to telemetryconfig * Defer telemetry decorator to handle errors * Fix CI failure --- iotedgedev/__init__.py | 1 + iotedgedev/cli.py | 23 +++++ iotedgedev/decorators.py | 75 ++++++++++++++++ iotedgedev/deploymentmanifest.py | 23 ++--- iotedgedev/dockercls.py | 4 +- iotedgedev/envvars.py | 28 +++--- iotedgedev/module.py | 17 ++-- iotedgedev/modules.py | 9 +- iotedgedev/output.py | 2 +- iotedgedev/simulator.py | 4 +- iotedgedev/telemetry.py | 147 +++++++++++++++++++++++++++++++ iotedgedev/telemetryconfig.py | 84 ++++++++++++++++++ iotedgedev/telemetryuploader.py | 60 +++++++++++++ iotedgedev/utility.py | 13 +-- setup.py | 7 +- tests/test_config.py | 28 ++++++ tests/test_decorators.py | 32 +++++++ tests/test_envvars.py | 12 +-- 18 files changed, 496 insertions(+), 73 deletions(-) create mode 100644 iotedgedev/decorators.py create mode 100644 iotedgedev/telemetry.py create mode 100644 iotedgedev/telemetryconfig.py create mode 100644 iotedgedev/telemetryuploader.py create mode 100644 tests/test_config.py create mode 100644 tests/test_decorators.py diff --git a/iotedgedev/__init__.py b/iotedgedev/__init__.py index 6cf46262..babbf047 100644 --- a/iotedgedev/__init__.py +++ b/iotedgedev/__init__.py @@ -5,3 +5,4 @@ __author__ = 'Jon Gallant' __email__ = 'info@jongallant.com' __version__ = '0.81.0' +__AIkey__ = '95b20d64-f54f-4de3-8ad5-165a75a6c6fe' diff --git a/iotedgedev/cli.py b/iotedgedev/cli.py index 420b64ca..2fe5b451 100644 --- a/iotedgedev/cli.py +++ b/iotedgedev/cli.py @@ -11,6 +11,7 @@ from fstrings import f from .azurecli import AzureCli +from .decorators import with_telemetry from .dockercls import Docker from .edge import Edge from .envvars import EnvVars @@ -35,26 +36,31 @@ @click.group(context_settings=CONTEXT_SETTINGS, cls=OrganizedGroup) @click.version_option() +@with_telemetry def main(): pass @main.group(context_settings=CONTEXT_SETTINGS, help="Manage IoT Edge solutions", order=1) +@with_telemetry def solution(): pass @main.group(context_settings=CONTEXT_SETTINGS, help="Manage IoT Edge simulator", order=1) +@with_telemetry def simulator(): pass @main.group(context_settings=CONTEXT_SETTINGS, help="Manage IoT Hub and IoT Edge devices", order=1) +@with_telemetry def iothub(): pass @main.group(context_settings=CONTEXT_SETTINGS, help="Manage Docker", order=1) +@with_telemetry def docker(): pass @@ -78,6 +84,7 @@ def docker(): required=False, type=click.Choice(["csharp", "nodejs", "python", "csharpfunction"]), help="Specify the template used to create the default module") +@with_telemetry def create(name, module, template): utility = Utility(envvars, output) sol = Solution(output, utility) @@ -91,6 +98,7 @@ def create(name, module, template): help="Create a new IoT Edge solution and provision Azure resources", # hack to prevent Click truncating help messages short_help="Create a new IoT Edge solution and provision Azure resources") +@with_telemetry def init(): utility = Utility(envvars, output) @@ -107,6 +115,7 @@ def init(): @solution.command(context_settings=CONTEXT_SETTINGS, help="Push, deploy, start, monitor") @click.pass_context +@with_telemetry def e2e(ctx): ctx.invoke(init) envvars.load(force=True) @@ -127,6 +136,7 @@ def e2e(ctx): default="csharp", show_default=True, help="Specify the template used to create the new module") +@with_telemetry def add(name, template): mod = Modules(envvars, output) mod.add(name, template) @@ -152,6 +162,7 @@ def add(name, template): is_flag=True, help="Deploy modules to Edge device using deployment.json in the config folder") @click.pass_context +@with_telemetry def build(ctx, push, do_deploy): mod = Modules(envvars, output) mod.build_push(no_push=not push) @@ -179,6 +190,7 @@ def build(ctx, push, do_deploy): is_flag=True, help="Inform the push command to not build modules images before pushing to container registry") @click.pass_context +@with_telemetry def push(ctx, do_deploy, no_build): mod = Modules(envvars, output) mod.push(no_build=no_build) @@ -191,6 +203,7 @@ def push(ctx, do_deploy, no_build): @solution.command(context_settings=CONTEXT_SETTINGS, help="Deploy solution to IoT Edge device") +@with_telemetry def deploy(): edge = Edge(envvars, output, azure_cli) edge.deploy() @@ -203,6 +216,7 @@ def deploy(): help="Expand environment variables and placeholders in *.template.json and copy to config folder", # hack to prevent Click truncating help messages short_help="Expand environment variables and placeholders in *.template.json and copy to config folder") +@with_telemetry def genconfig(): mod = Modules(envvars, output) mod.build_push(no_build=True, no_push=True) @@ -221,6 +235,7 @@ def genconfig(): required=False, default=socket.getfqdn(), show_default=True) +@with_telemetry def setup_simulator(gateway_host): sim = Simulator(envvars, output) sim.setup(gateway_host) @@ -265,6 +280,7 @@ def setup_simulator(gateway_host): default=53000, show_default=True, help="Port of the service for sending message.") +@with_telemetry def start_simulator(solution, build, verbose, inputs, port): sim = Simulator(envvars, output) if solution or not inputs: @@ -279,6 +295,7 @@ def start_simulator(solution, build, verbose, inputs, port): @simulator.command(context_settings=CONTEXT_SETTINGS, name="stop", help="Stop IoT Edge simulator") +@with_telemetry def stop_simulator(): sim = Simulator(envvars, output) sim.stop() @@ -302,6 +319,7 @@ def stop_simulator(): "-o", help="Specify the output file to save the credentials. If the file exists, its content will be overwritten.", required=False) +@with_telemetry def modulecred(local, output_file): sim = Simulator(envvars, output) sim.modulecred(local, output_file) @@ -315,6 +333,7 @@ def modulecred(local, output_file): "-t", required=False, help="Specify number of seconds to monitor for messages") +@with_telemetry def monitor(timeout): utility = Utility(envvars, output) ih = IoTHub(envvars, utility, output, azure_cli) @@ -550,6 +569,7 @@ def header_and_default(header, default, default2=None): is_flag=True, prompt='Update the .env file with connection strings?', help='If True, the current .env will be updated with the IoT Hub and Device connection strings.') +@with_telemetry def setup_iothub(credentials, service_principal, subscription, @@ -571,6 +591,7 @@ def setup_iothub(credentials, "Also, update config files to use CONTAINER_REGISTRY_* instead of the Microsoft Container Registry. See CONTAINER_REGISTRY environment variables.", short_help="Pull Edge runtime images from MCR and push to your specified container registry", name="setup") +@with_telemetry def setup_registry(): utility = Utility(envvars, output) dock = Docker(envvars, utility, output) @@ -600,6 +621,7 @@ def setup_registry(): required=False, is_flag=True, help="Remove all the images") +@with_telemetry def clean(module, container, image): utility = Utility(envvars, output) dock = Docker(envvars, utility, output) @@ -632,6 +654,7 @@ def clean(module, container, image): required=False, is_flag=True, help="Save EdgeAgent, EdgeHub and each Edge module logs to LOGS_PATH.") +@with_telemetry def log(show, save): utility = Utility(envvars, output) dock = Docker(envvars, utility, output) diff --git a/iotedgedev/decorators.py b/iotedgedev/decorators.py new file mode 100644 index 00000000..f89e1e67 --- /dev/null +++ b/iotedgedev/decorators.py @@ -0,0 +1,75 @@ +import hashlib +import sys +from functools import wraps + + +def with_telemetry(func): + @wraps(func) + def _wrapper(*args, **kwargs): + from . import telemetry + from .telemetryconfig import TelemetryConfig + + config = TelemetryConfig() + config.check_firsttime() + params = parse_params(*args, **kwargs) + telemetry.start(func.__name__, params) + try: + value = func(*args, **kwargs) + telemetry.success() + telemetry.flush() + return value + except Exception as e: + from .output import Output + Output().error('Error: {0}'.format(str(e))) + telemetry.fail(str(e), 'Command failed') + telemetry.flush() + sys.exit(1) + + return _wrapper + + +def suppress_all_exceptions(fallback_return=None): + """We need to suppress exceptions for some internal functions such as those related to telemetry. + They should not be visible to users. + """ + def _decorator(func): + @wraps(func) + def _wrapped_func(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception: + if fallback_return: + return fallback_return + else: + pass + + return _wrapped_func + + return _decorator + + +@suppress_all_exceptions() +def parse_params(*args, **kwargs): + """Record the parameter keys and whether the values are None""" + params = [] + for key, value in kwargs.items(): + is_none = '=' + if value is not None: + is_none = '!=' + params.append('{0}{1}None'.format(key, is_none)) + return params + + +def hash256_result(func): + """Secure the return string of the annotated function with SHA256 algorithm. If the annotated + function doesn't return string or return None, raise ValueError.""" + @wraps(func) + def _decorator(*args, **kwargs): + val = func(*args, **kwargs) + if not val: + raise ValueError('Return value is None') + elif not isinstance(val, str): + raise ValueError('Return value is not string') + hash_object = hashlib.sha256(val.encode('utf-8')) + return str(hash_object.hexdigest()) + return _decorator diff --git a/iotedgedev/deploymentmanifest.py b/iotedgedev/deploymentmanifest.py index 19767a9d..c6f30398 100644 --- a/iotedgedev/deploymentmanifest.py +++ b/iotedgedev/deploymentmanifest.py @@ -6,7 +6,6 @@ import json import os import shutil -import sys class DeploymentManifest: @@ -18,17 +17,18 @@ def __init__(self, envvars, output, utility, path, is_template): self.is_template = is_template self.json = json.loads(self.utility.get_file_contents(path, expandvars=True)) except FileNotFoundError: - self.output.error('Deployment manifest template file "{0}" not found'.format(path)) if is_template: deployment_manifest_path = envvars.DEPLOYMENT_CONFIG_FILE_PATH if os.path.exists(deployment_manifest_path): + self.output.error('Deployment manifest template file "{0}" not found'.format(path)) if output.confirm('Would you like to make a copy of the deployment manifest file "{0}" as the deployment template file?'.format(deployment_manifest_path), default=True): shutil.copyfile(deployment_manifest_path, path) self.json = json.load(open(envvars.DEPLOYMENT_CONFIG_FILE_PATH)) envvars.save_envvar("DEPLOYMENT_CONFIG_TEMPLATE_FILE", path) + else: + raise FileNotFoundError('Deployment manifest file "{0}" not found'.format(path)) else: - self.output.error('Deployment manifest file "{0}" not found'.format(path)) - sys.exit(1) + raise FileNotFoundError('Deployment manifest file "{0}" not found'.format(path)) def add_module_template(self, module_name): """Add a module template to the deployment manifest with amd64 as the default platform""" @@ -46,8 +46,7 @@ def add_module_template(self, module_name): try: self.utility.nested_set(self.get_module_content(), ["$edgeAgent", "properties.desired", "modules", module_name], json.loads(new_module)) except KeyError as err: - self.output.error("Missing key {0} in file {1}".format(err, self.path)) - sys.exit(1) + raise KeyError("Missing key {0} in file {1}".format(err, self.path)) self.add_default_route(module_name) @@ -59,8 +58,7 @@ def add_default_route(self, module_name): try: self.utility.nested_set(self.get_module_content(), ["$edgeHub", "properties.desired", "routes", new_route_name], new_route) except KeyError as err: - self.output.error("Missing key {0} in file {1}".format(err, self.path)) - sys.exit(1) + raise KeyError("Missing key {0} in file {1}".format(err, self.path)) def get_user_modules(self): """Get user modules from deployment manifest""" @@ -68,8 +66,7 @@ def get_user_modules(self): modules = self.get_desired_property("$edgeAgent", "modules") return list(modules.keys()) except KeyError as err: - self.output.error("Missing key {0} in file {1}".format(err, self.path)) - sys.exit(1) + raise KeyError("Missing key {0} in file {1}".format(err, self.path)) def get_system_modules(self): """Get system modules from deployment manifest""" @@ -77,8 +74,7 @@ def get_system_modules(self): modules = self.get_desired_property("$edgeAgent", "systemModules") return list(modules.keys()) except KeyError as err: - self.output.error("Missing key {0} in file {1}".format(err, self.path)) - sys.exit(1) + raise KeyError("Missing key {0} in file {1}".format(err, self.path)) def get_modules_to_process(self): """Get modules to process from deployment manifest template""" @@ -96,8 +92,7 @@ def get_modules_to_process(self): modules_to_process.append((module_dir, module_platform)) return modules_to_process except KeyError as err: - self.output.error("Missing key {0} in file {1}".format(err, self.path)) - sys.exit(1) + raise KeyError("Missing key {0} in file {1}".format(err, self.path)) def get_desired_property(self, module, prop): return self.get_module_content()[module]["properties.desired"][prop] diff --git a/iotedgedev/dockercls.py b/iotedgedev/dockercls.py index 8856b2c9..c55f4d4a 100644 --- a/iotedgedev/dockercls.py +++ b/iotedgedev/dockercls.py @@ -42,9 +42,7 @@ def init_local_registry(self, local_server): parts = local_server.split(":") if len(parts) < 2: - self.output.error("You must specific a port for your local registry server. Expected: 'localhost:5000'. Found: " + - local_server) - sys.exit() + raise ValueError("You must specific a port for your local registry server. Expected: 'localhost:5000'. Found: " + local_server) port = parts[1] ports = {'{0}/tcp'.format(port): int(port)} diff --git a/iotedgedev/envvars.py b/iotedgedev/envvars.py index 1b45f6b5..f180959f 100644 --- a/iotedgedev/envvars.py +++ b/iotedgedev/envvars.py @@ -112,9 +112,7 @@ def load(self, force=False): self.IOTHUB_CONNECTION_INFO = IoTHubConnectionString(self.IOTHUB_CONNECTION_STRING) except Exception as ex: - self.output.error("Unable to parse IOTHUB_CONNECTION_STRING Environment Variable. Please ensure that you have the right connection string set.") - self.output.error(str(ex)) - sys.exit(-1) + raise ValueError("Unable to parse IOTHUB_CONNECTION_STRING Environment Variable. Please ensure that you have the right connection string set. {0}".format(str(ex))) try: self.DEVICE_CONNECTION_STRING = self.get_envvar("DEVICE_CONNECTION_STRING") @@ -123,9 +121,7 @@ def load(self, force=False): self.DEVICE_CONNECTION_INFO = DeviceConnectionString(self.DEVICE_CONNECTION_STRING) except Exception as ex: - self.output.error("Unable to parse DEVICE_CONNECTION_STRING Environment Variable. Please ensure that you have the right connection string set.") - self.output.error(str(ex)) - sys.exit(-1) + raise ValueError("Unable to parse DEVICE_CONNECTION_STRING Environment Variable. Please ensure that you have the right connection string set. {0}".format(str(ex))) self.get_registries() @@ -155,11 +151,10 @@ def load(self, force=False): else: self.DOCKER_HOST = None except Exception as ex: - self.output.error( - "Environment variables not configured correctly. Run `iotedgedev solution create` to create a new solution with sample .env file. " - "Please see README for variable configuration options. Tip: You might just need to restart your command prompt to refresh your Environment Variables.") - self.output.error("Variable that caused exception: " + str(ex)) - sys.exit(-1) + msg = "Environment variables not configured correctly. Run `iotedgedev solution create` to create a new solution with sample .env file. " + "Please see README for variable configuration options. Tip: You might just need to restart your command prompt to refresh your Environment Variables. " + "Variable that caused exception: {0}".format(str(ex)) + raise ValueError(msg) self.clean() @@ -188,8 +183,7 @@ def get_envvar(self, key, required=False, default=None, altkeys=None): break if required and not val: - self.output.error("Environment Variable {0} not set. Either add to .env file or to your system's Environment Variables".format(key)) - sys.exit(-1) + raise ValueError("Environment Variable {0} not set. Either add to .env file or to your system's Environment Variables".format(key)) # if we have a val return it, if not and we have a default then return default, otherwise return None. if val: @@ -202,8 +196,7 @@ def get_envvar(self, key, required=False, default=None, altkeys=None): def verify_envvar_has_val(self, key, value): if not value: - self.output.error("Environment Variable {0} not set. Either add to .env file or to your system's Environment Variables".format(key)) - sys.exit(-1) + raise ValueError("Environment Variable {0} not set. Either add to .env file or to your system's Environment Variables".format(key)) def get_envvar_key_if_val(self, key): if key in os.environ and os.environ.get(key): @@ -220,8 +213,7 @@ def save_envvar(self, key, value): dotenv_path = os.path.join(os.getcwd(), dotenv_file) set_key(dotenv_path, key, value) except Exception: - self.output.error(f("Could not update the environment variable {key} in file {dotenv_path}")) - sys.exit(-1) + raise IOError(f("Could not update the environment variable {key} in file {dotenv_path}")) def get_registries(self): registries = {} @@ -248,7 +240,7 @@ def get_registries(self): registries[token] = {'username': '', 'password': ''} registries[token][subkey] = self.get_envvar(key) - # store parsed values as a dicitonary of containerregistry objects + # store parsed values as a dictionary of containerregistry objects for key, value in registries.items(): self.CONTAINER_REGISTRY_MAP[key] = ContainerRegistry(value['server'], value['username'], value['password']) diff --git a/iotedgedev/module.py b/iotedgedev/module.py index e0276d19..b4ce6dd8 100644 --- a/iotedgedev/module.py +++ b/iotedgedev/module.py @@ -1,6 +1,5 @@ import json import os -import sys class Module(object): @@ -17,17 +16,13 @@ def load_module_json(self): if os.path.exists(self.module_json_file): try: self.file_json_content = json.loads(self.utility.get_file_contents(self.module_json_file, expandvars=True)) - - self.module_language = self.file_json_content.get( - "language").lower() - except: - self.output.error( - "Error while loading module.json file : {0}".format(self.module_json_file)) - + self.module_language = self.file_json_content.get("language").lower() + except KeyError as e: + raise KeyError("Error parsing {0} from module.json file : {1}".format(str(e), self.module_json_file)) + except IOError: + raise IOError("Error loading module.json file : {0}".format(self.module_json_file)) else: - self.output.error( - "No module.json file found. module.json file is required in the root of your module folder") - sys.exit() + raise FileNotFoundError("No module.json file found. module.json file is required in the root of your module folder") @property def language(self): diff --git a/iotedgedev/modules.py b/iotedgedev/modules.py index 0823df91..ed45f984 100644 --- a/iotedgedev/modules.py +++ b/iotedgedev/modules.py @@ -24,14 +24,11 @@ def add(self, name, template): self.utility.ensure_dir(cwd) if name.startswith("_") or name.endswith("_"): - self.output.error("Module name cannot start or end with the symbol _") - return + raise ValueError("Module name cannot start or end with the symbol _") elif not re.match("^[a-zA-Z0-9_]+$", name): - self.output.error("Module name can only contain alphanumeric characters and the symbol _") - return + raise ValueError("Module name can only contain alphanumeric characters and the symbol _") elif os.path.exists(os.path.join(cwd, name)): - self.output.error("Module \"{0}\" already exists under {1}".format(name, os.path.abspath(self.envvars.MODULES_PATH))) - return + raise ValueError("Module \"{0}\" already exists under {1}".format(name, os.path.abspath(self.envvars.MODULES_PATH))) deployment_manifest = DeploymentManifest(self.envvars, self.output, self.utility, self.envvars.DEPLOYMENT_CONFIG_TEMPLATE_FILE, True) diff --git a/iotedgedev/output.py b/iotedgedev/output.py index 049ec59a..7dc860e8 100644 --- a/iotedgedev/output.py +++ b/iotedgedev/output.py @@ -47,7 +47,7 @@ def line(self): def echo(self, text, color="", dim=False): try: click.secho(text, fg=color, dim=dim) - except: + except Exception: print(text) def confirm(self, text, default=False, abort=True): diff --git a/iotedgedev/simulator.py b/iotedgedev/simulator.py index 0704162b..de76f2a9 100644 --- a/iotedgedev/simulator.py +++ b/iotedgedev/simulator.py @@ -1,5 +1,4 @@ import os -import sys from .modules import Modules from .utility import Utility @@ -34,8 +33,7 @@ def start_solution(self, verbose=True, build=False): mod.build() if not os.path.exists(self.envvars.DEPLOYMENT_CONFIG_FILE_PATH): - self.output.error("Deployment manifest {0} not found. Please build the solution before starting IoT Edge simulator.".format(self.envvars.DEPLOYMENT_CONFIG_FILE_PATH)) - sys.exit(1) + raise FileNotFoundError("Deployment manifest {0} not found. Please build the solution before starting IoT Edge simulator.".format(self.envvars.DEPLOYMENT_CONFIG_FILE_PATH)) self.output.header("Starting IoT Edge Simulator in Solution Mode") diff --git a/iotedgedev/telemetry.py b/iotedgedev/telemetry.py new file mode 100644 index 00000000..61da340d --- /dev/null +++ b/iotedgedev/telemetry.py @@ -0,0 +1,147 @@ +import datetime +import json +import os +import platform +import subprocess +import sys +import uuid +from collections import defaultdict +from functools import wraps + +from . import telemetryuploader +from .telemetryconfig import TelemetryConfig +from .decorators import hash256_result, suppress_all_exceptions + +PRODUCT_NAME = 'iotedgedev' + + +class TelemetrySession(object): + def __init__(self, correlation_id=None): + self.start_time = None + self.end_time = None + self.correlation_id = correlation_id or str(uuid.uuid4()) + self.command = 'command_name' + self.parameters = [] + self.result = 'None' + self.result_summary = None + self.exception = None + self.extra_props = {} + self.machineId = self._get_hash_mac_address() + self.events = defaultdict(list) + + def generate_payload(self): + props = { + 'EventId': str(uuid.uuid4()), + 'CorrelationId': self.correlation_id, + 'MachineId': self.machineId, + 'ProductName': PRODUCT_NAME, + 'ProductVersion': _get_core_version(), + 'CommandName': self.command, + 'OS.Type': platform.system().lower(), + 'OS.Version': platform.version().lower(), + 'Result': self.result, + 'StartTime': str(self.start_time), + 'EndTime': str(self.end_time), + 'Parameters': ','.join(self.parameters) + } + + if self.result_summary: + props['ResultSummary'] = self.result_summary + + if self.exception: + props['Exception'] = self.exception + + self.events[_get_AI_key()].append({ + 'name': '{}/command'.format(PRODUCT_NAME), + 'properties': props + }) + + payload = json.dumps(self.events) + return _remove_symbols(payload) + + @suppress_all_exceptions() + @hash256_result + def _get_hash_mac_address(self): + s = '' + for index, c in enumerate(hex(uuid.getnode())[2:].upper()): + s += c + if index % 2: + s += '-' + + s = s.strip('-') + return s + + +_session = TelemetrySession() + + +def _user_agrees_to_telemetry(func): + @wraps(func) + def _wrapper(*args, **kwargs): + config = TelemetryConfig() + if not config.get_boolean(config.DEFAULT_DIRECT, config.TELEMETRY_SECTION): + return None + return func(*args, **kwargs) + + return _wrapper + + +@suppress_all_exceptions() +def start(cmdname, params=[]): + _session.command = cmdname + _session.start_time = datetime.datetime.utcnow() + if params is not None: + _session.parameters.extend(params) + + +@suppress_all_exceptions() +def success(): + _session.result = 'Success' + + +@suppress_all_exceptions() +def fail(exception, summary): + _session.exception = exception + _session.result = 'Fail' + _session.result_summary = summary + + +@_user_agrees_to_telemetry +@suppress_all_exceptions() +def flush(): + # flush out current information + _session.end_time = datetime.datetime.utcnow() + + payload = _session.generate_payload() + if payload: + _upload_telemetry_with_user_agreement(payload) + + # reset session fields, retaining correlation id and application + _session.__init__(correlation_id=_session.correlation_id) + + +@suppress_all_exceptions(fallback_return=None) +def _get_core_version(): + from iotedgedev import __version__ as core_version + return core_version + + +@suppress_all_exceptions() +def _get_AI_key(): + from iotedgedev import __AIkey__ as key + return key + + +# This includes a final user-agreement-check; ALL methods sending telemetry MUST call this. +@_user_agrees_to_telemetry +@suppress_all_exceptions() +def _upload_telemetry_with_user_agreement(payload, **kwargs): + # Call telemetry uploader as a subprocess to prevent blocking iotedgedev process + subprocess.Popen([sys.executable, os.path.realpath(telemetryuploader.__file__), payload], **kwargs) + + +def _remove_symbols(s): + if isinstance(s, str): + for c in '$%^&|': + s = s.replace(c, '_') + return s diff --git a/iotedgedev/telemetryconfig.py b/iotedgedev/telemetryconfig.py new file mode 100644 index 00000000..aec28d22 --- /dev/null +++ b/iotedgedev/telemetryconfig.py @@ -0,0 +1,84 @@ +import os + +from six.moves import configparser + +from .decorators import suppress_all_exceptions + +PRIVACY_STATEMENT = """ +Welcome to iotedgedev! +------------------------- +Telemetry +--------- +The iotedgedev collects usage data in order to improve your experience. +The data is anonymous and does not include commandline argument values. +The data is collected by Microsoft. + +You can change your telemetry settings by updating '{0}' to 'no' in {1} +""" + + +class TelemetryConfig(object): + DEFAULT_DIRECT = 'DEFAULT' + FIRSTTIME_SECTION = 'firsttime' + TELEMETRY_SECTION = 'collect_telemetry' + + def __init__(self): + self.config_parser = configparser.ConfigParser({ + self.FIRSTTIME_SECTION: 'yes' + }) + self.setup() + + @suppress_all_exceptions() + def setup(self): + config_path = self.get_config_path() + config_folder = os.path.dirname(config_path) + if not os.path.exists(config_folder): + os.makedirs(config_folder) + if not os.path.exists(config_path): + self.dump() + else: + self.load() + self.dump() + + @suppress_all_exceptions() + def load(self): + with open(self.get_config_path(), 'r') as f: + self.config_parser.readfp(f) + + @suppress_all_exceptions() + def dump(self): + with open(self.get_config_path(), 'w') as f: + self.config_parser.write(f) + + @suppress_all_exceptions() + def get(self, direct, section): + return self.config_parser.get(direct, section) + + @suppress_all_exceptions() + def get_boolean(self, direct, section): + return self.config_parser.getboolean(direct, section) + + @suppress_all_exceptions() + def set(self, direct, section, val): + if val is not None: + self.config_parser.set(direct, section, val) + self.dump() + + @suppress_all_exceptions() + def check_firsttime(self): + if self.get(self.DEFAULT_DIRECT, self.FIRSTTIME_SECTION) != 'no': + self.set(self.DEFAULT_DIRECT, self.FIRSTTIME_SECTION, 'no') + print(PRIVACY_STATEMENT.format(self.TELEMETRY_SECTION, self.get_config_path())) + self.set(self.DEFAULT_DIRECT, self.TELEMETRY_SECTION, 'yes') + self.dump() + + @suppress_all_exceptions() + def get_config_path(self): + config_folder = self.get_config_folder() + if config_folder: + return os.path.join(config_folder, 'setting.ini') + return None + + @suppress_all_exceptions() + def get_config_folder(self): + return os.path.join(os.path.expanduser("~"), '.iotedgedev') diff --git a/iotedgedev/telemetryuploader.py b/iotedgedev/telemetryuploader.py new file mode 100644 index 00000000..9fc7eb6e --- /dev/null +++ b/iotedgedev/telemetryuploader.py @@ -0,0 +1,60 @@ +try: + # Python 2.x + import urllib2 as HTTPClient +except ImportError: + # Python 3.x + import urllib.request as HTTPClient + +import json +import sys + +import six +from applicationinsights import TelemetryClient +from applicationinsights.channel import (SynchronousQueue, SynchronousSender, + TelemetryChannel) +from applicationinsights.exceptions import enable + +from iotedgedev.decorators import suppress_all_exceptions + + +class LimitedRetrySender(SynchronousSender): + def __init__(self): + super(LimitedRetrySender, self).__init__() + + def send(self, data_to_send): + """ Override the default resend mechanism in SenderBase. Stop resend when it fails.""" + request_payload = json.dumps([a.write() for a in data_to_send]) + + request = HTTPClient.Request(self._service_endpoint_uri, bytearray(request_payload, 'utf-8'), + {'Accept': 'application/json', 'Content-Type': 'application/json; charset=utf-8'}) + try: + HTTPClient.urlopen(request, timeout=10) + except Exception: + pass + + +@suppress_all_exceptions() +def upload(data_to_save): + data_to_save = json.loads(data_to_save) + + for instrumentation_key in data_to_save: + client = TelemetryClient(instrumentation_key=instrumentation_key, + telemetry_channel=TelemetryChannel(queue=SynchronousQueue(LimitedRetrySender()))) + enable(instrumentation_key) + for record in data_to_save[instrumentation_key]: + name = record['name'] + raw_properties = record['properties'] + properties = {} + measurements = {} + for k, v in raw_properties.items(): + if isinstance(v, six.string_types): + properties[k] = v + else: + measurements[k] = v + client.track_event(name, properties, measurements) + client.flush() + + +if __name__ == '__main__': + # If user doesn't agree to upload telemetry, this scripts won't be executed. The caller should control. + upload(sys.argv[1]) diff --git a/iotedgedev/utility.py b/iotedgedev/utility.py index 4d94841d..3cc0826a 100644 --- a/iotedgedev/utility.py +++ b/iotedgedev/utility.py @@ -1,13 +1,12 @@ import fnmatch import os import subprocess -import sys from base64 import b64decode, b64encode from hashlib import sha256 from hmac import HMAC from time import time -from .compat import PY3 +from .compat import PY3 from .deploymentmanifest import DeploymentManifest from .moduletype import ModuleType @@ -32,8 +31,7 @@ def exe_proc(self, params, shell=False, cwd=None, suppress_out=False): self.output.procout(self.decode(stdout_data)) if proc.returncode != 0: - self.output.error(self.decode(stderr_data)) - sys.exit() + raise Exception(self.decode(stderr_data)) def call_proc(self, params, shell=False, cwd=None): try: @@ -47,8 +45,7 @@ def check_dependency(self, params, description, shell=False): try: self.exe_proc(params, shell=shell, suppress_out=True) except FileNotFoundError: - self.output.error("{0} is required by the Azure IoT Edge Dev Tool. For installation instructions, see https://aka.ms/iotedgedevwiki.".format(description)) - sys.exit(-1) + raise FileNotFoundError("{0} is required by the Azure IoT Edge Dev Tool. For installation instructions, see https://aka.ms/iotedgedevwiki.".format(description)) def is_dir_empty(self, name): if os.path.exists(name): @@ -137,9 +134,7 @@ def set_config(self, force=False, replacements=None): config_files = self.get_config_files() if len(config_files) == 0: - self.output.info( - "Unable to find config files in solution root directory") - sys.exit() + raise FileNotFoundError("Unable to find config files in solution root directory") # Expand envars and rewrite to config/ for config_file in config_files: diff --git a/setup.py b/setup.py index b88d7155..bab57f93 100644 --- a/setup.py +++ b/setup.py @@ -4,9 +4,10 @@ """The setup script.""" import atexit from subprocess import check_call -from setuptools import setup, find_packages -from setuptools.command.install import install + +from setuptools import find_packages, setup from setuptools.command.develop import develop +from setuptools.command.install import install def _execute(): @@ -45,6 +46,8 @@ def run(self): 'azure-cli-resource', 'azure-cli-cloud', 'iotedgehubdev', + 'six', + 'applicationinsights', 'commentjson' ] diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 00000000..ea4c3af2 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,28 @@ +import os + +import pytest + +from iotedgedev.telemetryconfig import TelemetryConfig + +pytestmark = pytest.mark.unit + + +def test_firsttime(request): + config = TelemetryConfig() + + def clean(): + config_path = config.get_config_path() + if os.path.exists(config_path): + os.remove(config_path) + request.addfinalizer(clean) + + clean() + config = TelemetryConfig() + + assert config.get(config.DEFAULT_DIRECT, config.FIRSTTIME_SECTION) == 'yes' + assert config.get(config.DEFAULT_DIRECT, config.TELEMETRY_SECTION) is None + + config.check_firsttime() + + assert config.get(config.DEFAULT_DIRECT, config.FIRSTTIME_SECTION) == 'no' + assert config.get(config.DEFAULT_DIRECT, config.TELEMETRY_SECTION) == 'yes' diff --git a/tests/test_decorators.py b/tests/test_decorators.py new file mode 100644 index 00000000..52982a3f --- /dev/null +++ b/tests/test_decorators.py @@ -0,0 +1,32 @@ +import pytest + +from iotedgedev.decorators import suppress_all_exceptions + +pytestmark = pytest.mark.unit + + +@pytest.fixture +def error_function(): + def _error_function(exception, fallback_return): + @suppress_all_exceptions(fallback_return=fallback_return) + def _inner_error_function(exception): + if not exception: + return 'Everything is OK' + else: + raise exception() + return _inner_error_function(exception) + return _error_function + + +def test_suppress_all_exceptions(error_function): + err_fn = error_function(Exception, 'fallback') + assert err_fn == 'fallback' + + err_fn = error_function(None, 'fallback') + assert err_fn == 'Everything is OK' + + err_fn = error_function(ImportError, 'fallback for ImportError') + assert err_fn == 'fallback for ImportError' + + err_fn = error_function(None, None) + assert err_fn == 'Everything is OK' diff --git a/tests/test_envvars.py b/tests/test_envvars.py index dd96be1f..038da273 100644 --- a/tests/test_envvars.py +++ b/tests/test_envvars.py @@ -179,9 +179,9 @@ def test_default_container_registry_password_value_exists_or_returns_empty_strin def test_container_registry_server_key_missing_sys_exit(): - with pytest.raises(SystemExit): - output = Output() - envvars = EnvVars(output) + output = Output() + envvars = EnvVars(output) + with pytest.raises(ValueError): envvars.get_envvar("CONTAINER_REGISTRY_SERVERUNITTEST", required=True) @@ -199,9 +199,9 @@ def clean(): def test_container_registry_server_value_missing_sys_exit(setup_test_env): - with pytest.raises(SystemExit): - output = Output() - envvars = EnvVars(output) + output = Output() + envvars = EnvVars(output) + with pytest.raises(ValueError): envvars.get_envvar("CONTAINER_REGISTRY_SERVERUNITTEST", required=True)