From 6de6c30b8f0d385087bd8a5a627a5f653a68e927 Mon Sep 17 00:00:00 2001 From: Jon Gallant Date: Fri, 29 Dec 2017 16:39:34 -0800 Subject: [PATCH] Init Commit --- .gitignore | 79 +++ .travis.yml | 29 + HISTORY.rst | 8 + LICENSE | 11 + MANIFEST.in | 8 + Makefile | 87 +++ README.md | 394 +++++++++++ iotedgedev/__init__.py | 7 + iotedgedev/cli.py | 138 ++++ iotedgedev/iotedgedev.py | 639 ++++++++++++++++++ iotedgedev/template/.env.tmp | 29 + iotedgedev/template/.gitignore | 9 + iotedgedev/template/config/modules.json | 74 ++ iotedgedev/template/config/runtime.json | 40 ++ .../template/modules/filter-module/.gitignore | 30 + .../filter-module/Docker/arm32v7/Dockerfile | 9 + .../filter-module/Docker/linux-x64/Dockerfile | 9 + .../Docker/linux-x64/Dockerfile.debug | 15 + .../Docker/windows-nano/Dockerfile | 9 + .../template/modules/filter-module/Program.cs | 174 +++++ .../filter-module/filter-module.csproj | 22 + iotedgedev/template/template.zip | Bin 0 -> 10788 bytes requirements.txt | 5 + requirements_dev.txt | 10 + setup.cfg | 21 + setup.py | 68 ++ tests/__init__.py | 3 + tests/test_iotedgedev.py | 86 +++ tox.ini | 27 + travis_pypi_setup.py | 127 ++++ 30 files changed, 2167 insertions(+) create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 HISTORY.rst create mode 100644 LICENSE create mode 100644 MANIFEST.in create mode 100644 Makefile create mode 100644 README.md create mode 100644 iotedgedev/__init__.py create mode 100644 iotedgedev/cli.py create mode 100644 iotedgedev/iotedgedev.py create mode 100644 iotedgedev/template/.env.tmp create mode 100644 iotedgedev/template/.gitignore create mode 100644 iotedgedev/template/config/modules.json create mode 100644 iotedgedev/template/config/runtime.json create mode 100644 iotedgedev/template/modules/filter-module/.gitignore create mode 100644 iotedgedev/template/modules/filter-module/Docker/arm32v7/Dockerfile create mode 100644 iotedgedev/template/modules/filter-module/Docker/linux-x64/Dockerfile create mode 100644 iotedgedev/template/modules/filter-module/Docker/linux-x64/Dockerfile.debug create mode 100644 iotedgedev/template/modules/filter-module/Docker/windows-nano/Dockerfile create mode 100644 iotedgedev/template/modules/filter-module/Program.cs create mode 100644 iotedgedev/template/modules/filter-module/filter-module.csproj create mode 100644 iotedgedev/template/template.zip create mode 100644 requirements.txt create mode 100644 requirements_dev.txt create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/test_iotedgedev.py create mode 100644 tox.ini create mode 100644 travis_pypi_setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..39c3de99 --- /dev/null +++ b/.gitignore @@ -0,0 +1,79 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# pyenv python configuration file +.python-version + +**/.vs +**/.vscode +**/bin +**/obj +**/out +.env +venv +/logs +/config +/build + + +py36 +.pypirc +test_project +README \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..6cfa93e8 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,29 @@ +# Config file for automatic testing at travis-ci.org +# This file will be regenerated if you run travis_pypi_setup.py + +language: python +python: + - 3.5 + - 3.4 + - 3.3 + - 2.7 + - 2.6 + +# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors +install: pip install -U tox-travis + +# command to run tests, e.g. python setup.py test +script: tox + +# After you create the Github repo and add it to Travis, run the +# travis_pypi_setup.py script to finish PyPI deployment setup +deploy: + provider: pypi + distributions: sdist bdist_wheel + user: jonbgallant + password: + secure: PLEASE_REPLACE_ME + on: + tags: true + repo: jonbgallant/azure-iot-edge-dev-tool + python: 2.7 diff --git a/HISTORY.rst b/HISTORY.rst new file mode 100644 index 00000000..00f5ac37 --- /dev/null +++ b/HISTORY.rst @@ -0,0 +1,8 @@ +======= +History +======= + +0.1.0 (2017-12-29) +------------------ + +* First release on PyPI. diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..66088920 --- /dev/null +++ b/LICENSE @@ -0,0 +1,11 @@ + +MIT License + +Copyright (c) 2017, Jon Gallant + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..507fd6c1 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,8 @@ +include HISTORY.rst +include LICENSE +include README.md +include iotedgedev/template/template.zip + +recursive-include tests * +recursive-exclude * __pycache__ +recursive-exclude * *.py[co] \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..3053f983 --- /dev/null +++ b/Makefile @@ -0,0 +1,87 @@ +.PHONY: clean clean-test clean-pyc clean-build docs help +.DEFAULT_GOAL := help +define BROWSER_PYSCRIPT +import os, webbrowser, sys +try: + from urllib import pathname2url +except: + from urllib.request import pathname2url + +webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) +endef +export BROWSER_PYSCRIPT + +define PRINT_HELP_PYSCRIPT +import re, sys + +for line in sys.stdin: + match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) + if match: + target, help = match.groups() + print("%-20s %s" % (target, help)) +endef +export PRINT_HELP_PYSCRIPT +BROWSER := python -c "$$BROWSER_PYSCRIPT" + +help: + @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) + +clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts + + +clean-build: ## remove build artifacts + rm -fr build/ + rm -fr dist/ + rm -fr .eggs/ + find . -name '*.egg-info' -exec rm -fr {} + + find . -name '*.egg' -exec rm -f {} + + +clean-pyc: ## remove Python file artifacts + find . -name '*.pyc' -exec rm -f {} + + find . -name '*.pyo' -exec rm -f {} + + find . -name '*~' -exec rm -f {} + + find . -name '__pycache__' -exec rm -fr {} + + +clean-test: ## remove test and coverage artifacts + rm -fr .tox/ + rm -f .coverage + rm -fr htmlcov/ + +lint: ## check style with flake8 + flake8 iotedgedev tests + +test: ## run tests quickly with the default Python + + python setup.py test + +test-all: ## run tests on every Python version with tox + tox + +coverage: ## check code coverage quickly with the default Python + coverage run --source iotedgedev setup.py test + coverage report -m + coverage html + $(BROWSER) htmlcov/index.html + +docs: ## generate Sphinx HTML documentation, including API docs + rm -f docs/iotedgedev.rst + rm -f docs/modules.rst + sphinx-apidoc -o docs/ iotedgedev + $(MAKE) -C docs clean + $(MAKE) -C docs html + $(BROWSER) docs/_build/html/index.html + +servedocs: docs ## compile the docs watching for changes + watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . + +release: clean ## package and upload a release + python setup.py sdist upload + python setup.py bdist_wheel upload + +dist: clean ## builds source and wheel package + python setup.py sdist + python setup.py bdist_wheel + ls -l dist + +install: clean ## install the package to the active Python's site-packages + python setup.py install diff --git a/README.md b/README.md new file mode 100644 index 00000000..c4beb278 --- /dev/null +++ b/README.md @@ -0,0 +1,394 @@ +# Azure IoT Edge Dev Tool + +The Azure IoT Edge Dev Tool **greatly simplifies** your Azure IoT Edge development process. It has everything you need to get started and helps with your day-to-day Edge development. + +You will be able to do all of the following with simple one line commands. + +1. Create a new Edge project: `iotedgedev project --create` +1. Build, Push and Deploy modules: `iotedgedev modules --build --deploy` +1. Setup and Start the Edge Runtime: `iotedgedev runtime --setup --start` +1. View and Save Docker log files: `iotedgedev docker --logs` +1. Use a Custom Container Registry: `iotedgedev docker --setup-registry` + +The project was created by Microsofties that work with Edge customers, who have found it very helpful. We hope to get a variation of this officially supported by the Azure IoT team in the near future. Your contributions and feedback to this project will help us build an amazing developer experience, so please do not hesitate to participate. + +Please see [Azure IoT Edge Dev Resources](https://github.com/jonbgallant/azure-iot-edge-dev) for links to official docs and other Edge dev information. + +## Setup +### Azure Setup +1. [Create Azure IoT Hub](https://docs.microsoft.com/en-us/azure/iot-hub/iot-hub-csharp-csharp-getstarted#create-an-iot-hub) +1. [Create Azure Container Registry](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-get-started-portal) + - Make sure you enable Admin Access when you create the Azure Container Registry +1. Create Edge Device using the Azure Portal + +You can also deploy the IoT Hub and Container Registry with this **Deploy to Azure** template: + +[![Azure Deployment](https://azuredeploy.net/deploybutton.png)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2Fjonbgallant%2Fazure-iot-edge-dev-tool%2Fmaster%2Fassets%2Fdeploy%2FARMDeployment%2Fazuredeploy.json) + +### Dev Machine Setup + +Here's what you need to do to get `iotedgedev` running on your dev machine. If you are using a seperate Edge device, like a Raspberry Pi, you do not need to run all of these steps on your Edge device, you can just use `iotedgectl` directly . See the [Edge Device Setup](#edge-device-setup) section below for more information on setting up your Edge device. + +> Note: See the ["Test Coverage"](#test-coverage) section below to see what this module has been tested with. + +1. Install **[Docker](https://docs.docker.com/engine/installation/)** + - Switch to Linux Containers if you are running Windows. + + **Do not** install via `sudo apt install docker.io`. Use the proper steps for [CE here](https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/#install-docker-ce), or use the [convenience script](https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/#install-using-the-convenience-script). + +1. Install **Python 2.7 or Python 3** + + - **Linux** - `sudo apt install python-pip` or `sudo apt install python3-pip` + - **Windows** - [Install from Python's website](https://www.python.org/downloads/) + +1. Install **[.NET Core SDK](https://www.microsoft.com/net/core#windowscmd)** + - The .NET Core SDK does not run on ARM, so you do not need to install this on Raspberry Pi. + +1. Install **Dependencies** + + > You can also run under a Python Virtual Environment. See the [Python Virtual Environment Setup](#python-virtual-environment-setup) instructions below for details on how to set that up. + + 1. System Dependencies + 1. Mac: + ``` + sudo easy_install pip + brew install libffi + ``` + + 1. Raspberry Pi: + ``` + sudo pip install --upgrade setuptools pip + sudo apt install python2.7-dev libffi-dev libssl-dev -y + ``` +1. Install **`azure-iot-edge-dev-tool`** + + > You do not need to run this on the Raspberry Pi Edge device. See the [Edge Device Setup](#edge-device-setup) section below for more information on setting up your Edge device. + + 1. Via Pip - Use this if you want to use the tool as is. + ``` + pip install -U azure-iot-edge-dev-tool + ``` + + 1. Via GitHub - Use this if you want to make changes to this tool. + + 1. Clone or Fork this Repository + + > Replace `project-name` with the name of your project. + + `git clone https://github.com/jonbgallant/azure-iot-edge-dev-tool.git project-name` + + 1. Install Dependencies + + > Use 'sudo' for Mac/Linux/RaspberryPi. + + `pip install -U -r requirements.txt` + + 1. Install iotedgedev Module + + The following command will make `iotedgedev` available in your terminal. + + From the root of repo: + + `pip install -e .` + +## Usage + +Here's how you create a project, build and deploy our modules, and then setup and start the Edge runtime. + +### Step 1: Create Azure IoT Edge Project + +The following command will setup the folder structure required for this module + +> Replace `edge-project` with the name of your project. Use `.` to create in the current folder. + +``` +iotedgedev project --create edge-project +``` + +#### Folder Structure + +When you create a new project, it will have the following contents: + +1. **config folder** - Contains sample config files for both modules and runtime. + +1. **build folder** - Contains the files outputted by the .NET Core SDK. + +1. **modules folders** - Contains all of the modules for your Edge project. + - The iotedgedev module assumes that you'll structure your Dockerfiles exactly like the filter-module sample. Have a Docker folder in the root of the project, then subfolders within that to support multiple Docker files. + + > It is important that you follow this structure or the module will not work. Please make suggestions or fork this project if you would like a different behavior. + +1. **logs folder** - Contains all the Docker log files for the Runtime and your modules. + +1. **.env.tmp** - Contains all the required Environment Variables for Edge Project. Rename to .env and add your settings. + +1. **.gitigore.tmp** - A suggested .gitignore file for your Edge projects. Rename to .gitignore if you'd like to use it. + +### Step 2: Update Environment Variables + +The settings used for this module are stored in a .env file in the root of your project. System or User Environment Variables take precedence over values in .env file. + +1. Rename `.env.tmp` to `.env` or run the following command: + + `cp .env.tmp .env` + +2. Open `.env` and set variables + + 1. Runtime Home Directory + + - For Linux/Raspberry Pi, change: `RUNTIME_HOME_DIR="/etc/azure-iot-edge"` + + 1. Active Modules + + - You can tell `iotedgedev` which modules you want to build by including them in the `ACTIVE_MODULES` setting. Comma separated. + + 1. Active Docker Directories + + - You can tell `iotedgedev` which Docker files to build and push by including the Dockerfile's parent folder in the `ACTIVE_DOCKER_DIRS` setting. + + 1. At a minimum, you need to set the following values, which you you can get from your Azure account: + ``` + IOTHUB_NAME="iot hub name" + IOTHUB_KEY="iot hub key" + DEVICE_CONNECTION_STRING="edge device connection string" + EDGE_DEVICE_ID="edge device id" + RUNTIME_HOST_NAME="the computer name that your edge will run on" + ACTIVE_MODULES="filter-module" + ACTIVE_DOCKER_DIRS="arm32v7,linux-x64" + CONTAINER_REGISTRY_SERVER="" + CONTAINER_REGISTRY_USERNAME="" + CONTAINER_REGISTRY_PASSWORD="" + ``` +1. Update Config + + If you are running on Raspberry Pi you need to use the arm32v7 Dockerfile. Open `config/modules.json`, find the `filter-module` line and replace `linux-x64` with `arm32v7`. + + Replace this: + `"image": "${CONTAINER_REGISTRY_SERVER}/filter-module:linux-x64-${CONTAINER_TAG}",` + + With this: + `"image": "${CONTAINER_REGISTRY_SERVER}/filter-module:arm32v7-${CONTAINER_TAG}",` + +### Step 3: Build and Deploy Modules + +> Use `sudo` for Linux. You __will not__ be able to build on the Raspberry Pi, because the .NET Core SDK does not support ARM. You can build on an x86 based machine and deploy to Pi. + +``` +iotedgedev modules --build --deploy +``` + +The- `--build` command will build each module in the `modules` folder and push it to your container registry. The- `--deploy` command will apply the `build/modules.json` configuration file to your Edge device. + +You can configure what modules will be built and deployed using the `ACTIVE_MODULES` env var in the `.env` file. + +### Step 4: Setup and Start the Edge Runtime + +> Use 'sudo' for Linux/RaspberryPi + +``` +iotedgedev runtime --setup --start +``` + +The- `--setup` command will apply the `/build/runtime.json` file to your Edge device. The- `--start` command will start the Edge runtime. + +### Step 5: Monitor Messages + +You can use the [Device Explorer](https://github.com/Azure/azure-iot-sdk-csharp/releases/download/2017-12-2/SetupDeviceExplorer.msi) to monitor the messages that are sent to your IoT Hub. + +## Commands +The `iotedgedev` module has the following commands: + +**project** + +`iotedgedev project --help` +- `--create TEXT` Creates a new Azure IoT Edge project. Use `--create .` to create in current folder. Use `--create TEXT` to create in a subfolder. + +**runtime** + +`iotedgedev runtime --help` +- `--setup` Setup Edge Runtime using runtime.json in build/config directory. +- `--start` Starts Edge Runtime. Calls iotedgectl start. +- `--stop` Stops Edge Runtime. Calls iotedgectl stop. +- `--restart` Restarts Edge Runtime. Calls iotedgectl stop, removes module containers and images, calls iotedgectl setup (with --config-file) and then calls iotedgectl start. +- `--status` Edge Runtime Status. Calls iotedgectl status. + +**modules** + +`iotedgedev modules --help` +- `--build` Builds and pushes modules specified in ACTIVE_MODULES Environment Variable to specified container registry. +- `--deploy` Deploys modules to Edge device using modules.json in build/config directory. + +**docker** + +`iotedgedev docker --help` +- `--setup-registry` Pulls Edge Runtime from Docker Hub and pushes to your specified container registry. Also, updates config files to use CONTAINER_REGISTRY_* instead of the Microsoft Docker hub. See CONTAINER_REGISTRY Environment Variables. +- `--clean` Removes all the Docker containers and Images. +- `--remove-modules` Removes only the edge modules Docker containers and images specified in ACTIVE_MODULES, not edgeAgent or edgeHub. +- `--remove-containers` Removes all the Docker containers +- `--remove-images` Removes all the Docker images. +- `--logs` Opens a new terminal window for edgeAgent, edgeHub and each edge module and saves to LOGS_PATH. You can configure the terminal command with LOGS_CMD. +- `--show-logs` Opens a new terminal window for edgeAgent, edgeHub and each edge module. You can configure the terminal command with LOGS_CMD. +- `--save-logs` Saves edgeAgent, edgeHub and each edge module logs to LOGS_PATH. + +**iotedgedev commands** + +- `--version` Show the version and exit. +- `--set-config` Expands Environment Variables in /config/*.json and copies to /build/config. + +### Setup Container Registry + +You can also use `iotedgedev` to host the Edge runtime from your own Azure Container Registry or a Local Container Registry. Set the `.env` values for your Container Registry and run the following command. It will pull all the Edge containers from Dockerhub, tag them and upload them to the container registry you have specified in `.env`. + +> Use 'sudo' for Linux/RaspberryPi + +``` +iotedgedev docker --setup-registry +``` + + +### View Docker Logs + +#### Show Logs +The iotedgedev module also include a "Show Logs" command that will open a new command prompt for each module it finds in your Edge config. Just run the following command: + +> Note: I haven't figured out how to launch new SSH windows in a reliable way. It's in the backlog. For now, you must be on the desktop of the machine to run this command. + +``` +iotedgedev docker --show-logs +``` + +You can configure the logs command in the `.env` file with the `LOGS_CMD` setting. The `.env.tmp` file provides two options, one for [ConEmu](https://conemu.github.io/) and one for Cmd.exe. + +#### Save Logs + +You can also output the logs to the LOGS_PATH directory. The following command will output all the logs and add them to an `edge-logs.zip` file that you can send to the Azure IoT support team if they request it. + +``` +iotedgedev docker --save-logs +``` + +#### Both Show and Save Logs + +Run the following to show and save logs with a single command + +``` +iotedgedev docker --logs +``` + +### Local Docker Registry Setup + +Instead of using a cloud based container registry, you can use a local Docker registry. Here's how to get it setup. + +1. Set `CONTAINER_REGISTRY_SERVER` in .env to `localhost:5000`. You can enter a different port if you'd like to. +1. Add `localhost:5000` and `127.0.0.1:5000` to Docker -> Settings -> Daemon -> Insecure Registries + +`iotedgedev` will look for `localhost` in your setting and take care of the rest for you. + +## Edge Device Setup + +The `iotedgedev` module is intended to help with Edge development and doesn't necessarily need to be taken on as a dependency in production or integration environments, where you'll likely want to use the `iotedgectl` module directly. You can use `iotedgedev` to generate your runtime.json file on your dev machine, copy that to your Edge device and then use the following command to setup and start your Edge Runtime. + +``` +iotedgectl setup --config-file runtime.json +iotedgectl start +``` + +Having said that, there's nothing stopping you from deploying `iotedgedev` to your Edge device. It may be helpful if you want to run the `iotedgedev docker --clean` command to clean up Docker containers and images. Or if you want to run `iotedgedev docker --show-logs` to see all the log files on the device or `iotedgedev docker --save-logs` to output to the LOGS_PATH directory. + +> Please note that the .NET Core SDK does not support ARM, so you will not be able to run `modules --build` or `modules --deploy` directly on a Raspberry Pi. + +### Raspberry Pi + +Whether you use `iotedgedev` or directly use `iotedgecgtl` on the Raspberry Pi, you will still need to run the following commands before you run the Edge Runtime. + + ``` + sudo pip install --upgrade setuptools pip + sudo apt install python2.7-dev libffi-dev libssl-dev -y + sudo pip install -U azure-iot-edge-dev-tool + ``` + +## Python Virtual Environment Setup + +You can run `iotedgedev` inside a Python Vritual Environment. + +1. Install virtualenv + + `pip install virtualenv` + +1. Create virtualenv + + Execute the following from the root of this repository. + + `virtualenv venv` + + > venv is just a project name that can be anything you want, but we recommend sticking with venv because the .gitignore file excludes it. + +1. Activate the virtualenv + + Windows: `venv\Scripts\activate.bat` + + Posix: `source venv/bin/activate` + +1. Install Dependencies + + Continue with the instructions above starting with the [Dev Machine Setup](#dev-machine-setup) -> Install Dependencies. + +1. Deactivate the virtualenv + + When you are done with your virtualenv, you can deactivate it with the follow command: + + `deactivate` + +## Test Coverage + +This module has been tested with the following: +- Windows 10 Fall Creators Update +- Ubuntu 16.04 +- Mac Sierra 10.12.6 +- Raspberry Pi with Raspbian Stretch (**.NET Core Runtime Only**, .NET Core SDK not supported on ARM.) - You cannot use Raspberry Pi as a Edge dev machine, but it can host the Edge runtime. +- Python 2.7.13 and Python 3.6.3 +- Docker Version 17.09.1-ce-win42 (14687), Channel: stable, 3176a6a + +## Troubleshooting + +1. Invalid Reference Format + ``` + 500 Server Error: Internal Server Error for url: http+docker://localunixsocket/v1.30/images + 500 Server Error: Internal Server Error ("invalid reference format") + ``` + + Solution: You likely installed Docker via `sudo apt install docker.io`. Use the proper steps for [CE here](https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/#install-docker-ce), or use the [convenience script](https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/#install-using-the-convenience-script). + +1. Permissions Error + + ``` + The directory '/home/user/.cache/pip/http' or its parent directory is not owned by the current user and the cache has been disabled. Please check the permissions and owner of that directory. If executing pip with sudo, you may want sudo's -H flag. + ``` + + Solution: Run pip install with -H `sudo -H pip install -U -r requirements.txt` + +1. Latest Docker Image Not Pulled from Registry + + By design, Docker only pulls images that have been changed. When you are developing you tend to push new images on a frequent basis, which Docker will not pull unless it has a unique tag. You could assign every push a new tag, or you can simply run the following command on the Edge device. + + ``` + iotedgedev runtime --restart + ``` + + This stops the runtime, removes your ACTIVE_MODULES containers and images, calls setup and then starts the runtime. Docker will then be forced to pull the imageS from the registry because it does not have them locally anymore. + +1. 404 Client Error: Not Found ("No such container: edgeAgent") + + I occasionally see this when running `iotedgedev runtime --restart`, but I have never seen it cause any issues. LMK if you see any issues because of it. + +## Backlog + +Please see the [GitHub project page](https://github.com/jonbgallant/azure-iot-edge-dev-tool/projects) for backlog tasks. + +## Issues + +Please use the [GitHub issues page](https://github.com/jonbgallant/azure-iot-edge-dev-tool/issues) to report any issues. + +## Contributing + +Please fork, branch and pull-request any changes you'd like to make. \ No newline at end of file diff --git a/iotedgedev/__init__.py b/iotedgedev/__init__.py new file mode 100644 index 00000000..8bf27b08 --- /dev/null +++ b/iotedgedev/__init__.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- + +"""Top-level package for Azure IoT Edge Dev Tool.""" + +__author__ = 'Jon Gallant' +__email__ = 'info@jongallant.com' +__version__ = '0.46.0' diff --git a/iotedgedev/cli.py b/iotedgedev/cli.py new file mode 100644 index 00000000..5a74dc10 --- /dev/null +++ b/iotedgedev/cli.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- + +"""Console script for iotedgedev.""" +from __future__ import absolute_import + +import click +from .iotedgedev import Docker +from .iotedgedev import Modules +from .iotedgedev import Runtime +from .iotedgedev import Project +from .iotedgedev import Utility +from .iotedgedev import EnvVars +from .iotedgedev import Output + + +output = Output() +envvars = EnvVars(output) +utility = Utility(envvars, output) +dock = Docker(envvars, utility, output) +mod = Modules(envvars, utility, output, dock) +run = Runtime(envvars, utility, output, dock) +proj = Project(output) + + +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) + + +@click.group(context_settings=CONTEXT_SETTINGS, invoke_without_command=True) +@click.version_option() +@click.option('--set-config', default=False, required=False, is_flag=True, help="Expands Environment Variables in /config/*.json and copies to /build/config.") +def main(set_config): + if(set_config): + utility.set_config() + else: + ctx = click.get_current_context() + if ctx.invoked_subcommand == None: + click.echo(ctx.get_help()) + + +@click.command(context_settings=CONTEXT_SETTINGS) +@click.option('--create', default=".", required=False, help="Creates a new Azure IoT Edge project. Use `--create .` to create in current folder. Use `--create TEXT` to create in a subfolder.") +def project(create): + if create: + proj.create(create) + + +@click.command(context_settings=CONTEXT_SETTINGS) +@click.option('--build', default=False, required=False, is_flag=True, help="Builds and pushes modules specified in ACTIVE_MODULES Environment Variable to specified container registry.") +@click.option('--deploy', default=False, required=False, is_flag=True, help="Deploys modules to Edge device using modules.json in build/config directory.") +def modules(build, deploy): + envvars.check() + utility.set_config() + dock.init_registry() + + if build: + mod.build() + + if deploy: + mod.deploy() + + +@click.command(context_settings=CONTEXT_SETTINGS) +@click.option('--setup', default=False, required=False, is_flag=True, help="Setup Edge Runtime using runtime.json in build/config directory.") +@click.option('--start', default=False, required=False, is_flag=True, help="Starts Edge Runtime. Calls iotedgectl start.") +@click.option('--stop', default=False, required=False, is_flag=True, help="Stops Edge Runtime. Calls iotedgectl stop.") +@click.option('--restart', default=False, required=False, is_flag=True, help="Restarts Edge Runtime. Calls iotedgectl stop, removes module containers and images, calls iotedgectl setup (with --config-file) and then calls iotedgectl start.") +@click.option('--status', default=False, required=False, is_flag=True, help="Edge Runtime Status. Calls iotedgectl status.") +def runtime(setup, start, stop, restart, status): + envvars.check() + + utility.set_config() + + if setup: + run.setup() + + if start: + run.start() + + if stop: + run.stop() + + if restart: + run.stop() + dock.remove_modules() + run.setup() + run.start() + + if status: + run.status() + + +@click.command(context_settings=CONTEXT_SETTINGS) +@click.option('--setup-registry', default=False, required=False, is_flag=True, help="Pulls Edge Runtime from Docker Hub and pushes to your specified container registry. Also, updates config files to use CONTAINER_REGISTRY_* instead of the Microsoft Docker hub. See CONTAINER_REGISTRY Environment Variables.") +@click.option('--clean', default=False, required=False, is_flag=True, help="Removes all the Docker containers and Images.") +@click.option('--remove-modules', default=False, required=False, is_flag=True, help="Removes only the edge modules Docker containers and images specified in ACTIVE_MODULES, not edgeAgent or edgeHub.") +@click.option('--remove-containers', default=False, required=False, is_flag=True, help="Removes all the Docker containers") +@click.option('--remove-images', default=False, required=False, is_flag=True, help="Removes all the Docker images.") +@click.option('--logs', default=False, required=False, is_flag=True, help="Opens a new terminal window for edgeAgent, edgeHub and each edge module and saves to LOGS_PATH. You can configure the terminal command with LOGS_CMD.") +@click.option('--show-logs', default=False, required=False, is_flag=True, help="Opens a new terminal window for edgeAgent, edgeHub and each edge module. You can configure the terminal command with LOGS_CMD.") +@click.option('--save-logs', default=False, required=False, is_flag=True, help="Saves edgeAgent, edgeHub and each edge module logs to LOGS_PATH.") +def docker(setup_registry, clean, remove_modules, remove_containers, remove_images, logs, show_logs, save_logs): + + envvars.check() + + utility.set_config() + + if setup_registry: + dock.setup_registry() + + if clean: + remove_containers = True + remove_images = True + + if remove_modules: + dock.remove_modules() + + if remove_containers: + dock.remove_containers() + + if remove_images: + dock.remove_images() + + if logs: + show_logs = True + save_logs = True + + if show_logs or save_logs: + dock.handle_logs_cmd(show_logs, save_logs) + + +main.add_command(runtime) +main.add_command(modules) +main.add_command(docker) +main.add_command(project) + + +if __name__ == "__main__": + main() diff --git a/iotedgedev/iotedgedev.py b/iotedgedev/iotedgedev.py new file mode 100644 index 00000000..7284780b --- /dev/null +++ b/iotedgedev/iotedgedev.py @@ -0,0 +1,639 @@ +# -*- coding: utf-8 -*- + +from __future__ import absolute_import +import requests +import uuid +import docker +import os +import subprocess +import sys +import json +import click +import zipfile +from base64 import b64encode, b64decode +from hashlib import sha256 +from time import time +import fnmatch +import inspect +from hmac import HMAC +from shutil import copyfile +from dotenv import load_dotenv + +dotenv_path = os.path.join(os.getcwd(), '.env') +load_dotenv(dotenv_path) + +if sys.version_info.major >= 3: + from urllib.parse import quote, urlencode +else: + from urllib import quote, urlencode + +# self.output.info("Python Version: " + sys.version) + + +class Output: + + def info(self, text): + click.secho(text, fg='yellow') + + def error(self, text): + click.secho(text, fg='red') + + def header(self, text): + click.secho("======== {0} ========".format(text).upper(), fg='white') + + def footer(self, text): + self.info(text.upper()) + self.line() + + def procout(self, text): + click.secho(text, dim=True) + + def line(self): + click.secho("") + + +class EnvVars: + def __init__(self, output): + self.output = output + + def check(self): + try: + self.IOTHUB_NAME = os.environ["IOTHUB_NAME"] + self.IOTHUB_KEY = os.environ["IOTHUB_KEY"] + self.DEVICE_CONNECTION_STRING = os.environ["DEVICE_CONNECTION_STRING"] + self.EDGE_DEVICE_ID = os.environ["EDGE_DEVICE_ID"] + self.RUNTIME_HOST_NAME = os.environ["RUNTIME_HOST_NAME"] + self.ACTIVE_MODULES = os.environ["ACTIVE_MODULES"] + self.ACTIVE_DOCKER_DIRS = os.environ["ACTIVE_DOCKER_DIRS"] + self.CONTAINER_REGISTRY_SERVER = os.environ["CONTAINER_REGISTRY_SERVER"] + self.CONTAINER_REGISTRY_USERNAME = os.environ["CONTAINER_REGISTRY_USERNAME"] + self.CONTAINER_REGISTRY_PASSWORD = os.environ["CONTAINER_REGISTRY_PASSWORD"] + self.IOTHUB_POLICY_NAME = os.environ["IOTHUB_POLICY_NAME"] + self.CONTAINER_TAG = os.environ["CONTAINER_TAG"] + self.RUNTIME_TAG = os.environ["RUNTIME_TAG"] + self.RUNTIME_VERBOSITY = os.environ["RUNTIME_VERBOSITY"] + self.RUNTIME_HOME_DIR = os.environ["RUNTIME_HOME_DIR"] + self.MODULES_CONFIG_FILE = os.environ["MODULES_CONFIG_FILE"] + self.RUNTIME_CONFIG_FILE = os.environ["RUNTIME_CONFIG_FILE"] + self.LOGS_PATH = os.environ["LOGS_PATH"] + self.MODULES_PATH = os.environ["MODULES_PATH"] + self.IOT_REST_API_VERSION = os.environ["IOT_REST_API_VERSION"] + self.DOTNET_VERBOSITY = os.environ["DOTNET_VERBOSITY"] + self.LOGS_CMD = os.environ["LOGS_CMD"] + except Exception as e: + self.output.error( + "Environment variables not configured correctly. Run `iotedgedev project --create [name]` to create a new project with sample .env file. Please see README for variable configuration options.") + self.output.error("Variable that caused exception: " + str(e)) + sys.exit() + + +class Utility: + def __init__(self, envvars, output): + self.envvars = envvars + self.output = output + + def exe_proc(self, params, shell=False): + proc = subprocess.Popen( + params, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell) + + stdout_data, stderr_data = proc.communicate() + if stdout_data != "": + self.output.procout(self.decode(stdout_data)) + + if proc.returncode != 0: + self.output.error(self.decode(stderr_data)) + sys.exit() + + def find_files(self, directory, pattern): + # find all files in directory that match the pattern. + for root, dirs, files in os.walk(directory): + for basename in files: + if fnmatch.fnmatch(basename, pattern): + filename = os.path.join(root, basename) + yield filename + + def get_iot_hub_sas_token(self, uri, key, policy_name, expiry=3600): + ttl = time() + expiry + sign_key = "%s\n%d" % ((quote(uri)), int(ttl)) + signature = b64encode( + HMAC(b64decode(key), sign_key.encode("utf-8"), sha256).digest()) + + rawtoken = { + "sr": uri, + "sig": signature, + "se": str(int(ttl)) + } + + if policy_name is not None: + rawtoken["skn"] = policy_name + + return "SharedAccessSignature " + urlencode(rawtoken) + + def get_file_contents(self, file): + with open(file, "r") as file: + return file.read() + + def decode(self, val): + return val.decode("utf-8").strip() + + def get_config_files(self): + config_dir = "config" + + # Create config dir if it doesn't exist + if not os.path.exists(config_dir): + os.makedirs(config_dir) + + # Get all config files in \config dir. + return [os.path.join(config_dir, f) for f in os.listdir( + config_dir) if f.endswith(".json")] + + def get_active_modules(self): + return [module.strip() for module in self.envvars.ACTIVE_MODULES.split(",") if module] + + def set_config(self): + self.output.header("PROCESSING CONFIG FILES") + + build_config_dir = os.path.join("build", "config") + + # Create config dir if it doesn't exist + if not os.path.exists(build_config_dir): + os.makedirs(build_config_dir) + + config_files = self.get_config_files() + + if len(config_files) == 0: + self.output.info("Unable to find config files in config directory") + sys.exit() + + # Expand envars and rewrite to \build\config + for config_file in config_files: + + build_config_file = os.path.join( + build_config_dir, os.path.basename(config_file)) + + self.output.info("Expanding '{0}' to '{1}'".format( + config_file, build_config_file)) + + config_file_expanded = os.path.expandvars( + self.get_file_contents(config_file)) + + with open(build_config_file, "w") as config_file_build: + config_file_build.write(config_file_expanded) + + self.output.line() + + +class Project: + def __init__(self, output): + self.output = output + + def create(self, name): + self.output.header("CREATING AZURE IOT EDGE PROJECT") + + try: + template_zip = os.path.join(os.path.split( + __file__)[0], "template", "template.zip") + except Exception as e: + self.output.error("Error while trying to load template.zip") + self.output.error(e) + + if name == ".": + name = "" + + zipf = zipfile.ZipFile(template_zip) + zipf.extractall(name) + + self.output.footer("Azure IoT Edge project created") + + +class Runtime: + def __init__(self, envvars, utility, output, dock): + self.envvars = envvars + self.utility = utility + self.dock = dock + self.output = output + + def start(self): + self.output.header("Starting Edge Runtime") + self.utility.exe_proc(["iotedgectl", "--verbose", + self.envvars.RUNTIME_VERBOSITY, "start"]) + + def stop(self): + self.output.header("Stopping Edge Runtime") + self.utility.exe_proc(["iotedgectl", "--verbose", + self.envvars.RUNTIME_VERBOSITY, "stop"]) + + def setup(self): + self.output.header("Setting Up Edge Runtime") + self.utility.exe_proc(["iotedgectl", "--verbose", self.envvars.RUNTIME_VERBOSITY, + "setup", "--config-file", self.envvars.RUNTIME_CONFIG_FILE]) + + def status(self): + self.output.header("Getting Edge Runtime Status") + self.utility.exe_proc(["iotedgectl", "--verbose", self.envvars.RUNTIME_VERBOSITY, + "status"]) + + +class Modules: + def __init__(self, envvars, utility, output, dock): + self.envvars = envvars + self.utility = utility + self.output = output + self.dock = dock + + def build(self): + self.output.header("BUILDING MODULES") + + # Get all the modules to build as specified in config. + modules_to_process = self.utility.get_active_modules() + + for module in os.listdir(self.envvars.MODULES_PATH): + + if len(modules_to_process) == 0 or modules_to_process[0] == "*" or module in modules_to_process: + + module_dir = os.path.join(self.envvars.MODULES_PATH, module) + + self.output.info("BUILDING MODULE: {0}".format(module_dir)) + + # 1. dotnet restore + # Removing restore as it will now be auto called by build. + #self.output.info("Step 1: Restore Module: " + module) + # self.utility.exe_proc(["dotnet", "restore", module_dir, + # "-v", self.envvars.DOTNET_VERBOSITY]) + + # 2. dotnet build + + # Find first proj file in module dir and use it. + project_files = [os.path.join(module_dir, f) for f in os.listdir( + module_dir) if f.endswith("proj")] + + if len(project_files) == 0: + self.output.info("No project file found for module.") + continue + + #self.output.info("Building project file: " + project_files[0]) + + self.utility.exe_proc(["dotnet", "build", project_files[0], + "-v", self.envvars.DOTNET_VERBOSITY]) + + # 3. Get all docker files in project + docker_files = self.utility.find_files( + module_dir, "Dockerfile*") + + docker_dirs_process = [docker_dir.strip() + for docker_dir in self.envvars.ACTIVE_DOCKER_DIRS.split(",") if docker_dir] + + # 4. Process each Dockerfile found + for docker_file in docker_files: + + docker_file_parent_folder = os.path.basename( + os.path.dirname(docker_file)) + + if len(docker_dirs_process) == 0 or docker_dirs_process[0] == "*" or docker_file_parent_folder in docker_dirs_process: + + self.output.info( + "PROCESSING DOCKER FILE: " + docker_file) + + docker_file_name = os.path.basename(docker_file) + + # assume /Docker/{runtime}/Dockerfile folder structure + # image name will be the same as the module folder name, filter-module + # tag will be {runtime}{ext}{container_tag}, i.e. linux-x64-debug-jong + # runtime is the Dockerfile immediate parent folder name + # ext is Dockerfile extension for example with Dockerfile.debug, debug is the mod + # CONTAINER_TAG is env var + + # i.e. when found: filter-module/Docker/linux-x64/Dockerfile.debug and CONTAINER_TAG = jong + # we'll get: filtermodule:linux-x64-debug-jong + + runtime = os.path.basename( + os.path.dirname(docker_file)) + ext = "" if os.path.splitext(docker_file)[ + 1] == "" else "-" + os.path.splitext(docker_file)[1][1:] + container_tag = "" if self.envvars.CONTAINER_TAG == "" else "-" + \ + self.envvars.CONTAINER_TAG + + tag_name = runtime + ext + container_tag + + # construct the build output path + build_path = os.path.join( + os.getcwd(), "build", "modules", module, runtime) + if not os.path.exists(build_path): + os.makedirs(build_path) + + # self.output.info(build_path) + + # dotnet publish + self.output.info( + "PUBLISHING PROJECT: " + project_files[0]) + + self.utility.exe_proc(["dotnet", "publish", project_files[0], "-f", "netcoreapp2.0", + "-o", build_path, "-v", self.envvars.DOTNET_VERBOSITY]) + + # copy Dockerfile to publish dir + build_dockerfile = os.path.join( + build_path, docker_file_name) + + copyfile(docker_file, build_dockerfile) + + image_source_name = "{0}:{1}".format( + module, tag_name).lower() + image_destination_name = "{0}/{1}:{2}".format( + self.envvars.CONTAINER_REGISTRY_SERVER, module, tag_name).lower() + + # cd to the build output to build the docker image + self.output.info( + "BUILDING DOCKER IMAGE: " + image_source_name) + + project_dir = os.getcwd() + os.chdir(build_path) + build_result = self.dock.docker_client.images.build( + tag=image_source_name, path=".", dockerfile=docker_file_name) + self.output.info( + "DOCKER IMAGE DETAILS: {0}".format(build_result)) + + os.chdir(project_dir) + + # tag the image + self.output.info("TAGGING DOCKER IMAGE WITH: {0}, {1}".format( + image_source_name, image_destination_name)) + + tag_result = self.dock.docker_api.tag( + image=image_source_name, repository=image_destination_name) + #self.output.info("Docker Tag Result: {0}".format(tag_result)) + + # push to container registry + self.output.info( + "PUSHING DOCKER IMAGE TO: " + image_destination_name) + + for line in self.dock.docker_client.images.push(repository=image_destination_name, stream=True): + self.output.procout(self.utility.decode(line)) + + self.output.footer("BUILD COMPLETE") + + def deploy(self): + self.output.header("DEPLOYING MODULES") + self.deploy_device_configuration( + self.envvars.IOTHUB_NAME, self.envvars.IOTHUB_KEY, + self.envvars.EDGE_DEVICE_ID, self.envvars.MODULES_CONFIG_FILE, + self.envvars.IOTHUB_POLICY_NAME, self.envvars.IOT_REST_API_VERSION) + self.output.footer("DEPLOY COMPLETE") + + def deploy_device_configuration(self, iothub_name, iothub_key, device_id, config_file, iothub_policy_name, api_version): + resource_uri = iothub_name + ".azure-devices.net" + token_expiration_period = 60 + deploy_uri = "https://{0}/devices/{1}/applyConfigurationContent?api-version={2}".format( + resource_uri, device_id, api_version) + iot_hub_sas_token = self.utility.get_iot_hub_sas_token( + resource_uri, iothub_key, iothub_policy_name, token_expiration_period) + + deploy_response = requests.post(deploy_uri, + headers={ + "Authorization": iot_hub_sas_token, + "Content-Type": "application/json" + }, + data=self.utility.get_file_contents( + config_file) + ) + + # self.output.info(deploy_uri) + # self.output.info(deploy_response.status_code) + # self.output.info(deploy_response.text) + + if deploy_response.status_code == 204: + self.output.info( + "Edge Device configuration successfully deployed to '{0}'.".format(device_id)) + else: + self.output.error( + "There was an error applying the configuration. You should see an error message above that indicates the issue.") + + +class Docker: + + def __init__(self, envvars, utility, output): + self.envvars = envvars + self.utility = utility + self.output = output + + self.docker_client = docker.from_env() + self.docker_api = docker.APIClient() + + def init_registry(self): + + self.output.header("INITIALIZING CONTAINER REGISTRY") + self.output.info("REGISTRY: " + self.envvars.CONTAINER_REGISTRY_SERVER) + + if "localhost" in self.envvars.CONTAINER_REGISTRY_SERVER: + self.init_local_registry() + + self.login_registry() + self.output.line() + + def init_local_registry(self): + + parts = self.envvars.CONTAINER_REGISTRY_SERVER.split(":") + + if len(parts) < 2: + self.output.error("You must specific a port for your local registry server. Expected: 'localhost:5000'. Found: " + + self.envvars.CONTAINER_REGISTRY_SERVER) + sys.exit() + + port = parts[1] + ports = {'{0}/tcp'.format(port): int(port)} + + try: + self.output.info("Looking for local 'registry' container") + self.docker_client.containers.get("registry") + self.output.info("Found local 'registry' container") + except docker.errors.NotFound: + self.output.info("Local 'registry' container not found") + + try: + self.output.info("Looking for local 'registry' image") + self.docker_client.images.get("registry:2") + self.output.info("Local 'registry' image found") + except docker.errors.ImageNotFound: + self.output.info("Local 'registry' image not found") + self.output.info("Pulling 'registry' image") + self.docker_client.images.pull("registry", tag="2") + + self.output.info("Running registry container") + self.docker_client.containers.run( + "registry:2", detach=True, name="registry", ports=ports, restart_policy={"Name": "always"}) + + def login_registry(self): + try: + + if "localhost" in self.envvars.CONTAINER_REGISTRY_SERVER: + client_login_status = self.docker_client.login( + self.envvars.CONTAINER_REGISTRY_SERVER) + + api_login_status = self.docker_api.login( + self.envvars.CONTAINER_REGISTRY_SERVER) + else: + + client_login_status = self.docker_client.login(registry=self.envvars.CONTAINER_REGISTRY_SERVER, + username=self.envvars.CONTAINER_REGISTRY_USERNAME, password=self.envvars.CONTAINER_REGISTRY_PASSWORD) + + api_login_status = self.docker_api.login(registry=self.envvars.CONTAINER_REGISTRY_SERVER, + username=self.envvars.CONTAINER_REGISTRY_USERNAME, password=self.envvars.CONTAINER_REGISTRY_PASSWORD) + + if api_login_status["Status"] == 'Login Succeeded' and client_login_status["Status"] == "Login Succeeded": + self.output.info( + "Successfully logged into container registry: " + self.envvars.CONTAINER_REGISTRY_SERVER) + else: + raise ValueError(str(client_login_status) + + str(api_login_status)) + except Exception as ex: + self.output.error( + "ERROR: Could not login to Container Registry. Please verify your credentials in CONTAINER_REGISTRY_ environment variables.") + self.output.error(str(ex)) + + def setup_registry(self): + self.output.header("Setting up Container Registry") + self.init_registry() + self.output.info("Pushing Edge Runtime to Container Registry") + image_names = ["azureiotedge-agent", "azureiotedge-hub", + "azureiotedge-simulated-temperature-sensor"] + + for image_name in image_names: + + microsoft_image_name = "microsoft/{0}:{1}".format( + image_name, self.envvars.RUNTIME_TAG) + + container_registry_image_name = "{0}/{1}:{2}".format( + self.envvars.CONTAINER_REGISTRY_SERVER, image_name, self.envvars.RUNTIME_TAG) + + for line in self.docker_api.pull(microsoft_image_name, stream=True): + self.output.procout(self.utility.decode(line)) + + tag_result = self.docker_api.tag( + image=microsoft_image_name, repository=container_registry_image_name) + + #self.output.info("Tag Result: {0}".format(tag_result)) + + # push to container registry + for line in self.docker_api.push(repository=container_registry_image_name, stream=True): + self.output.procout(self.utility.decode(line)) + + self.setup_registry_in_config(image_names) + + self.utility.set_config() + + self.output.footer("Container Registry Setup Complete") + + def setup_registry_in_config(self, image_names): + self.output.info( + "Replacing 'microsoft/' with '{CONTAINER_REGISTRY_SERVER}/' in config files.") + + # Replace microsoft/ with ${CONTAINER_REGISTRY_SERVER} + for config_file in self.utility.get_config_files(): + config_file_contents = self.utility.get_file_contents(config_file) + for image_name in image_names: + config_file_contents = config_file_contents.replace( + "microsoft/" + image_name, "${CONTAINER_REGISTRY_SERVER}/" + image_name) + + with open(config_file, "w") as config_file_build: + config_file_build.write(config_file_contents) + + def remove_modules(self): + self.output.info( + "Removing Edge Modules Containers and Images from Docker") + + for module in self.utility.get_active_modules(): + self.output.info("Searching for {0} Containers".format(module)) + containers = self.docker_client.containers.list( + filters={"name": module}) + for container in containers: + self.output.info("Removing Container: " + str(container)) + container.remove(force=True) + + self.output.info("Searching for {0} Images".format(module)) + for image in self.docker_client.images.list(): + if module in str(image): + self.output.info("Removing Module Image: " + str(image)) + self.docker_client.images.remove( + image=image.id, force=True) + + def remove_containers(self): + self.output.info("Removing Containers....") + containers = self.docker_client.containers.list(all=True) + self.output.info("Found {0} Containers".format(len(containers))) + for container in containers: + self.output.info("Removing Container: {0}:{1}".format( + container.id, container.name)) + container.remove(force=True) + self.output.info("Containers Removed") + + def remove_images(self): + self.output.info("Removing Dangling Images....") + images = self.docker_client.images.list( + all=True, filters={"dangling": True}) + self.output.info("Found {0} Images".format(len(images))) + + for image in images: + self.output.info("Removing Image: {0}".format(str(image.id))) + self.docker_client.images.remove(image=image.id, force=True) + self.output.info("Images Removed") + + self.output.info("Removing Images....") + images = self.docker_client.images.list() + self.output.info("Found {0} Images".format(len(images))) + + for image in images: + self.output.info("Removing Image: {0}".format(str(image.id))) + self.docker_client.images.remove(image=image.id, force=True) + self.output.info("Images Removed") + + def handle_logs_cmd(self, show, save): + + # self.output.info(self.envvars.MODULES_CONFIG_FILE) + modules_config = json.load(open(self.envvars.MODULES_CONFIG_FILE)) + + props = modules_config["moduleContent"]["$edgeAgent"]["properties.desired"] + + self.process_logs(show, save, props["systemModules"]) + self.process_logs(show, save, props["modules"]) + + if save: + self.zip_logs() + + def process_logs(self, show, save, modules): + # Create LOGS_PATH dir if it doesn't exist + if save and not os.path.exists(self.envvars.LOGS_PATH): + os.makedirs(self.envvars.LOGS_PATH) + + for module in modules: + if show: + try: + command = self.envvars.LOGS_CMD.format(module) + os.system(command) + except Exception as e: + self.output.error( + "Error while trying to open module log '{0}' with command '{1}'. Try iotedgedev docker --save-logs instead.".format(module, command)) + self.output.error(e) + if save: + try: + self.utility.exe_proc(["docker", "logs", module, ">", + os.path.join(self.envvars.LOGS_PATH, module + ".log")], True) + except: + self.output.error( + "Error while trying to save module log file '{0}'".format(module)) + self.output.error(e) + + def zip_logs(self): + log_files = [os.path.join(self.envvars.LOGS_PATH, f) + for f in os.listdir(self.envvars.LOGS_PATH) if f.endswith(".log")] + zip_path = os.path.join(self.envvars.LOGS_PATH, 'edge-logs.zip') + + self.output.info("Creating {0} file".format(zip_path)) + + zipf = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) + + for log_file in log_files: + self.output.info("Adding {0} to zip". format(log_file)) + zipf.write(log_file) + + zipf.close() + + self.output.info("Log files successfully saved to: " + zip_path) diff --git a/iotedgedev/template/.env.tmp b/iotedgedev/template/.env.tmp new file mode 100644 index 00000000..79421155 --- /dev/null +++ b/iotedgedev/template/.env.tmp @@ -0,0 +1,29 @@ +IOTHUB_NAME="iot hub name" +IOTHUB_KEY="iot hub key" +DEVICE_CONNECTION_STRING="edge device connection string" +EDGE_DEVICE_ID="edge device id" +RUNTIME_HOST_NAME="the computer name that your edge will run on" +ACTIVE_MODULES="filter-module" +ACTIVE_DOCKER_DIRS="arm32v7,linux-x64" +CONTAINER_REGISTRY_SERVER="" +CONTAINER_REGISTRY_USERNAME="" +CONTAINER_REGISTRY_PASSWORD="" + +IOTHUB_POLICY_NAME="iothubowner" +CONTAINER_TAG="latest" +RUNTIME_TAG="1.0-preview" +RUNTIME_VERBOSITY="INFO" +#INFO, DEBUG +RUNTIME_HOME_DIR="c:\\ProgramData\\azure-iot-edge" +#RUNTIME_HOME_DIR="/etc/azure-iot-edge" + +MODULES_CONFIG_FILE="build/config/modules.json" +RUNTIME_CONFIG_FILE="build/config/runtime.json" +LOGS_PATH="logs" +MODULES_PATH="modules" +IOT_REST_API_VERSION="2017-11-08-preview" +DOTNET_VERBOSITY="q" +#q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] +#LOGS_CMD="docker logs {0} -f -new_console:sV" +LOGS_CMD="start /B start cmd.exe @cmd /k docker logs {0} -f" + \ No newline at end of file diff --git a/iotedgedev/template/.gitignore b/iotedgedev/template/.gitignore new file mode 100644 index 00000000..7ed87a93 --- /dev/null +++ b/iotedgedev/template/.gitignore @@ -0,0 +1,9 @@ +**/.vs +**/.vscode +**/bin +**/obj +**/out +build +.env +venv +logs \ No newline at end of file diff --git a/iotedgedev/template/config/modules.json b/iotedgedev/template/config/modules.json new file mode 100644 index 00000000..09d3c9b0 --- /dev/null +++ b/iotedgedev/template/config/modules.json @@ -0,0 +1,74 @@ +{ + "moduleContent": { + "$edgeAgent": { + "properties.desired": { + "schemaVersion": "1.0", + "runtime": { + "type": "docker", + "settings": { + "minDockerVersion": "v1.25", + "loggingOptions": "" + } + }, + "systemModules": { + "edgeAgent": { + "type": "docker", + "settings": { + "image": "microsoft/azureiotedge-agent:${RUNTIME_TAG}", + "createOptions": "" + } + }, + "edgeHub": { + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "microsoft/azureiotedge-hub:${RUNTIME_TAG}", + "createOptions": "" + } + } + }, + "modules": { + "temp-sensor-module": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "microsoft/azureiotedge-simulated-temperature-sensor:${RUNTIME_TAG}", + "createOptions": "" + } + }, + "filter-module": { + "version": "1.0", + "type": "docker", + "status": "running", + "restartPolicy": "always", + "settings": { + "image": "${CONTAINER_REGISTRY_SERVER}/filter-module:linux-x64-${CONTAINER_TAG}", + "createOptions": "" + } + } + } + } + }, + "$edgeHub": { + "properties.desired": { + "schemaVersion": "1.0", + "routes": { + "sensorToFilter": "FROM /messages/modules/temp-sensor-module/outputs/temperatureOutput INTO BrokeredEndpoint(\"/modules/filter-module/inputs/input1\")", + "filterToIoTHub": "FROM /messages/modules/filter-module/outputs/output1 INTO $upstream" + }, + "storeAndForwardConfiguration": { + "timeToLiveSecs": 7200 + } + } + }, + "filter-module": { + "properties.desired": { + "schemaVersion": "1.0", + "TemperatureThreshold": 25 + } + } + } +} \ No newline at end of file diff --git a/iotedgedev/template/config/runtime.json b/iotedgedev/template/config/runtime.json new file mode 100644 index 00000000..0779ae95 --- /dev/null +++ b/iotedgedev/template/config/runtime.json @@ -0,0 +1,40 @@ +{ + "deployment": { + "docker": { + "edgeRuntimeImage": "microsoft/azureiotedge-agent:${RUNTIME_TAG}", + "loggingOptions": { + "log-driver": "json-file", + "log-opts": { + "max-size": "10m" + } + }, + "registries": [ + { + "address": "${CONTAINER_REGISTRY_SERVER}", + "password": "${CONTAINER_REGISTRY_PASSWORD}", + "username": "${CONTAINER_REGISTRY_USERNAME}" + } + ], + "uri": "unix:///var/run/docker.sock" + }, + "type": "docker" + }, + "deviceConnectionString": "${DEVICE_CONNECTION_STRING}", + "homeDir": "${RUNTIME_HOME_DIR}", + "hostName": "${RUNTIME_HOST_NAME}", + "logLevel": "info", + "schemaVersion": "1", + "security": { + "certificates": { + "option": "selfSigned", + "preInstalled": { + "deviceCACertificateFilePath": "", + "serverCertificateFilePath": "" + }, + "selfSigned": { + "forceNoPasswords": true, + "forceRegenerate": false + } + } + } +} \ No newline at end of file diff --git a/iotedgedev/template/modules/filter-module/.gitignore b/iotedgedev/template/modules/filter-module/.gitignore new file mode 100644 index 00000000..fe6af155 --- /dev/null +++ b/iotedgedev/template/modules/filter-module/.gitignore @@ -0,0 +1,30 @@ +# .NET Core +project.lock.json +project.fragment.lock.json +artifacts/ +**/Properties/launchSettings.json + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc diff --git a/iotedgedev/template/modules/filter-module/Docker/arm32v7/Dockerfile b/iotedgedev/template/modules/filter-module/Docker/arm32v7/Dockerfile new file mode 100644 index 00000000..df426299 --- /dev/null +++ b/iotedgedev/template/modules/filter-module/Docker/arm32v7/Dockerfile @@ -0,0 +1,9 @@ +FROM microsoft/dotnet:2.0.0-runtime-stretch-arm32v7 + +ARG EXE_DIR=. + +WORKDIR /app + +COPY $EXE_DIR/ ./ + +CMD ["dotnet", "filter-module.dll"] \ No newline at end of file diff --git a/iotedgedev/template/modules/filter-module/Docker/linux-x64/Dockerfile b/iotedgedev/template/modules/filter-module/Docker/linux-x64/Dockerfile new file mode 100644 index 00000000..7c28e52b --- /dev/null +++ b/iotedgedev/template/modules/filter-module/Docker/linux-x64/Dockerfile @@ -0,0 +1,9 @@ +FROM microsoft/dotnet:2.0.0-runtime + +ARG EXE_DIR=. + +WORKDIR /app + +COPY $EXE_DIR/ ./ + +CMD ["dotnet", "filter-module.dll"] \ No newline at end of file diff --git a/iotedgedev/template/modules/filter-module/Docker/linux-x64/Dockerfile.debug b/iotedgedev/template/modules/filter-module/Docker/linux-x64/Dockerfile.debug new file mode 100644 index 00000000..fe4026b5 --- /dev/null +++ b/iotedgedev/template/modules/filter-module/Docker/linux-x64/Dockerfile.debug @@ -0,0 +1,15 @@ +FROM microsoft/dotnet:2.0.0-runtime + +ARG EXE_DIR=. + +WORKDIR /app + +COPY $EXE_DIR/ ./ + +RUN apt-get update + +RUN apt-get install -y unzip procps + +RUN curl -sSL https://aka.ms/getvsdbgsh | bash /dev/stdin -v latest -l ~/vsdbg + +CMD ["dotnet", "filter-module.dll"] \ No newline at end of file diff --git a/iotedgedev/template/modules/filter-module/Docker/windows-nano/Dockerfile b/iotedgedev/template/modules/filter-module/Docker/windows-nano/Dockerfile new file mode 100644 index 00000000..7fe30e93 --- /dev/null +++ b/iotedgedev/template/modules/filter-module/Docker/windows-nano/Dockerfile @@ -0,0 +1,9 @@ +FROM microsoft/dotnet:2.0.0-runtime-nanoserver-1709 + +ARG EXE_DIR=. + +WORKDIR /app + +COPY $EXE_DIR/ ./ + +CMD ["dotnet", "filter-module.dll"] \ No newline at end of file diff --git a/iotedgedev/template/modules/filter-module/Program.cs b/iotedgedev/template/modules/filter-module/Program.cs new file mode 100644 index 00000000..a0227212 --- /dev/null +++ b/iotedgedev/template/modules/filter-module/Program.cs @@ -0,0 +1,174 @@ +namespace filter_module { + using System.Collections.Generic; + using System.IO; + using System.Runtime.InteropServices; + using System.Runtime.Loader; + using System.Security.Cryptography.X509Certificates; + using System.Text; + using System.Threading.Tasks; + using System.Threading; + using System; + using Microsoft.Azure.Devices.Client.Transport.Mqtt; + using Microsoft.Azure.Devices.Client; + using Microsoft.Azure.Devices.Shared; + using Newtonsoft.Json; + + class Program { + static int counter; + static int temperatureThreshold { get; set; } = 25; + + class MessageBody { + public Machine machine { get; set; } + public Ambient ambient { get; set; } + public string timeCreated { get; set; } + } + + class Machine { + public double temperature { get; set; } + public double pressure { get; set; } + } + + class Ambient { + public double temperature { get; set; } + public int humidity { get; set; } + } + + static void Main (string[] args) { + // The Edge runtime gives us the connection string we need -- it is injected as an environment variable + string connectionString = Environment.GetEnvironmentVariable ("EdgeHubConnectionString"); + + // Cert verification is not yet fully functional when using Windows OS for the container + bool bypassCertVerification = RuntimeInformation.IsOSPlatform (OSPlatform.Windows); + if (!bypassCertVerification) InstallCert (); + Init (connectionString, bypassCertVerification).Wait (); + + // Wait until the app unloads or is cancelled + var cts = new CancellationTokenSource (); + AssemblyLoadContext.Default.Unloading += (ctx) => cts.Cancel (); + Console.CancelKeyPress += (sender, cpe) => cts.Cancel (); + WhenCancelled (cts.Token).Wait (); + } + + /// + /// Handles cleanup operations when app is cancelled or unloads + /// + public static Task WhenCancelled (CancellationToken cancellationToken) { + var tcs = new TaskCompletionSource (); + cancellationToken.Register (s => ((TaskCompletionSource) s).SetResult (true), tcs); + return tcs.Task; + } + + /// + /// Add certificate in local cert store for use by client for secure connection to IoT Edge runtime + /// + static void InstallCert () { + string certPath = Environment.GetEnvironmentVariable ("EdgeModuleCACertificateFile"); + if (string.IsNullOrWhiteSpace (certPath)) { + // We cannot proceed further without a proper cert file + Console.WriteLine ($"Missing path to certificate collection file: {certPath}"); + throw new InvalidOperationException ("Missing path to certificate file."); + } else if (!File.Exists (certPath)) { + // We cannot proceed further without a proper cert file + Console.WriteLine ($"Missing path to certificate collection file: {certPath}"); + throw new InvalidOperationException ("Missing certificate file."); + } + X509Store store = new X509Store (StoreName.Root, StoreLocation.CurrentUser); + store.Open (OpenFlags.ReadWrite); + store.Add (new X509Certificate2 (X509Certificate2.CreateFromCertFile (certPath))); + Console.WriteLine ("Added Cert: " + certPath); + store.Close (); + } + + /// + /// Initializes the DeviceClient and sets up the callback to receive + /// messages containing temperature information + /// + static async Task Init (string connectionString, bool bypassCertVerification = false) { + + MqttTransportSettings mqttSetting = new MqttTransportSettings (TransportType.Mqtt_Tcp_Only); + // During dev you might want to bypass the cert verification. It is highly recommended to verify certs systematically in production + if (bypassCertVerification) { + mqttSetting.RemoteCertificateValidationCallback = (sender, certificate, chain, sslPolicyErrors) => true; + } + ITransportSettings[] settings = { mqttSetting }; + + // Open a connection to the Edge runtime + DeviceClient ioTHubModuleClient = DeviceClient.CreateFromConnectionString (connectionString, settings); + await ioTHubModuleClient.OpenAsync (); + Console.WriteLine ("IoT Hub module client initialized."); + + // Register callback to be called when a message is received by the module + // await ioTHubModuleClient.SetImputMessageHandlerAsync("input1", PipeMessage, iotHubModuleClient); + + // Attach callback for Twin desired properties updates + await ioTHubModuleClient.SetDesiredPropertyUpdateCallbackAsync (onDesiredPropertiesUpdate, null); + + // Register callback to be called when a message is received by the module + await ioTHubModuleClient.SetInputMessageHandlerAsync ("input1", FilterMessages, ioTHubModuleClient); + } + + static Task onDesiredPropertiesUpdate (TwinCollection desiredProperties, object userContext) { + try { + Console.WriteLine ("Desired property change:"); + Console.WriteLine (JsonConvert.SerializeObject (desiredProperties)); + + if (desiredProperties["TemperatureThreshold"] != null) + temperatureThreshold = desiredProperties["TemperatureThreshold"]; + + } catch (AggregateException ex) { + foreach (Exception exception in ex.InnerExceptions) { + Console.WriteLine (); + Console.WriteLine ("Error when receiving desired property: {0}", exception); + } + } catch (Exception ex) { + Console.WriteLine (); + Console.WriteLine ("Error when receiving desired property: {0}", ex.Message); + } + return Task.CompletedTask; + } + + static async Task FilterMessages (Message message, object userContext) { + int counterValue = Interlocked.Increment (ref counter); + + try { + DeviceClient deviceClient = (DeviceClient) userContext; + + byte[] messageBytes = message.GetBytes (); + string messageString = Encoding.UTF8.GetString (messageBytes); + Console.WriteLine ($"Received message {counterValue}: [{messageString}]"); + + // Get message body + var messageBody = JsonConvert.DeserializeObject (messageString); + + if (messageBody != null && messageBody.machine.temperature > temperatureThreshold) { + Console.WriteLine ($"Machine temperature {messageBody.machine.temperature} " + + $"exceeds threshold {temperatureThreshold}"); + var filteredMessage = new Message (messageBytes); + foreach (KeyValuePair prop in message.Properties) { + filteredMessage.Properties.Add (prop.Key, prop.Value); + } + + filteredMessage.Properties.Add ("MessageType", "Alert"); + await deviceClient.SendEventAsync ("output1", filteredMessage); + } + + // Indicate that the message treatment is completed + return MessageResponse.Completed; + } catch (AggregateException ex) { + foreach (Exception exception in ex.InnerExceptions) { + Console.WriteLine (); + Console.WriteLine ("Error in sample: {0}", exception); + } + // Indicate that the message treatment is not completed + DeviceClient deviceClient = (DeviceClient) userContext; + return MessageResponse.Abandoned; + } catch (Exception ex) { + Console.WriteLine (); + Console.WriteLine ("Error in sample: {0}", ex.Message); + // Indicate that the message treatment is not completed + DeviceClient deviceClient = (DeviceClient) userContext; + return MessageResponse.Abandoned; + } + } + } +} \ No newline at end of file diff --git a/iotedgedev/template/modules/filter-module/filter-module.csproj b/iotedgedev/template/modules/filter-module/filter-module.csproj new file mode 100644 index 00000000..539a786a --- /dev/null +++ b/iotedgedev/template/modules/filter-module/filter-module.csproj @@ -0,0 +1,22 @@ + + + Exe + netcoreapp2.0 + + + + True + + + + + + + + + + + + + + \ No newline at end of file diff --git a/iotedgedev/template/template.zip b/iotedgedev/template/template.zip new file mode 100644 index 0000000000000000000000000000000000000000..6a742d365033485f555b9bb32865c3d9c6fa12e4 GIT binary patch literal 10788 zcmb_i2Rzm7+ds!V$j;2n$lfy}8QGDY7U$TGb8PBlNA}7pl+5s$A%v{#t;k*t*&<}V zhaQiE>iPfQ_dPy8H$L}$eZSYZulxR87fn@kjI#g$00c;aC8`d;bFAP%0|5Hb_x}O_ z@R-7!c^qwQ-L#D1$udMcqp5qwb#7$cqo3|sg6%<^uq-1$C8bRhx!y3T^9_Tn%GRCr zbMo2g>Tjw|%s;my;C`+Xwu50e9fA?mvA8+eip&-rMYGv*3v`_tnfQqJUEiZa%%@lL8h?ow5}fJAUgDJ%=2UtCEPe;v^1*FhNn`r6G<64u2>=W zcph;;_$CdE5Pec>Oz|6<@K$IFPah5u(E`+0^AlbZG~9X%?yq>f~eKO zFVqumk7{(FH^~>?(r)MFnaCtq$XrQUD44tst~UM4)k^3b`_LOz_6{Mp9-xGtq&b|H zsS~IAoh_)Zspws(c39X$oOahaVzoXkX=aVx~2dIlof<@LQUjgLJ_D9XhoUoDpD z?N+=H(E)Z)Z4*cU|aOp;6$sU=||nh@=O-j*KRmuG@fiL~pV z7*!~KpMo$lj^vOJ?tVO(TRK{r!{GL&EDij)00xju_pPP#_)Zr%lR>m0yAVi7>@{-5sa2Ztm5;m`*7U={B*JuR5$<7`$12X-oZ$6mPZk=Dp@?UE zizN#HKiKRbrH0{$Gi#V|4nA>kM_{AR@P1= z8kX=A1SK!!CyhE2Uw!SUK#I8+Qjs{)exE#;Rj~8U=IFc$@y4ZCyv{c(+m#4er#Q{@ zXNh0J=o139C2*${%eY0;HsUOn7rWMGtBWrRDm}b{yCJ4uEf%`-BI7##)7{3?H zVe)2hoTaa#DWt>Uk?>GKldr5uAw%!XeT9w*xpi3H~98!2YwiYAiy#*zBWPR8Dl^IfDz+Z zeAzp}94&2358}&7TRkkEkEl|A#>2mhF23isgWLmHL^%;KJ}ZmEq()}gI|TX&CNlYE z@dHL1OoRf8%&pRrVJnS<+0ojv{Y%<-YU1(HIA0lfcfI1!STRiTl{?ScFhZ5_K^M8X zF=8vUc4aRk`RLC0eZ%%l>f8NM@9i?!%3pgl+??Qm@f{#v=H~A0q3btBav8d^*+#8%xcFHo0etAvTxvoj7@L{WliC7p1~T zq|~%pl>r_aLEN3O9H7CnW&@C-c_+LHu_v zLYY-29(ybUlB3KrvxGXD+H?P0^v@a)AWzs|l}E1|JqbA%2N{qIzg=^bsMR}aaO)V( zV2!`lxb~sl$IZWBmJ+(2oT+;w8w`Es5#U?U`bC384d!!CTDne3pHrnh+czETr=Ann zn^&ka?;O8-qr4>F!a@v}y-~mYt=8K~%I^`?Z)~g?#Vh=L6THz1!xJjQIqE}(ZF*Nt zUPfX{N}TJv?83@PPNL3oLujPy``m4w3ste;wou6e1{uPp%a57rsebf!iH_s+GrPPw zGVMu=Q2$mtuqEsts{jBe*Ztlq`|IRQ!A?g9$m!KQe?|A+tq5Yd6EPk^uuALP<-36GK_vB8n9}Q z{0vu)eMCU;)Q1SX7{UiJPcG!~bA9UI~U+b@o>_b7yeBjs3pwFmD^QB2*oH@Mf_EX zozvo-k&8W%5C4BI1jQ`Dt++mXM1V!_z5EAsvsdnMt!id*rzxrM*sz}mpT{6!vfJ@^ z)=1@7rh3g&P<_b<9>A%Qn7)Fet@en|*(4z}MXbC9`{P(e%hPY(K^veSPk^o?>v&`J zTi6e!COq#)D`0t)NDUAhSaW?17t6bneeHT{3(lb>KmZM-6212tgIbZ~fyjL?`oB?v z$7Db9bv4vg(fNrgzv!O{QhdZtheubhN1Ve+e2qB5@li<)ltkbWro#3-){v@(Uq}?oNNa(+kgAy+xt^3;ubKQ5h zj-^zHaKzsmuRvE`h4~zE8?pMz(6Lm0-?Axgr6MpVP|+Komw3w~(&SWcWc$*@8|TTU zXWu#M86t^C(yZNYRsLGx>U5p~eag8{cA?lbrBua<4&XXc!nFA56TwT%TCe42f8n}@ zd-$|@3H43b&ZI0P8V`zp+eSNzp8Zo}L~-M;8!~cskyn`CTj$uh!vk@!wTD|(>a@aY z`G_j?XD-PTlh{_{6wTnAzXl@Hr3bM|oqHK@lSIOBNRmUkil9hxNRFkR`{}^kT*tjz zi_f4`NagSOAazW)bb`C= z22g4RUv>#k=Zh7?$FN`}wupJ}e`h^(IwZ;^%gNBYHxgVc9Gg5rj>m!sbcKe0R=TUp zMr+#f{I;wUGe0-Qxe)NV2G-lOQ7^3mO|#ea9ACY?apjHn*_-uQnX>W$9F)lgIlC>i z+hR?g%l8}rwR zHdr(M^2mJoJX~*N%YD`RpU^ygO#S|FgoCaAu#VxkA-%uu(+xHBQ+xLWg}V7=^EE*XY<$w_i%TmwXq>@m`w0>^!d!*R~iy<^Tm5Rvd5}f0a_xJ4T z1M*f`0=-^~Vnx<$MwrAJ+D8rj;D+9EZPmpIYPlFySI6=g!YC_H6J;D#rDc81ESZ`H zuW8yE{@14`SymqtCLX)E4iM0Z3(;me@aDUH@1sn`y?sv(;z%f3(-zy#Pg>Nr^qsIN zxX{J${M@{V;?=s0MzFZNvof{0Lt6?*QN+;diw)oPOM`6~9*-qP)>&N8G#a;Ux=L#4 z*UIgO@HB!DoeBiJeiAef8{()9RdfjAKrWH43#9w0WdH5|XB57@n@hN#LiQgWDlE(=1~!8TLd3xkV*z6!6A`euh`6wz5Lj4TjNe2+(2P%53<43y z$3@|GD^b-K$fb11&G7%o?PvPqkwaeew}B@EsTtIs$q5^H!AFuK+pgBULq48#q1X5y*5(N>nMed&KPdw z0ktL=T3&vcsPfdi*%brIPJ*}TmNh8}T>JhJ$M4xNs0{qjw;&%+^s;l_NPZ|ugV!7}A)SD>7~JvWs2s#C=E;vUHx6ch2S z6uDO~-zYds{Hm_#%hCWrXrAyAC3da2D1Ez2>>0HZzo-66oGCFPk{glMphZs!D=~Ep zrE?v^v8>vIej3u(w@4+{4eynHrk?EU$*_e+J6h9Iv)JV(gpk0_Aaa#GGGCVNQMTsz z$A!lza9r@YMW0abW1du|u$xNrQct<9s*vo%Y+F~q--t;p51gt`?U^V4eau(7H=}}Q zHDY*4QuiJxS~Gg3eS*i&jfeU;UN!_*Y>z-57Cc#%>kv^GGW2(`8C(t3#3_~AHkjZQ z5-RG%*-lqWakg#L08Mr7O87FO=dI|}Q<@``CuygY0w|-InDJD0s@xh&Z$GxvFJ1XE za36mTR)Ln@8g9;Mhk-Q`b3Gv-DvD)PC*~p+D=^1#frc~&$4Az^noO(~e=5>TlEkn% zO2@9gc-(c4yf;u&eKVh?w0WTqK|BJTT8_@B_a?^KVz{FwF{C&(*f`tENdBBMiI%P# z9BIJOiklC8d_mc3tS4duN8+~6DsaoN$$<9U+FUqybgrZEwTz^po8&4i&}$~s==Jlu zK2&tg=wH>&N58}O6I_{Fx?9T~lAp9ofA8tktPK{p*;~8rOaqj)`?K<72bH7qWNLMH zZt?{vcEOTx#&@4UM{UQL`mf&RRMISQv}d%RL{ZyO0a-Iji=%IPD)c3>GSj@YENd!= z&kkcC)Xim@R&_TzlNywov@NJm32BjWz4cA9U_!MqHrR2E^=XGY-OKn_{>lcCi`wxd ztE}T+=r1oP)DGE*GVs@^8${I*HdkC(kJD?E?&=dph@O z(*!^kFOX*?nB5yYb}UgeGWo`!f{lLQV%McRf);g3zPM4kug#~aQYUr77%W!Rt(f3i zicVQQT)^Rp`MW;gNj(iOx4{WK100`beEiFUgx(_`8`^>kX7YKfKjK&o&CoQEMy9r2 zL7&V{rP5d4_SPksj8kx;sv}Y+C0nk?PnM3x!p18Do?#^~8>|kYwIqwryo(3% zfR0_8`4TVd)mzO)PLhTS&+*m#c*-insu>@BF15B zewV>X50(){lMCjncURtUzzBVd=`%CLQo}jXJEyKnQI|f;pc#s$P#M5+=nT`z`Fe5tCg#`v7DxI7bsDcYdZ9SsWL(%JIb$lLp_^x zhn~{v!Eo1H(H8Z&f|o;L^f5M1*wEj^_`lF2vZ&zI)N?l#6y%P;e{`1YsR6l}DfB^w*fsCbQhR+f$*b4D;DSY3ErR5)kt?E2>#sLKVAYjGjXhITfkb&EMlK8;9ib8#gd_l96+~y_FL!l1q>>x9esQ z_Z{kaD|Fc-;!}UjR}nddoS^%o%6%ohM8(dUc&*zQ|^`_i(-14Nc~XES9k3BOx*831yutYpTwRV;~!PF5nu4ZoS-Ngdoy? z*&gOwuOx&9x4AD}j0Tzmm)8yUlC!NOMmDTO$}jiK=OtGUDK}fakBPYPP%+%Va7TQ{ zL;r`p!H>V5?;sWmdw2yN0aaf2Xl64GWD=5x-%5AlCVACQtWliw8k?i8c##C*YASp? zaZdMa$U@PrB!>HuoTYx33n+l)19isrMq0@{keTc3;I2pRH{;9E-|iH{6<53vRoxQK zGiG0fl`{6Is%R>j*I36$ZGCvm`R{xCyjQ8i&C$_!UAoJrujvWQ;0}o%^6;ixlBVX> zE$RKv{hp44{>~WUYm+Gj1(&q+O|*p%G(jVH@!hF0cK&r{5B1z%1WRcwol7NjGjh); z$-QAp5%3)TnsGt2eJt}W%=OBL7;`p};TZrr-u4^ZXKR6j4V9vOq~c^dg7>O} z>sEGCIb&fmB79WQ}VdG<3xW_EXku|DleHqXGNoewUC znjxa89THC+p2I2cB*eUVMbl?=ALdqZm8qCx>zQHIXh~rTYaye+Q@-`ik~K)aBPBhrW;6-K7Wj4^Qg* zMvKYppaTfkRx042af zv;nf{*$ec%jK3<0=mi?fS1s<3sFF{&wNNE%BHKtuwNq5t;XAM*JEW}e9=usYPugtwY&-NRBy3**|++sB&pyVN$wCIVVR{xhLxz*Z@?y-AK8A_r*{JyD--u zD1SKOw7xlIjMLq-L&5t-T{{x|_a*2o%v=WZ{sFTH(f-_k=3HK#yMw@^?) z^pF+wzuBWUR8TaX^Jzy+3r|yu%Uf*>%d9C*fF*!Bzbm++mjj6;=oR-2u$uXf@t_0`^zE zjKdh(k6IwI8u9BG`diCG*rSpxDgn=XuO`^-dx2f9QwKqONCef*C-uH?Oz-Z0BXHD; zKn1?^V&Fj2!!fh}{VB~)H!M&!C&&92&5wY8tvf)AerkUM`&E58(0xFq{}hGO?FLi| zL;o8JKbsGzw(y*ObwC8bv~)n^&?bL!0>?2YY@PWRTO6zMqhfobj~pm{I38|*BWw`h zbT0ta2y*!U4fbzbF{BuC&#@P{CNq0rfkQT^EKb+%QCUzC{Eo%3%!(`{?N{zljvMFqs4R)g9vrZf#W_A6 zV|l9Li^_8UU%`HTNc?C0g+JGSQMp28tCB{LuN@7&EeCFYLg;nC9o5~ZYrm*mlYh(g rs1l5dD0y{Y?Hdviz%h8ReV5L6{JniK_y3a)K#Tm^Pf@Kj`@j7UZC~f! literal 0 HcmV?d00001 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..e51c2084 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +docker==2.6.0 +python-dotenv +requests +azure-iot-edge-runtime-ctl + diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 00000000..53b0d1b6 --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,10 @@ +pip +bumpversion==0.5.3 +wheel==0.29.0 +watchdog==0.8.3 +flake8==2.6.0 +tox==2.3.1 +coverage==4.1 +Sphinx==1.4.8 +cryptography==1.7 +PyYAML==3.11 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..e7565aff --- /dev/null +++ b/setup.cfg @@ -0,0 +1,21 @@ +[bumpversion] +current_version = 0.46.0 +commit = True +tag = True + +[bumpversion:file:setup.py] +search = version='{current_version}' +replace = version='{new_version}' + +[bumpversion:file:iotedgedev/__init__.py] +search = __version__ = '{current_version}' +replace = __version__ = '{new_version}' + +[bdist_wheel] +universal = 1 + +[flake8] +exclude = docs + +[aliases] + diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..f8fe9762 --- /dev/null +++ b/setup.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""The setup script.""" +import sys +from os import path + +from setuptools import setup, find_packages + +#with open('README.rst') as readme_file: +# readme = readme_file.read() + +with open('HISTORY.rst') as history_file: + history = history_file.read() + +requirements = [ + 'Click>=6.0', + 'docker==2.6.0', + 'python-dotenv', + 'requests', + 'azure-iot-edge-runtime-ctl' +] + +setup_requirements = [ + # TODO(jonbgallant): put setup requirements (distutils extensions, etc.) here +] + +test_requirements = [ + # TODO: put package test requirements here +] + + +setup( + name='azure-iot-edge-dev-tool', + version='0.46.0', + description="The Azure IoT Edge Dev Tool module greatly simplifies your Edge development process by automating many of your routine manual tasks, such as building, deploying, pushing modules and configuring your Edge Runtime.", + long_description="See https://github.com/jonbgallant/azure-iot-edge-dev-tool for usage instructions.", + author="Jon Gallant", + author_email='info@jongallant.com', + url='https://github.com/jonbgallant/azure-iot-edge-dev-tool', + packages=find_packages(include=['iotedgedev']), + entry_points={ + 'console_scripts': [ + 'iotedgedev=iotedgedev.cli:main' + ] + }, + include_package_data=True, + install_requires=requirements, + license="MIT license", + zip_safe=False, + keywords='azure iot edge dev tool', + classifiers=[ + 'Development Status :: 2 - Pre-Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Natural Language :: English', + "Programming Language :: Python :: 2", + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + ], + test_suite='tests', + tests_require=test_requirements, + setup_requires=setup_requirements, +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e5ca9621 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +"""Unit test package for iotedgedev.""" diff --git a/tests/test_iotedgedev.py b/tests/test_iotedgedev.py new file mode 100644 index 00000000..eeb152f5 --- /dev/null +++ b/tests/test_iotedgedev.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""Tests for `iotedgedev` package.""" + + +import unittest +import os +import shutil +from click.testing import CliRunner + +from iotedgedev import iotedgedev +from iotedgedev import cli + +from dotenv import load_dotenv, find_dotenv +load_dotenv(find_dotenv()) + +project = "test_project" +root_dir = os.getcwd() + +class TestIotedgedev(unittest.TestCase): + + @classmethod + def setUpClass(self): + """SETUP""" + print("SETTING UP TEST PROJECT") + try: + runner = CliRunner() + result = runner.invoke(cli.main, ['project', '--create', project]) + #print(result.output) + #assert result.exit_code == 0 + #assert 'Azure IoT Edge project created' in result.output + + shutil.copyfile('.env', os.path.join(os.getcwd(), project, '.env')) + os.chdir(project) + + except Exception as e: + print(e) + + @classmethod + def tearDownClass(self): + """TEARDOWN""" + os.chdir("..") + #shutil.rmtree(os.path.join(root_dir, project), ignore_errors=True) + + def test_version(self): + """VERSION""" + runner = CliRunner() + result = runner.invoke(cli.main, ['--version']) + print(result.output) + assert result.exit_code == 0 + assert 'version' in result.output + + def test_version(self): + """HELP""" + runner = CliRunner() + help_result = runner.invoke(cli.main, ['--help']) + assert help_result.exit_code == 0 + assert 'Show this message and exit.' in help_result.output + + def test_modules_build_deploy(self): + runner = CliRunner() + result = runner.invoke(cli.main, ['modules', '--build']) + print(result.output) + assert result.exit_code == 0 + assert '0 Error(s)' in result.output + result = runner.invoke(cli.main, ['modules', '--deploy']) + print(result.output) + assert result.exit_code == 0 + assert 'Edge Device configuration successfully deployed' in result.output + + # TODO: Figure out why tox messes with the paths. + #def test_runtime_setup(self): + # runner = CliRunner() + # result = runner.invoke(cli.main, ['runtime', '--setup']) + # print(result.output) + # assert result.exit_code == 0 + # assert 'Runtime setup successfully.' in result.output + + def test_docker_logs(self): + runner = CliRunner() + result = runner.invoke(cli.main, ['docker', '--save-logs']) + print(result.output) + assert result.exit_code == 0 + assert 'Log files successfully saved' in result.output + \ No newline at end of file diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..a02088f6 --- /dev/null +++ b/tox.ini @@ -0,0 +1,27 @@ +[tox] +envlist = py27, py36 + +[travis] +python = + 3.6: py36 + 2.7: py27 + +#[testenv:flake8] +#basepython=python +#deps=flake8 +#commands=flake8 iotedgedev + +[testenv] +setenv = + PYTHONPATH = {toxinidir} + +commands = + python setup.py test + +passenv = APPDATA ProgramFiles USERPROFILE + + +; If you want to make tox run the tests with the same versions, create a +; requirements.txt with the pinned versions and uncomment the following lines: +; deps = +; -r{toxinidir}/requirements.txt diff --git a/travis_pypi_setup.py b/travis_pypi_setup.py new file mode 100644 index 00000000..a1b7d560 --- /dev/null +++ b/travis_pypi_setup.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +"""Update encrypted deploy password in Travis config file.""" + + +from __future__ import print_function +import base64 +import json +import os +from getpass import getpass +import yaml +from cryptography.hazmat.primitives.serialization import load_pem_public_key +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.asymmetric.padding import PKCS1v15 + + +try: + from urllib import urlopen +except ImportError: + from urllib.request import urlopen + + +GITHUB_REPO = 'jonbgallant/azure-iot-edge-dev-tool' +TRAVIS_CONFIG_FILE = os.path.join( + os.path.dirname(os.path.abspath(__file__)), '.travis.yml') + + +def load_key(pubkey): + """Load public RSA key. + + Work around keys with incorrect header/footer format. + + Read more about RSA encryption with cryptography: + https://cryptography.io/latest/hazmat/primitives/asymmetric/rsa/ + """ + try: + return load_pem_public_key(pubkey.encode(), default_backend()) + except ValueError: + # workaround for https://github.com/travis-ci/travis-api/issues/196 + pubkey = pubkey.replace('BEGIN RSA', 'BEGIN').replace('END RSA', 'END') + return load_pem_public_key(pubkey.encode(), default_backend()) + + +def encrypt(pubkey, password): + """Encrypt password using given RSA public key and encode it with base64. + + The encrypted password can only be decrypted by someone with the + private key (in this case, only Travis). + """ + key = load_key(pubkey) + encrypted_password = key.encrypt(password, PKCS1v15()) + return base64.b64encode(encrypted_password) + + +def fetch_public_key(repo): + """Download RSA public key Travis will use for this repo. + + Travis API docs: http://docs.travis-ci.com/api/#repository-keys + """ + keyurl = 'https://api.travis-ci.org/repos/{0}/key'.format(repo) + data = json.loads(urlopen(keyurl).read().decode()) + if 'key' not in data: + errmsg = "Could not find public key for repo: {}.\n".format(repo) + errmsg += "Have you already added your GitHub repo to Travis?" + raise ValueError(errmsg) + return data['key'] + + +def prepend_line(filepath, line): + """Rewrite a file adding a line to its beginning.""" + with open(filepath) as f: + lines = f.readlines() + + lines.insert(0, line) + + with open(filepath, 'w') as f: + f.writelines(lines) + + +def load_yaml_config(filepath): + """Load yaml config file at the given path.""" + with open(filepath) as f: + return yaml.load(f) + + +def save_yaml_config(filepath, config): + """Save yaml config file at the given path.""" + with open(filepath, 'w') as f: + yaml.dump(config, f, default_flow_style=False) + + +def update_travis_deploy_password(encrypted_password): + """Put `encrypted_password` into the deploy section of .travis.yml.""" + config = load_yaml_config(TRAVIS_CONFIG_FILE) + + config['deploy']['password'] = dict(secure=encrypted_password) + + save_yaml_config(TRAVIS_CONFIG_FILE, config) + + line = ('# This file was autogenerated and will overwrite' + ' each time you run travis_pypi_setup.py\n') + prepend_line(TRAVIS_CONFIG_FILE, line) + + +def main(args): + """Add a PyPI password to .travis.yml so that Travis can deploy to PyPI. + + Fetch the Travis public key for the repo, and encrypt the PyPI password + with it before adding, so that only Travis can decrypt and use the PyPI + password. + """ + public_key = fetch_public_key(args.repo) + password = args.password or getpass('PyPI password: ') + update_travis_deploy_password(encrypt(public_key, password.encode())) + print("Wrote encrypted password to .travis.yml -- you're ready to deploy") + + +if '__main__' == __name__: + import argparse + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--repo', default=GITHUB_REPO, + help='GitHub repo (default: %s)' % GITHUB_REPO) + parser.add_argument('--password', + help='PyPI password (will prompt if not provided)') + + args = parser.parse_args() + main(args)