diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0c84ad7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,49 @@ + +# Ignore everything +** + +# ONVIF broker is built using a `docker build` command, so +# the source code needs to be available to the DotNet container +# build +!samples/brokers/onvif-video-broker + +# The streaming app is built using a `docker build` command, so +# the source code needs to be available to the container +# build +!samples/apps/video-streaming-app + +# The Rust binaries are not built with a `docker build` +# command (they are built using Cargo Cross, which I think +# uses a docker run). Because of this, the Rust src and +# Cargo.toml files can be ignored by docker. + + +# Cross-build binaries need to be available +# It is not clear to me why !target/*/*/controller +# does not work here, but it doesn't seem to. So +# for now, explicitly specifying each cross-build +# target and configuration path. +!target/x86_64-unknown-linux-gnu/debug/controller +!target/x86_64-unknown-linux-gnu/release/controller +!target/x86_64-unknown-linux-gnu/debug/agent +!target/x86_64-unknown-linux-gnu/release/agent +!target/x86_64-unknown-linux-gnu/debug/udev-video-broker +!target/x86_64-unknown-linux-gnu/release/udev-video-broker +!target/aarch64-unknown-linux-gnu/debug/controller +!target/aarch64-unknown-linux-gnu/release/controller +!target/aarch64-unknown-linux-gnu/debug/agent +!target/aarch64-unknown-linux-gnu/release/agent +!target/aarch64-unknown-linux-gnu/debug/udev-video-broker +!target/aarch64-unknown-linux-gnu/release/udev-video-broker +!target/arm-unknown-linux-gnueabihf/debug/controller +!target/arm-unknown-linux-gnueabihf/release/controller +!target/arm-unknown-linux-gnueabihf/debug/agent +!target/arm-unknown-linux-gnueabihf/release/agent +!target/arm-unknown-linux-gnueabihf/debug/udev-video-broker +!target/arm-unknown-linux-gnueabihf/release/udev-video-broker + +# Cross toml file needs to be available for making the cross build containers +!Cross.toml + +# Container image license to be copied into every container +!build/container-images-legal-notice.md \ No newline at end of file diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..4ec3b06 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,3 @@ +# https://help.github.com/en/articles/about-code-owners#codeowners-syntax + +* @bfjelds @kate-goldenring @jiria @britel \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..5e98234 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,32 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**Output of `kubectl get pods,akrii,akric -o wide`** + +**Kubernetes Version: [e.g. Native Kubernetes 1.19, MicroK8s 1.19, Minikube 1.19, K3s]** + +**To Reproduce** +Steps to reproduce the behavior: +1. Create cluster using '...' +2. Install Akri with the Helm command '...' +3. '...' + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Logs (please share snips of applicable logs)** + - To get the logs of any pod, run `kubectl get logs ` + - To get the logs of a pod that has already terminated, `kubectl get logs --previous` + - If you believe that the problem is with the Kubelet, run `journalctl -u kubelet` or `journalctl -u snap.microk8s.daemon-kubelet` if you are using a MicroK8s cluster. + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..4439246 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,23 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: feature +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Is your feature request related to a way you would like Akri extended? Please describe.** +A clear and concise description of what cool feature Akri lacks. Ex. I'd love Akri to support discovery of Bluetooth devices. + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/actions/build-component-multi-arch/action.yml b/.github/actions/build-component-multi-arch/action.yml new file mode 100644 index 0000000..b0c30c3 --- /dev/null +++ b/.github/actions/build-component-multi-arch/action.yml @@ -0,0 +1,27 @@ +name: 'build-component-multi-arch' +description: 'Build Akri Component Container' +inputs: + container_registry_base_url: + description: Azure Container Registry + required: true + container_registry_username: + description: Azure Container Registry name + required: true + container_registry_password: + description: Azure Container Registry password + required: true + container_name: + description: Component container name + required: true + container_prefix: + description: Container prefix (i.e. 'myacr.acr.io/foo' for myacr.acr.io/foo/container:label) + required: true + makefile_component_name: + description: Component prefix used by Makefile + required: true + github_event_name: + description: Specify the github event name (push, pull_request, release, etc) + required: true +runs: + using: 'node12' + main: 'main.js' diff --git a/.github/actions/build-component-multi-arch/main.js b/.github/actions/build-component-multi-arch/main.js new file mode 100644 index 0000000..7c600a5 --- /dev/null +++ b/.github/actions/build-component-multi-arch/main.js @@ -0,0 +1,62 @@ +const exec = require('@actions/exec'); +const core = require('@actions/core'); +const child_process = require('child_process'); +const fs = require("fs"); + +async function shell_cmd(cmd) { + return await new Promise((resolve, reject) => { + child_process.exec(cmd, function(error, stdout, stderr) { + if (error) { + console.log(`... error=${error}`) + reject(error) + } + + if (stderr) { + console.log(`... stderr=${stderr.trim()}`) + } + + console.log(`... stdout=${stdout.trim()}`) + resolve(stdout.trim()); + }); + }); +} + +(async () => { + try { + console.log(`Start main.js`) + + var dev_suffix = (core.getInput('github_event_name') == "release") ? "" : "-dev"; + const versioned_label = `v${fs.readFileSync('./version.txt').toString().trim()}${dev_suffix}`; + const latest_label = `latest${dev_suffix}`; + console.log(`Use labels: versioned=${versioned_label} latest=${latest_label}`); + + console.log(`Login into Container Registry user=${core.getInput('container_registry_username')} repo=${core.getInput('container_registry_base_url')}`); + await shell_cmd(`echo "${core.getInput('container_registry_password')}" | docker login -u ${core.getInput('container_registry_username')} --password-stdin ${core.getInput('container_registry_base_url')}`); + + process.env.DOCKER_CLI_EXPERIMENTAL = `enabled` + process.env.PREFIX = `${core.getInput('container_prefix')}` + process.env.LABEL_PREFIX = `${versioned_label}` + + console.log(`echo Create multi-arch versioned manifest`) + await exec.exec(`make ${core.getInput('makefile_component_name')}-docker-multi-arch-create`) + + console.log(`echo Inspect multi-arch versioned manifest`) + await exec.exec(`docker manifest inspect ${core.getInput('container_prefix')}/${core.getInput('container_name')}:${versioned_label}`) + + console.log(`echo Push multi-arch versioned manifest`) + await exec.exec(`make ${core.getInput('makefile_component_name')}-docker-multi-arch-push`) + + process.env.LABEL_PREFIX = `${latest_label}` + + console.log(`echo Create multi-arch latest manifest`) + await exec.exec(`make ${core.getInput('makefile_component_name')}-docker-multi-arch-create`) + + console.log(`echo Inspect multi-arch latest manifest`) + await exec.exec(`docker manifest inspect ${core.getInput('container_prefix')}/${core.getInput('container_name')}:${latest_label}`) + + console.log(`echo Push multi-arch latest manifest`) + await exec.exec(`make ${core.getInput('makefile_component_name')}-docker-multi-arch-push`) + } catch (error) { + core.setFailed(error); + } +})(); \ No newline at end of file diff --git a/.github/actions/build-component-multi-arch/package.json b/.github/actions/build-component-multi-arch/package.json new file mode 100644 index 0000000..40b3297 --- /dev/null +++ b/.github/actions/build-component-multi-arch/package.json @@ -0,0 +1,10 @@ +{ + "private": true, + "name": "build-component-multi-arch", + "dependencies": { + "@actions/core": "^1.2.2", + "@actions/exec": "1.0.2", + "child_process": "^1.0.2", + "fs": "^8.1.0" + } + } \ No newline at end of file diff --git a/.github/actions/build-component-per-arch/action.yml b/.github/actions/build-component-per-arch/action.yml new file mode 100644 index 0000000..f44b98b --- /dev/null +++ b/.github/actions/build-component-per-arch/action.yml @@ -0,0 +1,42 @@ +name: 'build-component-per-arch' +description: 'Build Akri Component Container' +inputs: + container_registry_base_url: + description: Azure Container Registry + required: true + container_registry_username: + description: Azure Container Registry name + required: true + container_registry_password: + description: Azure Container Registry password + required: true + container_name: + description: Component container name + required: true + container_prefix: + description: Container prefix (i.e. 'myacr.acr.io/foo' for myacr.acr.io/foo/container:label) + required: true + makefile_component_name: + description: Component prefix used by Makefile + required: true + platform: + description: Platform to build (amd64|arm64|arm32) + required: true + build_rust: + description: Specify whether rust is being built + required: true + github_event_name: + description: Specify the github event name (push, pull_request, release, etc) + required: true + github_ref: + description: Specify the github ref + required: true + github_event_action: + description: Specify the github event action (i.e. closed) + required: true + github_merged: + description: Specify whether a PR has been merged + required: true +runs: + using: 'node12' + main: 'main.js' diff --git a/.github/actions/build-component-per-arch/main.js b/.github/actions/build-component-per-arch/main.js new file mode 100644 index 0000000..a13ef0a --- /dev/null +++ b/.github/actions/build-component-per-arch/main.js @@ -0,0 +1,106 @@ +const exec = require('@actions/exec'); +const core = require('@actions/core'); +const child_process = require('child_process'); +const fs = require("fs"); + +async function shell_cmd(cmd) { + return await new Promise((resolve, reject) => { + child_process.exec(cmd, function(error, stdout, stderr) { + if (error) { + console.log(`... error=${error}`) + reject(error) + } + + if (stderr) { + console.log(`... stderr=${stderr.trim()}`) + } + + console.log(`... stdout=${stdout.trim()}`) + resolve(stdout.trim()); + }); + }); +} + +(async () => { + try { + console.log(`Start main.js`) + + console.log(`Use multiarch/qemu-user-static to configure cross-plat`); + await shell_cmd('docker run --rm --privileged multiarch/qemu-user-static --reset -p yes'); + + var dev_suffix = (core.getInput('github_event_name') == "release") ? "" : "-dev"; + const versioned_label = `v${fs.readFileSync('./version.txt').toString().trim()}${dev_suffix}`; + const latest_label = `latest${dev_suffix}`; + console.log(`Use labels: versioned=${versioned_label} latest=${latest_label}`); + + var push_containers = 0; + if (core.getInput('github_event_name') == 'release') push_containers = 1; + else if (core.getInput('github_event_name') == 'push' && + core.getInput('github_ref') == 'refs/heads/main') push_containers = 1; + else if (core.getInput('github_event_name') == 'pull_request' && + core.getInput('github_event_action') == 'closed' && + core.getInput('github_ref') == 'refs/heads/main' && + core.getInput('github_merged') == 'true') push_containers = 1; + else console.log(`Not pushing containers ... event: ${core.getInput('github_event_name')}, ref: ${core.getInput('github_ref')}, action: ${core.getInput('github_event_action')}, merged: ${core.getInput('github_merged')}`); + console.log(`Push containers: ${push_containers}`); + + var makefile_target_suffix = ""; + switch (core.getInput('platform')) { + case "amd64": makefile_target_suffix = "amd64"; break; + case "arm32v7": makefile_target_suffix = "arm32"; break; + case "arm64v8": makefile_target_suffix = "arm64"; break; + default: + core.setFailed(`Failed with unknown platform: ${core.getInput('platform')}`) + return + } + console.log(`Makefile build target suffix: ${makefile_target_suffix}`) + + console.log(`Login into Container Registry user=${core.getInput('container_registry_username')} repo=${core.getInput('container_registry_base_url')}`); + await shell_cmd(`echo "${core.getInput('container_registry_password')}" | docker login -u ${core.getInput('container_registry_username')} --password-stdin ${core.getInput('container_registry_base_url')}`); + + if (core.getInput('build_rust') == '1') { + console.log(`Install Rust`) + child_process.execSync(`curl https://sh.rustup.rs | sh -s -- -y --default-toolchain=1.41.0`); + const bindir = `${process.env.HOME}/.cargo/bin`; + process.env.PATH = `${process.env.PATH}:${bindir}`; + + console.log(`Check cargo version`) + await shell_cmd('cargo --version') + console.log(`Install Cross`) + await shell_cmd('make install-cross') + await shell_cmd('cross --version') + console.log(`Cross compile: akri-cross-build-${makefile_target_suffix}`) + await exec.exec(`make akri-cross-build-${makefile_target_suffix}`) + } else { + console.log(`Not building Rust: ${core.getInput('build_rust')}`) + } + + process.env.PREFIX = `${core.getInput('container_prefix')}` + + console.log(`Build the versioned container: make ${core.getInput('makefile_component_name')}-build-${makefile_target_suffix}`) + process.env.LABEL_PREFIX = `${versioned_label}` + await exec.exec(`make ${core.getInput('makefile_component_name')}-build-${makefile_target_suffix}`) + + console.log(`Build the latest container: make ${core.getInput('makefile_component_name')}-build-${makefile_target_suffix}`) + process.env.LABEL_PREFIX = `${latest_label}` + await exec.exec(`make ${core.getInput('makefile_component_name')}-build-${makefile_target_suffix}`) + + const image_name = `${core.getInput('container_prefix')}/${core.getInput('container_name')}:${versioned_label}-${core.getInput('platform')}` + console.log(`Check that container contains container-images-legal-notice.md: ${image_name}`) + await shell_cmd(`docker run ${image_name} find container-images-legal-notice.md | wc -l | grep -v 0`) + + if (push_containers == "1") { + console.log(`Push the versioned container: make ${core.getInput('makefile_component_name')}-docker-per-arch-${makefile_target_suffix}`) + process.env.LABEL_PREFIX = `${versioned_label}` + await exec.exec(`make ${core.getInput('makefile_component_name')}-docker-per-arch-${makefile_target_suffix}`) + + console.log(`Push the latest container: make ${core.getInput('makefile_component_name')}-docker-per-arch-${makefile_target_suffix}`) + process.env.LABEL_PREFIX = `${latest_label}` + await exec.exec(`make ${core.getInput('makefile_component_name')}-docker-per-arch-${makefile_target_suffix}`) + } else { + console.log(`Not pushing containers: ${push_containers}`) + } + } catch (error) { + core.setFailed(error); + } +})(); \ No newline at end of file diff --git a/.github/actions/build-component-per-arch/package.json b/.github/actions/build-component-per-arch/package.json new file mode 100644 index 0000000..d3f346b --- /dev/null +++ b/.github/actions/build-component-per-arch/package.json @@ -0,0 +1,10 @@ +{ + "private": true, + "name": "build-component-per-arch", + "dependencies": { + "@actions/core": "^1.2.2", + "@actions/exec": "1.0.2", + "child_process": "^1.0.2", + "fs": "^8.1.0" + } + } \ No newline at end of file diff --git a/.github/actions/build-intermediate/action.yml b/.github/actions/build-intermediate/action.yml new file mode 100644 index 0000000..aabf7ea --- /dev/null +++ b/.github/actions/build-intermediate/action.yml @@ -0,0 +1,39 @@ +name: 'build-component-per-arch' +description: 'Build Akri Component Container' +inputs: + container_registry_base_url: + description: Azure Container Registry + required: true + container_registry_username: + description: Azure Container Registry name + required: true + container_registry_password: + description: Azure Container Registry password + required: true + container_name: + description: Container name + required: true + container_prefix: + description: Container prefix (i.e. 'myacr.acr.io/foo' for myacr.acr.io/foo/container:label) + required: true + makefile_component_name: + description: Component prefix used by Makefile + required: true + platform: + description: Platform to build (amd64|arm64|arm32) + required: true + github_event_name: + description: Specify the github event name (push, pull_request, release, etc) + required: true + github_ref: + description: Specify the github ref + required: true + github_event_action: + description: Specify the github event action (i.e. closed) + required: true + github_merged: + description: Specify whether a PR has been merged + required: true +runs: + using: 'node12' + main: 'main.js' diff --git a/.github/actions/build-intermediate/main.js b/.github/actions/build-intermediate/main.js new file mode 100644 index 0000000..0869c78 --- /dev/null +++ b/.github/actions/build-intermediate/main.js @@ -0,0 +1,70 @@ +const exec = require('@actions/exec'); +const core = require('@actions/core'); +const child_process = require('child_process'); +const fs = require("fs"); + +async function shell_cmd(cmd) { + return await new Promise((resolve, reject) => { + child_process.exec(cmd, function(error, stdout, stderr) { + if (error) { + console.log(`... error=${error}`) + reject(error) + } + + if (stderr) { + console.log(`... stderr=${stderr.trim()}`) + } + + console.log(`... stdout=${stdout.trim()}`) + resolve(stdout.trim()); + }); + }); +} + +(async () => { + try { + console.log(`Start main.js`) + + console.log(`Use multiarch/qemu-user-static to configure cross-plat`); + child_process.execSync('docker run --rm --privileged multiarch/qemu-user-static --reset -p yes'); + + var push_containers = 0; + if (core.getInput('github_event_name') == 'release') push_containers = 1; + else if (core.getInput('github_event_name') == 'push' && + core.getInput('github_ref') == 'refs/heads/main') push_containers = 1; + else if (core.getInput('github_event_name') == 'pull_request' && + core.getInput('github_event_action') == 'closed' && + core.getInput('github_ref') == 'refs/heads/main' && + core.getInput('github_merged') == 'true') push_containers = 1; + else console.log(`Not pushing containers ... event: ${core.getInput('github_event_name')}, ref: ${core.getInput('github_ref')}, action: ${core.getInput('github_event_action')}, merged: ${core.getInput('github_merged')}`); + console.log(`Push containers: ${push_containers}`); + + var makefile_target_suffix = ""; + switch (core.getInput('platform')) { + case "amd64": makefile_target_suffix = "amd64"; break; + case "arm32v7": makefile_target_suffix = "arm32"; break; + case "arm64v8": makefile_target_suffix = "arm64"; break; + default: + core.setFailed(`Failed with unknown platform: ${core.getInput('platform')}`) + return + } + console.log(`Makefile build target suffix: ${makefile_target_suffix}`) + + console.log(`Login into Container Registry user=${core.getInput('container_registry_username')} repo=${core.getInput('container_registry_base_url')}`); + await shell_cmd(`echo "${core.getInput('container_registry_password')}" | docker login -u ${core.getInput('container_registry_username')} --password-stdin ${core.getInput('container_registry_base_url')}`); + + process.env.PREFIX = `${core.getInput('container_prefix')}` + + console.log(`Build the versioned container: make ${core.getInput('makefile_component_name')}-build-${makefile_target_suffix}`) + await exec.exec(`make ${core.getInput('makefile_component_name')}-build-${makefile_target_suffix}`) + + if (push_containers == "1") { + console.log(`Push the versioned container: make ${core.getInput('makefile_component_name')}-docker-per-arch-${makefile_target_suffix}`) + await exec.exec(`make ${core.getInput('makefile_component_name')}-docker-per-arch-${makefile_target_suffix}`) + } else { + console.log(`Not pushing containers: ${push_containers}`) + } + } catch (error) { + core.setFailed(error); + } +})(); \ No newline at end of file diff --git a/.github/actions/build-intermediate/package.json b/.github/actions/build-intermediate/package.json new file mode 100644 index 0000000..3504049 --- /dev/null +++ b/.github/actions/build-intermediate/package.json @@ -0,0 +1,10 @@ +{ + "private": true, + "name": "build-intermediate", + "dependencies": { + "@actions/core": "^1.2.2", + "@actions/exec": "1.0.2", + "child_process": "^1.0.2", + "fs": "^8.1.0" + } + } \ No newline at end of file diff --git a/.github/workflows/build-agent-container.yml b/.github/workflows/build-agent-container.yml new file mode 100644 index 0000000..771df18 --- /dev/null +++ b/.github/workflows/build-agent-container.yml @@ -0,0 +1,107 @@ +name: Build Agent + +on: + push: + branches: [ main ] + paths: + - .github/actions/build-component-per-arch/** + - .github/actions/build-component-multi-arch/** + - .github/workflows/build-agent-container.yml + - build/containers/Dockerfile.agent + - agent/** + - shared/** + - build/setup.sh + - version.txt + - build/akri-containers.mk + - Makefile + pull_request: + branches: [ main ] + paths: + - .github/actions/build-component-per-arch/** + - .github/actions/build-component-multi-arch/** + - .github/workflows/build-agent-container.yml + - build/containers/Dockerfile.agent + - agent/** + - shared/** + - build/setup.sh + - version.txt + - build/akri-containers.mk + - Makefile + release: + types: + - published + +env: + AKRI_COMPONENT: agent + MAKEFILE_COMPONENT: agent + +jobs: + + per-arch: + runs-on: ubuntu-latest + timeout-minutes: 40 + strategy: + matrix: + arch: + - arm64v8 + # - arm32v7 + - amd64 + + steps: + - name: Checkout repo + uses: actions/checkout@v2 + + - name: Prepare To Install + uses: actions/setup-node@v1 + with: + node-version: 12 + - name: Install Deps + run: | + yarn install + yarn add @actions/core @actions/github @actions/exec fs + + - name: Run Per-Arch component build for ${{ env.AKRI_COMPONENT }} + uses: ./.github/actions/build-component-per-arch + with: + github_event_name: ${{ github.event_name }} + github_ref: ${{ github.ref }} + github_event_action: ${{ github.event.action }} + github_merged: ${{ github.event.pull_request.merged }} + container_name: ${{ env.AKRI_COMPONENT }} + container_prefix: ghcr.io/deislabs/akri + container_registry_base_url: ghcr.io + container_registry_username: ${{ secrets.crUsername }} + container_registry_password: ${{ secrets.crPassword }} + makefile_component_name: ${{ env.MAKEFILE_COMPONENT }} + platform: ${{ matrix.arch }} + build_rust: "1" + + multi-arch: + if: (github.event_name == 'release') || (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true && github.ref != 'refs/heads/main') + needs: per-arch + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Checkout repo + uses: actions/checkout@v2 + + - name: Prepare To Install + uses: actions/setup-node@v1 + with: + node-version: 12 + - name: Install Deps + run: | + yarn install + yarn add @actions/core @actions/github @actions/exec fs + + - name: Run Multi-Arch component build for ${{ env.AKRI_COMPONENT }} + uses: ./.github/actions/build-component-multi-arch + with: + github_event_name: ${{ github.event_name }} + container_name: ${{ env.AKRI_COMPONENT }} + container_prefix: ghcr.io/deislabs/akri + container_registry_base_url: ghcr.io + container_registry_username: ${{ secrets.crUsername }} + container_registry_password: ${{ secrets.crPassword }} + makefile_component_name: ${{ env.MAKEFILE_COMPONENT }} diff --git a/.github/workflows/build-controller-container.yml b/.github/workflows/build-controller-container.yml new file mode 100644 index 0000000..44aa2eb --- /dev/null +++ b/.github/workflows/build-controller-container.yml @@ -0,0 +1,106 @@ +name: Build Controller + +on: + push: + branches: [ main ] + paths: + - .github/actions/build-component-per-arch/** + - .github/actions/build-component-multi-arch/** + - .github/workflows/build-controller-container.yml + - build/containers/Dockerfile.controller + - controller/** + - shared/** + - build/setup.sh + - version.txt + - build/akri-containers.mk + - Makefile + pull_request: + branches: [ main ] + paths: + - .github/actions/build-component-per-arch/** + - .github/actions/build-component-multi-arch/** + - .github/workflows/build-controller-container.yml + - build/containers/Dockerfile.controller + - controller/** + - shared/** + - build/setup.sh + - version.txt + - build/akri-containers.mk + - Makefile + release: + types: + - published + +env: + AKRI_COMPONENT: controller + MAKEFILE_COMPONENT: controller + +jobs: + + per-arch: + runs-on: ubuntu-latest + timeout-minutes: 40 + strategy: + matrix: + arch: + - arm64v8 + # - arm32v7 + - amd64 + steps: + - name: Checkout repo + uses: actions/checkout@v2 + + - name: Prepare To Install + uses: actions/setup-node@v1 + with: + node-version: 12 + - name: Install Deps + run: | + yarn install + yarn add @actions/core @actions/github @actions/exec fs + + - name: Run Per-Arch component build for ${{ env.AKRI_COMPONENT }} + uses: ./.github/actions/build-component-per-arch + with: + github_event_name: ${{ github.event_name }} + github_ref: ${{ github.ref }} + github_event_action: ${{ github.event.action }} + github_merged: ${{ github.event.pull_request.merged }} + container_name: ${{ env.AKRI_COMPONENT }} + container_prefix: ghcr.io/deislabs/akri + container_registry_base_url: ghcr.io + container_registry_username: ${{ secrets.crUsername }} + container_registry_password: ${{ secrets.crPassword }} + makefile_component_name: ${{ env.MAKEFILE_COMPONENT }} + platform: ${{ matrix.arch }} + build_rust: "1" + + multi-arch: + if: (github.event_name == 'release') || (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true && github.ref != 'refs/heads/main') + needs: per-arch + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Checkout repo + uses: actions/checkout@v2 + + - name: Prepare To Install + uses: actions/setup-node@v1 + with: + node-version: 12 + - name: Install Deps + run: | + yarn install + yarn add @actions/core @actions/github @actions/exec fs + + - name: Run Multi-Arch component build for ${{ env.AKRI_COMPONENT }} + uses: ./.github/actions/build-component-multi-arch + with: + github_event_name: ${{ github.event_name }} + container_name: ${{ env.AKRI_COMPONENT }} + container_prefix: ghcr.io/deislabs/akri + container_registry_base_url: ghcr.io + container_registry_username: ${{ secrets.crUsername }} + container_registry_password: ${{ secrets.crPassword }} + makefile_component_name: ${{ env.MAKEFILE_COMPONENT }} diff --git a/.github/workflows/build-onvif-video-broker-container.yml b/.github/workflows/build-onvif-video-broker-container.yml new file mode 100644 index 0000000..161ea8c --- /dev/null +++ b/.github/workflows/build-onvif-video-broker-container.yml @@ -0,0 +1,103 @@ +name: Build ONVIF Broker (.NET) + +on: + push: + branches: [ main ] + paths: + - .github/actions/build-component-per-arch/** + - .github/actions/build-component-multi-arch/** + - .github/workflows/build-onvif-video-broker-container.yml + - build/containers/Dockerfile.onvif-video-broker + - samples/brokers/onvif-video-broker/** + - version.txt + - build/akri-containers.mk + - Makefile + pull_request: + branches: [ main ] + paths: + - .github/actions/build-component-per-arch/** + - .github/actions/build-component-multi-arch/** + - .github/workflows/build-onvif-video-broker-container.yml + - build/containers/Dockerfile.onvif-video-broker + - samples/brokers/onvif-video-broker/** + - version.txt + - build/akri-containers.mk + - Makefile + release: + types: + - published + +env: + AKRI_COMPONENT: onvif-video-broker + MAKEFILE_COMPONENT: onvif + +jobs: + + per-arch: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + matrix: + arch: + - arm64v8 + # - arm32v7 + - amd64 + + steps: + - name: Checkout repo + uses: actions/checkout@v2 + + - name: Prepare To Install + uses: actions/setup-node@v1 + with: + node-version: 12 + - name: Install Deps + run: | + yarn install + yarn add @actions/core @actions/github @actions/exec fs + + - name: Run Per-Arch component build for ${{ env.AKRI_COMPONENT }} + uses: ./.github/actions/build-component-per-arch + with: + github_event_name: ${{ github.event_name }} + github_ref: ${{ github.ref }} + github_event_action: ${{ github.event.action }} + github_merged: ${{ github.event.pull_request.merged }} + container_name: ${{ env.AKRI_COMPONENT }} + container_prefix: ghcr.io/deislabs/akri + container_registry_base_url: ghcr.io + container_registry_username: ${{ secrets.crUsername }} + container_registry_password: ${{ secrets.crPassword }} + makefile_component_name: ${{ env.MAKEFILE_COMPONENT }} + platform: ${{ matrix.arch }} + build_rust: "0" + + multi-arch: + if: (github.event_name == 'release') || (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true && github.ref != 'refs/heads/main') + needs: per-arch + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Checkout repo + uses: actions/checkout@v2 + + - name: Prepare To Install + uses: actions/setup-node@v1 + with: + node-version: 12 + - name: Install Deps + run: | + yarn install + yarn add @actions/core @actions/github @actions/exec fs + + - name: Run Multi-Arch component build for ${{ env.AKRI_COMPONENT }} + uses: ./.github/actions/build-component-multi-arch + with: + github_event_name: ${{ github.event_name }} + container_name: ${{ env.AKRI_COMPONENT }} + container_prefix: ghcr.io/deislabs/akri + container_registry_base_url: ghcr.io + container_registry_username: ${{ secrets.crUsername }} + container_registry_password: ${{ secrets.crPassword }} + makefile_component_name: ${{ env.MAKEFILE_COMPONENT }} diff --git a/.github/workflows/build-opencv-base-container.yml b/.github/workflows/build-opencv-base-container.yml new file mode 100644 index 0000000..cb7bc25 --- /dev/null +++ b/.github/workflows/build-opencv-base-container.yml @@ -0,0 +1,84 @@ +name: Build OpenCV Base + +on: + push: + branches: [ main ] + paths: + - .github/actions/build-intermediate/** + - .github/workflows/build-opencv-base-container.yml + - build/containers/intermediate/Dockerfile.opencvsharp-build + - build/intermediate-containers.mk + - Makefile + pull_request: + branches: [ main ] + paths: + - .github/actions/build-intermediate/** + - .github/workflows/build-opencv-base-container.yml + - build/containers/intermediate/Dockerfile.opencvsharp-build + - build/intermediate-containers.mk + - Makefile + +env: + AKRI_COMPONENT: opencvsharp-build + MAKEFILE_COMPONENT: opencv-base + +jobs: + + per-arch: + if: >- + !contains(github.event.pull_request.title, '[IGNORE INTERMEDIATE BUILDS]') && + !contains(github.event.commits[0].message, '[IGNORE INTERMEDIATE BUILDS]') + runs-on: ubuntu-latest + strategy: + matrix: + arch: + - arm64v8 + # - arm32v7 + - amd64 + steps: + - name: Checkout repo + uses: actions/checkout@v2 + + - name: Akri intermediate builds are LONG running and should only be run when absolutely needed + if: >- + !contains(github.event.pull_request.title, '[ALLOW INTERMEDIATE BUILDS]') && + !contains(github.event.commits[0].message, '[ALLOW INTERMEDIATE BUILDS]') + run: | + echo "Akri intermediate builds are LONG running and should only be run when absolutely needed." + echo "Add [IGNORE INTERMEDIATE BUILDS] to first commit message to skip building intermediate containers." + echo "Add [ALLOW INTERMEDIATE BUILDS] to first commit message if needed." + exit 1 + + # Only run build version change check if PR title and Merge commit message do NOT contain "[SAME VERSION]" + - if: >- + github.event_name == 'pull_request' && + !contains(github.event.pull_request.title, '[SAME VERSION]') && + !contains(github.event.commits[0].message, '[SAME VERSION]') + name: Ensure that ${{ env.AKRI_COMPONENT }} version has changed + run: | + git fetch origin main + git diff origin/main -- ./build/intermediate-containers.mk | grep "BUILD_OPENCV_BASE_VERSION = " | wc -l | grep -v 0 + + - name: Prepare To Install + uses: actions/setup-node@v1 + with: + node-version: 12 + - name: Install Deps + run: | + yarn install + yarn add @actions/core @actions/github @actions/exec fs + + - name: Run Per-Arch component build for ${{ env.AKRI_COMPONENT }} + uses: ./.github/actions/build-intermediate + with: + github_event_name: ${{ github.event_name }} + github_ref: ${{ github.ref }} + github_event_action: ${{ github.event.action }} + github_merged: ${{ github.event.pull_request.merged }} + container_name: ${{ env.AKRI_COMPONENT }} + container_prefix: ghcr.io/deislabs/akri + container_registry_base_url: ghcr.io + container_registry_username: ${{ secrets.crUsername }} + container_registry_password: ${{ secrets.crPassword }} + makefile_component_name: ${{ env.MAKEFILE_COMPONENT }} + platform: ${{ matrix.arch }} diff --git a/.github/workflows/build-rust-crossbuild-container.yml b/.github/workflows/build-rust-crossbuild-container.yml new file mode 100644 index 0000000..d78b167 --- /dev/null +++ b/.github/workflows/build-rust-crossbuild-container.yml @@ -0,0 +1,84 @@ +name: Build Rust CrossBuild + +on: + push: + branches: [ main ] + paths: + - .github/actions/build-intermediate/** + - .github/workflows/build-rust-crossbuild-container.yml + - build/containers/intermediate/Dockerfile.rust-crossbuild-* + - build/intermediate-containers.mk + - Makefile + pull_request: + branches: [ main ] + paths: + - .github/actions/build-intermediate/** + - .github/workflows/build-rust-crossbuild-container.yml + - build/containers/intermediate/Dockerfile.rust-crossbuild-* + - build/intermediate-containers.mk + - Makefile + +env: + AKRI_COMPONENT: rust-crossbuild + MAKEFILE_COMPONENT: rust-crossbuild + +jobs: + + per-arch: + if: >- + !contains(github.event.pull_request.title, '[IGNORE INTERMEDIATE BUILDS]') && + !contains(github.event.commits[0].message, '[IGNORE INTERMEDIATE BUILDS]') + runs-on: ubuntu-latest + strategy: + matrix: + arch: + - arm64v8 + # - arm32v7 + - amd64 + steps: + - name: Checkout repo + uses: actions/checkout@v2 + + - name: Akri intermediate builds are LONG running and should only be run when absolutely needed + if: >- + !contains(github.event.pull_request.title, '[ALLOW INTERMEDIATE BUILDS]') && + !contains(github.event.commits[0].message, '[ALLOW INTERMEDIATE BUILDS]') + run: | + echo "Akri intermediate builds are LONG running and should only be run when absolutely needed." + echo "Add [IGNORE INTERMEDIATE BUILDS] to first commit message to skip building intermediate containers." + echo "Add [ALLOW INTERMEDIATE BUILDS] to first commit message if needed." + exit 1 + + # Only run build version change check if PR title and Merge commit message do NOT contain "[SAME VERSION]" + - if: >- + github.event_name == 'pull_request' && + !contains(github.event.pull_request.title, '[SAME VERSION]') && + !contains(github.event.commits[0].message, '[SAME VERSION]') + name: Ensure that ${{ env.AKRI_COMPONENT }} version has changed + run: | + git fetch origin main + git diff origin/main -- ./build/intermediate-containers.mk | grep "BUILD_RUST_CROSSBUILD_VERSION = " | wc -l | grep -v 0 + + - name: Prepare To Install + uses: actions/setup-node@v1 + with: + node-version: 12 + - name: Install Deps + run: | + yarn install + yarn add @actions/core @actions/github @actions/exec fs + + - name: Run Per-Arch component build for ${{ env.AKRI_COMPONENT }} + uses: ./.github/actions/build-intermediate + with: + github_event_name: ${{ github.event_name }} + github_ref: ${{ github.ref }} + github_event_action: ${{ github.event.action }} + github_merged: ${{ github.event.pull_request.merged }} + container_name: ${{ env.AKRI_COMPONENT }} + container_prefix: ghcr.io/deislabs/akri + container_registry_base_url: ghcr.io + container_registry_username: ${{ secrets.crUsername }} + container_registry_password: ${{ secrets.crPassword }} + makefile_component_name: ${{ env.MAKEFILE_COMPONENT }} + platform: ${{ matrix.arch }} diff --git a/.github/workflows/build-udev-video-broker-container.yml b/.github/workflows/build-udev-video-broker-container.yml new file mode 100644 index 0000000..cccd479 --- /dev/null +++ b/.github/workflows/build-udev-video-broker-container.yml @@ -0,0 +1,107 @@ +name: Build UDEV Broker + +on: + push: + branches: [ main ] + paths: + - .github/actions/build-component-per-arch/** + - .github/actions/build-component-multi-arch/** + - .github/workflows/build-udev-video-broker-container.yml + - build/containers/Dockerfile.udev-video-broker + - samples/brokers/udev-video-broker/** + - shared/** + - build/setup.sh + - version.txt + - build/akri-containers.mk + - Makefile + pull_request: + branches: [ main ] + paths: + - .github/actions/build-component-per-arch/** + - .github/actions/build-component-multi-arch/** + - .github/workflows/build-udev-video-broker-container.yml + - build/containers/Dockerfile.udev-video-broker + - samples/brokers/udev-video-broker/** + - shared/** + - build/setup.sh + - version.txt + - build/akri-containers.mk + - Makefile + release: + types: + - published + +env: + AKRI_COMPONENT: udev-video-broker + MAKEFILE_COMPONENT: udev + +jobs: + + per-arch: + runs-on: ubuntu-latest + timeout-minutes: 40 + strategy: + matrix: + arch: + - arm64v8 + # - arm32v7 + - amd64 + + steps: + - name: Checkout repo + uses: actions/checkout@v2 + + - name: Prepare To Install + uses: actions/setup-node@v1 + with: + node-version: 12 + - name: Install Deps + run: | + yarn install + yarn add @actions/core @actions/github @actions/exec fs + + - name: Run Per-Arch component build for ${{ env.AKRI_COMPONENT }} + uses: ./.github/actions/build-component-per-arch + with: + github_event_name: ${{ github.event_name }} + github_ref: ${{ github.ref }} + github_event_action: ${{ github.event.action }} + github_merged: ${{ github.event.pull_request.merged }} + container_name: ${{ env.AKRI_COMPONENT }} + container_prefix: ghcr.io/deislabs/akri + container_registry_base_url: ghcr.io + container_registry_username: ${{ secrets.crUsername }} + container_registry_password: ${{ secrets.crPassword }} + makefile_component_name: ${{ env.MAKEFILE_COMPONENT }} + platform: ${{ matrix.arch }} + build_rust: "1" + + multi-arch: + if: (github.event_name == 'release') || (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true && github.ref != 'refs/heads/main') + needs: per-arch + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Checkout repo + uses: actions/checkout@v2 + + - name: Prepare To Install + uses: actions/setup-node@v1 + with: + node-version: 12 + - name: Install Deps + run: | + yarn install + yarn add @actions/core @actions/github @actions/exec fs + + - name: Run Multi-Arch component build for ${{ env.AKRI_COMPONENT }} + uses: ./.github/actions/build-component-multi-arch + with: + github_event_name: ${{ github.event_name }} + container_name: ${{ env.AKRI_COMPONENT }} + container_prefix: ghcr.io/deislabs/akri + container_registry_base_url: ghcr.io + container_registry_username: ${{ secrets.crUsername }} + container_registry_password: ${{ secrets.crPassword }} + makefile_component_name: ${{ env.MAKEFILE_COMPONENT }} diff --git a/.github/workflows/build-video-streaming-app-container.yml b/.github/workflows/build-video-streaming-app-container.yml new file mode 100644 index 0000000..4734d2b --- /dev/null +++ b/.github/workflows/build-video-streaming-app-container.yml @@ -0,0 +1,103 @@ +name: Build Video Streaming App Container + +on: + push: + branches: [ main ] + paths: + - .github/actions/build-component-per-arch/** + - .github/actions/build-component-multi-arch/** + - .github/workflows/build-video-streaming-app-container.yml + - build/containers/Dockerfile.video-streaming-app + - samples/apps/video-streaming-app/** + - version.txt + - build/akri-containers.mk + - Makefile + pull_request: + branches: [ main ] + paths: + - .github/actions/build-component-per-arch/** + - .github/actions/build-component-multi-arch/** + - .github/workflows/build-video-streaming-app-container.yml + - build/containers/Dockerfile.video-streaming-app + - samples/apps/video-streaming-app/** + - version.txt + - build/akri-containers.mk + - Makefile + release: + types: + - published + +env: + AKRI_COMPONENT: video-streaming-app + MAKEFILE_COMPONENT: streaming + +jobs: + + per-arch: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + matrix: + arch: + - arm64v8 + # - arm32v7 + - amd64 + + steps: + - name: Checkout repo + uses: actions/checkout@v2 + + - name: Prepare To Install + uses: actions/setup-node@v1 + with: + node-version: 12 + - name: Install Deps + run: | + yarn install + yarn add @actions/core @actions/github @actions/exec fs + + - name: Run Per-Arch component build for ${{ env.AKRI_COMPONENT }} + uses: ./.github/actions/build-component-per-arch + with: + github_event_name: ${{ github.event_name }} + github_ref: ${{ github.ref }} + github_event_action: ${{ github.event.action }} + github_merged: ${{ github.event.pull_request.merged }} + container_name: ${{ env.AKRI_COMPONENT }} + container_prefix: ghcr.io/deislabs/akri + container_registry_base_url: ghcr.io + container_registry_username: ${{ secrets.crUsername }} + container_registry_password: ${{ secrets.crPassword }} + makefile_component_name: ${{ env.MAKEFILE_COMPONENT }} + platform: ${{ matrix.arch }} + build_rust: "0" + + multi-arch: + if: (github.event_name == 'release') || (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true && github.ref != 'refs/heads/main') + needs: per-arch + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Checkout repo + uses: actions/checkout@v2 + + - name: Prepare To Install + uses: actions/setup-node@v1 + with: + node-version: 12 + - name: Install Deps + run: | + yarn install + yarn add @actions/core @actions/github @actions/exec fs + + - name: Run Multi-Arch component build for ${{ env.AKRI_COMPONENT }} + uses: ./.github/actions/build-component-multi-arch + with: + github_event_name: ${{ github.event_name }} + container_name: ${{ env.AKRI_COMPONENT }} + container_prefix: ghcr.io/deislabs/akri + container_registry_base_url: ghcr.io + container_registry_username: ${{ secrets.crUsername }} + container_registry_password: ${{ secrets.crPassword }} + makefile_component_name: ${{ env.MAKEFILE_COMPONENT }} diff --git a/.github/workflows/check-rust.yml b/.github/workflows/check-rust.yml new file mode 100644 index 0000000..1d70214 --- /dev/null +++ b/.github/workflows/check-rust.yml @@ -0,0 +1,55 @@ +name: Check Rust + +on: + push: + branches: [ main ] + paths: + - .github/workflows/check-rust.yml + - '**.rs' + - '**/Cargo.toml' + - '**/Cargo.lock' + - build/setup.sh + pull_request: + branches: [ main ] + paths: + - .github/workflows/check-rust.yml + - '**.rs' + - '**/Cargo.toml' + - '**/Cargo.lock' + - build/setup.sh + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - uses: actions/checkout@v2 + - name: Rust install + uses: actions-rs/toolchain@v1 + with: + toolchain: 1.41.0 + override: true + components: clippy, rustfmt + - name: Install linux requirement + run: ./build/setup.sh + - name: Check rust format + run: cargo fmt --all -- --check + - name: Check clippy + run: cargo clippy --all + - name: Run check + run: cargo check + - name: Check for H2 patch not being applied + run: | + cargo check >& h2-check.txt + cat h2-check.txt | grep "h2" | grep "was not used in the crate graph" | wc -l | grep 0 + - name: Run tests + run: cargo test + - name: Run tests --ignored + run: cargo test -- --ignored + - name: Run doc + run: cargo doc --no-deps diff --git a/.github/workflows/check-versioning.yml b/.github/workflows/check-versioning.yml new file mode 100644 index 0000000..b545cef --- /dev/null +++ b/.github/workflows/check-versioning.yml @@ -0,0 +1,45 @@ +name: Check versioning + +on: + push: + branches: [ main ] + paths-ignore: + - '.gitignore' + - 'LICENSE' + - '**.md' + - .vscode/** + - docs/** + pull_request: + branches: [ main ] + paths-ignore: + - '.gitignore' + - 'LICENSE' + - '**.md' + - .vscode/** + - docs/** + release: + types: + - published + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - uses: actions/checkout@v2 + + # Only run version check for PRs. If PR title does NOT contain "[SAME VERSION]", then ensure that + # version.txt is different from what is in main. + - if: github.event_name == 'pull_request' && contains(github.event.pull_request.title, '[SAME VERSION]') == false + name: Run version check + run: ./version.sh -c + # If PR title does contain "[SAME VERSION]", then do not check that version.txt is different from + # what is in main. + - if: github.event_name == 'pull_request' && contains(github.event.pull_request.title, '[SAME VERSION]') == true + name: Run version check + run: ./version.sh -c -s diff --git a/.github/workflows/run-helm.yml b/.github/workflows/run-helm.yml new file mode 100644 index 0000000..fb767d8 --- /dev/null +++ b/.github/workflows/run-helm.yml @@ -0,0 +1,86 @@ +name: Helm + +on: + push: + branches: [ main ] + paths: + - .github/workflows/run-helm.yml + - deployment/** + - version.txt + pull_request: + branches: [ main ] + paths: + - .github/workflows/run-helm.yml + - deployment/** + - version.txt + release: + types: + - published + +env: + helm_version: "3.2.1" + +jobs: + helm: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v2 + - uses: azure/setup-helm@v1 + + - name: Lint helm chart + run: helm lint deployment/helm + + - name: Cache version.txt + run: | + mkdir -p /tmp/helm/repo + cp version.txt /tmp/helm/repo/version.txt + + - name: Create new helm package + run: helm package --dependency-update --destination /tmp/helm/repo deployment/helm + + - name: Inspect helm package + run: helm inspect all $(find /tmp/helm/repo -name "akri-*.tgz") + + - name: Upload new helm package as artifact + if: (github.event_name == 'release') || (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true && github.ref != 'refs/heads/main') + uses: actions/upload-artifact@v2 + with: + name: charts + path: /tmp/helm/repo + + - name: Checkout gh-pages + if: (github.event_name == 'release') || (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true && github.ref != 'refs/heads/main') + uses: actions/checkout@v2 + with: + ref: gh-pages + + - name: Get new chart from artifact path + if: (github.event_name == 'release') || (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true && github.ref != 'refs/heads/main') + shell: bash + run: | + mv /tmp/helm/repo/* . + find . + + - name: Create new merged helm chart index + if: (github.event_name == 'release') || (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true && github.ref != 'refs/heads/main') + run: helm repo index --url https://deislabs.github.io/akri --merge index.yaml . + + - name: Upload new merged helm chart index as artifact + if: (github.event_name == 'release') || (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true && github.ref != 'refs/heads/main') + uses: actions/upload-artifact@v2 + with: + name: index + path: index.yaml + + - name: Push gh-pages + if: (github.event_name == 'release') || (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged == true && github.ref != 'refs/heads/main') + shell: bash + run: | + git config user.name "${GITHUB_ACTOR}" + git config user.email "${GITHUB_ACTOR}@users.noreply.github.com" + git add akri-$(cat version.txt).tgz + git add index.yaml + git status + echo "Update Helm Repo for version $(cat version.txt)" | git commit --file - + git push https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git gh-pages diff --git a/.github/workflows/run-tarpaulin.yml b/.github/workflows/run-tarpaulin.yml new file mode 100644 index 0000000..5b7c73b --- /dev/null +++ b/.github/workflows/run-tarpaulin.yml @@ -0,0 +1,39 @@ +name: Tarpaulin Code Coverage + +on: + push: + branches: [ main ] + paths: + - .github/workflows/check-rust.yml + - '**.rs' + - '**/Cargo.toml' + - build/setup.sh + pull_request: + branches: [ main ] + paths: + - .github/workflows/check-rust.yml + - '**.rs' + - '**/Cargo.toml' + - build/setup.sh + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + - uses: actions/checkout@v2 + - name: Create tarpaulin instance + run: docker create --network host --security-opt seccomp=unconfined -v "${PWD}:/volume" xd009642/tarpaulin:0.12.2 bash -c "echo 'sleep 20m; echo bye' > /tmp/keep_alive.sh; chmod 777 /tmp/keep_alive.sh; /tmp/keep_alive.sh" > container_id.txt + - name: Start tarpaulin instance + run: docker start $(cat container_id.txt) + - name: Install linux requirement in tarpaulin instance + run: docker exec $(cat container_id.txt) sh -c "./build/setup.sh" + - name: Install rust requirements in tarpaulin instance + run: docker exec $(cat container_id.txt) sh -c "rustup component add rustfmt" + - name: Run tarpaulin + run: docker exec $(cat container_id.txt) sh -c "RUST_LOG=trace cargo tarpaulin -v --all-features --out Xml" diff --git a/.github/workflows/run-test-cases.yml b/.github/workflows/run-test-cases.yml new file mode 100644 index 0000000..3b15c24 --- /dev/null +++ b/.github/workflows/run-test-cases.yml @@ -0,0 +1,106 @@ +name: Run End-to-End Scenario + +on: + workflow_dispatch: + pull_request: + branches: [ main ] + paths: + - test/run-end-to-end.py + - test/run-conservation-of-broker-pod.py + - test/run-helm-install-delete.py + - test/shared_test_code.py + - .github/workflows/run-test-cases.yml + push: + branches: [ main ] + paths: + - version.txt + +jobs: + test-cases: + runs-on: ubuntu-18.04 + strategy: + fail-fast: false + matrix: + test-case: + - end-to-end + - helm-install-delete + - conservation-of-broker-pods + include: + - test-case: end-to-end + test-file: test/run-end-to-end.py + - test-case: helm-install-delete + test-file: test/run-helm-install-delete.py + - test-case: conservation-of-broker-pods + test-file: test/run-conservation-of-broker-pod.py + + steps: + - name: Checkout repository content + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install Python kubernetes dependency + run: | + python -m pip install --upgrade pip + pip install kubernetes + - name: Install MicroK8s + run: sudo snap install microk8s --classic --channel=1.18/stable + - name: wait for Microk8s to be ready + run: sudo microk8s status --wait-ready + - name: Add user to MicroK8s + run: sudo usermod -a -G microk8s $USER + - name: make user kube owner + run: | + sudo ls -la $HOME/.kube + echo sudo chown $(id -u):$(id -g) $HOME/.kube + sudo chown -f -R $USER $HOME/.kube --verbose + - name: Enable Helm + run: | + sudo sh -c "microk8s.kubectl config view --raw >~/.kube/config" + sudo cat ~/.kube/config + sudo microk8s.enable helm3 + - name: MicroK8s enable DNS + run: sudo microk8s.enable dns + - name: MicroK8s allow privileged pods + run: | + echo "--allow-privileged=true" | sudo tee -a /var/snap/microk8s/current/args/kube-apiserver + - name: MicroK8s stop + run: sudo microk8s.stop + - name: MicroK8s start + run: sudo microk8s.start + # Check for error with wait ready in next step + continue-on-error: true + - name: MicroK8s wait ready + run: sudo microk8s.status --wait-ready + - name: Label master node + run: sudo microk8s kubectl label node ${HOSTNAME,,} node-role.kubernetes.io/master= --overwrite=true + - name: Apply Docker secret + run: sudo microk8s kubectl create secret docker-registry regcred --docker-server=ghcr.io --docker-username=${{ secrets.crUsername }} --docker-password=${{ secrets.crPassword }} + - name: Add Akri Helm Chart + run: helm repo add akri-helm-charts https://deislabs.github.io/akri/ + - if: github.event_name == 'push' + name: Set sleep duration before running script to 1500 + run: echo 1500 > /tmp/sleep_duration.txt + - if: github.event_name != 'push' + name: Use main version for non-push + run: | + git fetch origin main + git show origin/main:version.txt > /tmp/version_to_test.txt + - if: github.event_name == 'push' + name: Use current version for push + run: cat version.txt > /tmp/version_to_test.txt + - name: Execute test script ${{ matrix.test-file }} + run: python ${{ matrix.test-file }} + - name: Upload Agent log as artifact + if: always() + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.test-case }}-agent-log + path: /tmp/agent_log.txt + - name: Upload controller log as artifact + if: always() + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.test-case }}-controller-log + path: /tmp/controller_log.txt \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1a2cebd --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +**/target/ +**/obj +**/bin +**/cobertura.xml diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..9969376 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,3218 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "addr2line" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6a2d3371669ab3ca9797670853d61402b03d0b4b9ebf33d677dfa720203072" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" + +[[package]] +name = "agent" +version = "0.0.33" +dependencies = [ + "akri-shared", + "async-stream", + "async-trait", + "blake2", + "cfg-if", + "chrono", + "env_logger", + "failure", + "futures 0.1.29", + "futures 0.3.5", + "futures-core", + "futures-util", + "hyper 0.11.27", + "k8s-openapi", + "kube", + "log 0.4.8", + "mime", + "mockall", + "pest", + "pest_derive", + "prost", + "rand 0.7.3", + "regex", + "serde", + "serde_derive", + "serde_json", + "serde_yaml", + "sxd-document", + "sxd-xpath", + "tokio 0.2.21", + "tokio-core", + "tonic", + "tonic-build", + "tower", + "udev", + "uuid", + "xml-rs", + "yaserde", + "yaserde_derive", +] + +[[package]] +name = "aho-corasick" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86" +dependencies = [ + "memchr", +] + +[[package]] +name = "akri-shared" +version = "0.0.33" +dependencies = [ + "async-trait", + "bytes 0.5.6", + "either", + "env_logger", + "failure", + "futures 0.1.29", + "futures 0.3.5", + "futures-util", + "hyper 0.11.27", + "hyper 0.13.7", + "k8s-openapi", + "kube", + "log 0.4.8", + "mockall", + "rand 0.7.3", + "serde", + "serde_derive", + "serde_json", + "serde_yaml", + "sxd-document", + "sxd-xpath", + "tokio 0.2.21", + "tokio-core", + "tokio-signal", +] + +[[package]] +name = "anyhow" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bb70cc08ec97ca5450e6eba421deeea5f172c0fc61f78b5357b2a8e8be195f" + +[[package]] +name = "arc-swap" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034" + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" + +[[package]] +name = "async-compression" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9021768bcce77296b64648cc7a7460e3df99979b97ed5c925c38d1cc83778d98" +dependencies = [ + "bytes 0.5.6", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", +] + +[[package]] +name = "async-std" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d68a33ebc8b57800847d00787307f84a562224a14db069b0acefe4c2abbf5d" +dependencies = [ + "async-task", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "kv-log-macro", + "log 0.4.8", + "memchr", + "num_cpus", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab 0.4.2", + "smol", + "wasm-bindgen-futures", +] + +[[package]] +name = "async-stream" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22068c0c19514942eefcfd4daf8976ef1aad84e61539f95cd200c35202f80af5" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25f9db3b38af870bf7e5cc649167533b493928e50744e2c30ae350230b414670" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-task" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17772156ef2829aadc587461c7753af20b7e8db1529bc66855add962a3b35d3" + +[[package]] +name = "async-trait" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a265e3abeffdce30b2e26b7a11b222fe37c6067404001b434101457d0385eb92" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "autocfg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" + +[[package]] +name = "backtrace" +version = "0.3.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46254cf2fdcdf1badb5934448c1bcbe046a56537b3987d96c51a7afc5d03f293" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643" +dependencies = [ + "byteorder", + "safemem", +] + +[[package]] +name = "base64" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" +dependencies = [ + "byteorder", +] + +[[package]] +name = "base64" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" + +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "blake2" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94cb07b0da6a73955f8fb85d24c466778e70cda767a568229b104f0264089330" +dependencies = [ + "byte-tools", + "crypto-mac", + "digest", + "opaque-debug", +] + +[[package]] +name = "blake2b_simd" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + +[[package]] +name = "blocking" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d17efb70ce4421e351d61aafd90c16a20fb5bfe339fcdc32a86816280e62ce0" +dependencies = [ + "futures-channel", + "futures-util", + "once_cell", + "parking", + "waker-fn", +] + +[[package]] +name = "bumpalo" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" + +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + +[[package]] +name = "byteorder" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" + +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "iovec", +] + +[[package]] +name = "bytes" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" + +[[package]] +name = "cache-padded" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" + +[[package]] +name = "cc" +version = "1.0.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a06fb2e53271d7c279ec1efea6ab691c35a2ae67ec0d91d7acec0caf13b518" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "chrono" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c74d84029116787153e02106bf53e66828452a4b325cc8652b788b5967c0a0b6" +dependencies = [ + "num-integer", + "num-traits", + "serde", + "time", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "concurrent-queue" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83c06aff61f2d899eb87c379df3cbf7876f14471dcab474e0b6dc90ab96c080" +dependencies = [ + "cache-padded", +] + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "controller" +version = "0.0.33" +dependencies = [ + "akri-shared", + "async-std", + "async-trait", + "chrono", + "env_logger", + "failure", + "futures 0.3.5", + "k8s-openapi", + "kube", + "log 0.4.8", + "mockall", + "serde", + "serde_derive", + "serde_json", + "serde_yaml", + "tokio 0.2.21", +] + +[[package]] +name = "core-foundation" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" + +[[package]] +name = "crc32fast" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "lazy_static", + "maybe-uninit", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg", + "cfg-if", + "lazy_static", +] + +[[package]] +name = "crypto-mac" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "difference" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198" + +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array", +] + +[[package]] +name = "dirs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3" +dependencies = [ + "cfg-if", + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e93d7f5705de3e49895a2b5e0b8855a1c27f080192ae9c32a6432d50741a57a" +dependencies = [ + "libc", + "redox_users", + "winapi 0.3.9", +] + +[[package]] +name = "downcast" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb454f0228b18c7f4c3b0ebbee346ed9c52e7443b0999cd543ff3571205701d" + +[[package]] +name = "dtoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" + +[[package]] +name = "either" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" + +[[package]] +name = "encoding_rs" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8ac63f94732332f44fe654443c46f6375d1939684c17b0afb6cb56b0456e171" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "env_logger" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3" +dependencies = [ + "atty", + "humantime", + "log 0.4.8", + "regex", + "termcolor", +] + +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + +[[package]] +name = "fastrand" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36a9cb09840f81cd211e435d00a4e487edd263dc3c8ff815c32dd76ad668ebed" + +[[package]] +name = "fixedbitset" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" + +[[package]] +name = "flate2" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68c90b0fc46cf89d227cc78b40e494ff81287a92dd07631e5af0d06fe3cf885e" +dependencies = [ + "cfg-if", + "crc32fast", + "libc", + "miniz_oxide", +] + +[[package]] +name = "float-cmp" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da62c4f1b81918835a8c6a484a397775fff5953fe83529afd51b05f5c6a6617d" +dependencies = [ + "num-traits", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "fragile" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f8140122fa0d5dcb9fc8627cfce2b37cc1500f752636d46ea28bc26785c2f9" + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "futures" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b980f2816d6ee8673b6517b52cb0e808a180efc92e5c19d02cdda79066703ef" + +[[package]] +name = "futures" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e05b85ec287aac0dc34db7d4a569323df697f9c55b99b15d6b4ef8cde49f613" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f366ad74c28cca6ba456d95e6422883cfb4b252a83bed929c83abfdbbf2967d5" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59f5fff90fd5d971f936ad674802482ba441b6f09ba5e15fd8b39145582ca399" + +[[package]] +name = "futures-cpupool" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" +dependencies = [ + "futures 0.1.29", + "num_cpus", +] + +[[package]] +name = "futures-executor" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d6bb888be1153d3abeb9006b11b02cf5e9b209fda28693c31ae1e4e012e314" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de27142b013a8e869c14957e6d2edeef89e97c289e69d042ee3a49acd8b51789" + +[[package]] +name = "futures-macro" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0b5a30a4328ab5473878237c447333c093297bded83a4983d10f4deea240d39" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f2032893cb734c7a05d85ce0cc8b8c4075278e93b24b66f9de99d6eb0fa8acc" + +[[package]] +name = "futures-task" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb66b5f09e22019b1ab0830f7785bcea8e7a42148683f99214f73f8ec21a626" +dependencies = [ + "once_cell", +] + +[[package]] +name = "futures-timer" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1de7508b218029b0f01662ed8f61b1c964b3ae99d6f25462d0f55a595109df6" + +[[package]] +name = "futures-util" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8764574ff08b701a084482c3c7031349104b07ac897393010494beaa18ce32c6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab 0.4.2", +] + +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +dependencies = [ + "typenum", +] + +[[package]] +name = "getrandom" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" + +[[package]] +name = "h2" +version = "0.2.6" +source = "git+https://github.com/kate-goldenring/h2#7c7ef6a579c9ce2392787c5728f805ce10f74ddf" +dependencies = [ + "bytes 0.5.6", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.1", + "indexmap", + "slab 0.4.2", + "tokio 0.2.21", + "tokio-util 0.3.1", + "tracing", +] + +[[package]] +name = "heck" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3deed196b6e7f9e44a2ae8d94225d80302d81208b1bb673fd21fe634645c85a9" +dependencies = [ + "libc", +] + +[[package]] +name = "http" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ccf5ede3a895d8856620237b2f02972c1bbc78d2965ad7fe8838d4a0ed41f0" +dependencies = [ + "bytes 0.4.12", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" +dependencies = [ + "bytes 0.5.6", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +dependencies = [ + "bytes 0.5.6", + "http 0.2.1", +] + +[[package]] +name = "httparse" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + +[[package]] +name = "hyper" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34a590ca09d341e94cddf8e5af0bbccde205d5fbc2fa3c09dd67c7f85cea59d7" +dependencies = [ + "base64 0.9.3", + "bytes 0.4.12", + "futures 0.1.29", + "futures-cpupool", + "httparse", + "iovec", + "language-tags", + "log 0.4.8", + "mime", + "net2", + "percent-encoding 1.0.1", + "relay", + "time", + "tokio-core", + "tokio-io", + "tokio-proto", + "tokio-service", + "unicase", + "want 0.0.4", +] + +[[package]] +name = "hyper" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e68a8dd9716185d9e64ea473ea6ef63529252e3e27623295a0378a19665d5eb" +dependencies = [ + "bytes 0.5.6", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http 0.2.1", + "http-body", + "httparse", + "itoa", + "pin-project", + "socket2", + "time", + "tokio 0.2.21", + "tower-service", + "tracing", + "want 0.3.0", +] + +[[package]] +name = "hyper-tls" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" +dependencies = [ + "bytes 0.5.6", + "hyper 0.13.7", + "native-tls", + "tokio 0.2.21", + "tokio-tls", +] + +[[package]] +name = "idna" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c398b2b113b55809ceb9ee3e753fcbac793f1956663f3c36549c1346015c2afe" +dependencies = [ + "autocfg", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "itertools" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" + +[[package]] +name = "js-sys" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4b9172132a62451e56142bff9afc91c8e4a4500aa5b847da36815b63bfda916" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "k8s-openapi" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8eb97e4ea14cef484aa56f44e93653cb6faa351b5f130d38584b3184b6ef5d1" +dependencies = [ + "base64 0.10.1", + "bytes 0.4.12", + "chrono", + "http 0.1.21", + "percent-encoding 2.1.0", + "serde", + "serde-value", + "serde_json", + "url", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "kube" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21a8a53ca3e8cb9f6ebdf09cdd8943d08926da7ca8d5c1d89dee1ed0e8df0a2a" +dependencies = [ + "base64 0.11.0", + "chrono", + "dirs", + "either", + "futures 0.3.5", + "futures-timer", + "http 0.2.1", + "k8s-openapi", + "log 0.4.8", + "openssl", + "reqwest", + "serde", + "serde_derive", + "serde_json", + "serde_yaml", + "thiserror", + "time", + "url", +] + +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log 0.4.8", +] + +[[package]] +name = "language-tags" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9f8082297d534141b30c8d39e9b1773713ab50fdbe4ff30f750d063b3bfd701" + +[[package]] +name = "libudev-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c8469b4a23b962c1396b9b451dda50ef5b283e8dd309d69033475fa9b334324" +dependencies = [ + "libc", + "pkg-config", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" + +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" +dependencies = [ + "log 0.4.8", +] + +[[package]] +name = "log" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "memchr" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" + +[[package]] +name = "memoffset" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c198b026e1bbf08a937e94c6c60f9ec4a2267f5b0d2eec9c1b21b061ce2be55f" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0f75932c1f6cfae3c04000e40114adf955636e19040f9c0a2c380702aa1c7f" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" +dependencies = [ + "cfg-if", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log 0.4.8", + "miow 0.2.1", + "net2", + "slab 0.4.2", + "winapi 0.2.8", +] + +[[package]] +name = "mio-named-pipes" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656" +dependencies = [ + "log 0.4.8", + "mio", + "miow 0.3.5", + "winapi 0.3.9", +] + +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio", +] + +[[package]] +name = "miow" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "miow" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07b88fb9795d4d36d62a012dfbf49a8f5cf12751f36d31a9dbe66d528e58979e" +dependencies = [ + "socket2", + "winapi 0.3.9", +] + +[[package]] +name = "mockall" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b95a7e7cfbce0e99ebbf5356a085d3b5e320a7ef300f77cd50a7148aa362e7c2" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5a615a1ad92048ad5d9633251edb7492b8abc057d7a679a9898476aef173935" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "multimap" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8883adfde9756c1d30b0f519c9b8c502a94b41ac62f696453c37c7fc0a958ce" + +[[package]] +name = "native-tls" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b0d88c06fe90d5ee94048ba40409ef1d9315d86f6f38c2efdaad4fb50c58b2d" +dependencies = [ + "lazy_static", + "libc", + "log 0.4.8", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "net2" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ba7c918ac76704fb42afcbbb43891e72731f3dcca3bef2a19786297baf14af7" +dependencies = [ + "cfg-if", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-integer" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" + +[[package]] +name = "once_cell" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d" + +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + +[[package]] +name = "openssl" +version = "0.10.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d575eff3665419f9b83678ff2815858ad9d11567e082f5ac1814baba4e2bcb4" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "lazy_static", + "libc", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" + +[[package]] +name = "openssl-sys" +version = "0.9.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "ordered-float" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3741934be594d77de1c8461ebcbbe866f585ea616a9753aa78f2bdc69f0e4579" +dependencies = [ + "num-traits", +] + +[[package]] +name = "parking" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efcee3c6d23b94012e240525f131c6abaa9e5eeb8f211002d93beec3b7be350" + +[[package]] +name = "parking_lot" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" +dependencies = [ + "lock_api", + "parking_lot_core", + "rustc_version", +] + +[[package]] +name = "parking_lot_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" +dependencies = [ + "cfg-if", + "cloudabi", + "libc", + "redox_syscall", + "rustc_version", + "smallvec 0.6.13", + "winapi 0.3.9", +] + +[[package]] +name = "percent-encoding" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "peresil" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f658886ed52e196e850cfbbfddab9eaa7f6d90dd0929e264c31e5cec07e09e57" + +[[package]] +name = "pest" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +dependencies = [ + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "833d1ae558dc601e9a60366421196a8d94bc0ac980476d0b67e1d0988d72b2d0" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54be6e404f5317079812fc8f9f5279de376d8856929e21c184ecf6bbd692a11d" +dependencies = [ + "maplit", + "pest", + "sha-1", +] + +[[package]] +name = "petgraph" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "467d164a6de56270bd7c4d070df81d07beace25012d5103ced4e9ff08d6afdb7" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pin-project" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12e3a6cdbfe94a5e4572812a0201f8c0ed98c1c452c7b8563ce2276988ef9c17" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a0ffd45cf79d88737d7cc85bfd5d2894bee1139b356e616fe85dc389c61aaf7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282adbf10f2698a7a77f8e983a74b2d18176c19a7fd32a45446139ae7b02b715" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d36492546b6af1463394d46f0c834346f31548646f6ba10849802c9c9a27ac33" + +[[package]] +name = "ppv-lite86" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "237a5ed80e274dbc66f86bd59c1e25edc039660be53194b5fe0a482e0f2612ea" + +[[package]] +name = "predicates" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "347a1b6f0b21e636bc9872fb60b83b8e185f6f5516298b8238699f7f9a531030" +dependencies = [ + "difference", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06075c3a3e92559ff8929e7a280684489ea27fe44805174c3ebd9328dcb37178" + +[[package]] +name = "predicates-tree" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e63c4859013b38a76eca2414c64911fba30def9e3202ac461a2d22831220124" +dependencies = [ + "predicates-core", + "treeline", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4" + +[[package]] +name = "proc-macro-nested" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" + +[[package]] +name = "proc-macro2" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "prost" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce49aefe0a6144a45de32927c77bd2859a5f7677b55f220ae5b744e87389c212" +dependencies = [ + "bytes 0.5.6", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b10678c913ecbd69350e8535c3aef91a8676c0773fc1d7b95cdd196d7f2f26" +dependencies = [ + "bytes 0.5.6", + "heck", + "itertools", + "log 0.4.8", + "multimap", + "petgraph", + "prost", + "prost-types", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537aa19b95acde10a12fec4301466386f757403de4cd4e5b4fa78fb5ecb18f72" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1834f67c0697c001304b75be76f67add9c89742eda3a085ad8ee0bb38c3417aa" +dependencies = [ + "bytes 0.5.6", + "prost", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ac302d8f83c0c1974bf758f6b041c6c8ada916fbb44a609158ca8b064cc76c" +dependencies = [ + "libc", + "rand 0.4.6", +] + +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi 0.3.9", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom", + "libc", + "rand_chacha", + "rand_core 0.5.1", + "rand_hc", + "rand_pcg", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_pcg" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "redox_users" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b23093265f8d200fa7b4c2c76297f47e681c655f6f1285a8780d6a022f7431" +dependencies = [ + "getrandom", + "redox_syscall", + "rust-argon2", +] + +[[package]] +name = "regex" +version = "1.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-syntax" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" + +[[package]] +name = "relay" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1576e382688d7e9deecea24417e350d3062d97e32e45d70b1cde65994ff1489a" +dependencies = [ + "futures 0.1.29", +] + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "reqwest" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b82c9238b305f26f53443e3a4bc8528d64b8d0bee408ec949eb7bf5635ec680" +dependencies = [ + "async-compression", + "base64 0.12.3", + "bytes 0.5.6", + "encoding_rs", + "futures-core", + "futures-util", + "http 0.2.1", + "http-body", + "hyper 0.13.7", + "hyper-tls", + "js-sys", + "lazy_static", + "log 0.4.8", + "mime", + "mime_guess", + "native-tls", + "percent-encoding 2.1.0", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio 0.2.21", + "tokio-tls", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "rscam" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89056084211cd54924fedf2e2199b906409d1f795cfd8e7e3271061742457018" +dependencies = [ + "libc", +] + +[[package]] +name = "rust-argon2" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bc8af4bda8e1ff4932523b94d3dd20ee30a87232323eda55903ffd71d2fb017" +dependencies = [ + "base64 0.11.0", + "blake2b_simd", + "constant_time_eq", + "crossbeam-utils", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "safemem" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi 0.3.9", +] + +[[package]] +name = "scoped-tls" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332ffa32bf586782a3efaeb58f127980944bbc8c4d6913a86107ac2a5ab24b28" + +[[package]] +name = "scoped-tls" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "security-framework" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64808902d7d99f78eaddd2b4e2509713babc3dc3c85ad6f4c447680f3c01e535" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17bf11d99252f512695eb468de5516e5cf75455521e69dfe343f3b74e4748405" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "1.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3" + +[[package]] +name = "serde-value" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a65a7291a8a568adcae4c10a677ebcedbc6c9cec91c054dee2ce40b0e3290eb" +dependencies = [ + "ordered-float", + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3433e879a558dde8b5e8feb2a04899cf34fdde1fafb894687e52105fc1162ac3" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97" +dependencies = [ + "dtoa", + "itoa", + "serde", + "url", +] + +[[package]] +name = "serde_yaml" +version = "0.8.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae3e2dd40a7cdc18ca80db804b7f461a39bb721160a85c9a1fa30134bf3c02a5" +dependencies = [ + "dtoa", + "linked-hash-map", + "serde", + "yaml-rust", +] + +[[package]] +name = "sha-1" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" +dependencies = [ + "block-buffer", + "digest", + "fake-simd", + "opaque-debug", +] + +[[package]] +name = "signal-hook-registry" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f478ede9f64724c5d173d7bb56099ec3e2d9fc2774aac65d34b8b890405f41" +dependencies = [ + "arc-swap", + "libc", +] + +[[package]] +name = "slab" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b4fcaed89ab08ef143da37bc52adbcc04d4a69014f4c1208d6b51f0c47bc23" + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "smallvec" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8cbcd6df1e117c2210e13ab5109635ad68a929fcbb8964dc965b76cb5ee013" + +[[package]] +name = "smallvec" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6" +dependencies = [ + "maybe-uninit", +] + +[[package]] +name = "smol" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "620cbb3c6e34da57d3a248cda0cd01cd5848164dc062e764e65d06fe3ea7aed5" +dependencies = [ + "async-task", + "blocking", + "concurrent-queue", + "fastrand", + "futures-io", + "futures-util", + "libc", + "once_cell", + "scoped-tls 1.0.0", + "slab 0.4.2", + "socket2", + "wepoll-sys-stjepang", + "winapi 0.3.9", +] + +[[package]] +name = "socket2" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03088793f677dce356f3ccc2edb1b314ad191ab702a5de3faf49304f7e104918" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "winapi 0.3.9", +] + +[[package]] +name = "subtle" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee" + +[[package]] +name = "sxd-document" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94d82f37be9faf1b10a82c4bd492b74f698e40082f0f40de38ab275f31d42078" +dependencies = [ + "peresil", + "typed-arena", +] + +[[package]] +name = "sxd-xpath" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36e39da5d30887b5690e29de4c5ebb8ddff64ebd9933f98a01daaa4fd11b36ea" +dependencies = [ + "peresil", + "quick-error", + "sxd-document", +] + +[[package]] +name = "syn" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "936cae2873c940d92e697597c5eee105fb570cd5689c695806f672883653349b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "synstructure" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "take" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b157868d8ac1f56b64604539990685fa7611d8fa9e5476cf0c02cf34d32917c5" + +[[package]] +name = "tempfile" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +dependencies = [ + "cfg-if", + "libc", + "rand 0.7.3", + "redox_syscall", + "remove_dir_all", + "winapi 0.3.9", +] + +[[package]] +name = "termcolor" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "thiserror" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfdd070ccd8ccb78f4ad66bf1982dc37f620ef696c6b5028fe2ed83dd3d0d08" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd80fc12f73063ac132ac92aceea36734f04a1d93c1240c6944e23a3b8841793" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "time" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "tinyvec" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53953d2d3a5ad81d9f844a32f14ebb121f50b650cd59d0ee2a07cf13c617efed" + +[[package]] +name = "tokio" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.29", + "mio", + "num_cpus", + "tokio-codec", + "tokio-current-thread", + "tokio-executor", + "tokio-fs", + "tokio-io", + "tokio-reactor", + "tokio-sync", + "tokio-tcp", + "tokio-threadpool", + "tokio-timer", + "tokio-udp", + "tokio-uds", +] + +[[package]] +name = "tokio" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d099fa27b9702bed751524694adbe393e18b36b204da91eb1cbbbbb4a5ee2d58" +dependencies = [ + "bytes 0.5.6", + "fnv", + "futures-core", + "iovec", + "lazy_static", + "libc", + "memchr", + "mio", + "mio-named-pipes", + "mio-uds", + "num_cpus", + "pin-project-lite", + "signal-hook-registry", + "slab 0.4.2", + "tokio-macros", + "winapi 0.3.9", +] + +[[package]] +name = "tokio-codec" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b2998660ba0e70d18684de5d06b70b70a3a747469af9dea7618cc59e75976b" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.29", + "tokio-io", +] + +[[package]] +name = "tokio-core" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeeffbbb94209023feaef3c196a41cbcdafa06b4a6f893f68779bb5e53796f71" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.29", + "iovec", + "log 0.4.8", + "mio", + "scoped-tls 0.1.2", + "tokio 0.1.22", + "tokio-executor", + "tokio-io", + "tokio-reactor", + "tokio-timer", +] + +[[package]] +name = "tokio-current-thread" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" +dependencies = [ + "futures 0.1.29", + "tokio-executor", +] + +[[package]] +name = "tokio-executor" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" +dependencies = [ + "crossbeam-utils", + "futures 0.1.29", +] + +[[package]] +name = "tokio-fs" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297a1206e0ca6302a0eed35b700d292b275256f596e2f3fea7729d5e629b6ff4" +dependencies = [ + "futures 0.1.29", + "tokio-io", + "tokio-threadpool", +] + +[[package]] +name = "tokio-io" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.29", + "log 0.4.8", +] + +[[package]] +name = "tokio-macros" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-proto" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fbb47ae81353c63c487030659494b295f6cb6576242f907f203473b191b0389" +dependencies = [ + "futures 0.1.29", + "log 0.3.9", + "net2", + "rand 0.3.23", + "slab 0.3.0", + "smallvec 0.2.1", + "take", + "tokio-core", + "tokio-io", + "tokio-service", +] + +[[package]] +name = "tokio-reactor" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" +dependencies = [ + "crossbeam-utils", + "futures 0.1.29", + "lazy_static", + "log 0.4.8", + "mio", + "num_cpus", + "parking_lot", + "slab 0.4.2", + "tokio-executor", + "tokio-io", + "tokio-sync", +] + +[[package]] +name = "tokio-service" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24da22d077e0f15f55162bdbdc661228c1581892f52074fb242678d015b45162" +dependencies = [ + "futures 0.1.29", +] + +[[package]] +name = "tokio-signal" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0c34c6e548f101053321cba3da7cbb87a610b85555884c41b07da2eb91aff12" +dependencies = [ + "futures 0.1.29", + "libc", + "mio", + "mio-uds", + "signal-hook-registry", + "tokio-executor", + "tokio-io", + "tokio-reactor", + "winapi 0.3.9", +] + +[[package]] +name = "tokio-sync" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" +dependencies = [ + "fnv", + "futures 0.1.29", +] + +[[package]] +name = "tokio-tcp" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.29", + "iovec", + "mio", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-threadpool" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" +dependencies = [ + "crossbeam-deque", + "crossbeam-queue", + "crossbeam-utils", + "futures 0.1.29", + "lazy_static", + "log 0.4.8", + "num_cpus", + "slab 0.4.2", + "tokio-executor", +] + +[[package]] +name = "tokio-timer" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" +dependencies = [ + "crossbeam-utils", + "futures 0.1.29", + "slab 0.4.2", + "tokio-executor", +] + +[[package]] +name = "tokio-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" +dependencies = [ + "native-tls", + "tokio 0.2.21", +] + +[[package]] +name = "tokio-udp" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2a0b10e610b39c38b031a2fcab08e4b82f16ece36504988dcbd81dbba650d82" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.29", + "log 0.4.8", + "mio", + "tokio-codec", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-uds" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab57a4ac4111c8c9dbcf70779f6fc8bc35ae4b2454809febac840ad19bd7e4e0" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.29", + "iovec", + "libc", + "log 0.4.8", + "mio", + "mio-uds", + "tokio-codec", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-util" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "571da51182ec208780505a32528fc5512a8fe1443ab960b3f2f3ef093cd16930" +dependencies = [ + "bytes 0.5.6", + "futures-core", + "futures-sink", + "log 0.4.8", + "pin-project-lite", + "tokio 0.2.21", +] + +[[package]] +name = "tokio-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +dependencies = [ + "bytes 0.5.6", + "futures-core", + "futures-sink", + "log 0.4.8", + "pin-project-lite", + "tokio 0.2.21", +] + +[[package]] +name = "tonic" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08283643b1d483eb7f3fc77069e63b5cba3e4db93514b3d45470e67f123e4e48" +dependencies = [ + "async-stream", + "async-trait", + "base64 0.10.1", + "bytes 0.5.6", + "futures-core", + "futures-util", + "http 0.2.1", + "http-body", + "hyper 0.13.7", + "percent-encoding 1.0.1", + "pin-project", + "prost", + "prost-derive", + "tokio 0.2.21", + "tokio-util 0.2.0", + "tower", + "tower-balance", + "tower-load", + "tower-make", + "tower-service", + "tracing", + "tracing-futures", +] + +[[package]] +name = "tonic-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0436413ba71545bcc6c2b9a0f9d78d72deb0123c6a75ccdfe7c056f9930f5e52" +dependencies = [ + "proc-macro2", + "prost-build", + "quote", + "syn", +] + +[[package]] +name = "tower" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3169017c090b7a28fce80abaad0ab4f5566423677c9331bb320af7e49cfe62" +dependencies = [ + "futures-core", + "tower-buffer", + "tower-discover", + "tower-layer", + "tower-limit", + "tower-load-shed", + "tower-retry", + "tower-service", + "tower-timeout", + "tower-util", +] + +[[package]] +name = "tower-balance" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a792277613b7052448851efcf98a2c433e6f1d01460832dc60bef676bc275d4c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project", + "rand 0.7.3", + "slab 0.4.2", + "tokio 0.2.21", + "tower-discover", + "tower-layer", + "tower-load", + "tower-make", + "tower-ready-cache", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-buffer" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4887dc2a65d464c8b9b66e0e4d51c2fd6cf5b3373afc72805b0a60bce00446a" +dependencies = [ + "futures-core", + "pin-project", + "tokio 0.2.21", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-discover" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f6b5000c3c54d269cc695dff28136bb33d08cbf1df2c48129e143ab65bf3c2a" +dependencies = [ + "futures-core", + "pin-project", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a35d656f2638b288b33495d1053ea74c40dc05ec0b92084dd71ca5566c4ed1dc" + +[[package]] +name = "tower-limit" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92c3040c5dbed68abffaa0d4517ac1a454cd741044f33ab0eefab6b8d1361404" +dependencies = [ + "futures-core", + "pin-project", + "tokio 0.2.21", + "tower-layer", + "tower-load", + "tower-service", +] + +[[package]] +name = "tower-load" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cc79fc3afd07492b7966d7efa7c6c50f8ed58d768a6075dd7ae6591c5d2017b" +dependencies = [ + "futures-core", + "log 0.4.8", + "pin-project", + "tokio 0.2.21", + "tower-discover", + "tower-service", +] + +[[package]] +name = "tower-load-shed" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f021e23900173dc315feb4b6922510dae3e79c689b74c089112066c11f0ae4e" +dependencies = [ + "futures-core", + "pin-project", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-make" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce50370d644a0364bf4877ffd4f76404156a248d104e2cc234cd391ea5cdc965" +dependencies = [ + "tokio 0.2.21", + "tower-service", +] + +[[package]] +name = "tower-ready-cache" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4eabb6620e5481267e2ec832c780b31cad0c15dcb14ed825df5076b26b591e1f" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "log 0.4.8", + "tokio 0.2.21", + "tower-service", +] + +[[package]] +name = "tower-retry" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6727956aaa2f8957d4d9232b308fe8e4e65d99db30f42b225646e86c9b6a952" +dependencies = [ + "futures-core", + "pin-project", + "tokio 0.2.21", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-service" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" + +[[package]] +name = "tower-timeout" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "127b8924b357be938823eaaec0608c482d40add25609481027b96198b2e4b31e" +dependencies = [ + "pin-project", + "tokio 0.2.21", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-util" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1093c19826d33807c72511e68f73b4a0469a3f22c2bd5f7d5212178b4b89674" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "tower-service", +] + +[[package]] +name = "tracing" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2e2a2de6b0d5cbb13fc21193a2296888eaab62b6044479aafb3c54c01c29fcd" +dependencies = [ + "cfg-if", + "log 0.4.8", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0693bf8d6f2bf22c690fc61a9d21ac69efdbb894a17ed596b9af0f01e64b84b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94ae75f0d28ae10786f3b1895c55fe72e79928fd5ccdebb5438c75e93fec178f" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-futures" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +dependencies = [ + "pin-project", + "tracing", +] + +[[package]] +name = "treeline" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41" + +[[package]] +name = "try-lock" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2aa4715743892880f70885373966c83d73ef1b0838a664ef0c76fffd35e7c2" + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "typed-arena" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9b2228007eba4120145f785df0f6c92ea538f5a3635a612ecf4e334c8c1446d" + +[[package]] +name = "typenum" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" + +[[package]] +name = "ucd-trie" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" + +[[package]] +name = "udev" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24953d50a3bce0f5f5a9a2766567072dc9af8096f8c40ea81815da651066bc9f" +dependencies = [ + "libc", + "libudev-sys", +] + +[[package]] +name = "udev-video-broker" +version = "0.0.33" +dependencies = [ + "akri-shared", + "env_logger", + "futures 0.1.29", + "log 0.4.8", + "prost", + "rscam", + "tokio 0.2.21", + "tonic", + "tonic-build", +] + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb19cf769fa8c6a80a162df694621ebeb4dafb606470b2b2fce0be40a98a977" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "url" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb" +dependencies = [ + "idna", + "matches", + "percent-encoding 2.1.0", +] + +[[package]] +name = "uuid" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fde2f6a4bea1d6e007c4ad38c6839fa71cbb63b6dbf5b595aa38dc9b1093c11" +dependencies = [ + "rand 0.7.3", +] + +[[package]] +name = "vcpkg" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6454029bf181f092ad1b853286f23e2c507d8e8194d01d92da4a55c274a5508c" + +[[package]] +name = "version_check" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" + +[[package]] +name = "waker-fn" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9571542c2ce85ce642e6b58b3364da2fb53526360dfb7c211add4f5c23105ff7" + +[[package]] +name = "want" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a05d9d966753fa4b5c8db73fcab5eed4549cfe0e1e4e66911e5564a0085c35d1" +dependencies = [ + "futures 0.1.29", + "log 0.4.8", + "try-lock 0.1.0", +] + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log 0.4.8", + "try-lock 0.2.3", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasm-bindgen" +version = "0.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a634620115e4a229108b71bde263bb4220c483b3f07f5ba514ee8d15064c4c2" +dependencies = [ + "cfg-if", + "serde", + "serde_json", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e53963b583d18a5aa3aaae4b4c1cb535218246131ba22a71f05b518098571df" +dependencies = [ + "bumpalo", + "lazy_static", + "log 0.4.8", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dba48d66049d2a6cc8488702e7259ab7afc9043ad0dc5448444f46f2a453b362" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fcfd5ef6eec85623b4c6e844293d4516470d8f19cd72d0d12246017eb9060b8" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9adff9ee0e94b926ca81b57f57f86d5545cdcb1d259e21ec9bdd95b901754c75" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7b90ea6c632dd06fd765d44542e234d5e63d9bb917ecd64d79778a13bd79ae" + +[[package]] +name = "web-sys" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "863539788676619aac1a23e2df3655e96b32b0e05eb72ca34ba045ad573c625d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wepoll-sys-stjepang" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fd319e971980166b53e17b1026812ad66c6b54063be879eb182342b55284694" +dependencies = [ + "cc", +] + +[[package]] +name = "which" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724" +dependencies = [ + "libc", +] + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "xml-rs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b07db065a5cf61a7e4ba64f29e67db906fb1787316516c4e6e5ff0fea1efcd8a" + +[[package]] +name = "yaml-rust" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39f0c922f1a334134dc2f7a8b67dc5d25f0735263feec974345ff706bcf20b0d" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "yaserde" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe81055b36d926da70850a7c1a9bc72eff184659da29e008ce9f2af4bfee7c89" +dependencies = [ + "log 0.4.8", + "xml-rs", +] + +[[package]] +name = "yaserde_derive" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fcdf088bb87971689cf7aef14b123c5c85ca8d71267985090369e5eab16a514" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..9fbea78 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,7 @@ +# Patch to allow invalid authority headers provided by grpc-go / kubelet +# Issue to track: https://github.com/grpc/grpc-go/issues/2628 +[patch.crates-io] +h2 = { git = "https://github.com/kate-goldenring/h2", branch = "master"} + +[workspace] +members = ["shared", "controller", "agent", "samples/brokers/udev-video-broker"] \ No newline at end of file diff --git a/Cross.toml b/Cross.toml new file mode 100644 index 0000000..efecafd --- /dev/null +++ b/Cross.toml @@ -0,0 +1,14 @@ +[build.env] +passthrough = [ + "PKG_CONFIG_ALLOW_CROSS", + "RUST_LOG", +] + +[target.x86_64-unknown-linux-gnu] +image = "ghcr.io/deislabs/akri/rust-crossbuild:x86_64-unknown-linux-gnu-0.1.16-0.0.4" + +[target.arm-unknown-linux-gnueabihf] +image = "ghcr.io/deislabs/akri/rust-crossbuild:arm-unknown-linux-gnueabihf-0.1.16-0.0.4" + +[target.aarch64-unknown-linux-gnu] +image = "ghcr.io/deislabs/akri/rust-crossbuild:aarch64-unknown-linux-gnu-0.1.16-0.0.4" \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4b3ba9d --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Microsoft Corporation. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b0325c4 --- /dev/null +++ b/Makefile @@ -0,0 +1,32 @@ +BUILD_AMD64 ?= 1 +BUILD_ARM32 ?= 0 #skip building arm32 for now +BUILD_ARM64 ?= 1 + +REGISTRY ?= devcaptest.azurecr.io +UNIQUE_ID ?= $(USER) + +INTERMEDIATE_DOCKERFILE_DIR ?= build/containers/intermediate +DOCKERFILE_DIR ?= build/containers + +PREFIX ?= $(REGISTRY)/$(UNIQUE_ID) +VERSION=$(shell cat version.txt) +VERSION_LABEL=v$(VERSION) +LABEL_PREFIX ?= $(VERSION_LABEL) + +CACHE_OPTION ?= + +ENABLE_DOCKER_MANIFEST = DOCKER_CLI_EXPERIMENTAL=enabled + +AMD64_SUFFIX = amd64 +ARM32V7_SUFFIX = arm32v7 +ARM64V8_SUFFIX = arm64v8 + +AMD64_TARGET = x86_64-unknown-linux-gnu +ARM32V7_TARGET = arm-unknown-linux-gnueabihf +ARM64V8_TARGET = aarch64-unknown-linux-gnu + +# Intermediate container defines +include build/intermediate-containers.mk + +# Akri container defines +include build/akri-containers.mk \ No newline at end of file diff --git a/NOTICE.txt b/NOTICE.txt new file mode 100644 index 0000000..22a0979 --- /dev/null +++ b/NOTICE.txt @@ -0,0 +1,29 @@ +NOTICES + +This repository incorporates material as listed below or described in the code. + +flask-video-streaming + +https://github.com/miguelgrinberg/flask-video-streaming + +The MIT License (MIT) + +Copyright (c) 2014 Miguel Grinberg + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..b40cad3 --- /dev/null +++ b/README.md @@ -0,0 +1,36 @@ +# Akri +[![Check Rust](https://github.com/deislabs/akri/workflows/Check%20Rust/badge.svg?branch=main&event=push)](https://github.com/deislabs/akri/actions?query=workflow%3A%22Check+Rust%22) [![Tarpaulin Code Coverage](https://github.com/deislabs/akri/workflows/Tarpaulin%20Code%20Coverage/badge.svg?branch=main&event=push)](https://github.com/deislabs/akri/actions?query=workflow%3A%22Tarpaulin+Code+Coverage%22) [![Version](https://img.shields.io/badge/rustc-1.41.0-blue.svg)](https://blog.rust-lang.org/2020/01/30/Rust-1.41.0.html) + +[![Build Controller](https://github.com/deislabs/akri/workflows/Build%20Controller/badge.svg?branch=main&event=push)](https://github.com/deislabs/akri/actions?query=workflow%3A%22Build+Controller%22) [![Build Agent](https://github.com/deislabs/akri/workflows/Build%20Agent/badge.svg?branch=main&event=push)](https://github.com/deislabs/akri/actions?query=workflow%3A%22Build+Agent%22) [![Run End-to-End Scenario](https://github.com/deislabs/akri/workflows/Run%20End-to-End%20Scenario/badge.svg?branch=main&event=push)](https://github.com/deislabs/akri/actions?query=workflow%3A%22Run+End-to-End+Scenario%22) + +Akri lets you easily expose heterogeneous leaf devices (such as IP cameras and USB devices) as resources in a Kubernetes cluster, while also supporting the exposure of embedded hardware resources such as GPUs and FPGAs. Akri continually detects nodes that have access to these devices and schedules workloads based on them. + +Simply put: you name it, Akri finds it, you use it. + +## Why Akri +At the edge, there are a variety of sensors, controllers, and MCU class devices that are producing data and performing actions. For Kubernetes to be a viable edge computing solution, these heterogeneous “leaf devices” need to be easily utilized by Kubernetes clusters. However, many of these leaf devices are too small to run Kubernetes themselves. Akri is an open source project that exposes these leaf devices as resources in a Kubernetes cluster. It leverages and extends the Kubernetes [device plugin framework](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/), which was created with the cloud in mind and focuses on advertising static resources such as GPUs and other system hardware. Akri took this framework and applied it to the edge, where there is a diverse set of leaf devices with unique communication protocols and intermittent availability. + +Akri is made for the edge, **handling the dynamic appearance and disappearance of leaf devices**. Akri provides an abstraction layer similar to [CNI](https://github.com/containernetworking/cni), but instead of abstracting the underlying network details, it is removing the work of finding, utilizing, and monitoring the availability of the leaf device. An operator simply has to apply a Akri Configuration to a cluster, specifying the discovery protocol (say ONVIF) and the pod that should be deployed upon discovery (say a video frame server). Then, Akri does the rest. An operator can also allow multiple nodes to utilize a leaf device, thereby **providing high availability** in the case where a node goes offline. Furthermore, Akri will automatically create a Kubernetes service for each type of leaf device (or Akri Configuration), removing the need for an application to track the state of pods or nodes. + +Most importantly, Akri **was built to be extensible**. We currently have ONVIF and udev discovery protocols and are [working](./docs/roadmap.md#implement-additional-protocol-handlers) to implement [OPC UA](./docs/proposals/opcua.md), but more can be easily added by community members like you. The more protocols Akri can support, the wider an array of leaf devices Akri can discover. We are excited to work with you to build a more connected edge. + +## Architecture in Short +Akri’s architecture is made up of four key components: two custom resources, a device plugin implementation, and a custom controller. The first custom resource, the Akri Configuration, is where **you name it**. This tells Akri what kind of device it should look for. At this point, **Akri finds it**! Akri's device plugin implementation looks for the device and tracks its availability using Akri's second custom resource, the Akri Instance. Having found your device, the Akri Controller helps **you use it**. It sees each Akri Instance (which represents a leaf device) and deploys a ("broker") pod that knows how to connect to the resource and utilize it. + +Akri ONVIF Flow + +## Quick Start with a Demo +Try the [end to end demo](./docs/end-to-end-demo.md) of Akri to see Akri discover mock video cameras and a streaming app display the footage from those cameras. It includes instructions on K8s cluster setup. If you would like to perform the demo on a cluster of Raspberry Pi 4's, see the [Raspberry Pi 4 demo](./docs/rpi4-demo.md). + +## Documentation +- [Running Akri using our currently supported protocols](./docs/user-guide.md) +- [Akri architecture in depth](./docs/architecture.md) +- [How to extend Akri for protocols that haven't been supported yet](./docs/extensibility.md). +- Proposals for enhancements such as new protocol implementations can be found in the [proposals folder](./docs/proposals) + +## Roadmap +Akri was built to be extensible. We currently have ONVIF and udev discovery protocols and are [working](./docs/roadmap.md#implement-additional-protocol-handlers) to implement [OPC UA](./docs/proposals/opcua.md), but as a community, we hope to continuously support more protocols. We have created a [protocol implementation roadmap](./docs/roadmap.md#implement-additional-protocol-handlers) in order to prioritize development of protocols. If there is a protocol you feel we should prioritize, please [create an issue](https://github.com/deislabs/akri/issues/new/choose), or better yet, contribute the implementation! We are excited to work with you to build a more connected edge. + +## Contributing +This project welcomes contributions, whether by [creating new issues](https://github.com/deislabs/akri/issues/new/choose) or pull requests. See our [contributing document](./docs/contributing.md) on how to get started. \ No newline at end of file diff --git a/agent/Cargo.toml b/agent/Cargo.toml new file mode 100644 index 0000000..f0f8649 --- /dev/null +++ b/agent/Cargo.toml @@ -0,0 +1,50 @@ +[package] +name = "agent" +version = "0.0.33" +authors = ["Kate Goldenring ", ""] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +async-stream = "0.2" +async-trait = "0.1.0" +blake2 = "0.8.0" +chrono = "0.4.10" +cfg-if = "0.1" +env_logger = "0.6.1" +failure = "0.1.5" +futures = { version = "0.3.1", package = "futures" } +futures-core = "0.3" +futures-util = "0.3" +futures-old = { version = "0.1", package = "futures" } +hyper = "0.11" +kube = { version = "0.23.0", features = ["openapi"] } +k8s-openapi = { version = "0.6.0", features = ["v1_16"] } +log = "0.4" +mime = "0.3" +mockall = "0.6.0" +pest = "2.0" +pest_derive = "2.0" +prost = "0.6" +rand = "0.7" +regex = "1" +serde = "1.0.104" +serde_json = "1.0.45" +serde_yaml = "0.8.11" +serde_derive = "1.0.104" +akri-shared = { path = "../shared" } +sxd-document = "0.3.0" +sxd-xpath = "0.4.0" +tokio = { version = "0.2", features = ["full"] } +tokio-core = "0.1" +tonic = "0.1" +tower = "0.3" +udev = "0.4" +uuid = { version = "0.8.1", features = ["v4"] } +xml-rs = "0.8.0" +yaserde = "0.3.13" +yaserde_derive = "0.3.13" + +[build-dependencies] +tonic-build = "0.1.1" diff --git a/agent/README.md b/agent/README.md new file mode 100644 index 0000000..b086152 --- /dev/null +++ b/agent/README.md @@ -0,0 +1,25 @@ +# Introduction +This is the Akri Agent project. It is an implementation of a [Kubernetes device plugin](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/device-plugins/). + +# Design + +## Traits + +### Public +* **DiscoveryHandler** - This provides an abstraction to allow protocol specific code to handle discovery and provide details for Instance creation. Planned implementations of this trait include `OnvifDiscoveryHandler`, `UdevDiscoveryHandler`, `OpcuaDiscoveryHandler`, and `DebugEchoDiscoveryHandler`. +```Rust +#[async_trait] +pub trait DiscoveryHandler { + async fn discover(&self) -> Result, Error>; + fn are_shared(&self) -> Result; +} +``` + +### Private +* **EnvVarQuery** - This provides a mockable way to query for `get_discovery_handler` to query environment variables. +```Rust +trait EnvVarQuery { + fn get_env_var(&self, name: &'static str) -> Result; +} +``` + diff --git a/agent/build.rs b/agent/build.rs new file mode 100644 index 0000000..f76ecc5 --- /dev/null +++ b/agent/build.rs @@ -0,0 +1,8 @@ +/// This generates Device Plugin code (in v1beta1.rs) from pluginapi.proto +fn main() { + tonic_build::configure() + .build_client(true) + .out_dir("./src/util") + .compile(&["./proto/pluginapi.proto"], &["./proto"]) + .expect("failed to compile protos"); +} diff --git a/agent/proto/README.md b/agent/proto/README.md new file mode 100644 index 0000000..2f1bbcb --- /dev/null +++ b/agent/proto/README.md @@ -0,0 +1,5 @@ +## pluginapi.proto + +**Purpose:** Upon building, this protocol file auto-generates `../v1beta1.rs`, which contains structures and implementations for Device Plugin messages, client, and server. + +**Versioning:** This file is kubernetes Device Plugin protocol/API version **v1beta1** from kubernetes version **1.15**. Device Plugins declare their protocol version to kubelet when registering with it, as kubelet's Registration server and Device Plugin client should be built against the same version. Check for newer versions of v1beta1 protocol [here](https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1/api.proto); however, all versions of v1beta1 after 1.15 include Device Plugin Integration with Topology Manager via an additional `TopologyInfo` field in the `Device` struct. Topology support is not needed for this project and kubelet does not require it when registering a device. \ No newline at end of file diff --git a/agent/proto/pluginapi.proto b/agent/proto/pluginapi.proto new file mode 100644 index 0000000..883f9a9 --- /dev/null +++ b/agent/proto/pluginapi.proto @@ -0,0 +1,152 @@ + syntax = 'proto3'; + + package v1beta1; + + + // Registration is the service advertised by the Kubelet + // Only when Kubelet answers with a success code to a Register Request + // may Device Plugins start their service + // Registration may fail when device plugin version is not supported by + // Kubelet or the registered resourceName is already taken by another + // active device plugin. Device plugin is expected to terminate upon registration failure + service Registration { + rpc Register(RegisterRequest) returns (Empty) {} + } + + message DevicePluginOptions { + // Indicates if PreStartContainer call is required before each container start + bool pre_start_required = 1; + } + + message RegisterRequest { + // Version of the API the Device Plugin was built against + string version = 1; + // Name of the unix socket the device plugin is listening on + // PATH = path.Join(DevicePluginPath, endpoint) + string endpoint = 2; + // Schedulable resource name. As of now it's expected to be a DNS Label + string resource_name = 3; + // Options to be communicated with Device Manager + DevicePluginOptions options = 4; + } + + message Empty { + } + + // DevicePlugin is the service advertised by Device Plugins + service DevicePlugin { + // GetDevicePluginOptions returns options to be communicated with Device + // Manager + rpc GetDevicePluginOptions(Empty) returns (DevicePluginOptions) {} + + // ListAndWatch returns a stream of List of Devices + // Whenever a Device state change or a Device disapears, ListAndWatch + // returns the new list + rpc ListAndWatch(Empty) returns (stream ListAndWatchResponse) {} + + // Allocate is called during container creation so that the Device + // Plugin can run device specific operations and instruct Kubelet + // of the steps to make the Device available in the container + rpc Allocate(AllocateRequest) returns (AllocateResponse) {} + + // PreStartContainer is called, if indicated by Device Plugin during registeration phase, + // before each container start. Device plugin can run device specific operations + // such as reseting the device before making devices available to the container + rpc PreStartContainer(PreStartContainerRequest) returns (PreStartContainerResponse) {} + } + + // ListAndWatch returns a stream of List of Devices + // Whenever a Device state change or a Device disapears, ListAndWatch + // returns the new list + message ListAndWatchResponse { + repeated Device devices = 1; + } + + /* E.g: + * struct Device { + * ID: "GPU-fef8089b-4820-abfc-e83e-94318197576e", + * State: "Healthy", + *} */ + message Device { + // A unique ID assigned by the device plugin used + // to identify devices during the communication + // Max length of this field is 63 characters + string ID = 1; + // Health of the device, can be healthy or unhealthy, see constants.go + string health = 2; + } + + // - PreStartContainer is expected to be called before each container start if indicated by plugin during registration phase. + // - PreStartContainer allows kubelet to pass reinitialized devices to containers. + // - PreStartContainer allows Device Plugin to run device specific operations on + // the Devices requested + message PreStartContainerRequest { + repeated string devicesIDs = 1; + } + + // PreStartContainerResponse will be send by plugin in response to PreStartContainerRequest + message PreStartContainerResponse { + } + + // - Allocate is expected to be called during pod creation since allocation + // failures for any container would result in pod startup failure. + // - Allocate allows kubelet to exposes additional artifacts in a pod's + // environment as directed by the plugin. + // - Allocate allows Device Plugin to run device specific operations on + // the Devices requested + message AllocateRequest { + repeated ContainerAllocateRequest container_requests = 1; + } + + message ContainerAllocateRequest { + repeated string devicesIDs = 1; + } + + // AllocateResponse includes the artifacts that needs to be injected into + // a container for accessing 'deviceIDs' that were mentioned as part of + // 'AllocateRequest'. + // Failure Handling: + // if Kubelet sends an allocation request for dev1 and dev2. + // Allocation on dev1 succeeds but allocation on dev2 fails. + // The Device plugin should send a ListAndWatch update and fail the + // Allocation request + message AllocateResponse { + repeated ContainerAllocateResponse container_responses = 1; + } + + message ContainerAllocateResponse { + // List of environment variable to be set in the container to access one of more devices. + map envs = 1; + // Mounts for the container. + repeated Mount mounts = 2; + // Devices for the container. + repeated DeviceSpec devices = 3; + // Container annotations to pass to the container runtime + map annotations = 4; + } + + // Mount specifies a host volume to mount into a container. + // where device library or tools are installed on host and container + message Mount { + // Path of the mount within the container. + string container_path = 1; + // Path of the mount on the host. + string host_path = 2; + // If set, the mount is read-only. + bool read_only = 3; + } + + // DeviceSpec specifies a host device to mount into a container. + message DeviceSpec { + // Path of the device within the container. + string container_path = 1; + // Path of the device on the host. + string host_path = 2; + // Cgroups permissions of the device, candidates are one or more of + // * r - allows container to read from the specified device. + // * w - allows container to write to the specified device. + // * m - allows container to create device files that do not yet exist. + string permissions = 3; + } + + \ No newline at end of file diff --git a/agent/src/main.rs b/agent/src/main.rs new file mode 100644 index 0000000..849dcef --- /dev/null +++ b/agent/src/main.rs @@ -0,0 +1,60 @@ +#[macro_use] +extern crate log; +#[macro_use] +extern crate yaserde_derive; +#[macro_use] +extern crate serde_derive; + +extern crate pest; +#[macro_use] +extern crate pest_derive; + +extern crate hyper; +extern crate tokio_core; + +mod protocols; +mod util; + +use akri_shared::akri::API_NAMESPACE; +use env_logger; +use log::{info, trace}; +use std::time::Duration; +use util::{ + config_action, constants::SLOT_RECONCILIATION_SLOT_GRACE_PERIOD_SECS, + slot_reconciliation::periodic_slot_reconciliation, +}; + +/// This is the entry point for the Akri Agent. +/// It must be built on unix systems, since the underlying libraries for the `DevicePluginService` unix socket connection are unix only. +#[cfg(unix)] +#[tokio::main] +async fn main() -> Result<(), Box> { + println!("{} Agent start", API_NAMESPACE); + + println!( + "{} KUBERNETES_PORT found ... env_logger::init", + API_NAMESPACE + ); + env_logger::try_init()?; + trace!( + "{} KUBERNETES_PORT found ... env_logger::init finished", + API_NAMESPACE + ); + + let mut tasks = Vec::new(); + + tasks.push(tokio::spawn(async move { + let slot_grace_period = Duration::from_secs(SLOT_RECONCILIATION_SLOT_GRACE_PERIOD_SECS); + periodic_slot_reconciliation(slot_grace_period) + .await + .unwrap(); + })); + + tasks.push(tokio::spawn(async move { + config_action::do_config_watch().await.unwrap() + })); + + futures::future::try_join_all(tasks).await?; + info!("{} Agent end", API_NAMESPACE); + Ok(()) +} diff --git a/agent/src/protocols/debug_echo/discovery_handler.rs b/agent/src/protocols/debug_echo/discovery_handler.rs new file mode 100644 index 0000000..017e100 --- /dev/null +++ b/agent/src/protocols/debug_echo/discovery_handler.rs @@ -0,0 +1,59 @@ +use super::super::{DiscoveryHandler, DiscoveryResult}; +use akri_shared::akri::configuration::DebugEchoDiscoveryHandlerConfig; +use async_trait::async_trait; +use failure::Error; +use std::{collections::HashMap, fs}; + +/// File acting as an environment variable for testing discovery. +/// To mimic an instance going offline, kubectl exec into one of the akri-agent-daemonset pods +/// and echo "OFFLINE" > /tmp/debug-echo-availability.txt +/// To mimic a device coming back online, remove the word "OFFLINE" from the file +/// ie: echo "" > /tmp/debug-echo-availability.txt +pub const DEBUG_ECHO_AVAILABILITY_CHECK_PATH: &str = "/tmp/debug-echo-availability.txt"; +/// String to write into DEBUG_ECHO_AVAILABILITY_CHECK_PATH to make DebugEcho devices undiscoverable +pub const OFFLINE: &str = "OFFLINE"; + +/// `DebugEchoDiscoveryHandler` contains a `DebugEchoDiscoveryHandlerConfig` which has a +/// list of mock instances (`discovery_handler_config.descriptions`) and their sharability. +/// It mocks discovering the instances by inspecting the contents of the file at `DEBUG_ECHO_AVAILABILITY_CHECK_PATH`. +/// If the file contains "OFFLINE", it won't discover any of the instances, else it discovers them all. +#[derive(Debug)] +pub struct DebugEchoDiscoveryHandler { + discovery_handler_config: DebugEchoDiscoveryHandlerConfig, +} + +impl DebugEchoDiscoveryHandler { + pub fn new(discovery_handler_config: &DebugEchoDiscoveryHandlerConfig) -> Self { + DebugEchoDiscoveryHandler { + discovery_handler_config: discovery_handler_config.clone(), + } + } +} + +#[async_trait] +impl DiscoveryHandler for DebugEchoDiscoveryHandler { + async fn discover(&self) -> Result, Error> { + let availability = + fs::read_to_string(DEBUG_ECHO_AVAILABILITY_CHECK_PATH).unwrap_or_default(); + trace!( + "discover -- DebugEcho capabilities visible? {}", + !availability.contains(OFFLINE) + ); + // If the device is offline, return an empty list of instance info + if availability.contains(OFFLINE) { + Ok(Vec::new()) + } else { + Ok(self + .discovery_handler_config + .descriptions + .iter() + .map(|description| { + DiscoveryResult::new(description, HashMap::new(), self.are_shared().unwrap()) + }) + .collect::>()) + } + } + fn are_shared(&self) -> Result { + Ok(self.discovery_handler_config.shared) + } +} diff --git a/agent/src/protocols/debug_echo/mod.rs b/agent/src/protocols/debug_echo/mod.rs new file mode 100644 index 0000000..edff99f --- /dev/null +++ b/agent/src/protocols/debug_echo/mod.rs @@ -0,0 +1,4 @@ +mod discovery_handler; +pub use self::discovery_handler::{ + DebugEchoDiscoveryHandler, DEBUG_ECHO_AVAILABILITY_CHECK_PATH, OFFLINE, +}; diff --git a/agent/src/protocols/mod.rs b/agent/src/protocols/mod.rs new file mode 100644 index 0000000..1be62eb --- /dev/null +++ b/agent/src/protocols/mod.rs @@ -0,0 +1,173 @@ +use akri_shared::{ + akri::configuration::ProtocolHandler, + os::env_var::{ActualEnvVarQuery, EnvVarQuery}, +}; +use async_trait::async_trait; +use blake2::digest::{Input, VariableOutput}; +use blake2::VarBlake2b; +use failure::Error; +use std::collections::HashMap; + +#[derive(Debug, Clone)] +pub struct DiscoveryResult { + pub digest: String, + pub properties: HashMap, +} +impl DiscoveryResult { + fn new(id_to_digest: &str, properties: HashMap, shared: bool) -> Self { + let mut id_to_digest = id_to_digest.to_string(); + // For unshared devices, include node hostname in id_to_digest so instances have unique names + if !shared { + id_to_digest = format!( + "{}{}", + &id_to_digest, + std::env::var("AGENT_NODE_NAME").unwrap() + ); + } + let mut hasher = VarBlake2b::new(3).unwrap(); + hasher.input(id_to_digest); + let digest = hasher + .vec_result() + .iter() + .map(|num| format!("{:02x}", num)) + .collect::>() + .join(""); + DiscoveryResult { digest, properties } + } +} + +/// DiscoveryHandler describes anything that can find available instances and define +/// whether they are shared. +/// +/// DiscoveryHandler provides an abstraction to help in Instance +/// creation: search/find for instances, specify whether the instance +/// should be shared, etc. +/// +/// # Examples +/// +/// ``` +/// pub struct SampleDiscoveryHandler {} +/// #[async_trait] +/// impl DiscoveryHandler for SampleDiscoveryHandler { +/// async fn discover(&self) -> Result, failure::Error> { +/// Ok(Vec::new()) +/// } +/// fn are_shared(&self) -> Result { +/// Ok(true) +/// } +/// } +/// ``` +#[async_trait] +pub trait DiscoveryHandler { + async fn discover(&self) -> Result, Error>; + fn are_shared(&self) -> Result; +} + +pub mod debug_echo; +mod onvif; +mod opcua; +mod udev; + +pub fn get_discovery_handler( + discovery_handler_config: &ProtocolHandler, +) -> Result, Error> { + let query_var_set = ActualEnvVarQuery {}; + inner_get_discovery_handler(discovery_handler_config, &query_var_set) +} + +fn inner_get_discovery_handler( + discovery_handler_config: &ProtocolHandler, + query: &impl EnvVarQuery, +) -> Result, Error> { + match discovery_handler_config { + ProtocolHandler::onvif(onvif) => Ok(Box::new(onvif::OnvifDiscoveryHandler::new(&onvif))), + ProtocolHandler::udev(udev) => Ok(Box::new(udev::UdevDiscoveryHandler::new(&udev))), + ProtocolHandler::opcua(opcua) => Ok(Box::new(opcua::OpcuaDiscoveryHandler::new(&opcua))), + ProtocolHandler::debugEcho(dbg) => match query.get_env_var("ENABLE_DEBUG_ECHO") { + Ok(_) => Ok(Box::new(debug_echo::DebugEchoDiscoveryHandler::new(dbg))), + _ => Err(failure::format_err!("No protocol configured")), + }, + } +} + +#[cfg(test)] +mod test { + use super::*; + use akri_shared::{ + akri::configuration::{Configuration, ProtocolHandler}, + os::env_var::MockEnvVarQuery, + }; + use std::env::VarError; + + #[tokio::test] + async fn test_inner_get_discovery_handler() { + let mock_query = MockEnvVarQuery::new(); + + let onvif_json = r#"{"onvif":{}}"#; + let deserialized: ProtocolHandler = serde_json::from_str(onvif_json).unwrap(); + assert!(inner_get_discovery_handler(&deserialized, &mock_query).is_ok()); + + let udev_json = r#"{"udev":{"udevRules":[]}}"#; + let deserialized: ProtocolHandler = serde_json::from_str(udev_json).unwrap(); + assert!(inner_get_discovery_handler(&deserialized, &mock_query).is_ok()); + + let opcua_json = r#"{"opcua":{}}"#; + let deserialized: ProtocolHandler = serde_json::from_str(opcua_json).unwrap(); + let discovery_handler = inner_get_discovery_handler(&deserialized, &mock_query).unwrap(); + assert!( + discovery_handler.discover().await.is_err(), + "OPC protocol handler not implemented" + ); + + let json = r#"{}"#; + assert!(serde_json::from_str::(json).is_err()); + } + + #[tokio::test] + async fn test_udev_discover_no_rules() { + let mock_query = MockEnvVarQuery::new(); + + let json = r#"{"udev":{"udevRules":[]}}"#; + let deserialized: ProtocolHandler = serde_json::from_str(json).unwrap(); + let discovery_handler = inner_get_discovery_handler(&deserialized, &mock_query).unwrap(); + assert_eq!(discovery_handler.discover().await.unwrap().len(), 0); + } + + #[tokio::test] + async fn test_factory_for_debug_echo_when_no_env_var_set() { + let json = r#"{"protocol":{"debugEcho":{"descriptions":["foo1"],"shared":true}}}"#; + let deserialized: Configuration = serde_json::from_str(json).unwrap(); + + let mut mock_query_without_var_set = MockEnvVarQuery::new(); + mock_query_without_var_set + .expect_get_env_var() + .returning(|_| Err(VarError::NotPresent)); + if inner_get_discovery_handler(&deserialized.protocol, &mock_query_without_var_set).is_ok() + { + panic!("protocol configuration as debugEcho should return error when 'ENABLE_DEBUG_ECHO' env var is not set") + } + + let mut mock_query_with_var_set = MockEnvVarQuery::new(); + mock_query_with_var_set + .expect_get_env_var() + .returning(|_| Ok("1".to_string())); + let pi = DiscoveryResult::new(&"foo1".to_string(), HashMap::new(), true); + let debug_echo_discovery_handler = + inner_get_discovery_handler(&deserialized.protocol, &mock_query_with_var_set).unwrap(); + assert_eq!(true, debug_echo_discovery_handler.are_shared().unwrap()); + assert_eq!( + 1, + debug_echo_discovery_handler.discover().await.unwrap().len() + ); + assert_eq!( + pi.digest, + debug_echo_discovery_handler + .discover() + .await + .unwrap() + .get(0) + .unwrap() + .digest + ); + } +} diff --git a/agent/src/protocols/onvif/discovery_handler.rs b/agent/src/protocols/onvif/discovery_handler.rs new file mode 100644 index 0000000..b46b1b8 --- /dev/null +++ b/agent/src/protocols/onvif/discovery_handler.rs @@ -0,0 +1,511 @@ +use super::super::{DiscoveryHandler, DiscoveryResult}; +use super::discovery_impl::util; +use akri_shared::akri::configuration::{FilterList, FilterType, OnvifDiscoveryHandlerConfig}; +use akri_shared::onvif::device_info::{ + OnvifQuery, OnvifQueryImpl, ONVIF_DEVICE_IP_ADDRESS_LABEL_ID, + ONVIF_DEVICE_MAC_ADDRESS_LABEL_ID, ONVIF_DEVICE_SERVICE_URL_LABEL_ID, +}; +use async_trait::async_trait; +use failure::Error; +use std::{collections::HashMap, time::Duration}; + +/// `OnvifDiscoveryHandler` discovers the onvif instances as described by the filters `discover_handler_config.ip_addresses`, +/// `discover_handler_config.mac_addresses`, and `discover_handler_config.scopes`. +/// The instances it discovers are always shared. +#[derive(Debug)] +pub struct OnvifDiscoveryHandler { + discovery_handler_config: OnvifDiscoveryHandlerConfig, +} + +impl OnvifDiscoveryHandler { + pub fn new(discovery_handler_config: &OnvifDiscoveryHandlerConfig) -> Self { + OnvifDiscoveryHandler { + discovery_handler_config: discovery_handler_config.clone(), + } + } + + fn execute_filter(filter_list: Option<&FilterList>, filter_against: &[String]) -> bool { + if filter_list.is_none() { + return false; + } + let filter_action = filter_list.as_ref().unwrap().action.clone(); + let filter_count = filter_list + .unwrap() + .items + .iter() + .filter(|pattern| { + filter_against + .iter() + .filter(|filter_against_item| filter_against_item.contains(*pattern)) + .count() + > 0 + }) + .count(); + + if FilterType::Include == filter_action { + filter_count == 0 + } else { + filter_count != 0 + } + } + + async fn apply_filters( + &self, + device_service_uris: Vec, + onvif_query: &impl OnvifQuery, + ) -> Result, failure::Error> { + let mut result = Vec::new(); + for device_service_url in device_service_uris.iter() { + trace!("apply_filters - device service url {}", &device_service_url); + let (ip_address, mac_address) = match onvif_query + .get_device_ip_and_mac_address(&device_service_url) + .await + { + Ok(ip_and_mac) => ip_and_mac, + Err(e) => { + error!("apply_filters - error getting ip and mac address: {}", e); + continue; + } + }; + + // Evaluate camera ip address against ip filter if provided + let ip_address_as_vec = vec![ip_address.clone()]; + if OnvifDiscoveryHandler::execute_filter( + self.discovery_handler_config.ip_addresses.as_ref(), + &ip_address_as_vec, + ) { + continue; + } + + // Evaluate camera mac address against mac filter if provided + let mac_address_as_vec = vec![mac_address.clone()]; + if OnvifDiscoveryHandler::execute_filter( + self.discovery_handler_config.mac_addresses.as_ref(), + &mac_address_as_vec, + ) { + continue; + } + + let ip_and_mac_joined = format!("{}-{}", &ip_address, &mac_address); + + // Evaluate camera scopes against scopes filter if provided + let device_scopes = match onvif_query.get_device_scopes(&device_service_url).await { + Ok(scopes) => scopes, + Err(e) => { + error!("apply_filters - error getting scopes: {}", e); + continue; + } + }; + if OnvifDiscoveryHandler::execute_filter( + self.discovery_handler_config.scopes.as_ref(), + &device_scopes, + ) { + continue; + } + + let mut properties = HashMap::new(); + properties.insert( + ONVIF_DEVICE_SERVICE_URL_LABEL_ID.to_string(), + device_service_url.to_string(), + ); + properties.insert(ONVIF_DEVICE_IP_ADDRESS_LABEL_ID.into(), ip_address); + properties.insert(ONVIF_DEVICE_MAC_ADDRESS_LABEL_ID.into(), mac_address); + + trace!( + "apply_filters - returns DiscoveryResult ip/mac: {:?}, props: {:?}", + &ip_and_mac_joined, + &properties + ); + result.push(DiscoveryResult::new( + &ip_and_mac_joined, + properties, + self.are_shared().unwrap(), + )) + } + Ok(result) + } +} + +#[async_trait] +impl DiscoveryHandler for OnvifDiscoveryHandler { + async fn discover(&self) -> Result, failure::Error> { + let onvif_query = OnvifQueryImpl {}; + + info!("discover - filters:{:?}", &self.discovery_handler_config,); + let discovered_onvif_cameras = util::simple_onvif_discover(Duration::from_secs( + self.discovery_handler_config.discovery_timeout_seconds as u64, + )) + .await?; + info!("discover - discovered:{:?}", &discovered_onvif_cameras,); + let filtered_onvif_cameras = self + .apply_filters(discovered_onvif_cameras, &onvif_query) + .await; + info!("discover - filtered:{:?}", &filtered_onvif_cameras); + filtered_onvif_cameras + } + fn are_shared(&self) -> Result { + Ok(true) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use akri_shared::onvif::device_info::test_onvif::MockOnvifQueryImpl; + + struct IpAndMac { + mock_uri: &'static str, + mock_ip: &'static str, + mock_mac: &'static str, + } + + struct Scope { + mock_uri: &'static str, + mock_scope: &'static str, + } + + fn configure_scenario( + mock: &mut MockOnvifQueryImpl, + ip_and_mac: Option, + scope: Option, + ) { + if let Some(ip_and_mac_) = ip_and_mac { + configure_get_device_ip_and_mac_address( + mock, + &ip_and_mac_.mock_uri, + &ip_and_mac_.mock_ip, + &ip_and_mac_.mock_mac, + ) + } + if let Some(scope_) = scope { + configure_get_device_scopes(mock, &scope_.mock_uri, &scope_.mock_scope) + } + } + + fn configure_get_device_ip_and_mac_address( + mock: &mut MockOnvifQueryImpl, + uri: &'static str, + ip: &'static str, + mac: &'static str, + ) { + mock.expect_get_device_ip_and_mac_address() + .times(1) + .withf(move |u| u == uri) + .returning(move |_| Ok((ip.to_string(), mac.to_string()))); + } + + fn configure_get_device_scopes( + mock: &mut MockOnvifQueryImpl, + uri: &'static str, + scope: &'static str, + ) { + mock.expect_get_device_scopes() + .times(1) + .withf(move |u| u == uri) + .returning(move |_| Ok(vec![scope.to_string()])); + } + + #[tokio::test] + async fn test_apply_filters_no_filters() { + let mock_uri = "device_uri"; + + let mut mock = MockOnvifQueryImpl::new(); + configure_scenario( + &mut mock, + Some(IpAndMac { + mock_uri: "device_uri", + mock_ip: "mock.ip", + mock_mac: "mock:mac", + }), + Some(Scope { + mock_uri: "device_uri", + mock_scope: "mock.scope", + }), + ); + + let onvif = OnvifDiscoveryHandler::new(&OnvifDiscoveryHandlerConfig { + ip_addresses: None, + mac_addresses: None, + scopes: None, + discovery_timeout_seconds: 1, + }); + let instances = onvif + .apply_filters(vec![mock_uri.to_string()], &mock) + .await + .unwrap(); + + assert_eq!(1, instances.len()); + } + + #[tokio::test] + async fn test_apply_filters_include_ip_exist() { + let mock_uri = "device_uri"; + let mock_ip = "mock.ip"; + + let mut mock = MockOnvifQueryImpl::new(); + configure_scenario( + &mut mock, + Some(IpAndMac { + mock_uri, + mock_ip, + mock_mac: "mock:mac", + }), + Some(Scope { + mock_uri, + mock_scope: "mock.scope", + }), + ); + + let onvif = OnvifDiscoveryHandler::new(&OnvifDiscoveryHandlerConfig { + ip_addresses: Some(FilterList { + action: FilterType::Include, + items: vec![mock_ip.to_string()], + }), + mac_addresses: None, + scopes: None, + discovery_timeout_seconds: 1, + }); + let instances = onvif + .apply_filters(vec![mock_uri.to_string()], &mock) + .await + .unwrap(); + + assert_eq!(1, instances.len()); + } + + #[tokio::test] + async fn test_apply_filters_include_ip_nonexist() { + let mock_uri = "device_uri"; + + let mut mock = MockOnvifQueryImpl::new(); + configure_scenario( + &mut mock, + Some(IpAndMac { + mock_uri, + mock_ip: "mock.ip", + mock_mac: "mock:mac", + }), + None, + ); + + let onvif = OnvifDiscoveryHandler::new(&OnvifDiscoveryHandlerConfig { + ip_addresses: Some(FilterList { + action: FilterType::Include, + items: vec!["nonexist.ip".to_string()], + }), + mac_addresses: None, + scopes: None, + discovery_timeout_seconds: 1, + }); + let instances = onvif + .apply_filters(vec![mock_uri.to_string()], &mock) + .await + .unwrap(); + + assert_eq!(0, instances.len()); + } + + #[tokio::test] + async fn test_apply_filters_exclude_ip_nonexist() { + let mock_uri = "device_uri"; + + let mut mock = MockOnvifQueryImpl::new(); + configure_scenario( + &mut mock, + Some(IpAndMac { + mock_uri, + mock_ip: "mock.ip", + mock_mac: "mock:mac", + }), + Some(Scope { + mock_uri, + mock_scope: "mock.scope", + }), + ); + + let onvif = OnvifDiscoveryHandler::new(&OnvifDiscoveryHandlerConfig { + ip_addresses: Some(FilterList { + action: FilterType::Exclude, + items: vec!["nonexist.ip".to_string()], + }), + mac_addresses: None, + scopes: None, + discovery_timeout_seconds: 1, + }); + let instances = onvif + .apply_filters(vec![mock_uri.to_string()], &mock) + .await + .unwrap(); + + assert_eq!(1, instances.len()); + } + + #[tokio::test] + async fn test_apply_filters_exclude_ip_exist() { + let mock_uri = "device_uri"; + let mock_ip = "mock.ip"; + + let mut mock = MockOnvifQueryImpl::new(); + configure_scenario( + &mut mock, + Some(IpAndMac { + mock_uri, + mock_ip, + mock_mac: "mock:mac", + }), + None, + ); + + let onvif = OnvifDiscoveryHandler::new(&OnvifDiscoveryHandlerConfig { + ip_addresses: Some(FilterList { + action: FilterType::Exclude, + items: vec![mock_ip.to_string()], + }), + mac_addresses: None, + scopes: None, + discovery_timeout_seconds: 1, + }); + let instances = onvif + .apply_filters(vec![mock_uri.to_string()], &mock) + .await + .unwrap(); + + assert_eq!(0, instances.len()); + } + + #[tokio::test] + async fn test_apply_filters_include_mac_exist() { + let mock_uri = "device_uri"; + let mock_mac = "mock:mac"; + + let mut mock = MockOnvifQueryImpl::new(); + configure_scenario( + &mut mock, + Some(IpAndMac { + mock_uri, + mock_ip: "mock.ip", + mock_mac, + }), + Some(Scope { + mock_uri, + mock_scope: "mock.scope", + }), + ); + + let onvif = OnvifDiscoveryHandler::new(&OnvifDiscoveryHandlerConfig { + ip_addresses: None, + mac_addresses: Some(FilterList { + action: FilterType::Include, + items: vec![mock_mac.to_string()], + }), + scopes: None, + discovery_timeout_seconds: 1, + }); + let instances = onvif + .apply_filters(vec![mock_uri.to_string()], &mock) + .await + .unwrap(); + + assert_eq!(1, instances.len()); + } + + #[tokio::test] + async fn test_apply_filters_include_mac_nonexist() { + let mock_uri = "device_uri"; + + let mut mock = MockOnvifQueryImpl::new(); + configure_scenario( + &mut mock, + Some(IpAndMac { + mock_uri, + mock_ip: "mock.ip", + mock_mac: "mock:mac", + }), + None, + ); + + let onvif = OnvifDiscoveryHandler::new(&OnvifDiscoveryHandlerConfig { + ip_addresses: None, + mac_addresses: Some(FilterList { + action: FilterType::Include, + items: vec!["nonexist:mac".to_string()], + }), + scopes: None, + discovery_timeout_seconds: 1, + }); + let instances = onvif + .apply_filters(vec![mock_uri.to_string()], &mock) + .await + .unwrap(); + + assert_eq!(0, instances.len()); + } + + #[tokio::test] + async fn test_apply_filters_exclude_mac_nonexist() { + let mock_uri = "device_uri"; + + let mut mock = MockOnvifQueryImpl::new(); + configure_scenario( + &mut mock, + Some(IpAndMac { + mock_uri, + mock_ip: "mock.ip", + mock_mac: "mock:mac", + }), + Some(Scope { + mock_uri, + mock_scope: "mock.scope", + }), + ); + + let onvif = OnvifDiscoveryHandler::new(&OnvifDiscoveryHandlerConfig { + ip_addresses: None, + mac_addresses: Some(FilterList { + action: FilterType::Exclude, + items: vec!["nonexist:mac".to_string()], + }), + scopes: None, + discovery_timeout_seconds: 1, + }); + let instances = onvif + .apply_filters(vec![mock_uri.to_string()], &mock) + .await + .unwrap(); + + assert_eq!(1, instances.len()); + } + + #[tokio::test] + async fn test_apply_filters_exclude_mac_exist() { + let mock_uri = "device_uri"; + let mock_mac = "mock:mac"; + + let mut mock = MockOnvifQueryImpl::new(); + configure_scenario( + &mut mock, + Some(IpAndMac { + mock_uri, + mock_ip: "mock.ip", + mock_mac, + }), + None, + ); + + let onvif = OnvifDiscoveryHandler::new(&OnvifDiscoveryHandlerConfig { + ip_addresses: None, + mac_addresses: Some(FilterList { + action: FilterType::Exclude, + items: vec![mock_mac.to_string()], + }), + scopes: None, + discovery_timeout_seconds: 1, + }); + let instances = onvif + .apply_filters(vec![mock_uri.to_string()], &mock) + .await + .unwrap(); + + assert_eq!(0, instances.len()); + } +} diff --git a/agent/src/protocols/onvif/discovery_impl.rs b/agent/src/protocols/onvif/discovery_impl.rs new file mode 100644 index 0000000..e506dc1 --- /dev/null +++ b/agent/src/protocols/onvif/discovery_impl.rs @@ -0,0 +1,395 @@ +mod to_serialize { + use super::common::*; + use std::io::Write; + use yaserde::YaSerialize; + + #[derive(Default, PartialEq, Debug, YaSerialize)] + #[yaserde(prefix = "s", namespace = "s: http://www.w3.org/2003/05/soap-envelope")] + pub struct Envelope { + #[yaserde(prefix = "s", rename = "Header")] + pub header: Header, + + #[yaserde(prefix = "s", rename = "Body")] + pub body: Body, + } + + #[derive(Default, PartialEq, Debug, YaSerialize)] + #[yaserde( + prefix = "s", + namespace = "s: http://www.w3.org/2003/05/soap-envelope", + namespace = "d: http://schemas.xmlsoap.org/ws/2005/04/discovery" + )] + pub struct Body { + #[yaserde(prefix = "d", rename = "Probe")] + pub probe: Probe, + } + + #[derive(Default, PartialEq, Debug, YaSerialize)] + #[yaserde( + prefix = "s", + namespace = "s: http://www.w3.org/2003/05/soap-envelope", + namespace = "w: http://schemas.xmlsoap.org/ws/2004/08/addressing" + )] + pub struct Header { + #[yaserde(prefix = "w", rename = "MessageID")] + pub message_id: String, + + #[yaserde(prefix = "w", rename = "To")] + pub reply_to: String, + + #[yaserde(prefix = "w", rename = "Action")] + pub action: String, + } +} + +mod to_deserialize { + use super::common::*; + use std::io::Read; + use yaserde::YaDeserialize; + + #[derive(Default, PartialEq, Debug, YaDeserialize)] + #[yaserde(prefix = "s", namespace = "s: http://www.w3.org/2003/05/soap-envelope")] + pub struct Envelope { + #[yaserde(prefix = "s", rename = "Header")] + pub header: Header, + + #[yaserde(prefix = "s", rename = "Body")] + pub body: Body, + } + + #[derive(Default, PartialEq, Debug, YaDeserialize)] + #[yaserde( + prefix = "s", + namespace = "s: http://www.w3.org/2003/05/soap-envelope", + namespace = "d: http://schemas.xmlsoap.org/ws/2005/04/discovery" + )] + pub struct Body { + #[yaserde(prefix = "d", rename = "ProbeMatches")] + pub probe_matches: ProbeMatches, + } + + #[derive(Default, PartialEq, Debug, YaDeserialize)] + #[yaserde( + prefix = "s", + namespace = "s: http://www.w3.org/2003/05/soap-envelope", + namespace = "w: http://schemas.xmlsoap.org/ws/2004/08/addressing" + )] + pub struct Header { + #[yaserde(prefix = "w", rename = "RelatesTo")] + pub relates_to: String, + } +} + +#[allow(dead_code)] +pub mod probe_types { + pub const DEVICE_NAMESPACE_PREFIX: &str = "devwsdl"; + pub const NETWORK_VIDEO_TRANSMITTER_NAMESPACE_PREFIX: &str = "netwsdl"; + pub const DEVICE_NAMESPACE_DESCRIPTOR: &str = "devwsdl: http://www.onvif.org/ver10/device/wsdl"; + pub const NETWORK_VIDEO_TRANSMITTER_NAMESPACE_DESCRIPTOR: &str = + "netwsdl: http://www.onvif.org/ver10/network/wsdl"; + pub const DEVICE: &str = "devwsdl:Device"; + pub const NETWORK_VIDEO_TRANSMITTER: &str = "netwsdl:NetworkVideoTransmitter"; +} + +mod common { + use std::io::{Read, Write}; + use yaserde::{YaDeserialize, YaSerialize}; + + #[derive(Default, PartialEq, Debug, YaDeserialize, YaSerialize)] + #[yaserde( + prefix = "d", + namespace = "d: http://schemas.xmlsoap.org/ws/2005/04/discovery", + namespace = probe_typews::NETWORK_VIDEO_TRANSMITTER_NAMESPACE_DESCRIPTOR, + namespace = probe_typews::DEVICE_NAMESPACE_DESCRIPTOR + )] + pub struct Probe { + #[yaserde(prefix = "d", rename = "Types")] + pub probe_types: Vec, + } + + #[derive(Default, PartialEq, Debug, YaDeserialize, YaSerialize)] + #[yaserde( + prefix = "d", + namespace = "d: http://schemas.xmlsoap.org/ws/2005/04/discovery", + namespace = "wsa: http://schemas.xmlsoap.org/ws/2004/08/addressing" + )] + pub struct ProbeMatch { + #[yaserde(prefix = "d", rename = "XAddrs")] + pub xaddrs: String, + #[yaserde(prefix = "wsa", rename = "EndpointReference")] + pub endpoint_reference: String, + #[yaserde(prefix = "d", rename = "Types")] + pub probe_types: Vec, + #[yaserde(prefix = "d", rename = "Scopes")] + pub scopes: Vec, + #[yaserde(prefix = "d", rename = "MetadataVersion")] + pub metadata_version: String, + } + + #[derive(Default, PartialEq, Debug, YaDeserialize, YaSerialize)] + #[yaserde( + prefix = "d", + namespace = "d: http://schemas.xmlsoap.org/ws/2005/04/discovery" + )] + pub struct ProbeMatches { + #[yaserde(prefix = "d", rename = "ProbeMatch")] + pub probe_match: Vec, + } +} + +pub mod util { + use super::{common, probe_types, to_deserialize, to_serialize}; + use log::{error, info, trace}; + use std::{ + net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}, + sync::{Arc, Mutex}, + }; + use tokio::{ + io::ErrorKind, + sync::{mpsc, mpsc::error::TryRecvError}, + time, + time::Duration, + }; + + fn create_onvif_discovery_message(uuid_string: &str) -> String { + let probe_types: Vec = vec![probe_types::NETWORK_VIDEO_TRANSMITTER.into()]; + let envelope = to_serialize::Envelope { + header: to_serialize::Header { + message_id: uuid_string.into(), + action: "http://schemas.xmlsoap.org/ws/2005/04/discovery/Probe".into(), + reply_to: "urn:schemas-xmlsoap-org:ws:2005:04:discovery".into(), + }, + body: to_serialize::Body { + probe: common::Probe { probe_types }, + }, + }; + let envelope_as_string = yaserde::ser::to_string(&envelope).unwrap(); + trace!( + "create_onvif_discovery_message - discovery message: {:?}", + &envelope_as_string + ); + envelope_as_string + } + + #[cfg(test)] + mod serialize_tests { + use super::*; + + #[test] + fn test_create_onvif_discovery_message() { + let _ = env_logger::builder().is_test(true).try_init(); + + let uuid_str = format!("uuid:{}", uuid::Uuid::new_v4()); + let expected_msg = format!( + "{}urn:schemas-xmlsoap-org:ws:2005:04:discoveryhttp://schemas.xmlsoap.org/ws/2005/04/discovery/Probenetwsdl:NetworkVideoTransmitter", + &uuid_str + ); + assert_eq!(expected_msg, create_onvif_discovery_message(&uuid_str)); + } + } + + fn get_device_uris_from_discovery_response(discovery_response: &str) -> Vec { + let response_envelope = + yaserde::de::from_str::(&discovery_response); + // The response envelope follows this format: + // + // https://10.0.0.1:5357/svc + // https://10.0.0.2:5357/svc + // https://10.0.0.3:5357/svc + // + response_envelope + .unwrap() + .body + .probe_matches + .probe_match + .iter() + .flat_map(|probe_match| probe_match.xaddrs.split_whitespace()) + .map(|addr| addr.to_string()) + .collect::>() + } + + #[cfg(test)] + mod deserialize_tests { + use super::*; + + #[test] + fn test_get_device_uris_from_discovery_response() { + let _ = env_logger::builder().is_test(true).try_init(); + + let uris = vec!["uri_one".to_string(), "uri_two".to_string()]; + let response = format!( + "\nurn:uuid:2bc6f06c-5566-7788-99ac-0012414fb745uuid:7b1d26aa-b02e-4ad2-8aab-4c928298ee0chttp://schemas.xmlsoap.org/ws/2004/08/addressing/role/anonymoushttp://schemas.xmlsoap.org/ws/2005/04/discovery/ProbeMatchesurn:uuid:10919da4-5566-7788-99aa-0012414fb745dn:NetworkVideoTransmitteronvif://www.onvif.org/type/video_encoder onvif://www.onvif.org/type/audio_encoder onvif://www.onvif.org/hardware/IPC-model onvif://www.onvif.org/location/country/china onvif://www.onvif.org/name/NVT onvif://www.onvif.org/Profile/Streaming {}10", + &uris.join(" ") + ); + assert_eq!(uris, get_device_uris_from_discovery_response(&response)); + } + } + + pub async fn simple_onvif_discover(timeout: Duration) -> Result, failure::Error> { + let (mut discovery_timeout_tx, mut discovery_timeout_rx) = mpsc::channel(2); + let (mut discovery_cancel_tx, mut discovery_cancel_rx) = mpsc::channel(2); + let shared_devices = Arc::new(Mutex::new(Vec::new())); + + let uuid_str = format!("uuid:{}", uuid::Uuid::new_v4()); + trace!("simple_onvif_discover - for {}", &uuid_str); + + let thread_devices = shared_devices.clone(); + tokio::spawn(async move { + trace!( + "simple_onvif_discover - spawned thread enter for {}", + &uuid_str + ); + + const LOCAL_IPV4_ADDR: Ipv4Addr = Ipv4Addr::UNSPECIFIED; + const LOCAL_PORT: u16 = 0; + let local_socket_addr = SocketAddr::new(IpAddr::V4(LOCAL_IPV4_ADDR), LOCAL_PORT); + + // WS-Discovery multicast ip and port selected from available standard + // options. See https://en.wikipedia.org/wiki/WS-Discovery + const MULTI_IPV4_ADDR: Ipv4Addr = Ipv4Addr::new(239, 255, 255, 250); + const MULTI_PORT: u16 = 3702; + let multi_socket_addr = SocketAddr::new(IpAddr::V4(MULTI_IPV4_ADDR), MULTI_PORT); + + trace!( + "simple_onvif_discover - binding to: {:?}", + local_socket_addr + ); + let socket = UdpSocket::bind(local_socket_addr).unwrap(); + socket + .set_write_timeout(Some(Duration::from_millis(200))) + .unwrap(); + socket + .set_read_timeout(Some(Duration::from_millis(200))) + .unwrap(); + trace!( + "simple_onvif_discover - joining multicast: {:?} {:?}", + &MULTI_IPV4_ADDR, + &LOCAL_IPV4_ADDR + ); + socket + .join_multicast_v4(&MULTI_IPV4_ADDR, &LOCAL_IPV4_ADDR) + .unwrap(); + + let envelope_as_string = create_onvif_discovery_message(&uuid_str); + match socket.send_to(&envelope_as_string.as_bytes(), multi_socket_addr) { + Ok(_) => { + loop { + let mut buf = vec![0; 16 * 1024]; + match socket.recv_from(&mut buf) { + Ok((len, _)) => { + let broadcast_response_as_string = + String::from_utf8_lossy(&buf[..len]).to_string(); + trace!( + "simple_onvif_discover - response: {:?}", + broadcast_response_as_string + ); + + get_device_uris_from_discovery_response( + &broadcast_response_as_string, + ) + .iter() + .for_each(|device_uri| { + trace!( + "simple_onvif_discover - device_uri parsed from response: {:?}", + device_uri + ); + thread_devices.lock().unwrap().push(device_uri.to_string()); + trace!( + "simple_onvif_discover - thread_devices: {:?}", + thread_devices.lock().unwrap() + ); + }); + } + Err(e) => match e.kind() { + ErrorKind::WouldBlock | ErrorKind::TimedOut => { + match discovery_cancel_rx.try_recv() { + Err(TryRecvError::Closed) | Ok(_) => { + trace!("simple_onvif_discover - recv_from error ... timeout signalled/disconnected (stop collecting responses): {:?}", e); + break; + } + Err(TryRecvError::Empty) => { + trace!("simple_onvif_discover - recv_from error ... no timeout (continue collecting responses): {:?}", e); + // continue looping + } + } + } + e => { + error!("simple_onvif_discover - recv_from error: {:?}", e); + Err(e).unwrap() + } + }, + } + } + } + Err(e) => match e.kind() { + ErrorKind::WouldBlock | ErrorKind::TimedOut => { + trace!("simple_onvif_discover - send_to timeout: {:?}", e); + return; + } + e => { + error!("simple_onvif_discover - send_to error: {:?}", e); + Err(e).unwrap() + } + }, + } + + let _best_effort_send = discovery_timeout_tx.send(()).await; + trace!("simple_onvif_discover - spawned thread exit"); + }); + + // Wait for timeout for discovery thread + let discovery_timeout_rx_result = time::timeout( + Duration::from_secs(timeout.as_secs()), + discovery_timeout_rx.recv(), + ) + .await; + trace!( + "simple_onvif_discover - spawned thread finished or timeout: {:?}", + discovery_timeout_rx_result + ); + // Send cancel message to thread to ensure it doesn't hang around + let _best_effort_cancel = discovery_cancel_tx.send(()).await; + + let result_devices = shared_devices.lock().unwrap().clone(); + info!("simple_onvif_discover - devices: {:?}", result_devices); + Ok(result_devices) + } + + #[cfg(test)] + mod discovery_tests { + use super::*; + use std::{ + sync::{Arc, Mutex}, + time::{Duration, SystemTime}, + }; + + #[tokio::test(core_threads = 2)] + async fn test_timeout_for_simple_onvif_discover() { + let _ = env_logger::builder().is_test(true).try_init(); + + let timeout = Duration::from_secs(2); + let duration = Arc::new(Mutex::new(Duration::from_secs(5))); + + let thread_duration = duration.clone(); + tokio::spawn(async move { + let start = SystemTime::now(); + let _ignore = simple_onvif_discover(timeout).await.unwrap(); + let end = SystemTime::now(); + let mut inner_duration = thread_duration.lock().unwrap(); + *inner_duration = end.duration_since(start).unwrap(); + trace!( + "call to simple_onvif_discover took {} milliseconds", + inner_duration.as_millis() + ); + }); + + let wait_for_call_millis = timeout.as_secs() * 1000 + 200; + trace!("wait for {} milliseconds", wait_for_call_millis); + std::thread::sleep(Duration::from_millis(wait_for_call_millis)); + // validate that this ends in 2 seconds or less + trace!("duration to test: {}", duration.lock().unwrap().as_millis()); + // we could test for exactly 2 seconds here, but a little wiggle room seems reasonable + assert!(duration.lock().unwrap().as_millis() <= wait_for_call_millis.into()); + } + } +} diff --git a/agent/src/protocols/onvif/mod.rs b/agent/src/protocols/onvif/mod.rs new file mode 100644 index 0000000..5d3a2ee --- /dev/null +++ b/agent/src/protocols/onvif/mod.rs @@ -0,0 +1,3 @@ +mod discovery_handler; +mod discovery_impl; +pub use self::discovery_handler::OnvifDiscoveryHandler; diff --git a/agent/src/protocols/opcua/discovery_handler.rs b/agent/src/protocols/opcua/discovery_handler.rs new file mode 100644 index 0000000..e0de06e --- /dev/null +++ b/agent/src/protocols/opcua/discovery_handler.rs @@ -0,0 +1,24 @@ +use super::super::{DiscoveryHandler, DiscoveryResult}; +use akri_shared::akri::configuration::OpcuaDiscoveryHandlerConfig; +use async_trait::async_trait; +use failure::Error; + +/// `OnvifDiscoveryHandler` discovers the OPC instances. The instances it discovers are always shared. +#[derive(Debug)] +pub struct OpcuaDiscoveryHandler {} + +impl OpcuaDiscoveryHandler { + pub fn new(_discovery_handler_config: &OpcuaDiscoveryHandlerConfig) -> Self { + OpcuaDiscoveryHandler {} + } +} + +#[async_trait] +impl DiscoveryHandler for OpcuaDiscoveryHandler { + async fn discover(&self) -> Result, Error> { + Err(failure::format_err!("OPC protocol handler not implemented")) + } + fn are_shared(&self) -> Result { + Ok(true) + } +} diff --git a/agent/src/protocols/opcua/mod.rs b/agent/src/protocols/opcua/mod.rs new file mode 100644 index 0000000..765b4eb --- /dev/null +++ b/agent/src/protocols/opcua/mod.rs @@ -0,0 +1,2 @@ +mod discovery_handler; +pub use self::discovery_handler::OpcuaDiscoveryHandler; diff --git a/agent/src/protocols/udev/discovery_handler.rs b/agent/src/protocols/udev/discovery_handler.rs new file mode 100644 index 0000000..2e83cdf --- /dev/null +++ b/agent/src/protocols/udev/discovery_handler.rs @@ -0,0 +1,58 @@ +use super::super::{DiscoveryHandler, DiscoveryResult}; +use super::{discovery_impl, udev_enumerator, UDEV_DEVNODE_LABEL_ID}; +use akri_shared::akri::configuration::UdevDiscoveryHandlerConfig; +use async_trait::async_trait; +use failure::Error; +use std::collections::HashSet; + +/// `UdevDiscoveryHandler` discovers udev instances by parsing the udev rules in `discovery_handler_config.udev_rules`. +/// The instances it discovers are always unshared. +#[derive(Debug)] +pub struct UdevDiscoveryHandler { + discovery_handler_config: UdevDiscoveryHandlerConfig, +} + +impl UdevDiscoveryHandler { + pub fn new(discovery_handler_config: &UdevDiscoveryHandlerConfig) -> Self { + UdevDiscoveryHandler { + discovery_handler_config: discovery_handler_config.clone(), + } + } +} + +#[async_trait] +impl DiscoveryHandler for UdevDiscoveryHandler { + async fn discover(&self) -> Result, Error> { + let udev_rules = self.discovery_handler_config.udev_rules.clone(); + trace!("discover - for udev rules {:?}", udev_rules); + let mut devpaths: HashSet = HashSet::new(); + udev_rules.iter().for_each(|rule| { + let enumerator = udev_enumerator::create_enumerator(); + match discovery_impl::do_parse_and_find(enumerator, &rule) { + Ok(paths) => paths.into_iter().for_each(|path| { + devpaths.insert(path); + }), + Err(e) => error!( + "discover - for rule {} do_parse_and_find returned error {}", + rule, e + ), + } + }); + trace!( + "discover - mapping and returning devices at devpaths {:?}", + devpaths + ); + Ok(devpaths + .into_iter() + .map(|path| { + let mut properties = std::collections::HashMap::new(); + properties.insert(UDEV_DEVNODE_LABEL_ID.to_string(), path.clone()); + DiscoveryResult::new(&path, properties, self.are_shared().unwrap()) + }) + .collect::>()) + } + + fn are_shared(&self) -> Result { + Ok(false) + } +} diff --git a/agent/src/protocols/udev/discovery_impl.rs b/agent/src/protocols/udev/discovery_impl.rs new file mode 100644 index 0000000..c5ca5c6 --- /dev/null +++ b/agent/src/protocols/udev/discovery_impl.rs @@ -0,0 +1,652 @@ +extern crate udev; + +use super::udev_device::{ + get_devnode, get_devpath, get_driver, get_property_value, get_sysname, DeviceExt, +}; +use super::udev_enumerator::Enumerator; +use pest::iterators::Pair; +use pest::Parser; +use regex::Regex; + +const TAGS: &str = "TAGS"; + +#[derive(Parser)] +#[grammar = "protocols/udev/udev_rule_grammar.pest"] +pub struct UdevRuleParser; + +#[derive(Debug, PartialEq)] +pub struct UdevFilter<'a> { + field: Pair<'a, Rule>, + operation: Rule, + value: String, +} + +/// This parses the udev rule into UdevFilters and finds all devices that match those filters +pub fn do_parse_and_find( + enumerator: impl Enumerator, + udev_rule_string: &str, +) -> Result, failure::Error> { + let udev_filters = parse_udev_rule(udev_rule_string)?; + let devpaths = find_devices(enumerator, udev_filters)?; + trace!( + "do_parse_and_find - returning discovered devices with devpaths: {:?}", + devpaths + ); + Ok(devpaths) +} + +/// This parses a udev rule and returns a list of UdevFilter objects that specify which devices to search for. +/// This returns an error if the udev rule parameter does not fit the format specified in udev +/// man pages/wiki and therefore does not match the grammar specified in udev_rule_grammar.pest +/// A udev rule is made of a list of field-value pairs which have format field"value" +/// This function will only create UdevFilter objects for field-value pairs with supported fields and operations. +/// Udev discovery is only interested in match operations ("==", "!="), so all action ("=" , "+=" , "-=" , ":=") operations +/// will be ignored. +/// Udev discovery is only interested in match fields, so all action fields, such as TEST, are ignored +/// Some match fields that look up the device hierarchy, such as SUBSYSTEMS, are yet to be supported and are also ignored +fn parse_udev_rule(udev_rule_string: &str) -> Result, failure::Error> { + info!( + "parse_udev_rule - enter for udev rule string {}", + udev_rule_string + ); + let mut udev_filters: Vec = Vec::new(); + + // So long as parse succeeds, subsequent unwraps will not fails, since they are following the + // format specified in the grammar + let udev_rule = UdevRuleParser::parse(Rule::udev_rule, udev_rule_string)? + .next() // move to first rule within udev_rule aka inner_rule + .unwrap() // does not panic because udev_rule always has inner_rule + .into_inner() // go into inner_rule which has format { udev_filter ~ ("," ~ udev_filter)* } + .next() // move to first rule in inner_rule aka udev_filter + .unwrap(); // does not panic because inner_rule always has udev_filter + + trace!( + "parse_udev_rule - parsing udev_rule {:?}", + udev_rule.as_str() + ); + for udev_filter in udev_rule.into_inner() { + let mut inner_rules = udev_filter.into_inner(); + let field_pair = inner_rules.next().unwrap(); + let inner_field = field_pair.into_inner().next().unwrap(); + // Ignore unsupported fields + if inner_field.as_rule() == Rule::unsupported_field { + trace!( + "parse_udev_rule - unsupported field {}", + inner_field.into_inner().next().unwrap().as_str() + ); + continue; + } + + let operation = inner_rules + .next() + .unwrap() + .into_inner() + .next() + .unwrap() + .as_rule(); + let mut quoted_value = inner_rules.next().unwrap().into_inner(); + let value = quoted_value.next().unwrap().as_str(); + // ignore action operations + if operation != Rule::action_operation { + udev_filters.push(UdevFilter { + field: inner_field, + operation, + value: value.to_string(), + }); + } + } + Ok(udev_filters) +} + +/// This searches for devices that match the UdevFilters and returns their devpaths +fn find_devices( + enumerator: impl Enumerator, + udev_filters: Vec, +) -> std::io::Result> { + let mut enumerator = enumerator; + trace!("find_devices - enter with udev_filters {:?}", udev_filters); + + // Enumerator scans sys devices for its filters. Only certain filters can be applied to it. + // Divide device fields by type of filter than can be applied to Enumerator, if any + // (1) Enumerator can filter for field by equality/match + // (2) Enumerator can filter for field by inequality/nomatch + // (3) Enumerator cannot filter for field. Must manually filter by looking at each Device the filtered Enumerator returns. + let match_fields = vec![ + Rule::devpath, + Rule::kernel, + Rule::tag, + Rule::subsystem, + Rule::attribute, + Rule::property, + ]; + let nomatch_fields = vec![Rule::attribute, Rule::subsystem]; + + let mut match_udev_filters: Vec<&UdevFilter> = Vec::new(); + let mut nomatch_udev_filters: Vec<&UdevFilter> = Vec::new(); + let mut remaining_udev_filters: Vec<&UdevFilter> = Vec::new(); + + // Sort UdevFilters based off of which group they belong to + udev_filters.iter().for_each(|udev_filter| { + if udev_filter.operation == Rule::equality + && match_fields.contains(&udev_filter.field.as_rule()) + { + match_udev_filters.push(udev_filter); + } else if udev_filter.operation == Rule::inequality + && nomatch_fields.contains(&udev_filter.field.as_rule()) + { + nomatch_udev_filters.push(udev_filter); + } else { + remaining_udev_filters.push(udev_filter); + } + }); + + // Apply UdevFilters of groups in 1,2,3 order + filter_by_match_udev_filters(&mut enumerator, match_udev_filters); + filter_by_nomatch_udev_filters(&mut enumerator, nomatch_udev_filters); + let devices: Vec = enumerator.scan_devices()?.collect(); + let final_devices = filter_by_remaining_udev_filters(devices, remaining_udev_filters); + + let device_devpaths: Vec = final_devices + .into_iter() + .filter_map(|device| { + if let Some(devnode) = get_devnode(&device) { + Some(devnode.to_str().unwrap().to_string()) + } else { + None + } + }) + .collect(); + + Ok(device_devpaths) +} + +/// This adds equality filters to the Enumerator +fn filter_by_match_udev_filters(enumerator: &mut impl Enumerator, udev_filters: Vec<&UdevFilter>) { + trace!( + "enumerator_match_udev_filters - enter with udev_filters {:?}", + udev_filters + ); + for udev_filter in udev_filters { + match udev_filter.field.as_rule() { + Rule::devpath => { + let mut syspath: String = "/sys".to_owned(); + syspath.push_str(&udev_filter.value); + enumerator.add_syspath(&syspath).unwrap(); + } + Rule::kernel => { + enumerator.match_sysname(&udev_filter.value).unwrap(); + } + Rule::tag => { + enumerator.match_tag(&udev_filter.value).unwrap(); + } + Rule::subsystem => { + enumerator.match_subsystem(&udev_filter.value).unwrap(); + } + Rule::attribute => { + let key = udev_filter + .field + .clone() + .into_inner() + .next() + .unwrap() + .into_inner() + .next() + .unwrap() + .as_str(); + enumerator.match_attribute(key, &udev_filter.value).unwrap(); + } + Rule::property => { + let key = udev_filter + .field + .clone() + .into_inner() + .next() + .unwrap() + .into_inner() + .next() + .unwrap() + .as_str(); + enumerator.match_property(key, &udev_filter.value).unwrap(); + } + _ => { + error!("enumerator_match_udev_filters - encountered unsupported field"); + } + } + } +} + +/// This adds inequality filters to the Enumerator +fn filter_by_nomatch_udev_filters( + enumerator: &mut impl Enumerator, + udev_filters: Vec<&UdevFilter>, +) { + trace!( + "enumerator_nomatch_udev_filters - enter with udev_filters {:?}", + udev_filters + ); + for udev_filter in udev_filters { + match udev_filter.field.as_rule() { + Rule::attribute => { + let key = udev_filter + .field + .clone() + .into_inner() + .next() + .unwrap() + .into_inner() + .next() + .unwrap() + .as_str(); + enumerator + .nomatch_attribute(key, &udev_filter.value) + .unwrap(); + } + Rule::subsystem => { + enumerator.nomatch_subsystem(&udev_filter.value).unwrap(); + } + _ => { + error!("enumerator_nomatch_udev_filters - encountered unsupported field"); + } + } + } +} + +/// This iterates over devices returned by filtered Enumerator and inspects the device's fields to see if they match/don't match +/// the fields in the remaining UdevFilters that cound not be applied to Enumerator. +fn filter_by_remaining_udev_filters( + devices: Vec, + udev_filters: Vec<&UdevFilter>, +) -> Vec { + trace!( + "filter_by_remaining_udev_filters - enter with udev_filters {:?}", + udev_filters + ); + let mut mutable_devices = devices; + for udev_filter in udev_filters { + match udev_filter.field.as_rule() { + Rule::devpath => { + let re = Regex::new(&udev_filter.value).unwrap(); + // Filter for inequality. Equality already accounted for in filter_by_match_udev_filters + mutable_devices = mutable_devices + .into_iter() + .filter(|device| { + let devpath = get_devpath(device).to_str().unwrap(); + match re.find(devpath) { + Some(found_string) => { + found_string.start() != 0 || found_string.end() != devpath.len() + } + None => true, + } + }) + .collect(); + } + Rule::kernel => { + let re = Regex::new(&udev_filter.value).unwrap(); + // Filter for inequality. Equality already accounted for in filter_by_match_udev_filters + mutable_devices = mutable_devices + .into_iter() + .filter(|device| { + let sysname = get_sysname(device).to_str().unwrap(); + match re.find(sysname) { + Some(found_string) => { + found_string.start() != 0 || found_string.end() != sysname.len() + } + None => true, + } + }) + .collect(); + } + Rule::tag => { + let re = Regex::new(&udev_filter.value).unwrap(); + mutable_devices = mutable_devices + .into_iter() + .filter(|device| { + if let Some(tags) = get_property_value(device, TAGS) { + let tags = tags.to_str().unwrap().split(':'); + // Filter for inequality. Equality already accounted for in filter_by_match_udev_filters + // Return false if discover a tag that should be excluded + let mut include = true; + for tag in tags { + if let Some(found_string) = re.find(tag) { + if found_string.start() == 0 && found_string.end() == tag.len() + { + include = false; + break; + } + } + } + include + } else { + true + } + }) + .collect(); + } + Rule::property => { + let key = udev_filter + .field + .clone() + .into_inner() + .next() + .unwrap() + .into_inner() + .next() + .unwrap() + .as_str(); + let re = Regex::new(&udev_filter.value).unwrap(); + // Filter for inequality. Equality already accounted for in filter_by_match_udev_filters + mutable_devices = mutable_devices + .into_iter() + .filter(|device| { + if let Some(property_value) = get_property_value(device, key) { + let property_value_str = property_value.to_str().unwrap(); + match re.find(property_value_str) { + Some(found_string) => { + found_string.start() != 0 + || found_string.end() != property_value_str.len() + } + None => true, + } + } else { + true + } + }) + .collect(); + } + Rule::driver => { + let re = Regex::new(&udev_filter.value).unwrap(); + let is_equality = udev_filter.operation == Rule::equality; + mutable_devices = mutable_devices + .into_iter() + .filter(|device| match get_driver(device) { + Some(driver) => { + let driver = driver.to_str().unwrap(); + match re.find(driver) { + Some(found_string) => { + let is_match = found_string.start() == 0 + && found_string.end() == driver.len(); + (is_equality && is_match) || (!is_equality && !is_match) + } + None => !is_equality, + } + } + None => !is_equality, + }) + .collect(); + } + _ => { + error!("filter_by_remaining_udev_filters - encountered unsupported field"); + } + } + } + mutable_devices +} + +#[cfg(test)] +mod discovery_tests { + use super::super::udev_enumerator::{create_enumerator, MockEnumerator}; + use super::*; + use std::{ + collections::HashMap, + ffi::OsStr, + fs::File, + io::{prelude::*, BufReader}, + path::Path, + }; + + pub struct MockDevice<'a> { + pub devpath: String, + pub devnode: String, + pub sysname: String, + pub properties: std::collections::HashMap, + pub driver: Option<&'a OsStr>, + } + + impl<'a> DeviceExt for MockDevice<'a> { + fn mockable_devpath(&self) -> &OsStr { + OsStr::new(&self.devpath) + } + fn mockable_devnode(&self) -> Option<&Path> { + Some(Path::new(&self.devnode)) + } + fn mockable_sysname(&self) -> &OsStr { + OsStr::new(&self.sysname) + } + fn mockable_property_value(&self, property: &str) -> Option<&OsStr> { + if let Some(value) = self.properties.get(property) { + Some(OsStr::new(value)) + } else { + None + } + } + fn mockable_driver(&self) -> Option<&OsStr> { + self.driver + } + } + + #[test] + fn test_parse_udev_rule_detailed() { + let _ = env_logger::builder().is_test(true).try_init(); + let rule = "KERNEL==\"video[0-9]*\",SUBSYSTEMS==\"usb\", SUBSYSTEM==\"video4linux\", ATTR{idVendor}==\"05a9\", ATTRS{idProduct}==\"4519\", SYMLINK+=\"video-cam\""; + let udev_filters = parse_udev_rule(rule).unwrap(); + assert_eq!(udev_filters.len(), 3); + assert_eq!(udev_filters[0].field.as_str(), "KERNEL"); + assert_eq!(udev_filters[0].operation, Rule::equality); + assert_eq!(&udev_filters[0].value, "video[0-9]*"); + + assert_eq!(udev_filters[1].field.as_str(), "SUBSYSTEM"); + assert_eq!(udev_filters[1].operation, Rule::equality); + assert_eq!(&udev_filters[1].value, "video4linux"); + + assert_eq!(udev_filters[2].field.as_str(), "ATTR{idVendor}"); + assert_eq!(udev_filters[2].operation, Rule::equality); + assert_eq!(&udev_filters[2].value, "05a9"); + } + + #[test] + fn test_parse_udev_rule_error() { + // Throws error if unknown field (TYPO) + let rule = "KERNEL==\"video[0-9]*\", TYPO==\"blah\", ATTR{idVendor}==\"05a9\", ATTRS{idProduct}==\"4519\""; + assert!(parse_udev_rule(rule).is_err()); + + // Throws error if leading space + let rule = " KERNEL==\"video[0-9]*\", TYPO==\"blah\", ATTR{idVendor}==\"05a9\", ATTRS{idProduct}==\"4519\""; + assert!(parse_udev_rule(rule).is_err()); + } + + #[test] + fn test_parse_udev_rule_empty() { + // Assert that doesn't throw error on empty rules + let rule = ""; + let result = parse_udev_rule(rule); + assert!(result.is_ok()); + let udev_filters = result.unwrap(); + assert_eq!(udev_filters.len(), 0); + } + + #[test] + fn test_parse_udev_rule_from_file() { + let _ = env_logger::builder().is_test(true).try_init(); + let file_path = "../test/example.rules"; + let file = File::open(file_path).expect("no such file"); + let buf = BufReader::new(file); + let mut num_udev_filters: Vec = Vec::new(); + let lines: Vec = buf + .lines() + .map(|l| { + let unwrapped = l.expect("Could not parse line"); + num_udev_filters.push(unwrapped[0..1].parse::().unwrap()); + unwrapped[2..].to_string() + }) + .collect(); + for x in 0..lines.len() { + let line = &lines[x]; + let udev_filters = parse_udev_rule(line).unwrap(); + assert_eq!(udev_filters.len(), num_udev_filters[x]); + } + } + + #[test] + fn test_filter_by_match_udev_filters() { + let rule = "SUBSYSTEM==\"video4linux\", ATTR{someKey}==\"1000\", KERNEL==\"video0\", ENV{ID}==\"1\", TAG==\"some_tag\", DEVPATH==\"/devices/path\""; + let mut mock = MockEnumerator::new(); + mock.expect_match_subsystem() + .times(1) + .withf(move |value: &str| value == "video4linux") + .returning(|_| Ok(())); + mock.expect_match_attribute() + .times(1) + .withf(move |key: &str, value: &str| key == "someKey" && value == "1000") + .returning(|_, _| Ok(())); + mock.expect_match_sysname() + .times(1) + .withf(move |value: &str| value == "video0") + .returning(|_| Ok(())); + mock.expect_match_property() + .times(1) + .withf(move |key: &str, value: &str| key == "ID" && value == "1") + .returning(|_, _| Ok(())); + mock.expect_match_tag() + .times(1) + .withf(move |value: &str| value == "some_tag") + .returning(|_| Ok(())); + mock.expect_add_syspath() + .times(1) + .withf(move |value: &str| value == "/sys/devices/path") + .returning(|_| Ok(())); + let udev_filters = parse_udev_rule(rule).unwrap(); + let udev_filters: Vec<&UdevFilter> = udev_filters.iter().collect(); + filter_by_match_udev_filters(&mut mock, udev_filters); + } + + #[test] + fn test_filter_by_nomatch_udev_filters() { + let rule = "SUBSYSTEM!=\"usb\", ATTR{someKey}!=\"1000\""; + let mut mock = MockEnumerator::new(); + mock.expect_nomatch_subsystem() + .times(1) + .withf(move |value: &str| value == "usb") + .returning(|_| Ok(())); + mock.expect_nomatch_attribute() + .times(1) + .withf(move |key: &str, value: &str| key == "someKey" && value == "1000") + .returning(|_, _| Ok(())); + let udev_filters = parse_udev_rule(rule).unwrap(); + let udev_filters: Vec<&UdevFilter> = udev_filters.iter().collect(); + filter_by_nomatch_udev_filters(&mut mock, udev_filters); + } + + #[test] + fn test_filter_by_remaining_udev_filters() { + let rule = "KERNEL!=\"video0\", TAG!=\"tag_exclude\", ENV{ID}!=\"id_num\", TAG!=\"tag[3-9]\", DEVPATH!=\"/devices/path/exclude\", DRIVER!=\"exclude\""; + let mut include_properties = std::collections::HashMap::new(); + include_properties.insert("TAGS".to_string(), "tag0:tag_excluded:tag2".to_string()); + let mut tag_exclude_properties = std::collections::HashMap::new(); + tag_exclude_properties.insert("TAGS".to_string(), "tag3:other:tag2".to_string()); + let mut id_exclude_properties = std::collections::HashMap::new(); + id_exclude_properties.insert("ID".to_string(), "id_num".to_string()); + let mock_device_to_exclude0 = MockDevice { + devpath: "/devices/path/exclude".to_string(), + devnode: "/dev/exclude".to_string(), + sysname: "/sys/mock0".to_string(), + properties: HashMap::new(), + driver: Some(OsStr::new("include")), + }; + let mock_device_to_exclude1 = MockDevice { + devpath: "/devices/path/include".to_string(), + devnode: "/dev/exclude".to_string(), + sysname: "/sys/mock1".to_string(), + properties: HashMap::new(), + driver: Some(OsStr::new("exclude")), + }; + let mock_device_to_include1 = MockDevice { + devpath: "/devices/path/include".to_string(), + devnode: "/dev/include".to_string(), + sysname: "/sys/mock2".to_string(), + properties: include_properties, + driver: Some(OsStr::new("include")), + }; + let mock_device_to_exclude3 = MockDevice { + devpath: "/devices/path/include".to_string(), + devnode: "/dev/include".to_string(), + sysname: "/sys/mock3".to_string(), + properties: tag_exclude_properties, + driver: Some(OsStr::new("include")), + }; + let mock_device_to_include2 = MockDevice { + devpath: "/devices/path/include".to_string(), + devnode: "/dev/include".to_string(), + sysname: "/sys/mock4".to_string(), + properties: HashMap::new(), + driver: Some(OsStr::new("include")), + }; + let mock_device_to_exclude4 = MockDevice { + devpath: "/devices/path/include".to_string(), + devnode: "/dev/include".to_string(), + sysname: "/sys/mock5".to_string(), + properties: id_exclude_properties, + driver: Some(OsStr::new("include")), + }; + let devices = vec![ + mock_device_to_exclude0, + mock_device_to_exclude1, + mock_device_to_include1, + mock_device_to_exclude3, + mock_device_to_include2, + mock_device_to_exclude4, + ]; + let udev_filters = parse_udev_rule(rule).unwrap(); + let udev_filters: Vec<&UdevFilter> = udev_filters.iter().collect(); + let filtered_devices = filter_by_remaining_udev_filters(devices, udev_filters); + + assert_eq!(filtered_devices.len(), 2); + assert_eq!( + get_sysname(&filtered_devices[0]).to_str().unwrap(), + "/sys/mock2" + ); + assert_eq!( + get_sysname(&filtered_devices[1]).to_str().unwrap(), + "/sys/mock4" + ); + + let rule = "DRIVER==\"include\""; + let mock_device = MockDevice { + devpath: "/devices/path/include".to_string(), + devnode: "/dev/include".to_string(), + sysname: "/sys/mock3".to_string(), + properties: HashMap::new(), + driver: Some(OsStr::new("not_included")), + }; + let udev_filters = parse_udev_rule(rule).unwrap(); + let udev_filters: Vec<&UdevFilter> = udev_filters.iter().collect(); + let filtered_devices = filter_by_remaining_udev_filters(vec![mock_device], udev_filters); + assert_eq!(filtered_devices.len(), 0); + } + + // Only tests that proper match calls were made + #[test] + fn test_do_parse_and_find() { + let rule = "KERNEL==\"video[0-9]*\",ATTR{someKey}!=\"1000\", SUBSYSTEMS==\"usb\", SUBSYSTEM==\"video4linux\", SYMLINK+=\"video-cam\""; + let mut mock = MockEnumerator::new(); + mock.expect_match_subsystem() + .times(1) + .withf(move |value: &str| value == "video4linux") + .returning(|_| Ok(())); + mock.expect_nomatch_attribute() + .times(1) + .withf(move |key: &str, value: &str| key == "someKey" && value == "1000") + .returning(|_, _| Ok(())); + mock.expect_match_sysname() + .times(1) + .withf(move |value: &str| value == "video[0-9]*") + .returning(|_| Ok(())); + mock.expect_scan_devices().times(1).returning(|| { + let mut enumerator = create_enumerator(); + enumerator + .match_attribute("random", "attribute_that_should_not_be_found") + .unwrap(); + enumerator.scan_devices() + }); + assert_eq!(do_parse_and_find(mock, rule).unwrap().len(), 0); + } +} diff --git a/agent/src/protocols/udev/mod.rs b/agent/src/protocols/udev/mod.rs new file mode 100644 index 0000000..35c2680 --- /dev/null +++ b/agent/src/protocols/udev/mod.rs @@ -0,0 +1,125 @@ +mod discovery_handler; +mod discovery_impl; +pub use self::discovery_handler::UdevDiscoveryHandler; +pub const UDEV_DEVNODE_LABEL_ID: &str = "UDEV_DEVNODE"; + +pub mod udev_device { + extern crate udev; + use std::{ffi::OsStr, path::Path}; + + /// Extension Trait for udev::Device. Enables creation of MockDevice for testing. + pub trait DeviceExt { + fn mockable_devpath(&self) -> &OsStr; + fn mockable_devnode(&self) -> Option<&Path>; + fn mockable_sysname(&self) -> &OsStr; + fn mockable_property_value(&self, property: &str) -> Option<&OsStr>; + fn mockable_driver(&self) -> Option<&OsStr>; + } + + impl DeviceExt for udev::Device { + fn mockable_devpath(&self) -> &OsStr { + self.devpath() + } + fn mockable_devnode(&self) -> Option<&Path> { + self.devnode() + } + fn mockable_sysname(&self) -> &OsStr { + self.sysname() + } + fn mockable_property_value(&self, property: &str) -> Option<&OsStr> { + self.property_value(property) + } + fn mockable_driver(&self) -> Option<&OsStr> { + self.driver() + } + } + + pub fn get_devpath(device: &impl DeviceExt) -> &OsStr { + device.mockable_devpath() + } + + pub fn get_devnode(device: &impl DeviceExt) -> Option<&Path> { + device.mockable_devnode() + } + + pub fn get_sysname(device: &impl DeviceExt) -> &OsStr { + device.mockable_sysname() + } + + pub fn get_property_value<'a, 'b>( + device: &'a impl DeviceExt, + property: &'b str, + ) -> Option<&'a OsStr> { + device.mockable_property_value(property) + } + + pub fn get_driver(device: &impl DeviceExt) -> Option<&OsStr> { + device.mockable_driver() + } +} + +pub mod udev_enumerator { + extern crate udev; + use mockall::predicate::*; + use mockall::*; + + /// Wrap udev::Enumerator functions in a trait to inable mocking for testing. + #[automock] + pub trait Enumerator { + fn match_subsystem(&mut self, value: &str) -> std::io::Result<()>; + fn nomatch_subsystem(&mut self, value: &str) -> std::io::Result<()>; + fn match_attribute(&mut self, key: &str, value: &str) -> std::io::Result<()>; + fn nomatch_attribute(&mut self, key: &str, value: &str) -> std::io::Result<()>; + fn match_sysname(&mut self, value: &str) -> std::io::Result<()>; + fn match_property(&mut self, key: &str, value: &str) -> std::io::Result<()>; + fn match_tag(&mut self, value: &str) -> std::io::Result<()>; + fn add_syspath(&mut self, value: &str) -> std::io::Result<()>; + fn scan_devices(&mut self) -> std::io::Result; + } + + pub fn create_enumerator() -> impl Enumerator { + EnumeratorImpl::new() + } + + pub struct EnumeratorImpl { + inner_enumerator: udev::Enumerator, + } + + impl EnumeratorImpl { + fn new() -> Self { + EnumeratorImpl { + inner_enumerator: udev::Enumerator::new().unwrap(), + } + } + } + + impl Enumerator for EnumeratorImpl { + fn match_subsystem(&mut self, value: &str) -> std::io::Result<()> { + self.inner_enumerator.match_subsystem(value) + } + fn nomatch_subsystem(&mut self, value: &str) -> std::io::Result<()> { + self.inner_enumerator.nomatch_subsystem(value) + } + fn match_attribute(&mut self, key: &str, value: &str) -> std::io::Result<()> { + self.inner_enumerator.match_attribute(key, value) + } + fn nomatch_attribute(&mut self, key: &str, value: &str) -> std::io::Result<()> { + self.inner_enumerator.nomatch_attribute(key, value) + } + fn match_sysname(&mut self, value: &str) -> std::io::Result<()> { + self.inner_enumerator.match_sysname(value) + } + fn match_property(&mut self, key: &str, value: &str) -> std::io::Result<()> { + self.inner_enumerator.match_property(key, value) + } + fn match_tag(&mut self, value: &str) -> std::io::Result<()> { + self.inner_enumerator.match_tag(value) + } + fn add_syspath(&mut self, value: &str) -> std::io::Result<()> { + self.inner_enumerator.add_syspath(value) + } + fn scan_devices(&mut self) -> std::io::Result { + self.inner_enumerator.scan_devices() + } + } +} diff --git a/agent/src/protocols/udev/udev_rule_grammar.pest b/agent/src/protocols/udev/udev_rule_grammar.pest new file mode 100644 index 0000000..363c6cc --- /dev/null +++ b/agent/src/protocols/udev/udev_rule_grammar.pest @@ -0,0 +1,65 @@ +/// Grammar for parsing udev rules +WHITESPACE = _{ " " } +// if remove *, will throw error when empty string +udev_rule = { SOI ~ (inner_rule)* ~ EOI } +inner_rule = { udev_filter ~ ("," ~ udev_filter)* } +udev_filter = ${ field ~ operation ~ quoted_value } +field = { unsupported_field | devpath | kernel | tag | driver | subsystem | attribute | property } +action_field = { action | name | symlink | test | program | result | run | label | goto | import | options } +unsupported_field = { sysctl | kernels | subsystems | drivers | attributes | tags | constant | owner | group | mode | seclabel | action_field } +bounded_key = {"{" ~ key ~ "}"} +// remove ! on key and value rules if want to allow spaces between ""/{} and key/value (ie: { DEVPATH } vs {DEVPATH}) +key = !{ (ASCII_ALPHANUMERIC | SPACE_SEPARATOR | "$" | "." | "_" | "*" | "?" | "[" | "]" | "-" | "|" | "\\" | "/" )* } +value = !{ (ASCII_ALPHANUMERIC | SPACE_SEPARATOR | "$" | "." | "_" | "*" | ":" | "?" | "[" | "]" | "-" | "|" | "\\" | "/" | "%" | "{"| "}")* } +allowed_value_characters = { ASCII_DIGIT | ASCII_ALPHA | MARK | PUNCTUATION | SYMBOL | SPACE_SEPARATOR } +quoted_value = {"\"" ~ value ~ "\""} +operation = { equality | inequality | action_operation } +action_operation = { addition | removal | final_assignment | assignment } +equality = { "==" } +inequality = { "!=" } +assignment = { "=" } +addition = { "+=" } +removal = { "-=" } +final_assignment = { ":=" } + +// Supported fields +devpath = { "DEVPATH" } +kernel = { "KERNEL" } +tag = { "TAG" } +driver = { "DRIVER" } +subsystem = { "SUBSYSTEM" } +attribute = { "ATTR" ~ bounded_key } // {key} +property = { "ENV" ~ bounded_key } // {key} + +// +// Unsupported fields +// +// Actions +action = { "ACTION" } +name = { "NAME" } +symlink = { "SYMLINK" } +sysctl = { "SYSCTL" ~ bounded_key } // {kernel key} +test = { "TEST" ~ bounded_key } // {octal mode mask} +program = { "PROGRAM" } +result = { "RESULT" } +run = { "RUN" ~ bounded_key } // {type} where type = program | builtin +label = { "LABEL" } +goto = { "GOTO" } +import = { "IMPORT" ~ bounded_key } // {type} where type = program | builtin | file | db | cmdline | parent +options = { "OPTIONS" } + +// Suppport in V2 +kernels = { "KERNELS" } +subsystems = { "SUBSYSTEMS" } +drivers = { "DRIVERS" } +attributes = { "ATTRS" ~ bounded_key } +tags = { "TAGS" } +constant = { "CONST" ~ bounded_key } // {key} where key = "arch" | "virt" +owner = { "OWNER" } +group = { "GROUP" } +mode = { "MODE" } +seclabel = { "SECLABEL" ~ bounded_key } // {module} + + + + diff --git a/agent/src/util/config_action.rs b/agent/src/util/config_action.rs new file mode 100644 index 0000000..bb5bffa --- /dev/null +++ b/agent/src/util/config_action.rs @@ -0,0 +1,796 @@ +use super::super::protocols; +use super::{ + constants::{DISCOVERY_DELAY_SECS, SHARED_INSTANCE_OFFLINE_GRACE_PERIOD_SECS}, + device_plugin_service, + device_plugin_service::{ + get_device_instance_name, ConnectivityStatus, InstanceInfo, InstanceMap, + }, +}; +use akri_shared::{ + akri::{ + configuration::{Configuration, KubeAkriConfig, ProtocolHandler}, + API_CONFIGURATIONS, API_NAMESPACE, API_VERSION, + }, + k8s, + k8s::KubeInterface, +}; +use futures::StreamExt; +use kube::api::{Informer, RawApi, WatchEvent}; +use log::{info, trace}; +use std::{ + collections::HashMap, + sync::Arc, + time::{Duration, Instant}, +}; +use tokio::{ + sync::{broadcast, mpsc, Mutex}, + time::timeout, +}; + +type ConfigMap = Arc>>; + +/// Information for managing a Configuration, such as all applied Instances of that Configuration +/// and senders for ceasing to discover instances upon Configuration deletion. +#[derive(Debug)] +pub struct ConfigInfo { + instance_map: InstanceMap, + stop_discovery_sender: mpsc::Sender<()>, + finished_discovery_sender: broadcast::Sender<()>, +} + +/// This handles pre-existing Configurations and invokes an internal method that watches for Configuration events. +pub async fn do_config_watch() -> Result<(), Box> { + info!("do_config_watch - enter"); + let config_map: ConfigMap = Arc::new(Mutex::new(HashMap::new())); + let kube_interface = k8s::create_kube_interface(); + let mut tasks = Vec::new(); + + // Handle pre-existing configs + let pre_existing_configs = kube_interface.get_configurations().await?; + for config in pre_existing_configs { + let config_map = config_map.clone(); + tasks.push(tokio::spawn(async move { + handle_config_add(&config, config_map).await.unwrap(); + })); + } + + // Watch for new configs and changes + tasks.push(tokio::spawn(async move { + watch_for_config_changes(&kube_interface, config_map) + .await + .unwrap(); + })); + + futures::future::try_join_all(tasks).await?; + info!("do_config_watch - end"); + Ok(()) +} + +/// This watches for Configuration events +async fn watch_for_config_changes( + kube_interface: &impl KubeInterface, + config_map: ConfigMap, +) -> Result<(), Box> { + trace!("watch_for_config_changes - start"); + let akri_config_type = RawApi::customResource(API_CONFIGURATIONS) + .group(API_NAMESPACE) + .version(API_VERSION); + let informer = Informer::raw(kube_interface.get_kube_client(), akri_config_type) + .init() + .await?; + loop { + let mut configs = informer.poll().await?.boxed(); + + // Currently, this does not handle None except to break the + // while. + while let Some(event) = configs.next().await { + handle_config(kube_interface, event?, config_map.clone()).await? + } + } +} + +/// This takes an event off the Configuration stream and delegates it to the +/// correct function based on the event type. +async fn handle_config( + kube_interface: &impl KubeInterface, + event: WatchEvent, + config_map: ConfigMap, +) -> Result<(), Box> { + trace!("handle_config - something happened to a configuration"); + match event { + WatchEvent::Added(config) => { + info!( + "handle_config - added DevCapConfig {}", + config.metadata.name + ); + tokio::spawn(async move { + handle_config_add(&config, config_map).await.unwrap(); + }); + Ok(()) + } + WatchEvent::Deleted(config) => { + info!( + "handle_config - deleted DevCapConfig {}", + config.metadata.name, + ); + handle_config_delete(kube_interface, &config, config_map).await?; + Ok(()) + } + // If a config is updated, delete all associated instances and device plugins and then recreate them to reflect updated config + WatchEvent::Modified(config) => { + info!( + "handle_config - modified DevCapConfig {}", + config.metadata.name, + ); + handle_config_delete(kube_interface, &config, config_map.clone()).await?; + tokio::spawn(async move { + handle_config_add(&config, config_map).await.unwrap(); + }); + Ok(()) + } + WatchEvent::Error(ref e) => { + error!("handle_config - error for DevCapConfig: {}", e); + Ok(()) + } + } +} + +/// This handles added Congfiguration by creating a new ConfigInfo for it and adding it to the ConfigMap. +/// Then calls a function to continually observe the availability of instances associated with the Configuration. +async fn handle_config_add( + config: &KubeAkriConfig, + config_map: ConfigMap, +) -> Result<(), Box> { + let config_protocol = config.spec.protocol.clone(); + let discovery_handler = protocols::get_discovery_handler(&config_protocol).unwrap(); + let discovery_results = discovery_handler.discover().await.unwrap(); + let config_name = config.metadata.name.clone(); + let config_uid = config.metadata.uid.as_ref().unwrap().clone(); + let config_namespace = config.metadata.namespace.as_ref().unwrap().clone(); + info!( + "handle_config_add - entered for DevCapConfig {} with visible_instances={:?}", + config.metadata.name, &discovery_results + ); + // Create a new instance map for this config and add it to the config map + let instance_map: InstanceMap = Arc::new(Mutex::new(HashMap::new())); + // Channel capacity: should only ever be sent once upon config deletion + let (stop_discovery_sender, stop_discovery_receiver) = mpsc::channel(1); + // Channel capacity: should only ever be sent once upon receiving stop watching message + let (finished_discovery_sender, _) = broadcast::channel(1); + let config_info = ConfigInfo { + instance_map: instance_map.clone(), + stop_discovery_sender, + finished_discovery_sender: finished_discovery_sender.clone(), + }; + config_map + .lock() + .await + .insert(config_name.clone(), config_info); + + let kube_interface = k8s::create_kube_interface(); + let config_spec = config.spec.clone(); + // Keep discovering instances until the config is deleted, signaled by a message from handle_config_delete + tokio::spawn(async move { + let periodic_dicovery = PeriodicDiscovery { + config_name, + config_uid, + config_namespace, + config_spec, + config_protocol, + instance_map, + }; + periodic_dicovery + .do_periodic_discovery( + &kube_interface, + stop_discovery_receiver, + finished_discovery_sender, + ) + .await + .unwrap(); + }) + .await?; + Ok(()) +} + +/// This handles a deleted Congfiguration. First, it ceases to discover instances associated with the Configuration. +/// Then, for each of the Configuration's Instances, it signals the DevicePluginService to shutdown, +/// and deletes the Instance CRD. +pub async fn handle_config_delete( + kube_interface: &impl KubeInterface, + config: &KubeAkriConfig, + config_map: ConfigMap, +) -> Result<(), Box> { + trace!( + "handle_config_delete - for config {} telling do_periodic_discovery to end", + config.metadata.name + ); + // Send message to stop observing instances' availability and waits until response is received + if config_map + .lock() + .await + .get(&config.metadata.name) + .unwrap() + .stop_discovery_sender + .clone() + .send(()) + .await + .is_ok() + { + let mut finished_discovery_receiver = config_map + .lock() + .await + .get(&config.metadata.name) + .unwrap() + .finished_discovery_sender + .subscribe(); + finished_discovery_receiver.recv().await.unwrap(); + trace!( + "handle_config_delete - for config {} received message that do_periodic_discovery ended", + config.metadata.name + ); + } else { + trace!( + "handle_config_delete - for config {} do_periodic_discovery receiver has been dropped", + config.metadata.name + ); + } + + // Get map of instances for the Configuration and then remove Configuration from ConfigMap + let instance_map: InstanceMap; + { + let mut config_map_locked = config_map.lock().await; + instance_map = config_map_locked + .get(&config.metadata.name) + .unwrap() + .instance_map + .clone(); + config_map_locked.remove(&config.metadata.name); + } + + // Shutdown Instances' DevicePluginServices and delete the Instances + let mut instance_map_locked = instance_map.lock().await; + let instances_to_delete_map = instance_map_locked.clone(); + let namespace = config.metadata.namespace.as_ref().unwrap(); + for (instance_name, instance_info) in instances_to_delete_map { + trace!( + "handle_config_delete - found Instance {} associated with deleted config {} ... sending message to end list_and_watch", + instance_name, + config.metadata.name + ); + instance_info + .list_and_watch_message_sender + .send(device_plugin_service::ListAndWatchMessageKind::End) + .unwrap(); + instance_map_locked.remove(&instance_name); + try_delete_instance(kube_interface, &instance_name, &namespace).await?; + } + + Ok(()) +} + +/// This deletes an Instance unless it has already been deleted by another node +async fn try_delete_instance( + kube_interface: &impl KubeInterface, + instance_name: &str, + instance_namespace: &str, +) -> Result<(), Box> { + match kube_interface + .delete_instance(instance_name, &instance_namespace) + .await + { + Ok(()) => { + trace!("delete_instance - deleted Instance {}", instance_name); + Ok(()) + } + Err(e) => { + // Check if already was deleted else return error + if let Err(_e) = kube_interface + .find_instance(&instance_name, &instance_namespace) + .await + { + trace!( + "delete_instance - discovered Instance {} already deleted", + instance_name + ); + Ok(()) + } else { + Err(e) + } + } + } +} + +/// Information required for periodic discovery +struct PeriodicDiscovery { + config_name: String, + config_uid: String, + config_namespace: String, + config_spec: Configuration, + config_protocol: ProtocolHandler, + instance_map: InstanceMap, +} + +impl PeriodicDiscovery { + /// This is spawned as a task for each Configuration and continues to periodically run + /// until the Config is deleted, at which point, this function is signaled to stop. + /// Looks up which instances are currently visible to the node. Passes this list to a function that + /// updates the ConnectivityStatus of the Configuration's Instances or deletes Instance CRDs if needed. + /// If a new instance becomes visible that isn't in the Configuration's InstanceMap, + /// a DevicePluginService and Instance CRD are created for it, and it is added to the InstanceMap. + async fn do_periodic_discovery( + &self, + kube_interface: &impl KubeInterface, + mut stop_discovery_receiver: mpsc::Receiver<()>, + finished_discovery_sender: broadcast::Sender<()>, + ) -> Result<(), Box> { + trace!( + "do_periodic_discovery - start for config {}", + self.config_name + ); + let protocol = protocols::get_discovery_handler(&self.config_protocol)?; + let shared = protocol.are_shared()?; + loop { + trace!( + "do_periodic_discovery - loop iteration for config {}", + &self.config_name + ); + let discovery_results = protocol.discover().await?; + let config_name = self.config_name.clone(); + let currently_visible_instances: HashMap = + discovery_results + .iter() + .map(|discovery_result| { + let instance_name = + get_device_instance_name(&discovery_result.digest, &config_name); + (instance_name, discovery_result.clone()) + }) + .collect(); + + // Update the connectivity status of instances and return list of visible instances that don't have Instance CRDs + let new_discovery_results = self + .update_connectivity_status(kube_interface, ¤tly_visible_instances, shared) + .await?; + + // If there are newly visible instances associated with a Config, make a device plugin and Instance CRD for them + if !new_discovery_results.is_empty() { + for discovery_result in new_discovery_results { + let config_name = config_name.clone(); + let instance_name = + get_device_instance_name(&discovery_result.digest, &config_name); + trace!( + "do_periodic_discovery - new instance {} came online", + instance_name + ); + let instance_properties = discovery_result.properties.clone(); + let config_spec = self.config_spec.clone(); + let instance_map = self.instance_map.clone(); + if let Err(e) = device_plugin_service::build_device_plugin( + instance_name, + config_name, + self.config_uid.clone(), + self.config_namespace.clone(), + config_spec, + shared, + instance_properties, + instance_map, + ) + .await + { + error!("do_periodic_discovery - error {} building device plugin ... trying again on next iteration", e); + } + } + } + if timeout( + Duration::from_secs(DISCOVERY_DELAY_SECS), + stop_discovery_receiver.recv(), + ) + .await + .is_ok() + { + trace!("do_periodic_discovery - for config {} received message to end ... sending message that finished and returning Ok", config_name); + finished_discovery_sender.send(()).unwrap(); + return Ok(()); + }; + } + } + + /// Takes in a list of currently visible instances and either updates an Instance's ConnectivityStatus or deletes an Instance. + /// If an instance is no longer visible then it's ConnectivityStatus is changed to Offline(time now). + /// The associated DevicePluginService checks its ConnectivityStatus before sending a response back to kubelet + /// and will send all unhealthy devices if its status is Offline, preventing kubelet from allocating any more pods to it. + /// An Instance CRD is deleted and it's DevicePluginService shutdown if its: + /// (A) shared instance is still not visible after 5 minutes or (B) unshared instance is still not visible on the next visibility check. + /// An unshared instance will be offline for between DISCOVERY_DELAY_SECS - 2 x DISCOVERY_DELAY_SECS + async fn update_connectivity_status( + &self, + kube_interface: &impl KubeInterface, + currently_visible_instances: &HashMap, + shared: bool, + ) -> Result, Box> + { + let instance_map_clone = self.instance_map.lock().await.clone(); + // Find all visible instances that do not have Instance CRDs yet + let new_discovery_results: Vec = currently_visible_instances + .iter() + .filter(|(name, _)| !instance_map_clone.contains_key(*name)) + .map(|(_, p)| p.clone()) + .collect(); + + for (instance, instance_info) in instance_map_clone { + if currently_visible_instances.contains_key(&instance) { + let connectivity_status = instance_info.connectivity_status; + // If instance is visible, make sure connectivity status is (updated to be) Online + if let ConnectivityStatus::Offline(_instant) = connectivity_status { + trace!( + "update_connectivity_status - instance {} that was temporarily offline is back online", + instance + ); + let list_and_watch_message_sender = instance_info.list_and_watch_message_sender; + let updated_instance_info = InstanceInfo { + connectivity_status: ConnectivityStatus::Online, + list_and_watch_message_sender: list_and_watch_message_sender.clone(), + }; + self.instance_map + .lock() + .await + .insert(instance.clone(), updated_instance_info); + list_and_watch_message_sender + .send(device_plugin_service::ListAndWatchMessageKind::Continue) + .unwrap(); + } + trace!( + "update_connectivity_status - instance {} still online", + instance + ); + } else { + // If the instance is not visible: + // // If the instance has not already been labeled offline, label it + // // If the instance has already been labeled offline + // // // shared - remove instance from map if grace period has elaspsed without the instance coming back online + // // // unshared - remove instance from map + match instance_info.connectivity_status { + ConnectivityStatus::Online => { + let sender = instance_info.list_and_watch_message_sender.clone(); + let updated_instance_info = InstanceInfo { + connectivity_status: ConnectivityStatus::Offline(Instant::now()), + list_and_watch_message_sender: instance_info + .list_and_watch_message_sender, + }; + self.instance_map + .lock() + .await + .insert(instance.clone(), updated_instance_info); + trace!( + "update_connectivity_status - instance {} went offline ... starting timer and forcing list_and_watch to continue", + instance + ); + sender + .send(device_plugin_service::ListAndWatchMessageKind::Continue) + .unwrap(); + } + ConnectivityStatus::Offline(instant) => { + let time_offline = instant.elapsed().as_secs(); + // If instance has been offline for longer than the grace period or it is unshared, terminate the associated device plugin + if !shared || time_offline >= SHARED_INSTANCE_OFFLINE_GRACE_PERIOD_SECS { + trace!("update_connectivity_status - instance {} has been offline too long ... terminating DevicePluginService", instance); + device_plugin_service::terminate_device_plugin_service( + &instance, + self.instance_map.clone(), + ) + .await?; + try_delete_instance(kube_interface, &instance, &self.config_namespace) + .await?; + } + } + } + } + } + Ok(new_discovery_results) + } +} + +#[cfg(test)] +mod config_action_tests { + use super::*; + use akri_shared::k8s::test_kube::MockKubeImpl; + use protocols::debug_echo::{DEBUG_ECHO_AVAILABILITY_CHECK_PATH, OFFLINE}; + use std::{env, fs}; + use tokio::sync::broadcast; + + async fn build_instance_map( + config: &KubeAkriConfig, + visibile_discovery_results: &mut Vec, + list_and_watch_message_receivers: &mut Vec< + broadcast::Receiver, + >, + connectivity_status: ConnectivityStatus, + ) -> InstanceMap { + // Set env vars for getting instances + env::set_var("AGENT_NODE_NAME", "node-a"); + env::set_var("ENABLE_DEBUG_ECHO", "yes"); + let protocol = config.spec.protocol.clone(); + let discovery_handler = protocols::get_discovery_handler(&protocol).unwrap(); + let discovery_results = discovery_handler.discover().await.unwrap(); + *visibile_discovery_results = discovery_results.clone(); + let instance_map: InstanceMap = Arc::new(Mutex::new( + discovery_results + .iter() + .map(|instance_info| { + let (list_and_watch_message_sender, list_and_watch_message_receiver) = + broadcast::channel(2); + list_and_watch_message_receivers.push(list_and_watch_message_receiver); + let instance_name = + get_device_instance_name(&instance_info.digest, &config.metadata.name); + ( + instance_name, + InstanceInfo { + list_and_watch_message_sender, + connectivity_status: connectivity_status.clone(), + }, + ) + }) + .collect(), + )); + instance_map + } + + #[tokio::test] + async fn test_handle_config_delete() { + let _ = env_logger::builder().is_test(true).try_init(); + let path_to_config = "../test/json/config-a.json"; + let dcc_json = fs::read_to_string(path_to_config).expect("Unable to read file"); + let config: KubeAkriConfig = serde_json::from_str(&dcc_json).unwrap(); + let config_name = config.metadata.name.clone(); + let mut list_and_watch_message_receivers = Vec::new(); + let mut visible_discovery_results = Vec::new(); + let mut mock = MockKubeImpl::new(); + let instance_map: InstanceMap = build_instance_map( + &config, + &mut visible_discovery_results, + &mut list_and_watch_message_receivers, + ConnectivityStatus::Online, + ) + .await; + let (stop_discovery_sender, mut stop_discovery_receiver) = mpsc::channel(2); + let (finished_discovery_sender, _) = broadcast::channel(2); + let mut map: HashMap = HashMap::new(); + map.insert( + config_name.clone(), + ConfigInfo { + stop_discovery_sender, + instance_map: instance_map.clone(), + finished_discovery_sender: finished_discovery_sender.clone(), + }, + ); + let config_map: ConfigMap = Arc::new(Mutex::new(map)); + + mock.expect_delete_instance() + .times(2) + .returning(move |_, _| Ok(())); + tokio::spawn(async move { + handle_config_delete(&mock, &config, config_map.clone()) + .await + .unwrap(); + // Assert that config is removed from map after it has been deleted + assert!(!config_map.lock().await.contains_key(&config_name)); + }); + + // Assert that handle_config_delete tells do_periodic_discovery to end + assert!(stop_discovery_receiver.recv().await.is_some()); + // Mimic do_periodic_discovery's response + finished_discovery_sender.send(()).unwrap(); + + // Assert list_and_watch is signaled to end for every instance associated with a config + let mut tasks = Vec::new(); + for mut receiver in list_and_watch_message_receivers { + tasks.push(tokio::spawn(async move { + assert_eq!( + receiver.recv().await.unwrap(), + device_plugin_service::ListAndWatchMessageKind::End + ); + })); + } + futures::future::join_all(tasks).await; + + // Assert that all instances have been removed from the instance map + assert_eq!(instance_map.lock().await.len(), 0); + } + + // 1: ConnectivityStatus of all instances that go offline is changed from Online to Offline + // 2: ConnectivityStatus of shared instances that come back online in under 5 minutes is changed from Offline to Online + // 3: ConnectivityStatus of unshared instances that come back online before next periodic discovery is changed from Offline to Online + #[tokio::test] + async fn test_update_connectivity_status() { + let _ = env_logger::builder().is_test(true).try_init(); + let path_to_config = "../test/json/config-a.json"; + let dcc_json = fs::read_to_string(path_to_config).expect("Unable to read file"); + let config: KubeAkriConfig = serde_json::from_str(&dcc_json).unwrap(); + let config_name = config.metadata.name.clone(); + let mut list_and_watch_message_receivers = Vec::new(); + let mut visible_discovery_results = Vec::new(); + let mock = MockKubeImpl::new(); + + // + // 1: Assert that ConnectivityStatus of instance that are no longer visible is changed to Offline + // + let instance_map: InstanceMap = build_instance_map( + &config, + &mut visible_discovery_results, + &mut list_and_watch_message_receivers, + ConnectivityStatus::Online, + ) + .await; + let shared = true; + // discover returns an empty vector when instances are offline + let no_visible_instances: HashMap = HashMap::new(); + let periodic_dicovery = PeriodicDiscovery { + config_name: config_name.clone(), + config_uid: config.metadata.uid.as_ref().unwrap().clone(), + config_namespace: config.metadata.namespace.as_ref().unwrap().clone(), + config_spec: config.spec.clone(), + config_protocol: config.spec.protocol.clone(), + instance_map: instance_map.clone(), + }; + periodic_dicovery + .update_connectivity_status(&mock, &no_visible_instances, shared) + .await + .unwrap(); + let unwrapped_instance_map = instance_map.lock().await.clone(); + for (_, instance_info) in unwrapped_instance_map { + assert_ne!( + instance_info.connectivity_status, + ConnectivityStatus::Online + ); + } + + // + // 2: Assert that ConnectivityStatus of shared instances that come back online in <5 mins is changed to Online + // + let instance_map: InstanceMap = build_instance_map( + &config, + &mut visible_discovery_results, + &mut list_and_watch_message_receivers, + ConnectivityStatus::Offline(Instant::now()), + ) + .await; + let shared = true; + let currently_visible_instances: HashMap = + visible_discovery_results + .iter() + .map(|instance_info| { + let instance_name = + get_device_instance_name(&instance_info.digest, &config_name); + (instance_name, instance_info.clone()) + }) + .collect(); + let periodic_dicovery = PeriodicDiscovery { + config_name: config_name.clone(), + config_uid: config.metadata.uid.as_ref().unwrap().clone(), + config_namespace: config.metadata.namespace.as_ref().unwrap().clone(), + config_spec: config.spec.clone(), + config_protocol: config.spec.protocol.clone(), + instance_map: instance_map.clone(), + }; + periodic_dicovery + .update_connectivity_status(&mock, ¤tly_visible_instances, shared) + .await + .unwrap(); + let unwrapped_instance_map = instance_map.lock().await.clone(); + for (_, instance_info) in unwrapped_instance_map { + assert_eq!( + instance_info.connectivity_status, + ConnectivityStatus::Online + ); + } + + // + // 3: Assert that ConnectivityStatus of unshared instances that come back online before next visibility check is changed to Online + // + let instance_map: InstanceMap = build_instance_map( + &config, + &mut visible_discovery_results, + &mut list_and_watch_message_receivers, + ConnectivityStatus::Offline(Instant::now()), + ) + .await; + let shared = false; + let periodic_dicovery = PeriodicDiscovery { + config_name: config_name.clone(), + config_uid: config.metadata.uid.as_ref().unwrap().clone(), + config_namespace: config.metadata.namespace.as_ref().unwrap().clone(), + config_spec: config.spec.clone(), + config_protocol: config.spec.protocol.clone(), + instance_map: instance_map.clone(), + }; + periodic_dicovery + .update_connectivity_status(&mock, ¤tly_visible_instances, shared) + .await + .unwrap(); + let unwrapped_instance_map = instance_map.lock().await.clone(); + for (_, instance_info) in unwrapped_instance_map { + assert_eq!( + instance_info.connectivity_status, + ConnectivityStatus::Online + ); + } + } + + /// Checks the termination case for when an unshared instance is still offline upon the second periodic discovery + /// Must be run independently since writing "OFFLINE" to DEBUG_ECHO_AVAILABILITY_CHECK_PATH in order to emulate + /// offline devices can clobber other tests run in parallel that are looking for online devices. + /// Run with: cargo test -- test_do_periodic_discovery --ignored + #[tokio::test] + #[ignore] + async fn test_do_periodic_discovery() { + let _ = env_logger::builder().is_test(true).try_init(); + // Set env vars + env::set_var("AGENT_NODE_NAME", "node-a"); + env::set_var("ENABLE_DEBUG_ECHO", "yes"); + // Make each get_instances check return an empty list of instances + let path_to_config = "../test/json/config-a.json"; + let dcc_json = fs::read_to_string(path_to_config).expect("Unable to read file"); + let config: KubeAkriConfig = serde_json::from_str(&dcc_json).unwrap(); + let config_name = config.metadata.name.clone(); + let config_uid = config.metadata.uid.as_ref().unwrap().clone(); + let config_namespace = config.metadata.namespace.as_ref().unwrap().clone(); + let protocol = config.spec.protocol.clone(); + let mut visible_discovery_results = Vec::new(); + let mut list_and_watch_message_receivers = Vec::new(); + let (mut watch_periph_tx, watch_periph_rx) = mpsc::channel(2); + let (finished_watching_tx, mut finished_watching_rx) = broadcast::channel(2); + let mut mock = MockKubeImpl::new(); + + // Set ConnectivityStatus of all instances in InstanceMap initially to Offline + let instance_map: InstanceMap = build_instance_map( + &config, + &mut visible_discovery_results, + &mut list_and_watch_message_receivers, + ConnectivityStatus::Offline(Instant::now()), + ) + .await; + + // + // Assert that when an unshared instance is already offline it is terminated + // + mock.expect_delete_instance() + .times(2) + .returning(move |_, _| Ok(())); + let instance_map_clone = instance_map.clone(); + // Change instances to be offline + fs::write(DEBUG_ECHO_AVAILABILITY_CHECK_PATH, OFFLINE).unwrap(); + tokio::spawn(async move { + let periodic_dicovery = PeriodicDiscovery { + config_name: config_name.clone(), + config_uid: config_uid.clone(), + config_namespace: config_namespace.clone(), + config_spec: config.spec, + config_protocol: protocol, + instance_map: instance_map_clone, + }; + periodic_dicovery + .do_periodic_discovery(&mock, watch_periph_rx, finished_watching_tx) + .await + .unwrap(); + }); + let mut tasks = Vec::new(); + for mut receiver in list_and_watch_message_receivers { + tasks.push(tokio::spawn(async move { + assert_eq!( + receiver.recv().await.unwrap(), + device_plugin_service::ListAndWatchMessageKind::End + ); + })); + } + futures::future::join_all(tasks).await; + + // Assert that all instances have been removed from the instance map + assert_eq!(instance_map.lock().await.len(), 0); + + watch_periph_tx.send(()).await.unwrap(); + // Assert that replies saying finished watching + assert!(finished_watching_rx.recv().await.is_ok()); + + // Reset file to be online + fs::write(DEBUG_ECHO_AVAILABILITY_CHECK_PATH, "ONLINE").unwrap(); + } +} diff --git a/agent/src/util/constants.rs b/agent/src/util/constants.rs new file mode 100644 index 0000000..f3dc51b --- /dev/null +++ b/agent/src/util/constants.rs @@ -0,0 +1,35 @@ +/// For unshared devices, Healthy means the device is discoverable. +/// For shared devices, Healthy means the device is either unused or used by this node. +pub const HEALTHY: &str = "Healthy"; + +/// For unshared devices, Unhealthy means the device is not discoverable. +/// For shared devices, UnHealthy means that the device shared and used already by another node. +pub const UNHEALTHY: &str = "Unhealthy"; + +/// Current version of the API supported by kubelet. +pub const K8S_DEVICE_PLUGIN_VERSION: &str = "v1beta1"; + +/// DevicePluginPath is the folder the kubelet expects to find Device-Plugin sockets. Only privileged pods have access to this path. +#[cfg(not(test))] +pub const DEVICE_PLUGIN_PATH: &str = "/var/lib/kubelet/device-plugins/"; +/// Path for testing `DevicePluginService` +#[cfg(test)] +pub const DEVICE_PLUGIN_PATH: &str = "/tmp/device-plugins/"; + +/// Path of the Kubelet registry socket +pub const KUBELET_SOCKET: &str = "/var/lib/kubelet/device-plugins/kubelet.sock"; + +/// Maximum length of time `list_and_watch` will sleep before sending kubelet another list of virtual devices +pub const LIST_AND_WATCH_SLEEP_SECS: u64 = 60; + +/// Length of time to sleep between instance discovery checks +pub const DISCOVERY_DELAY_SECS: u64 = 10; + +/// Length of time a shared instance can be offline before it's `DevicePluginService` is shutdown. +pub const SHARED_INSTANCE_OFFLINE_GRACE_PERIOD_SECS: u64 = 300; + +/// Length of time to sleep between slot reconciliation checks +pub const SLOT_RECONCILIATION_CHECK_DELAY_SECS: u64 = 10; + +/// Length of time a slot can be unused before slot reconciliation relaims it +pub const SLOT_RECONCILIATION_SLOT_GRACE_PERIOD_SECS: u64 = 300; diff --git a/agent/src/util/crictl_containers.rs b/agent/src/util/crictl_containers.rs new file mode 100644 index 0000000..8a44179 --- /dev/null +++ b/agent/src/util/crictl_containers.rs @@ -0,0 +1,135 @@ +use akri_shared::akri::AKRI_SLOT_ANNOTATION_NAME; +use std::collections::{HashMap, HashSet}; + +/// Output from crictl query +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +struct CriCtlOutput { + containers: Vec, +} + +/// Container from crictl query +#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +struct CriCtlContainer { + annotations: HashMap, +} + +/// This gets the usage slots for an instance by getting the annotations that were stored at id `AKRI_SLOT_ANNOTATION_NAME` during allocate. +pub fn get_container_slot_usage(crictl_output: &str) -> HashSet { + match serde_json::from_str::(&crictl_output) { + Ok(crictl_output_parsed) => crictl_output_parsed + .containers + .iter() + .filter_map(|container| { + container + .annotations + .get(&AKRI_SLOT_ANNOTATION_NAME.to_string()) + }) + .map(|string_ref| string_ref.to_string()) + .collect(), + Err(e) => { + trace!( + "handle_crictl_output - failed to parse crictl output: {:?} => [{:?}]", + e, + &crictl_output + ); + HashSet::default() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn get_container_str(annotation: &str) -> String { + format!("{{ \ + \"id\": \"46afc04a13ac21d73ff93843efd39590d66927d9b5d743d239542cf2f6de703e\", \ + \"podSandboxId\": \"9094d7341170ecbc6fb0a6a72ba449c8ea98d3267c60e06d815d03102ca7a3e6\", \ + \"metadata\": {{ \ + \"name\": \"akri-agent\", \ + \"attempt\": 0 \ + }}, \ + \"image\": {{ \ + \"image\": \"akri.sh/agent@sha256:86bb6234353129bcae170cfc7db5ad5f282cfc3495555a39aa88042948491850\" \ + }}, \ + \"imageRef\": \"sha256:1305fb97b2db8e9aa715af6a6cd0711986da7935bcbb98f6363aaa5b86163072\", \ + \"state\": \"CONTAINER_RUNNING\", \ + \"createdAt\": \"1587749289000000000\", \ + \"labels\": {{ \ + \"io.kubernetes.container.name\": \"akri-agent\", \ + \"io.kubernetes.pod.name\": \"akri-agent-daemonset-lt2gc\", \ + \"io.kubernetes.pod.namespace\": \"default\", \ + \"io.kubernetes.pod.uid\": \"1ed0098d-8d6f-4001-8192-f690f9b8ae98\" \ + }}, \ + \"annotations\": {{ \ + {} \ + \"io.kubernetes.container.hash\": \"34d65174\", \ + \"io.kubernetes.container.restartCount\": \"0\", \ + \"io.kubernetes.container.terminationMessagePath\": \"/dev/termination-log\", \ + \"io.kubernetes.container.terminationMessagePolicy\": \"File\", \ + \"io.kubernetes.pod.terminationGracePeriod\": \"30\" \ + }} \ + }}", + annotation) + } + + #[test] + fn test_get_container_slot_usage() { + let _ = env_logger::builder().is_test(true).try_init(); + + // Empty output + assert_eq!(HashSet::::new(), get_container_slot_usage(r#""#)); + // Empty json output + assert_eq!(HashSet::::new(), get_container_slot_usage(r#"{}"#)); + // Expected output with no containers + assert_eq!( + HashSet::::new(), + get_container_slot_usage(r#"{\"containers\": []}"#) + ); + // Output with syntax error + assert_eq!( + HashSet::::new(), + get_container_slot_usage(r#"{ddd}"#) + ); // syntax error + // Expected output with no slot + assert_eq!( + HashSet::::new(), + get_container_slot_usage(&format!( + "{{ \"containers\": [ {} ] }}", + &get_container_str("") + )) + ); + // Expected output with slot (including unexpected property) + let mut expected = HashSet::new(); + expected.insert("foo".to_string()); + assert_eq!( + expected, + get_container_slot_usage(&format!( + "{{ \"ddd\": \"\", \"containers\": [ {} ] }}", + &get_container_str("\"akri.agent.slot\": \"foo\",") + )) + ); + // Expected output with slot + assert_eq!( + expected, + get_container_slot_usage(&format!( + "{{ \"containers\": [ {} ] }}", + &get_container_str("\"akri.agent.slot\": \"foo\",") + )) + ); + // Expected output with multiple containers + let mut expected_2 = HashSet::new(); + expected_2.insert("foo1".to_string()); + expected_2.insert("foo2".to_string()); + assert_eq!( + expected_2, + get_container_slot_usage(&format!( + "{{ \"containers\": [ {}, {} ] }}", + &get_container_str("\"akri.agent.slot\": \"foo1\","), + &get_container_str("\"akri.agent.slot\": \"foo2\","), + )) + ); + } +} diff --git a/agent/src/util/device_plugin_service.rs b/agent/src/util/device_plugin_service.rs new file mode 100644 index 0000000..f76a484 --- /dev/null +++ b/agent/src/util/device_plugin_service.rs @@ -0,0 +1,1684 @@ +use super::constants::{ + DEVICE_PLUGIN_PATH, HEALTHY, K8S_DEVICE_PLUGIN_VERSION, KUBELET_SOCKET, + LIST_AND_WATCH_SLEEP_SECS, UNHEALTHY, +}; +use super::v1beta1; +use super::v1beta1::{ + device_plugin_server::{DevicePlugin, DevicePluginServer}, + registration_client, AllocateRequest, AllocateResponse, DevicePluginOptions, Empty, + ListAndWatchResponse, PreStartContainerRequest, PreStartContainerResponse, +}; +use akri_shared::{ + akri::{ + configuration::{Configuration, ProtocolHandler}, + instance::Instance, + retry::{random_delay, MAX_INSTANCE_UPDATE_TRIES}, + AKRI_PREFIX, AKRI_SLOT_ANNOTATION_NAME, + }, + k8s, + k8s::KubeInterface, +}; +use futures::stream::TryStreamExt; +use log::{error, info, trace}; +use std::{ + collections::HashMap, + convert::TryFrom, + env, + path::Path, + sync::Arc, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; +use tokio::{ + net::UnixListener, + net::UnixStream, + sync::{broadcast, mpsc, Mutex}, + task, + time::{delay_for, timeout}, +}; +use tonic::{ + transport::{Endpoint, Server, Uri}, + Code, Request, Response, Status, +}; +use tower::service_fn; + +/// Message sent in channel to `list_and_watch`. +/// Dictates what action `list_and_watch` should take upon being awoken. +#[derive(PartialEq, Clone, Debug)] +pub enum ListAndWatchMessageKind { + /// Prematurely continue looping + Continue, + /// Stop looping + End, +} + +/// Describes the discoverability of an instance for this node +#[derive(PartialEq, Debug, Clone)] +pub enum ConnectivityStatus { + /// Was discovered + Online, + /// Could not be discovered. Instant contains time at which it was no longer discovered. + Offline(Instant), +} + +/// Contains an Instance's state +#[derive(Clone, Debug)] +pub struct InstanceInfo { + /// Sender to tell `list_and_watch` to either prematurely continue looping or end + pub list_and_watch_message_sender: broadcast::Sender, + /// Instance's `ConnectivityStatus` + pub connectivity_status: ConnectivityStatus, +} + +pub type InstanceMap = Arc>>; + +/// Kubernetes Device-Plugin for an Instance. +/// +/// `DevicePluginService` implements Kubernetes Device-Plugin v1beta1 API specification +/// defined in a public proto file (imported here at agent/proto/pluginapi.proto). +/// The code generated from pluginapi.proto can be found in `agent/src/util/v1beta1.rs`. +/// Each `DevicePluginService` has an associated Instance and Configuration. +/// Serves a unix domain socket, sending and receiving messages to/from kubelet. +/// Kubelet is its client, calling each of its methods. +#[derive(Clone)] +pub struct DevicePluginService { + /// Instance CRD name + instance_name: String, + /// Socket endpoint + endpoint: String, + /// Instance's Configuration + config: Configuration, + /// Name of Instance's Configuration CRD + config_name: String, + /// UID of Instance's Configuration CRD + config_uid: String, + /// Namespace of Instance's Configuration CRD + config_namespace: String, + /// Instance is [not]shared + shared: bool, + /// Hostname of node this Device Plugin is running on + node_name: String, + /// Information that must be communicated with broker. Stored in Instance CRD as metadata. + instance_properties: HashMap, + /// Map of all Instances that have the same Configuration CRD as this one + instance_map: InstanceMap, + /// Receiver for list_and_watch continue or end messages + /// Note: since the tonic grpc generated list_and_watch definition takes in &self, + /// using broadcast sender instead of mpsc receiver + /// Can clone broadcast sender and subscribe receiver to use in spawned thread in list_and_watch + list_and_watch_message_sender: broadcast::Sender, + /// Upon send, terminates function that acts as the shutdown signal for this service + server_ender_sender: mpsc::Sender<()>, +} + +#[tonic::async_trait] +impl DevicePlugin for DevicePluginService { + /// Returns options to be communicated with kubelet Device Manager + async fn get_device_plugin_options( + &self, + _request: Request, + ) -> Result, Status> { + trace!("get_device_plugin_options - kubelet called get_device_plugin_options"); + let resp = DevicePluginOptions { + pre_start_required: true, + }; + Ok(Response::new(resp)) + } + + type ListAndWatchStream = mpsc::Receiver>; + + /// Called by Kubelet right after the DevicePluginService registers with Kubelet. + /// Returns a stream of List of "virtual" Devices over a channel. + /// Since Kubernetes designed Device-Plugin so that multiple consumers can use a Device, + /// "virtual" Devices are reservation slots for using the Device or Instance in akri terms. + /// The number of "virtual" Devices (length of `ListAndWatchResponse`) is determined by Instance.capacity. + /// Whenever Instance state changes or an Instance disapears, `list_and_watch` returns the new list. + /// Runs until receives message to end due to Instance disappearing or Configuration being deleted. + async fn list_and_watch( + &self, + _request: Request, + ) -> Result, Status> { + info!( + "list_and_watch - kubelet called list_and_watch for instance {}", + self.instance_name + ); + let dps = Arc::new(self.clone()); + let mut list_and_watch_message_receiver = self.list_and_watch_message_sender.subscribe(); + + // Create a channel that list_and_watch can periodically send updates to kubelet on + let (mut kubelet_update_sender, kubelet_update_receiver) = mpsc::channel(4); + // Spawn thread so can send kubelet the receiving end of the channel to listen on + tokio::spawn(async move { + let mut keep_looping = true; + #[cfg(not(test))] + let kube_interface = Arc::new(k8s::create_kube_interface()); + + // Try to create an Instance CRD for this plugin and add it to the global InstanceMap else shutdown + #[cfg(not(test))] + { + if let Err(e) = try_create_instance(dps.clone(), kube_interface.clone()).await { + error!( + "list_and_watch - ending service because could not create instance {} with error {}", + dps.instance_name, + e + ); + dps.server_ender_sender.clone().send(()).await.unwrap(); + keep_looping = false; + } + } + + while keep_looping { + trace!( + "list_and_watch - loop iteration for Instance {}", + dps.instance_name + ); + + let virtual_devices: Vec; + #[cfg(test)] + { + virtual_devices = + build_unhealthy_virtual_devices(dps.config.capacity, &dps.instance_name); + } + #[cfg(not(test))] + { + virtual_devices = + build_list_and_watch_response(dps.clone(), kube_interface.clone()) + .await + .unwrap(); + } + + let resp = v1beta1::ListAndWatchResponse { + devices: virtual_devices, + }; + + // Send virtual devices list back to kubelet + if let Err(e) = kubelet_update_sender.send(Ok(resp)).await { + trace!( + "list_and_watch - for Instance {} kubelet no longer receiving with error {}", + dps.instance_name, + e + ); + // This means kubelet is down/has been restarted. Remove instance from instance map so + // do_periodic_discovery will create a new device plugin service for this instance. + dps.instance_map.lock().await.remove(&dps.instance_name); + dps.server_ender_sender.clone().send(()).await.unwrap(); + keep_looping = false; + } + // Sleep for LIST_AND_WATCH_SLEEP_SECS unless receive message to shutdown the server + // or continue (and send another list of devices) + match timeout( + Duration::from_secs(LIST_AND_WATCH_SLEEP_SECS), + list_and_watch_message_receiver.recv(), + ) + .await + { + Ok(message) => { + // If receive message to end list_and_watch, send list of unhealthy devices + // and shutdown the server by sending message on server_ender_sender channel + if message == Ok(ListAndWatchMessageKind::End) { + trace!( + "list_and_watch - for Instance {} received message to end", + dps.instance_name + ); + let devices = build_unhealthy_virtual_devices( + dps.config.capacity, + &dps.instance_name, + ); + kubelet_update_sender.send(Ok(v1beta1::ListAndWatchResponse { devices })) + .await + .unwrap(); + dps.server_ender_sender.clone().send(()).await.unwrap(); + keep_looping = false; + } + } + Err(_) => trace!( + "list_and_watch - for Instance {} did not receive a message for {} seconds ... continuing", dps.instance_name, LIST_AND_WATCH_SLEEP_SECS + ), + } + } + trace!("list_and_watch - for Instance {} ending", dps.instance_name); + }); + Ok(Response::new(kubelet_update_receiver)) + } + + /// Kubelet calls allocate during pod creation. + /// This means kubelet is trying to reserve a usage slot (virtual Device) of the Instance for this node. + /// Returns error if cannot reserve that slot. + async fn allocate( + &self, + requests: Request, + ) -> Result, Status> { + info!( + "allocate - kubelet called allocate for Instance {}", + self.instance_name + ); + let kube_interface = Arc::new(k8s::create_kube_interface()); + match self.internal_allocate(requests, kube_interface).await { + Ok(resp) => Ok(resp), + Err(e) => Err(e), + } + } + + /// Should never be called, as indicated by DevicePluginService during registration. + async fn pre_start_container( + &self, + _request: Request, + ) -> Result, Status> { + error!( + "pre_start_container - kubelet called pre_start_container for Instance {}", + self.instance_name + ); + Ok(Response::new(v1beta1::PreStartContainerResponse {})) + } +} + +impl DevicePluginService { + /// Called when kubelet is trying to reserve for this node a usage slot (or virtual device) of the Instance. + /// Tries to update Instance CRD to reserve the requested slot. If cannot reserve that slot, forces `list_and_watch` to continue + /// (sending kubelet the latest list of slots) and returns error, so kubelet will not schedule the pod to this node. + async fn internal_allocate( + &self, + requests: Request, + kube_interface: Arc, + ) -> Result, Status> { + let mut container_responses: Vec = Vec::new(); + + for request in requests.into_inner().container_requests { + trace!( + "internal_allocate - for Instance {} handling request {:?}", + &self.instance_name, + request, + ); + let mut akri_annotations = std::collections::HashMap::new(); + for device_usage_id in request.devices_i_ds { + trace!( + "internal_allocate - for Instance {} processing request for device usage slot id {}", + &self.instance_name, + device_usage_id + ); + + akri_annotations.insert( + AKRI_SLOT_ANNOTATION_NAME.to_string(), + device_usage_id.clone(), + ); + + if let Err(e) = try_update_instance_device_usage( + &device_usage_id, + &self.node_name, + &self.instance_name, + &self.config_namespace, + kube_interface.clone(), + ) + .await + { + trace!("internal_allocate - could not assign {} slot to {} node ... forcing list_and_watch to continue", device_usage_id, &self.node_name); + self.list_and_watch_message_sender + .send(ListAndWatchMessageKind::Continue) + .unwrap(); + return Err(e); + } + + trace!( + "internal_allocate - finished processing device_usage_id {}", + device_usage_id + ); + } + // Successfully reserved device_usage_slot[s] for this node. + // Add response to list of responses + let response = build_container_allocate_response( + akri_annotations, + &self.instance_properties, + &self.config.protocol, + ); + container_responses.push(response); + } + trace!( + "internal_allocate - for Instance {} returning responses", + &self.instance_name + ); + Ok(Response::new(v1beta1::AllocateResponse { + container_responses, + })) + } +} + +/// This returns the value that should be inserted at `device_usage_id` slot for an instance else an error. +/// # More details +/// Cases based on the usage slot (`device_usage_id`) value +/// 1. device_usage[id] == "" ... this means that the device is available for use +/// * return this node name +/// 2. device_usage[id] == self.nodeName ... this means THIS node previously used id, but the DevicePluginManager knows that this is no longer true +/// * return "" +/// 3. device_usage[id] == ... this means that we believe this device is in use by another node and should be marked unhealthy +/// * return error +/// 4. No corresponding id found ... this is an unknown error condition (BAD) +/// * return error +fn get_slot_value( + device_usage_id: &str, + node_name: &str, + instance: &Instance, +) -> Result { + if let Some(allocated_node) = instance.device_usage.get(device_usage_id) { + if allocated_node == "" { + Ok(node_name.to_string()) + } else if allocated_node == node_name { + Ok("".to_string()) + } else { + trace!("internal_allocate - request for device slot {} previously claimed by a diff node {} than this one {} ... indicates the device on THIS node must be marked unhealthy, invoking ListAndWatch ... returning failure, next scheduling should succeed!", device_usage_id, allocated_node, node_name); + Err(Status::new( + Code::Unknown, + "Requested device already in use", + )) + } + } else { + // No corresponding id found + trace!( + "internal_allocate - could not find {} id in device_usage", + device_usage_id + ); + Err(Status::new( + Code::Unknown, + "Could not find device usage slot", + )) + } +} + +/// This tries up to `MAX_INSTANCE_UPDATE_TRIES` to update the requested slot of the Instance with the appropriate value (either "" to clear slot or node_name). +/// It cannot be assumed that this will successfully update Instance on first try since Device Plugins on other nodes may be simultaneously trying to update the Instance. +/// This returns an error if slot does not need to be updated or `MAX_INSTANCE_UPDATE_TRIES` attempted. +async fn try_update_instance_device_usage( + device_usage_id: &str, + node_name: &str, + instance_name: &str, + instance_namespace: &str, + kube_interface: Arc, +) -> Result<(), Status> { + let mut instance: Instance; + for x in 0..MAX_INSTANCE_UPDATE_TRIES { + // Grab latest instance + match kube_interface + .find_instance(&instance_name, &instance_namespace) + .await + { + Ok(instance_object) => instance = instance_object.spec, + Err(_) => { + trace!( + "internal_allocate - could not find Instance {}", + instance_name + ); + return Err(Status::new( + Code::Unknown, + format!("Could not find Instance {}", instance_name), + )); + } + } + + // at this point, `value` should either be: + // * `node_name`: meaning that this node is claiming this slot + // * "": meaning this node previously claimed this slot, but kubelet + // knows that claim is no longer valid. In this case, reset the + // slot (which triggers each node to set the slot as Healthy) to + // allow a fair rescheduling of the workload + let value = get_slot_value(device_usage_id, node_name, &instance)?; + instance + .device_usage + .insert(device_usage_id.to_string(), value.clone()); + + match kube_interface + .update_instance(&instance, &instance_name, &instance_namespace) + .await + { + Ok(()) => { + if value == node_name { + return Ok(()); + } else { + return Err(Status::new(Code::Unknown, "Devices are in inconsistent state, updated device usage, please retry scheduling")); + } + } + Err(e) => { + if x == (MAX_INSTANCE_UPDATE_TRIES - 1) { + trace!("internal_allocate - update_instance returned error [{}] after max tries ... returning error", e); + return Err(Status::new(Code::Unknown, "Could not update Instance")); + } + } + } + random_delay().await; + } + Ok(()) +} + +/// This sets the volume mounts and environment variables according to the instance's protocol. +fn build_container_allocate_response( + annotations: HashMap, + instance_properties: &HashMap, + protocol: &ProtocolHandler, +) -> v1beta1::ContainerAllocateResponse { + let mut mounts: Vec = Vec::new(); + + // Set mounts according to protocol + match protocol { + ProtocolHandler::udev(_handler_config) => { + trace!("get_volumes_and_mounts - setting volumes and mounts for udev protocol"); + mounts = instance_properties + .iter() + .map(|(_id, devpath)| v1beta1::Mount { + container_path: devpath.clone(), + host_path: devpath.clone(), + read_only: true, + }) + .collect(); + } + _ => trace!("get_volumes_and_mounts - no mounts or volumes required by this protocol"), + } + + // Create response, setting environment variables to be an instance's properties (specified by protocol) + v1beta1::ContainerAllocateResponse { + annotations, + mounts, + envs: instance_properties.clone(), + ..Default::default() + } +} + +/// Try to find Instance CRD for this instance or create one and add it to the global InstanceMap +/// If a Config does not exist for this instance, return error. +/// This is most likely caused by deletion of a Config right after adding it, in which case +/// `handle_config_delete` fails to delete this instance because kubelet has yet to call `list_and_watch` +async fn try_create_instance( + dps: Arc, + kube_interface: Arc, +) -> Result<(), Box> { + // Make sure Configuration exists for instance + if let Err(e) = kube_interface + .find_configuration(&dps.config_name, &dps.config_namespace) + .await + { + error!( + "try_create_instance - no Configuration for device {} ... returning error", + dps.instance_name + ); + return Err(e); + } + + let device_usage: std::collections::HashMap = (0..dps.config.capacity) + .map(|x| (format!("{}-{}", dps.instance_name, x), "".to_string())) + .collect(); + let instance = Instance { + configuration_name: dps.config_name.clone(), + shared: dps.shared, + nodes: vec![dps.node_name.clone()], + device_usage, + metadata: dps.instance_properties.clone(), + rbac: "rbac".to_string(), + }; + + // Try up to MAX_INSTANCE_UPDATE_TRIES to create or update instance, breaking on success + for x in 0..MAX_INSTANCE_UPDATE_TRIES { + // First check if instance already exists + match kube_interface + .find_instance(&dps.instance_name, &dps.config_namespace) + .await + { + Ok(mut instance_object) => { + trace!( + "try_create_instance - discovered Instance {} already created", + dps.instance_name + ); + + // Check if instance's node list already contains this node, possibly due to device plugin failure and restart + if !instance_object.spec.nodes.contains(&dps.node_name) { + instance_object.spec.nodes.push(dps.node_name.clone()); + match kube_interface + .update_instance( + &instance_object.spec, + &instance_object.metadata.name, + &dps.config_namespace, + ) + .await + { + Ok(()) => { + trace!( + "try_create_instance - updated Instance {} to include {}", + dps.instance_name, + dps.node_name + ); + break; + } + Err(e) => { + trace!("try_create_instance - call to update_instance returned with error {} on try # {} of {}", e, x, MAX_INSTANCE_UPDATE_TRIES); + if x == (MAX_INSTANCE_UPDATE_TRIES - 1) { + return Err(e); + } + } + }; + } else { + break; + } + } + Err(_) => { + match kube_interface + .create_instance( + &instance, + &dps.instance_name, + &dps.config_namespace, + &dps.config_name, + &dps.config_uid, + ) + .await + { + Ok(()) => { + trace!( + "try_create_instance - created Instance with name {}", + dps.instance_name + ); + break; + } + Err(e) => { + trace!("try_create_instance - couldn't create instance with error {} on try # {} of {}", e, x, MAX_INSTANCE_UPDATE_TRIES); + if x == MAX_INSTANCE_UPDATE_TRIES - 1 { + return Err(e); + } + } + } + } + } + random_delay().await; + } + + // Successfully created or updated instance. Add it to instance_map. + dps.instance_map.lock().await.insert( + dps.instance_name.clone(), + InstanceInfo { + list_and_watch_message_sender: dps.list_and_watch_message_sender.clone(), + connectivity_status: ConnectivityStatus::Online, + }, + ); + + Ok(()) +} + +/// Returns list of "virtual" Devices and their health. +/// If the instance is offline, returns all unhealthy virtual Devices. +async fn build_list_and_watch_response( + dps: Arc, + kube_interface: Arc, +) -> Result, Box> { + info!( + "build_list_and_watch_response -- for Instance {} entered", + dps.instance_name + ); + + // If instance has been removed from map, send back all unhealthy device slots + if !dps + .instance_map + .lock() + .await + .contains_key(&dps.instance_name) + { + trace!("build_list_and_watch_response - Instance {} removed from map ... returning unhealthy devices", dps.instance_name); + return Ok(build_unhealthy_virtual_devices( + dps.config.capacity, + &dps.instance_name, + )); + } + // If instance is offline, send back all unhealthy device slots + if dps + .instance_map + .lock() + .await + .get(&dps.instance_name) + .unwrap() + .connectivity_status + != ConnectivityStatus::Online + { + trace!("build_list_and_watch_response - device for Instance {} is offline ... returning unhealthy devices", dps.instance_name); + return Ok(build_unhealthy_virtual_devices( + dps.config.capacity, + &dps.instance_name, + )); + } + + trace!( + "build_list_and_watch_response -- device for Instance {} is online", + dps.instance_name + ); + + match kube_interface + .find_instance(&dps.instance_name, &dps.config_namespace) + .await + { + Ok(kube_akri_instance) => Ok(build_virtual_devices( + &kube_akri_instance.spec.device_usage, + kube_akri_instance.spec.shared, + &dps.node_name, + )), + Err(_) => { + trace!("build_list_and_watch_response - could not find instance {} so returning unhealthy devices", dps.instance_name); + Ok(build_unhealthy_virtual_devices( + dps.config.capacity, + &dps.instance_name, + )) + } + } +} + +/// This builds a list of unhealthy virtual Devices. +fn build_unhealthy_virtual_devices(capacity: i32, instance_name: &str) -> Vec { + let mut devices: Vec = Vec::new(); + for x in 0..capacity { + let device = v1beta1::Device { + id: format!("{}-{}", instance_name, x), + health: UNHEALTHY.to_string(), + }; + trace!( + "build_unhealthy_virtual_devices -- for Instance {} reporting unhealthy devices for device with name [{}] and health: [{}]", + instance_name, + device.id, + device.health, + ); + devices.push(device); + } + devices +} + +/// This builds a list of virtual Devices, determining the health of each virtual Device as follows: +/// Healthy if it is available to be used by this node or Unhealthy if it is already taken by another node. +fn build_virtual_devices( + device_usage: &HashMap, + shared: bool, + node_name: &str, +) -> Vec { + let mut devices: Vec = Vec::new(); + for (device_name, allocated_node) in device_usage { + // Throw error if unshared resource is reserved by another node + if !shared && allocated_node != "" && allocated_node != node_name { + panic!("build_virtual_devices - unshared device reserved by a different node"); + } + // Advertise the device as Unhealthy if it is + // USED by !this_node && SHARED + let unhealthy = shared && allocated_node != "" && allocated_node != node_name; + let health = if unhealthy { + UNHEALTHY.to_string() + } else { + HEALTHY.to_string() + }; + trace!( + "build_virtual_devices - [shared = {}] device with name [{}] and health: [{}]", + shared, + device_name, + health + ); + devices.push(v1beta1::Device { + id: device_name.clone(), + health, + }); + } + devices +} + +/// This sends message to end `list_and_watch` and removes instance from InstanceMap. +/// Called when an instance has been offline for too long. +pub async fn terminate_device_plugin_service( + instance_name: &str, + instance_map: InstanceMap, +) -> Result<(), Box> { + let mut instance_map = instance_map.lock().await; + trace!( + "terminate_device_plugin_service -- forcing list_and_watch to end for Instance {}", + instance_name + ); + instance_map + .get(instance_name) + .unwrap() + .list_and_watch_message_sender + .send(ListAndWatchMessageKind::End) + .unwrap(); + + trace!( + "terminate_device_plugin_service -- removing Instance {} from instance_map", + instance_name + ); + instance_map.remove(instance_name); + Ok(()) +} + +/// This creates a new DevicePluginService for an instance and registers it with kubelet +pub async fn build_device_plugin( + instance_name: String, + config_name: String, + config_uid: String, + config_namespace: String, + config: Configuration, + shared: bool, + instance_properties: HashMap, + instance_map: InstanceMap, +) -> Result<(), Box> { + info!("build_device_plugin - entered for device {}", instance_name); + let capability_id: String = format!("{}/{}", AKRI_PREFIX, instance_name); + let unique_time = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH)?; + let device_endpoint: String = format!("{}-{}.sock", instance_name, unique_time.as_secs()); + let socket_path: String = format!( + "{}{}", + DEVICE_PLUGIN_PATH.to_string(), + device_endpoint.clone() + ); + // Channel capacity set to 6 because 3 possible senders (allocate, update_connectivity_status, and handle_config_delete) + // and and receiver only periodically checks channel + let (list_and_watch_message_sender, _) = broadcast::channel(6); + // Channel capacity set to 2 because worst case both register and list_and_watch send messages at same time and receiver is always listening + let (server_ender_sender, server_ender_receiver) = mpsc::channel(2); + let device_plugin_service = DevicePluginService { + instance_name: instance_name.clone(), + endpoint: device_endpoint.clone(), + config, + config_name: config_name.clone(), + config_uid: config_uid.clone(), + config_namespace: config_namespace.clone(), + shared, + node_name: env::var("AGENT_NODE_NAME")?, + instance_properties, + instance_map: instance_map.clone(), + list_and_watch_message_sender: list_and_watch_message_sender.clone(), + server_ender_sender: server_ender_sender.clone(), + }; + + serve( + device_plugin_service, + socket_path.clone(), + server_ender_receiver, + ) + .await?; + + register( + capability_id, + device_endpoint, + &instance_name, + server_ender_sender, + ) + .await?; + + Ok(()) +} + +/// This acts as a signal future to gracefully shutdown DevicePluginServer upon its completion. +/// Ends when it receives message from `list_and_watch`. +async fn shutdown_signal(mut server_ender_receiver: mpsc::Receiver<()>) { + match server_ender_receiver.recv().await { + Some(_) => trace!( + "shutdown_signal - received signal ... device plugin service gracefully shutting down" + ), + None => trace!("shutdown_signal - connection to server_ender_sender closed ... error"), + } +} + +// This serves DevicePluginServer +async fn serve( + device_plugin_service: DevicePluginService, + socket_path: String, + server_ender_receiver: mpsc::Receiver<()>, +) -> Result<(), Box> { + info!( + "serve - creating a device plugin server that will listen at: {}", + socket_path + ); + tokio::fs::create_dir_all(Path::new(&socket_path[..]).parent().unwrap()) + .await + .expect("Failed to create dir at socket path"); + let mut uds = UnixListener::bind(socket_path.clone()).expect("Failed to bind to socket path"); + let service = DevicePluginServer::new(device_plugin_service); + let socket_path_to_delete = socket_path.clone(); + task::spawn(async move { + Server::builder() + .add_service(service) + .serve_with_incoming_shutdown( + uds.incoming().map_ok(unix::UnixStream), + shutdown_signal(server_ender_receiver), + ) + .await + .unwrap(); + trace!( + "serve - gracefully shutdown ... deleting socket {}", + socket_path_to_delete + ); + // Socket may already be deleted in the case of kubelet restart + std::fs::remove_file(socket_path_to_delete).unwrap_or(()); + }); + + // Test that server is running, trying for at most 10 seconds + // Similar to grpc.timeout, which is yet to be implemented for tonic + // See issue: https://github.com/hyperium/tonic/issues/75 + let mut connected = false; + let start = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + let start_plus_10 = start + 10; + + while (SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs() + < start_plus_10) + && !connected + { + let path = socket_path.clone(); + if let Ok(_v) = Endpoint::try_from("lttp://[::]:50051")? + .connect_with_connector(service_fn(move |_: Uri| UnixStream::connect(path.clone()))) + .await + { + connected = true + } else { + delay_for(Duration::from_secs(1)).await + } + } + + if !connected { + error!( + "serve - could not connect to Device Plugin server on socket {}", + socket_path + ); + } + Ok(()) +} + +/// This registers DevicePlugin with kubelet. +/// During registration, the device plugin must send +/// (1) name of unix socket, +/// (2) Device-Plugin API it was built against (v1beta1), +/// (3) resource name akri.sh/device_id. +/// If registration request to kubelet fails, terminates DevicePluginService. +async fn register( + capability_id: String, + socket_name: String, + instance_name: &str, + mut server_ender_sender: mpsc::Sender<()>, +) -> Result<(), Box> { + info!( + "register - entered for Instance {} and socket_name: {}", + capability_id, socket_name + ); + let op = DevicePluginOptions { + pre_start_required: false, + }; + + // lttp://... is a fake uri that is unused (in service_fn) but neccessary for uds connection + let channel = Endpoint::try_from("lttp://[::]:50051")? + .connect_with_connector(service_fn(|_: Uri| UnixStream::connect(KUBELET_SOCKET))) + .await?; + let mut registration_client = registration_client::RegistrationClient::new(channel); + + let register_request = tonic::Request::new(v1beta1::RegisterRequest { + version: K8S_DEVICE_PLUGIN_VERSION.into(), + endpoint: socket_name, + resource_name: capability_id, + options: Some(op), + }); + trace!( + "register - before call to register with Kubelet at socket {}", + KUBELET_SOCKET + ); + + // If fail to register with kubelet, terminate device plugin + if registration_client + .register(register_request) + .await + .is_err() + { + trace!( + "register - failed to register Instance {} with kubelet ... terminating device plugin", + instance_name + ); + server_ender_sender.send(()).await?; + } + Ok(()) +} + +/// This creates an Instance's unique name +pub fn get_device_instance_name(id: &str, config_name: &str) -> String { + format!("{}-{}", config_name, &id) + .replace(".", "-") + .replace("/", "-") +} + +/// Module to enable UDS with tonic grpc. +/// This is unix only since the underlying UnixStream and UnixListener libraries are unix only. +#[cfg(unix)] +mod unix { + use std::{ + pin::Pin, + task::{Context, Poll}, + }; + + use tokio::io::{AsyncRead, AsyncWrite}; + use tonic::transport::server::Connected; + + #[derive(Debug)] + pub struct UnixStream(pub tokio::net::UnixStream); + + impl Connected for UnixStream {} + + impl AsyncRead for UnixStream { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut [u8], + ) -> Poll> { + Pin::new(&mut self.0).poll_read(cx, buf) + } + } + + impl AsyncWrite for UnixStream { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + Pin::new(&mut self.0).poll_write(cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.0).poll_flush(cx) + } + + fn poll_shutdown( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll> { + Pin::new(&mut self.0).poll_shutdown(cx) + } + } +} + +#[cfg(test)] +mod device_plugin_service_tests { + use super::super::v1beta1::device_plugin_client::DevicePluginClient; + use super::*; + use akri_shared::akri::configuration::KubeAkriConfig; + use akri_shared::{ + akri::instance::{Instance, KubeAkriInstance}, + k8s::test_kube::MockKubeImpl, + }; + use mockall::predicate::*; + use std::{ + fs, + io::{Error, ErrorKind}, + }; + + enum NodeName { + ThisNode, + OtherNode, + } + + // Need to be kept alive during tests + struct DevicePluginServiceReceivers { + list_and_watch_message_receiver: broadcast::Receiver, + server_ender_receiver: mpsc::Receiver<()>, + } + + fn configure_find_instance( + mock: &mut MockKubeImpl, + result_file: &'static str, + instance_name: String, + instance_namespace: String, + device_usage_node: &'static str, + node_name: NodeName, + ) { + let instance_name_clone = instance_name.clone(); + mock.expect_find_instance() + .times(1) + .withf(move |name: &str, namespace: &str| { + namespace == instance_namespace && name == instance_name + }) + .returning(move |_, _| { + let mut instance_json = + fs::read_to_string(result_file).expect("Unable to read file"); + let host_name = match node_name { + NodeName::ThisNode => "node-a", + NodeName::OtherNode => "other", + }; + instance_json = instance_json.replace("node-a", &host_name); + instance_json = instance_json.replace("config-a-b494b6", &instance_name_clone); + instance_json = + instance_json.replace("\":\"\"", &format!("\":\"{}\"", device_usage_node)); + let instance: KubeAkriInstance = serde_json::from_str(&instance_json).unwrap(); + Ok(instance) + }); + } + + fn create_device_plugin_service( + connectivity_status: ConnectivityStatus, + add_to_instance_map: bool, + ) -> (DevicePluginService, DevicePluginServiceReceivers) { + let path_to_config = "../test/json/config-a.json"; + let kube_akri_config_json = + fs::read_to_string(path_to_config).expect("Unable to read file"); + let kube_akri_config: KubeAkriConfig = + serde_json::from_str(&kube_akri_config_json).unwrap(); + let device_instance_name = + get_device_instance_name("b494b6", &kube_akri_config.metadata.name); + let unique_time = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH); + let device_endpoint: String = format!( + "{}-{}.sock", + device_instance_name, + unique_time.unwrap_or_default().as_secs() + ); + let (list_and_watch_message_sender, list_and_watch_message_receiver) = + broadcast::channel(4); + let (server_ender_sender, server_ender_receiver) = mpsc::channel(1); + + let mut map = HashMap::new(); + if add_to_instance_map { + let instance_info: InstanceInfo = InstanceInfo { + list_and_watch_message_sender: list_and_watch_message_sender.clone(), + connectivity_status, + }; + map.insert(device_instance_name.clone(), instance_info); + } + let instance_map: InstanceMap = Arc::new(Mutex::new(map)); + + let dps = DevicePluginService { + instance_name: device_instance_name, + endpoint: device_endpoint, + config: kube_akri_config.spec.clone(), + config_name: kube_akri_config.metadata.name, + config_uid: kube_akri_config.metadata.uid.unwrap(), + config_namespace: kube_akri_config.metadata.namespace.unwrap(), + shared: false, + node_name: "node-a".to_string(), + instance_properties: HashMap::new(), + instance_map, + list_and_watch_message_sender, + server_ender_sender, + }; + ( + dps, + DevicePluginServiceReceivers { + list_and_watch_message_receiver, + server_ender_receiver, + }, + ) + } + + fn check_devices(instance_name: String, devices: Vec) { + let capacity: usize = 5; + // update_virtual_devices_health returns devices in jumbled order (ie 2, 4, 1, 5, 3) + let expected_device_ids: Vec = (0..capacity) + .map(|x| format!("{}-{}", instance_name, x)) + .collect(); + assert_eq!(devices.len(), capacity); + // Can't use map on Device type + let device_ids: Vec = devices.into_iter().map(|device| device.id).collect(); + for device in expected_device_ids { + assert!(device_ids.contains(&device)); + } + } + + // Tests that instance names are formatted correctly + #[test] + fn test_get_device_instance_name() { + let instance_name1: String = "/dev/video0".to_string(); + let instance_name2: String = "10.1.2.3".to_string(); + assert_eq!( + "usb-camera--dev-video0", + get_device_instance_name(&instance_name1, &"usb-camera".to_string()) + ); + assert_eq!( + "ip-camera-10-1-2-3".to_string(), + get_device_instance_name(&instance_name2, &"ip-camera".to_string()) + ); + } + + fn configure_find_configuration( + mock: &mut MockKubeImpl, + config_name: String, + config_namespace: String, + ) { + mock.expect_find_configuration() + .times(1) + .withf(move |name: &str, namespace: &str| { + namespace == config_namespace && name == config_name + }) + .returning(move |_, _| { + let path_to_config = "../test/json/config-a.json"; + let kube_akri_config_json = + fs::read_to_string(path_to_config).expect("Unable to read file"); + let kube_akri_config: KubeAkriConfig = + serde_json::from_str(&kube_akri_config_json).unwrap(); + Ok(kube_akri_config) + }); + } + + // Tests that try_create_instance creates an instance + #[tokio::test] + async fn test_try_create_instance() { + let _ = env_logger::builder().is_test(true).try_init(); + let (device_plugin_service, _device_plugin_service_receivers) = + create_device_plugin_service(ConnectivityStatus::Online, false); + let mut mock = MockKubeImpl::new(); + configure_find_configuration( + &mut mock, + device_plugin_service.config_name.clone(), + device_plugin_service.config_namespace.clone(), + ); + let instance_name = device_plugin_service.instance_name.clone(); + let config_name = device_plugin_service.config_name.clone(); + let config_uid = device_plugin_service.config_uid.clone(); + let config_namespace = device_plugin_service.config_namespace.clone(); + mock.expect_find_instance() + .times(1) + .withf(move |name: &str, namespace: &str| { + namespace == config_namespace && name == instance_name + }) + .returning(move |_, _| { + let error = Error::new(ErrorKind::InvalidInput, "Configuration doesn't exist"); + Err(Box::new(error)) + }); + let instance_name = device_plugin_service.instance_name.clone(); + let config_namespace = device_plugin_service.config_namespace.clone(); + mock.expect_create_instance() + .withf(move |instance, name, namespace, owner_name, owner_uid| { + namespace == config_namespace + && name == instance_name + && instance.nodes.contains(&"node-a".to_string()) + && owner_name == config_name + && owner_uid == config_uid + }) + .returning(move |_, _, _, _, _| Ok(())); + + let dps = Arc::new(device_plugin_service); + assert!(try_create_instance(dps.clone(), Arc::new(mock)) + .await + .is_ok()); + assert!(dps + .instance_map + .lock() + .await + .contains_key(&dps.instance_name)); + } + + // Tests that try_create_instance updates already existing instance with this node + #[tokio::test] + async fn test_try_create_instance_already_created() { + let _ = env_logger::builder().is_test(true).try_init(); + let (device_plugin_service, _device_plugin_service_receivers) = + create_device_plugin_service(ConnectivityStatus::Online, false); + let mut mock = MockKubeImpl::new(); + configure_find_configuration( + &mut mock, + device_plugin_service.config_name.clone(), + device_plugin_service.config_namespace.clone(), + ); + configure_find_instance( + &mut mock, + "../test/json/local-instance.json", + device_plugin_service.instance_name.clone(), + device_plugin_service.config_namespace.clone(), + "", + NodeName::OtherNode, + ); + let instance_name = device_plugin_service.instance_name.clone(); + let config_namespace = device_plugin_service.config_namespace.clone(); + mock.expect_update_instance() + .times(1) + .withf(move |instance, name, namespace| { + namespace == config_namespace + && name == instance_name + && instance.nodes.contains(&"node-a".to_string()) + }) + .returning(move |_, _, _| Ok(())); + + let dps = Arc::new(device_plugin_service); + assert!(try_create_instance(dps.clone(), Arc::new(mock)) + .await + .is_ok()); + assert!(dps + .instance_map + .lock() + .await + .contains_key(&dps.instance_name)); + } + + // Test when instance already created and already contains this node. + // Should find the instance but not update it. + #[tokio::test] + async fn test_try_create_instance_already_created_no_update() { + let _ = env_logger::builder().is_test(true).try_init(); + let (device_plugin_service, _device_plugin_service_receivers) = + create_device_plugin_service(ConnectivityStatus::Online, false); + let mut mock = MockKubeImpl::new(); + configure_find_configuration( + &mut mock, + device_plugin_service.config_name.clone(), + device_plugin_service.config_namespace.clone(), + ); + configure_find_instance( + &mut mock, + "../test/json/local-instance.json", + device_plugin_service.instance_name.clone(), + device_plugin_service.config_namespace.clone(), + "", + NodeName::ThisNode, + ); + let dps = Arc::new(device_plugin_service); + assert!(try_create_instance(dps.clone(), Arc::new(mock)) + .await + .is_ok()); + assert!(dps + .instance_map + .lock() + .await + .contains_key(&dps.instance_name)); + } + + // Tests that try_create_instance returns error when trying to create an Instance for a Config that DNE + #[tokio::test] + async fn test_try_create_instance_no_config() { + let _ = env_logger::builder().is_test(true).try_init(); + let (device_plugin_service, _device_plugin_service_receivers) = + create_device_plugin_service(ConnectivityStatus::Online, false); + let config_name = device_plugin_service.config_name.clone(); + let config_namespace = device_plugin_service.config_namespace.clone(); + let mut mock = MockKubeImpl::new(); + mock.expect_find_configuration() + .times(1) + .withf(move |name: &str, namespace: &str| { + namespace == config_namespace && name == config_name + }) + .returning(move |_, _| { + let error = Error::new(ErrorKind::InvalidInput, "Configuration doesn't exist"); + Err(Box::new(error)) + }); + assert!( + try_create_instance(Arc::new(device_plugin_service), Arc::new(mock)) + .await + .is_err() + ); + } + + // Tests that try_create_instance error + #[tokio::test] + async fn test_try_create_instance_error() { + let _ = env_logger::builder().is_test(true).try_init(); + let (device_plugin_service, _device_plugin_service_receivers) = + create_device_plugin_service(ConnectivityStatus::Online, false); + let mut mock = MockKubeImpl::new(); + configure_find_configuration( + &mut mock, + device_plugin_service.config_name.clone(), + device_plugin_service.config_namespace.clone(), + ); + let instance_name = device_plugin_service.instance_name.clone(); + let config_name = device_plugin_service.config_name.clone(); + let config_uid = device_plugin_service.config_uid.clone(); + let config_namespace = device_plugin_service.config_namespace.clone(); + mock.expect_find_instance() + .times(MAX_INSTANCE_UPDATE_TRIES as usize) + .withf(move |name: &str, namespace: &str| { + namespace == config_namespace && name == instance_name + }) + .returning(move |_, _| Err(None.ok_or("failure")?)); + let instance_name = device_plugin_service.instance_name.clone(); + let config_namespace = device_plugin_service.config_namespace.clone(); + mock.expect_create_instance() + .times(MAX_INSTANCE_UPDATE_TRIES as usize) + .withf(move |instance, name, namespace, owner_name, owner_uid| { + namespace == config_namespace + && name == instance_name + && instance.nodes.contains(&"node-a".to_string()) + && owner_name == config_name + && owner_uid == config_uid + }) + .returning(move |_, _, _, _, _| Err(None.ok_or("failure")?)); + + let dps = Arc::new(device_plugin_service); + assert!(try_create_instance(dps.clone(), Arc::new(mock)) + .await + .is_err()); + assert!(!dps + .instance_map + .lock() + .await + .contains_key(&dps.instance_name)); + } + + // Tests list_and_watch by creating DevicePluginService and DevicePlugin client (emulating kubelet) + #[tokio::test] + async fn test_list_and_watch() { + let _ = env_logger::builder().is_test(true).try_init(); + let (device_plugin_service, device_plugin_service_receivers) = + create_device_plugin_service(ConnectivityStatus::Online, false); + let socket_path: String = format!( + "{}{}", + DEVICE_PLUGIN_PATH.to_string(), + device_plugin_service.endpoint.clone() + ); + let list_and_watch_message_sender = + device_plugin_service.list_and_watch_message_sender.clone(); + let instance_name = device_plugin_service.instance_name.clone(); + serve( + device_plugin_service, + socket_path.clone(), + device_plugin_service_receivers.server_ender_receiver, + ) + .await + .unwrap(); + let channel = Endpoint::try_from("lttp://[::]:50051") + .unwrap() + .connect_with_connector(service_fn(move |_: Uri| { + UnixStream::connect(socket_path.clone()) + })) + .await + .unwrap(); + let mut client = DevicePluginClient::new(channel); + let mut stream = client + .list_and_watch(Request::new(Empty {})) + .await + .unwrap() + .into_inner(); + list_and_watch_message_sender + .send(ListAndWatchMessageKind::End) + .unwrap(); + if let Some(list_and_watch_response) = stream.message().await.unwrap() { + assert_eq!( + list_and_watch_response.devices[0].id, + format!("{}-0", instance_name) + ); + }; + } + + #[tokio::test] + async fn test_build_virtual_devices() { + let mut device_usage: HashMap = HashMap::new(); + let mut expected_devices_nodea: HashMap = HashMap::new(); + let mut expected_devices_nodeb: HashMap = HashMap::new(); + let instance_name = "s0meH@sH"; + for x in 0..5 { + if x % 2 == 0 { + device_usage.insert(format!("{}-{}", instance_name, x), "nodeA".to_string()); + expected_devices_nodea + .insert(format!("{}-{}", instance_name, x), HEALTHY.to_string()); + expected_devices_nodeb + .insert(format!("{}-{}", instance_name, x), UNHEALTHY.to_string()); + } else { + device_usage.insert(format!("{}-{}", instance_name, x), "".to_string()); + expected_devices_nodea + .insert(format!("{}-{}", instance_name, x), HEALTHY.to_string()); + expected_devices_nodeb + .insert(format!("{}-{}", instance_name, x), HEALTHY.to_string()); + } + } + + // Test shared all healthy + let mut devices: Vec = + build_virtual_devices(&device_usage, true, &"nodeA".to_string()); + for device in devices { + assert_eq!( + expected_devices_nodea.get(&device.id).unwrap(), + &device.health + ); + } + + // Test unshared all healthy + devices = build_virtual_devices(&device_usage, false, &"nodeA".to_string()); + for device in devices { + assert_eq!( + expected_devices_nodea.get(&device.id).unwrap(), + &device.health + ); + } + + // Test shared some unhealthy (taken by another node) + devices = build_virtual_devices(&device_usage, true, &"nodeB".to_string()); + for device in devices { + assert_eq!( + expected_devices_nodeb.get(&device.id).unwrap(), + &device.health + ); + } + + // Test unshared panic. A different node should never be listed under any device usage slots + let result = std::panic::catch_unwind(|| { + build_virtual_devices(&device_usage, false, &"nodeB".to_string()) + }); + assert!(result.is_err()); + } + + // Tests when ConnectivityStatus is offline and unhealthy devices are returned + #[tokio::test] + async fn test_build_list_and_watch_response_offline() { + let _ = env_logger::builder().is_test(true).try_init(); + let (device_plugin_service, _device_plugin_service_receivers) = + create_device_plugin_service(ConnectivityStatus::Offline(Instant::now()), true); + let mock = MockKubeImpl::new(); + let devices = + build_list_and_watch_response(Arc::new(device_plugin_service), Arc::new(mock)) + .await + .unwrap(); + devices + .into_iter() + .for_each(|device| assert!(device.health == UNHEALTHY)); + } + + // Tests when instance has not yet been created for this device, all devices are returned as UNHEALTHY + #[tokio::test] + async fn test_build_list_and_watch_response_no_instance() { + let _ = env_logger::builder().is_test(true).try_init(); + let (device_plugin_service, _device_plugin_service_receivers) = + create_device_plugin_service(ConnectivityStatus::Online, true); + let instance_name = device_plugin_service.instance_name.clone(); + let instance_namespace = device_plugin_service.config_namespace.clone(); + let mut mock = MockKubeImpl::new(); + mock.expect_find_instance() + .times(1) + .withf(move |name: &str, namespace: &str| { + namespace == instance_namespace && name == instance_name + }) + .returning(move |_, _| { + let error = Error::new(ErrorKind::InvalidInput, "Instance doesn't exist"); + Err(Box::new(error)) + }); + let devices = + build_list_and_watch_response(Arc::new(device_plugin_service), Arc::new(mock)) + .await + .unwrap(); + devices + .into_iter() + .for_each(|device| assert!(device.health == UNHEALTHY)); + } + + // Test when instance has already been created and includes this node + #[tokio::test] + async fn test_build_list_and_watch_response_no_instance_update() { + let _ = env_logger::builder().is_test(true).try_init(); + let (device_plugin_service, _device_plugin_service_receivers) = + create_device_plugin_service(ConnectivityStatus::Online, true); + let instance_name = device_plugin_service.instance_name.clone(); + let instance_namespace = device_plugin_service.config_namespace.clone(); + let mut mock = MockKubeImpl::new(); + configure_find_instance( + &mut mock, + "../test/json/local-instance.json", + instance_name.clone(), + instance_namespace.clone(), + "", + NodeName::ThisNode, + ); + let devices = + build_list_and_watch_response(Arc::new(device_plugin_service), Arc::new(mock)) + .await + .unwrap(); + check_devices(instance_name, devices); + } + + // Test when device_usage[id] == "" + // internal_allocate should set device_usage[id] = m.nodeName, return + #[tokio::test] + async fn test_internal_allocate_success() { + let _ = env_logger::builder().is_test(true).try_init(); + let (device_plugin_service, mut device_plugin_service_receivers) = + create_device_plugin_service(ConnectivityStatus::Online, true); + let device_usage_id_slot = format!("{}-0", device_plugin_service.instance_name); + let device_usage_id_slot_2 = device_usage_id_slot.clone(); + let node_name = device_plugin_service.node_name.clone(); + let mut mock = MockKubeImpl::new(); + configure_find_instance( + &mut mock, + "../test/json/local-instance.json", + device_plugin_service.instance_name.clone(), + device_plugin_service.config_namespace.clone(), + "", + NodeName::ThisNode, + ); + mock.expect_update_instance() + .times(1) + .withf(move |instance_to_update: &Instance, _, _| { + instance_to_update + .device_usage + .get(&device_usage_id_slot) + .unwrap() + == &node_name + }) + .returning(move |_, _, _| Ok(())); + let devices_i_ds = vec![device_usage_id_slot_2]; + let container_requests = vec![v1beta1::ContainerAllocateRequest { devices_i_ds }]; + let requests = Request::new(AllocateRequest { container_requests }); + assert!(device_plugin_service + .internal_allocate(requests, Arc::new(mock),) + .await + .is_ok()); + assert!(device_plugin_service_receivers + .list_and_watch_message_receiver + .try_recv() + .is_err()); + } + + // Test when device_usage[id] == self.nodeName + // Expected behavior: internal_allocate should set device_usage[id] == "", invoke list_and_watch, and return error + #[tokio::test] + async fn test_internal_allocate_deallocate() { + let _ = env_logger::builder().is_test(true).try_init(); + let (device_plugin_service, mut device_plugin_service_receivers) = + create_device_plugin_service(ConnectivityStatus::Online, true); + let device_usage_id_slot = format!("{}-0", device_plugin_service.instance_name); + let device_usage_id_slot_2 = device_usage_id_slot.clone(); + let mut mock = MockKubeImpl::new(); + configure_find_instance( + &mut mock, + "../test/json/local-instance.json", + device_plugin_service.instance_name.clone(), + device_plugin_service.config_namespace.clone(), + "node-a", + NodeName::ThisNode, + ); + mock.expect_update_instance() + .times(1) + .withf(move |instance_to_update: &Instance, _, _| { + instance_to_update + .device_usage + .get(&device_usage_id_slot) + .unwrap() + == "" + }) + .returning(move |_, _, _| Ok(())); + let devices_i_ds = vec![device_usage_id_slot_2]; + let container_requests = vec![v1beta1::ContainerAllocateRequest { devices_i_ds }]; + let requests = Request::new(AllocateRequest { container_requests }); + match device_plugin_service + .internal_allocate(requests, Arc::new(mock)) + .await + { + Ok(_) => { + panic!("internal allocate is expected to fail due to devices being in bad state") + } + Err(e) => assert_eq!( + e.message(), + "Devices are in inconsistent state, updated device usage, please retry scheduling" + ), + } + assert_eq!( + device_plugin_service_receivers + .list_and_watch_message_receiver + .recv() + .await + .unwrap(), + ListAndWatchMessageKind::Continue + ); + } + + // Tests when device_usage[id] == + // Expected behavior: should invoke list_and_watch, and return error + #[tokio::test] + async fn test_internal_allocate_taken() { + let _ = env_logger::builder().is_test(true).try_init(); + let (device_plugin_service, mut device_plugin_service_receivers) = + create_device_plugin_service(ConnectivityStatus::Online, true); + let device_usage_id_slot = format!("{}-0", device_plugin_service.instance_name); + let mut mock = MockKubeImpl::new(); + configure_find_instance( + &mut mock, + "../test/json/local-instance.json", + device_plugin_service.instance_name.clone(), + device_plugin_service.config_namespace.clone(), + "other", + NodeName::ThisNode, + ); + let devices_i_ds = vec![device_usage_id_slot]; + let container_requests = vec![v1beta1::ContainerAllocateRequest { devices_i_ds }]; + let requests = Request::new(AllocateRequest { container_requests }); + match device_plugin_service + .internal_allocate(requests, Arc::new(mock)) + .await + { + Ok(_) => panic!( + "internal allocate is expected to fail due to requested device already being used" + ), + Err(e) => assert_eq!(e.message(), "Requested device already in use"), + } + assert_eq!( + device_plugin_service_receivers + .list_and_watch_message_receiver + .recv() + .await + .unwrap(), + ListAndWatchMessageKind::Continue + ); + } + + // Tests when instance does not have the requested device usage id + // Expected behavior: should invoke list_and_watch, and return error + #[tokio::test] + async fn test_internal_allocate_no_id() { + let _ = env_logger::builder().is_test(true).try_init(); + let (device_plugin_service, mut device_plugin_service_receivers) = + create_device_plugin_service(ConnectivityStatus::Online, true); + let device_usage_id_slot = format!("{}-100", device_plugin_service.instance_name); + let mut mock = MockKubeImpl::new(); + configure_find_instance( + &mut mock, + "../test/json/local-instance.json", + device_plugin_service.instance_name.clone(), + device_plugin_service.config_namespace.clone(), + "other", + NodeName::ThisNode, + ); + let devices_i_ds = vec![device_usage_id_slot]; + let container_requests = vec![v1beta1::ContainerAllocateRequest { devices_i_ds }]; + let requests = Request::new(AllocateRequest { container_requests }); + match device_plugin_service + .internal_allocate(requests, Arc::new(mock)) + .await + { + Ok(_) => { + panic!("internal allocate is expected to fail due to invalid device usage slot") + } + Err(e) => assert_eq!(e.message(), "Could not find device usage slot"), + } + assert_eq!( + device_plugin_service_receivers + .list_and_watch_message_receiver + .recv() + .await + .unwrap(), + ListAndWatchMessageKind::Continue + ); + } +} diff --git a/agent/src/util/mod.rs b/agent/src/util/mod.rs new file mode 100644 index 0000000..cdc5f45 --- /dev/null +++ b/agent/src/util/mod.rs @@ -0,0 +1,6 @@ +pub mod config_action; +pub mod constants; +pub mod crictl_containers; +mod device_plugin_service; +pub mod slot_reconciliation; +mod v1beta1; diff --git a/agent/src/util/slot_reconciliation.rs b/agent/src/util/slot_reconciliation.rs new file mode 100644 index 0000000..9a5f76f --- /dev/null +++ b/agent/src/util/slot_reconciliation.rs @@ -0,0 +1,762 @@ +use super::{constants::SLOT_RECONCILIATION_CHECK_DELAY_SECS, crictl_containers}; +use akri_shared::{akri::instance::Instance, k8s::KubeInterface}; +use async_trait::async_trait; +use k8s_openapi::api::core::v1::PodStatus; +use mockall::predicate::*; +use std::{ + collections::{HashMap, HashSet}, + sync::{Arc, Mutex}, + time::{Duration, Instant}, +}; +use tokio::process::Command; + +type SlotQueryResult = Result, Box>; + +// +// mockall and async_trait do not work effortlessly together ... to enable both, +// follow the example here: +// https://github.com/mibes/mockall-async/blob/53aec15219a720ef5ac483959ff8821cb7d656ae/src/main.rs +// +// When async traits are supported by Rust without the async_trait crate, we should +// add: +// #[automock] +// +#[async_trait] +pub trait SlotQuery { + async fn get_node_slots(&self) -> SlotQueryResult; +} + +/// Discovers which of an instance's usage slots are actively used by containers on this node +pub struct CriCtlSlotQuery { + pub crictl_path: String, + pub runtime_endpoint: String, + pub image_endpoint: String, +} + +#[async_trait] +impl SlotQuery for CriCtlSlotQuery { + /// Calls crictl to query container runtime in search of active containers and extracts their usage slots. + async fn get_node_slots(&self) -> SlotQueryResult { + match Command::new(&self.crictl_path) + .args(&[ + "--runtime-endpoint", + &self.runtime_endpoint, + "--image-endpoint", + &self.image_endpoint, + "ps", + "-v", + "--output", + "json", + ]) + .output() + .await + { + Ok(output) => { + if output.status.success() { + trace!("get_node_slots - crictl called successfully"); + let output_string = String::from_utf8_lossy(&output.stdout); + Ok(crictl_containers::get_container_slot_usage(&output_string)) + } else { + let output_string = String::from_utf8_lossy(&output.stderr); + Err(None.ok_or(format!( + "get_node_slots - Failed to call crictl: {:?}", + output_string + ))?) + } + } + Err(e) => { + trace!("get_node_slots - Command failed to call crictl: {:?}", e); + Err(e.into()) + } + } + } +} + +/// Makes sure Instance's `device_usage` accurately reflects actual usage. +pub struct DevicePluginSlotReconciler { + pub removal_slot_map: Arc>>, +} + +impl DevicePluginSlotReconciler { + pub async fn reconcile( + &self, + node_name: &str, + slot_grace_period: Duration, + slot_query: &impl SlotQuery, + kube_interface: &impl KubeInterface, + ) { + trace!( + "reconcile - thread iteration start [{:?}]", + self.removal_slot_map + ); + + let node_slot_usage = match slot_query.get_node_slots().await { + Ok(usage) => usage, + Err(e) => { + trace!("reconcile - get_node_slots failed: {:?}", e); + // If an error occurs in the crictl call, return early + // to avoid treating this error like crictl found no + // active containers. Currently, reconcile is a best + // effort approach. + return; + } + }; + trace!( + "reconcile - slots currently in use on this node: {:?}", + node_slot_usage + ); + + // Any slot found in use should be scrubbed from our list + node_slot_usage.iter().for_each(|slot| { + trace!("reconcile - remove slot from tracked slots: {:?}", slot); + self.removal_slot_map.lock().unwrap().remove(slot); + }); + trace!( + "reconcile - removal_slot_map after removing node_slot_usage: {:?}", + self.removal_slot_map + ); + + let instances = match kube_interface.get_instances().await { + Ok(instances) => instances, + Err(e) => { + trace!("reconcile - Failed to get instances: {:?}", e); + return; + } + }; + + let pods = match kube_interface + .find_pods_with_field(&format!("{}={}", "spec.nodeName", &node_name,)) + .await + { + Ok(pods) => { + trace!("reconcile - found {} pods on this node", pods.items.len()); + pods + } + Err(e) => { + trace!("reconcile - error finding pending pods: {}", e); + return; + } + }; + + // Check to see if there are any Pods on this Node that have + // Containers that are not ready. If there are any, we should + // wait for the Containers to be ready before cleaning any + // Instance device_usage + let any_unready_pods = pods.items.iter().any(|pod| { + pod.status + .as_ref() + .unwrap_or(&PodStatus::default()) + .conditions + .as_ref() + .unwrap_or(&Vec::new()) + .iter() + .any(|condition| condition.type_ == "ContainersReady" && condition.status != "True") + }); + if any_unready_pods { + trace!("reconcile - Pods with unready Containers exist on this node, we can't clean the slots yet"); + return; + } + + for instance in instances { + // Check Instance against list of slots that are being used by this node's + // current pods. If we find any missing, we should update the Instance for + // the actual slot usage. + let slots_missing_this_node_name = instance + .spec + .device_usage + .iter() + .filter_map(|(k, v)| { + if v != node_name && node_slot_usage.contains(k) { + // We need to add node_name to this slot IF + // the slot is not labeled with node_name AND + // there is a container using that slot on this node + Some(k.to_string()) + } else { + None + } + }) + .collect::>(); + + // Check Instance to find slots that are registered to this node, but + // there is no actual pod using the slot. We should update the Instance + // to clear the false usage. + // + // For slots that need to be cleaned, we should wait for a "grace + // period" prior to updating the Instance. + let slots_to_clean = instance + .spec + .device_usage + .iter() + .filter_map(|(k, v)| { + if v == node_name && !node_slot_usage.contains(k) { + // We need to clean this slot IF + // this slot is handled by this node AND + // there are no containers using that slot on this node + Some(k.to_string()) + } else { + None + } + }) + .filter(|slot_string| { + let mut local_slot_map = self.removal_slot_map.lock().unwrap(); + if let Some(time) = local_slot_map.get(slot_string) { + let now = Instant::now(); + match now.checked_duration_since(*time) { + Some(duration) => { + if duration > slot_grace_period { + trace!("reconcile - slot expired: [{:?}]", duration); + true // slot has been unoccupied beyond the grace period + } else { + false // still in grace period + } + } + None => { + false // still in grace period + } + } + } else { + trace!("reconcile - slot added to list: [Now]"); + local_slot_map.insert(slot_string.to_string(), Instant::now()); + false // do not remove this node just yet + } + }) + .collect::>(); + trace!( + "reconcile - these slots have no pods according to crictl AND have expired: {:?}", + &slots_to_clean + ); + + if !slots_to_clean.is_empty() || !slots_missing_this_node_name.is_empty() { + trace!( + "reconcile - update Instance slots_to_clean: {:?} slots_missing_this_node_name: {:?}", + slots_to_clean, + slots_missing_this_node_name + ); + let modified_device_usage = instance + .spec + .device_usage + .iter() + .map(|(slot, node)| { + ( + slot.to_string(), + if slots_missing_this_node_name.contains(slot) { + // Set this to node_name because there have been + // cases where a Pod is running (which corresponds + // to an Allocate call, but the Instance slot is empty. + node_name.into() + } else if slots_to_clean.contains(slot) { + // Set this to empty string because there is no + // Deallocate message from kubelet for us to know + // when a slot is no longer in use + "".into() + } else { + // This slot remains unchanged. + node.into() + }, + ) + }) + .collect::>(); + let modified_instance = Instance { + configuration_name: instance.spec.configuration_name.clone(), + metadata: instance.spec.metadata.clone(), + rbac: instance.spec.rbac.clone(), + shared: instance.spec.shared, + device_usage: modified_device_usage, + nodes: instance.spec.nodes.clone(), + }; + trace!("reconcile - update Instance from: {:?}", &instance.spec); + trace!("reconcile - update Instance to: {:?}", &modified_instance); + match kube_interface + .update_instance( + &modified_instance, + &instance.metadata.name, + &instance.metadata.namespace.unwrap(), + ) + .await + { + Ok(()) => { + slots_to_clean.iter().for_each(|slot| { + trace!("reconcile - remove {} from removal_slot_map", slot); + self.removal_slot_map.lock().unwrap().remove(slot); + }); + } + Err(e) => { + // If update fails, let the next iteration update the Instance. We + // may want to revisit this decision and add some retry logic + // here. + trace!("reconcile - update Instance failed: {:?}", e); + } + } + } + } + + trace!("reconcile - thread iteration end"); + } +} + +/// This periodically checks to make sure that all Instances' device_usage +/// accurately reflects the actual usage. +/// +/// The Kubernetes Device-Plugin implementation has no notifications for +/// when a Pod disappears (which should, in turn, free up a slot). Because +/// of this, if a Pod disappears, there will be a slot that Akri (and the +/// Kubernetes scheduler) falsely thinks is in use. +/// +/// To work around this, we have done 2 things: +/// 1. Each of Agent's device plugins add slot information to the Annotations +/// section of the Allocate response. +/// 2. periodic_slot_reconciliation will periodically call crictl to query the +/// container runtime in search of active Containers that have our slot +/// Annotations. This function will make sure that our Instance device_usage +/// accurately reflects the actual usage. +/// +/// It has rarely been seen, perhaps due to connectivity issues, that active +/// Containers with our Annotation are no longer in our Instance. This is a bug that +/// we are aware of, but haven't found yet. To address this, until a fix is found, +/// we will also make sure that any Container that exists with our Annotation will +/// be shown in our Instance device_usage. +pub async fn periodic_slot_reconciliation( + slot_grace_period: std::time::Duration, +) -> Result<(), Box> { + trace!("periodic_slot_reconciliation - start"); + let kube_interface = akri_shared::k8s::create_kube_interface(); + let node_name = std::env::var("AGENT_NODE_NAME").unwrap(); + let crictl_path = std::env::var("HOST_CRICTL_PATH").unwrap(); + let runtime_endpoint = std::env::var("HOST_RUNTIME_ENDPOINT").unwrap(); + let image_endpoint = std::env::var("HOST_IMAGE_ENDPOINT").unwrap(); + + let reconciler = DevicePluginSlotReconciler { + removal_slot_map: Arc::new(std::sync::Mutex::new(HashMap::new())), + }; + let slot_query = CriCtlSlotQuery { + crictl_path, + runtime_endpoint, + image_endpoint, + }; + + loop { + trace!("periodic_slot_reconciliation - iteration pre delay_for"); + tokio::time::delay_for(std::time::Duration::from_secs( + SLOT_RECONCILIATION_CHECK_DELAY_SECS, + )) + .await; + + trace!("periodic_slot_reconciliation - iteration call reconiler.reconcile"); + reconciler + .reconcile(&node_name, slot_grace_period, &slot_query, &kube_interface) + .await; + + trace!("periodic_slot_reconciliation - iteration end"); + } +} + +pub mod test_crictl { + use super::{SlotQuery, SlotQueryResult}; + use async_trait::async_trait; + use mockall::predicate::*; + use mockall::*; + + // + // mockall and async_trait do not work effortlessly together ... to enable both, + // follow the example here: + // https://github.com/mibes/mockall-async/blob/53aec15219a720ef5ac483959ff8821cb7d656ae/src/main.rs + // + // We can probably eliminate this when async traits are supported by Rust without + // the async_trait crate. + // + mock! { + pub SlotQueryImpl { + fn get_node_slots(&self) -> SlotQueryResult; + } + } + + #[async_trait] + impl SlotQuery for MockSlotQueryImpl { + async fn get_node_slots(&self) -> SlotQueryResult { + self.get_node_slots() + } + } +} + +#[cfg(test)] +mod reconcile_tests { + use super::test_crictl::MockSlotQueryImpl; + use super::*; + use akri_shared::{ + akri::instance::KubeAkriInstanceList, k8s::test_kube::MockKubeImpl, os::file, + }; + use k8s_openapi::api::core::v1::{PodSpec, PodStatus}; + use kube::api::{Object, ObjectList}; + + fn configure_get_node_slots( + mock: &mut MockSlotQueryImpl, + result: HashSet, + error: bool, + ) { + mock.expect_get_node_slots().times(1).returning(move || { + if !error { + Ok(result.clone()) + } else { + Err(None.ok_or("failure")?) + } + }); + } + + fn configure_get_instances(mock: &mut MockKubeImpl, result_file: &'static str) { + mock.expect_get_instances().times(1).returning(move || { + let instance_list_json = file::read_file_to_string(result_file); + let instance_list: KubeAkriInstanceList = + serde_json::from_str(&instance_list_json).unwrap(); + Ok(instance_list) + }); + } + + fn configure_find_pods_with_field( + mock: &mut MockKubeImpl, + selector: &'static str, + result_file: &'static str, + ) { + mock.expect_find_pods_with_field() + .times(1) + .withf(move |s| s == selector) + .returning(move |_| { + let pods_json = file::read_file_to_string(result_file); + let pods: ObjectList> = + serde_json::from_str(&pods_json).unwrap(); + Ok(pods) + }); + } + + struct NodeSlots { + node_slots: HashSet, + node_slots_error: bool, + } + + struct UpdateInstance { + expected_slot_1_node: &'static str, + expected_slot_5_node: &'static str, + } + + async fn configure_scnenario( + node_slots: NodeSlots, + instances_result_file: &'static str, + update_instance: Option, + grace_period: Duration, + reconciler: &DevicePluginSlotReconciler, + ) { + let mut slot_query = MockSlotQueryImpl::new(); + // slot_query to identify one slot used by this node + configure_get_node_slots( + &mut slot_query, + node_slots.node_slots, + node_slots.node_slots_error, + ); + + let mut kube_interface = MockKubeImpl::new(); + if !node_slots.node_slots_error { + // kube_interface to find Instance with node-a using slots: + // config-a-359973-1 & config-a-359973-3 + configure_get_instances(&mut kube_interface, instances_result_file); + // kube_interface to find no pods with unready containers + configure_find_pods_with_field( + &mut kube_interface, + "spec.nodeName=node-a", + "../test/json/running-pod-list-for-config-a-shared.json", + ); + if let Some(update_instance_) = update_instance { + trace!( + "expect_update_instance - slot1: {}, slot5: {}", + update_instance_.expected_slot_1_node, + update_instance_.expected_slot_5_node + ); + // kube_interface to update Instance + kube_interface + .expect_update_instance() + .times(1) + .withf(move |instance, name, namespace| { + name == "config-a-359973" + && namespace == "config-a-namespace" + && instance.nodes.len() == 3 + && instance.nodes.contains(&"node-a".to_string()) + && instance.nodes.contains(&"node-b".to_string()) + && instance.nodes.contains(&"node-c".to_string()) + && instance.device_usage["config-a-359973-0"] == "node-b" + && instance.device_usage["config-a-359973-1"] + == update_instance_.expected_slot_1_node + && instance.device_usage["config-a-359973-2"] == "node-b" + && instance.device_usage["config-a-359973-3"] == "node-a" + && instance.device_usage["config-a-359973-4"] == "node-c" + && instance.device_usage["config-a-359973-5"] + == update_instance_.expected_slot_5_node + }) + .returning(move |_, _, _| Ok(())); + } + } + + reconciler + .reconcile("node-a", grace_period, &slot_query, &kube_interface) + .await; + } + + #[tokio::test] + async fn test_reconcile_no_slots_to_reconcile() { + let _ = env_logger::builder().is_test(true).try_init(); + + let reconciler = DevicePluginSlotReconciler { + removal_slot_map: Arc::new(Mutex::new(HashMap::new())), + }; + configure_scnenario( + NodeSlots { + node_slots: HashSet::new(), + node_slots_error: false, + }, + "../test/json/shared-instance-list.json", + None, + Duration::from_secs(10), + &reconciler, + ) + .await; + } + + #[tokio::test] + async fn test_reconcile_get_slots_error() { + let _ = env_logger::builder().is_test(true).try_init(); + + let reconciler = DevicePluginSlotReconciler { + removal_slot_map: Arc::new(Mutex::new(HashMap::new())), + }; + configure_scnenario( + NodeSlots { + node_slots: HashSet::new(), + node_slots_error: true, + }, + "", + None, + Duration::from_secs(10), + &reconciler, + ) + .await; + } + + #[tokio::test] + async fn test_reconcile_slots_to_add() { + let _ = env_logger::builder().is_test(true).try_init(); + + let reconciler = DevicePluginSlotReconciler { + removal_slot_map: Arc::new(Mutex::new(HashMap::new())), + }; + + let grace_period = Duration::from_millis(100); + let mut node_slots = HashSet::new(); + node_slots.insert("config-a-359973-3".to_string()); + node_slots.insert("config-a-359973-5".to_string()); + configure_scnenario( + // slot_query to identify one slot used by this node + NodeSlots { + node_slots, + node_slots_error: false, + }, + // kube_interface to find Instance with node-a using slots: + // config-a-359973-1 & config-a-359973-3 + "../test/json/shared-instance-list-slots.json", + Some(UpdateInstance { + expected_slot_1_node: "node-a", + expected_slot_5_node: "node-a", + }), + grace_period, + &reconciler, + ) + .await; + + // Validate that the slot has been added to the list of "to be removed slots" + assert!(reconciler.removal_slot_map.lock().unwrap().len() == 1); + assert!(reconciler + .removal_slot_map + .lock() + .unwrap() + .contains_key("config-a-359973-1")); + } + + #[tokio::test] + async fn test_reconcile_slots_to_delete() { + let _ = env_logger::builder().is_test(true).try_init(); + + let reconciler = DevicePluginSlotReconciler { + removal_slot_map: Arc::new(Mutex::new(HashMap::new())), + }; + + let grace_period = Duration::from_millis(100); + let mut node_slots = HashSet::new(); + node_slots.insert("config-a-359973-3".to_string()); + configure_scnenario( + // slot_query to identify one slot used by this node + NodeSlots { + node_slots: node_slots.clone(), + node_slots_error: false, + }, + // kube_interface to find Instance with node-a using slots: + // config-a-359973-1 & config-a-359973-3 + "../test/json/shared-instance-list-slots.json", + None, + grace_period, + &reconciler, + ) + .await; + + // Validate that the slot has been added to the list of "to be removed slots" + assert!(reconciler.removal_slot_map.lock().unwrap().len() == 1); + assert!(reconciler + .removal_slot_map + .lock() + .unwrap() + .contains_key("config-a-359973-1")); + + // Wait for more than the grace period ... it short, so, just wait twice :) + std::thread::sleep(grace_period); + std::thread::sleep(grace_period); + + configure_scnenario( + // slot_query to identify one slot used by this node + NodeSlots { + node_slots: node_slots.clone(), + node_slots_error: false, + }, + // kube_interface to find Instance with node-a using slots: + // config-a-359973-1 & config-a-359973-3 + "../test/json/shared-instance-list-slots.json", + Some(UpdateInstance { + expected_slot_1_node: "", + expected_slot_5_node: "", + }), + grace_period, + &reconciler, + ) + .await; + + // Validate that the slot has been added to the list of "to be removed slots" + assert!(reconciler.removal_slot_map.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn test_reconcile_slots_to_delete_and_add() { + let _ = env_logger::builder().is_test(true).try_init(); + + let reconciler = DevicePluginSlotReconciler { + removal_slot_map: Arc::new(Mutex::new(HashMap::new())), + }; + + let grace_period = Duration::from_millis(100); + let mut node_slots = HashSet::new(); + node_slots.insert("config-a-359973-3".to_string()); + configure_scnenario( + // slot_query to identify one slot used by this node + NodeSlots { + node_slots, + node_slots_error: false, + }, + // kube_interface to find Instance with node-a using slots: + // config-a-359973-1 & config-a-359973-3 + "../test/json/shared-instance-list-slots.json", + None, + grace_period, + &reconciler, + ) + .await; + + // Validate that the slot has been added to the list of "to be removed slots" + assert!(reconciler.removal_slot_map.lock().unwrap().len() == 1); + assert!(reconciler + .removal_slot_map + .lock() + .unwrap() + .contains_key("config-a-359973-1")); + + // Wait for more than the grace period ... it short, so, just wait twice :) + std::thread::sleep(grace_period); + std::thread::sleep(grace_period); + + let mut node_slots_added = HashSet::new(); + node_slots_added.insert("config-a-359973-3".to_string()); + node_slots_added.insert("config-a-359973-5".to_string()); + configure_scnenario( + // slot_query to identify one slot used by this node + NodeSlots { + node_slots: node_slots_added, + node_slots_error: false, + }, + // kube_interface to find Instance with node-a using slots: + // config-a-359973-1 & config-a-359973-3 + "../test/json/shared-instance-list-slots.json", + Some(UpdateInstance { + expected_slot_1_node: "", + expected_slot_5_node: "node-a", + }), + grace_period, + &reconciler, + ) + .await; + + // Validate that the slot has been added to the list of "to be removed slots" + assert!(reconciler.removal_slot_map.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn test_reconcile_slots_to_delete_only_temporarily() { + let _ = env_logger::builder().is_test(true).try_init(); + + let reconciler = DevicePluginSlotReconciler { + removal_slot_map: Arc::new(Mutex::new(HashMap::new())), + }; + + let grace_period = Duration::from_millis(100); + let mut node_slots = HashSet::new(); + node_slots.insert("config-a-359973-3".to_string()); + configure_scnenario( + // slot_query to identify one slot used by this node + NodeSlots { + node_slots, + node_slots_error: false, + }, + // kube_interface to find Instance with node-a using slots: + // config-a-359973-1 & config-a-359973-3 + "../test/json/shared-instance-list-slots.json", + None, + grace_period, + &reconciler, + ) + .await; + + // Validate that the slot has been added to the list of "to be removed slots" + assert!(reconciler.removal_slot_map.lock().unwrap().len() == 1); + assert!(reconciler + .removal_slot_map + .lock() + .unwrap() + .contains_key("config-a-359973-1")); + + // Wait for more than the grace period ... it short, so, just wait twice :) + std::thread::sleep(grace_period); + std::thread::sleep(grace_period); + + let mut node_slots_added = HashSet::new(); + node_slots_added.insert("config-a-359973-1".to_string()); + node_slots_added.insert("config-a-359973-3".to_string()); + configure_scnenario( + // slot_query to identify two slots used by this node + NodeSlots { + node_slots: node_slots_added, + node_slots_error: false, + }, + // kube_interface to find Instance with node-a using slots: + // config-a-359973-1 & config-a-359973-3 + "../test/json/shared-instance-list-slots.json", + None, + grace_period, + &reconciler, + ) + .await; + + // Validate that the slot has been added to the list of "to be removed slots" + assert!(reconciler.removal_slot_map.lock().unwrap().is_empty()); + } +} diff --git a/agent/src/util/v1beta1.rs b/agent/src/util/v1beta1.rs new file mode 100644 index 0000000..2b179f0 --- /dev/null +++ b/agent/src/util/v1beta1.rs @@ -0,0 +1,638 @@ +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DevicePluginOptions { + /// Indicates if PreStartContainer call is required before each container start + #[prost(bool, tag = "1")] + pub pre_start_required: bool, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RegisterRequest { + /// Version of the API the Device Plugin was built against + #[prost(string, tag = "1")] + pub version: std::string::String, + /// Name of the unix socket the device plugin is listening on + /// PATH = path.Join(DevicePluginPath, endpoint) + #[prost(string, tag = "2")] + pub endpoint: std::string::String, + /// Schedulable resource name. As of now it's expected to be a DNS Label + #[prost(string, tag = "3")] + pub resource_name: std::string::String, + /// Options to be communicated with Device Manager + #[prost(message, optional, tag = "4")] + pub options: ::std::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Empty {} +/// ListAndWatch returns a stream of List of Devices +/// Whenever a Device state change or a Device disapears, ListAndWatch +/// returns the new list +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListAndWatchResponse { + #[prost(message, repeated, tag = "1")] + pub devices: ::std::vec::Vec, +} +/// E.g: +/// struct Device { +/// ID: "GPU-fef8089b-4820-abfc-e83e-94318197576e", +/// State: "Healthy", +///} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Device { + /// A unique ID assigned by the device plugin used + /// to identify devices during the communication + /// Max length of this field is 63 characters + #[prost(string, tag = "1")] + pub id: std::string::String, + /// Health of the device, can be healthy or unhealthy, see constants.go + #[prost(string, tag = "2")] + pub health: std::string::String, +} +/// - PreStartContainer is expected to be called before each container start if indicated by plugin during registration phase. +/// - PreStartContainer allows kubelet to pass reinitialized devices to containers. +/// - PreStartContainer allows Device Plugin to run device specific operations on +/// the Devices requested +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PreStartContainerRequest { + #[prost(string, repeated, tag = "1")] + pub devices_i_ds: ::std::vec::Vec, +} +/// PreStartContainerResponse will be send by plugin in response to PreStartContainerRequest +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PreStartContainerResponse {} +/// - Allocate is expected to be called during pod creation since allocation +/// failures for any container would result in pod startup failure. +/// - Allocate allows kubelet to exposes additional artifacts in a pod's +/// environment as directed by the plugin. +/// - Allocate allows Device Plugin to run device specific operations on +/// the Devices requested +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AllocateRequest { + #[prost(message, repeated, tag = "1")] + pub container_requests: ::std::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ContainerAllocateRequest { + #[prost(string, repeated, tag = "1")] + pub devices_i_ds: ::std::vec::Vec, +} +/// AllocateResponse includes the artifacts that needs to be injected into +/// a container for accessing 'deviceIDs' that were mentioned as part of +/// 'AllocateRequest'. +/// Failure Handling: +/// if Kubelet sends an allocation request for dev1 and dev2. +/// Allocation on dev1 succeeds but allocation on dev2 fails. +/// The Device plugin should send a ListAndWatch update and fail the +/// Allocation request +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AllocateResponse { + #[prost(message, repeated, tag = "1")] + pub container_responses: ::std::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ContainerAllocateResponse { + /// List of environment variable to be set in the container to access one of more devices. + #[prost(map = "string, string", tag = "1")] + pub envs: ::std::collections::HashMap, + /// Mounts for the container. + #[prost(message, repeated, tag = "2")] + pub mounts: ::std::vec::Vec, + /// Devices for the container. + #[prost(message, repeated, tag = "3")] + pub devices: ::std::vec::Vec, + /// Container annotations to pass to the container runtime + #[prost(map = "string, string", tag = "4")] + pub annotations: ::std::collections::HashMap, +} +/// Mount specifies a host volume to mount into a container. +/// where device library or tools are installed on host and container +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct Mount { + /// Path of the mount within the container. + #[prost(string, tag = "1")] + pub container_path: std::string::String, + /// Path of the mount on the host. + #[prost(string, tag = "2")] + pub host_path: std::string::String, + /// If set, the mount is read-only. + #[prost(bool, tag = "3")] + pub read_only: bool, +} +/// DeviceSpec specifies a host device to mount into a container. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeviceSpec { + /// Path of the device within the container. + #[prost(string, tag = "1")] + pub container_path: std::string::String, + /// Path of the device on the host. + #[prost(string, tag = "2")] + pub host_path: std::string::String, + /// Cgroups permissions of the device, candidates are one or more of + /// * r - allows container to read from the specified device. + /// * w - allows container to write to the specified device. + /// * m - allows container to create device files that do not yet exist. + #[prost(string, tag = "3")] + pub permissions: std::string::String, +} +#[doc = r" Generated client implementations."] +pub mod registration_client { + #![allow(unused_variables, dead_code, missing_docs)] + use tonic::codegen::*; + #[doc = " Registration is the service advertised by the Kubelet"] + #[doc = " Only when Kubelet answers with a success code to a Register Request"] + #[doc = " may Device Plugins start their service"] + #[doc = " Registration may fail when device plugin version is not supported by"] + #[doc = " Kubelet or the registered resourceName is already taken by another"] + #[doc = " active device plugin. Device plugin is expected to terminate upon registration failure"] + pub struct RegistrationClient { + inner: tonic::client::Grpc, + } + impl RegistrationClient { + #[doc = r" Attempt to create a new client by connecting to a given endpoint."] + pub async fn connect(dst: D) -> Result + where + D: std::convert::TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl RegistrationClient + where + T: tonic::client::GrpcService, + T::ResponseBody: Body + HttpBody + Send + 'static, + T::Error: Into, + ::Error: Into + Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_interceptor(inner: T, interceptor: impl Into) -> Self { + let inner = tonic::client::Grpc::with_interceptor(inner, interceptor); + Self { inner } + } + pub async fn register( + &mut self, + request: impl tonic::IntoRequest, + ) -> Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/v1beta1.Registration/Register"); + self.inner.unary(request.into_request(), path, codec).await + } + } + impl Clone for RegistrationClient { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } + } +} +#[doc = r" Generated client implementations."] +pub mod device_plugin_client { + #![allow(unused_variables, dead_code, missing_docs)] + use tonic::codegen::*; + #[doc = " DevicePlugin is the service advertised by Device Plugins"] + pub struct DevicePluginClient { + inner: tonic::client::Grpc, + } + impl DevicePluginClient { + #[doc = r" Attempt to create a new client by connecting to a given endpoint."] + pub async fn connect(dst: D) -> Result + where + D: std::convert::TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl DevicePluginClient + where + T: tonic::client::GrpcService, + T::ResponseBody: Body + HttpBody + Send + 'static, + T::Error: Into, + ::Error: Into + Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_interceptor(inner: T, interceptor: impl Into) -> Self { + let inner = tonic::client::Grpc::with_interceptor(inner, interceptor); + Self { inner } + } + #[doc = " GetDevicePluginOptions returns options to be communicated with Device"] + #[doc = " Manager"] + pub async fn get_device_plugin_options( + &mut self, + request: impl tonic::IntoRequest, + ) -> Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/v1beta1.DevicePlugin/GetDevicePluginOptions", + ); + self.inner.unary(request.into_request(), path, codec).await + } + #[doc = " ListAndWatch returns a stream of List of Devices"] + #[doc = " Whenever a Device state change or a Device disapears, ListAndWatch"] + #[doc = " returns the new list"] + pub async fn list_and_watch( + &mut self, + request: impl tonic::IntoRequest, + ) -> Result< + tonic::Response>, + tonic::Status, + > { + self.inner.ready().await.map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/v1beta1.DevicePlugin/ListAndWatch"); + self.inner + .server_streaming(request.into_request(), path, codec) + .await + } + #[doc = " Allocate is called during container creation so that the Device"] + #[doc = " Plugin can run device specific operations and instruct Kubelet"] + #[doc = " of the steps to make the Device available in the container"] + pub async fn allocate( + &mut self, + request: impl tonic::IntoRequest, + ) -> Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/v1beta1.DevicePlugin/Allocate"); + self.inner.unary(request.into_request(), path, codec).await + } + #[doc = " PreStartContainer is called, if indicated by Device Plugin during registeration phase,"] + #[doc = " before each container start. Device plugin can run device specific operations"] + #[doc = " such as reseting the device before making devices available to the container"] + pub async fn pre_start_container( + &mut self, + request: impl tonic::IntoRequest, + ) -> Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/v1beta1.DevicePlugin/PreStartContainer"); + self.inner.unary(request.into_request(), path, codec).await + } + } + impl Clone for DevicePluginClient { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } + } +} +#[doc = r" Generated server implementations."] +pub mod registration_server { + #![allow(unused_variables, dead_code, missing_docs)] + use tonic::codegen::*; + #[doc = "Generated trait containing gRPC methods that should be implemented for use with RegistrationServer."] + #[async_trait] + pub trait Registration: Send + Sync + 'static { + async fn register( + &self, + request: tonic::Request, + ) -> Result, tonic::Status>; + } + #[doc = " Registration is the service advertised by the Kubelet"] + #[doc = " Only when Kubelet answers with a success code to a Register Request"] + #[doc = " may Device Plugins start their service"] + #[doc = " Registration may fail when device plugin version is not supported by"] + #[doc = " Kubelet or the registered resourceName is already taken by another"] + #[doc = " active device plugin. Device plugin is expected to terminate upon registration failure"] + #[derive(Debug)] + #[doc(hidden)] + pub struct RegistrationServer { + inner: _Inner, + } + struct _Inner(Arc, Option); + impl RegistrationServer { + pub fn new(inner: T) -> Self { + let inner = Arc::new(inner); + let inner = _Inner(inner, None); + Self { inner } + } + pub fn with_interceptor(inner: T, interceptor: impl Into) -> Self { + let inner = Arc::new(inner); + let inner = _Inner(inner, Some(interceptor.into())); + Self { inner } + } + } + impl Service> for RegistrationServer { + type Response = http::Response; + type Error = Never; + type Future = BoxFuture; + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + let inner = self.inner.clone(); + match req.uri().path() { + "/v1beta1.Registration/Register" => { + struct RegisterSvc(pub Arc); + impl tonic::server::UnaryService for RegisterSvc { + type Response = super::Empty; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = self.0.clone(); + let fut = async move { inner.register(request).await }; + Box::pin(fut) + } + } + let inner = self.inner.clone(); + let fut = async move { + let interceptor = inner.1.clone(); + let inner = inner.0; + let method = RegisterSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = if let Some(interceptor) = interceptor { + tonic::server::Grpc::with_interceptor(codec, interceptor) + } else { + tonic::server::Grpc::new(codec) + }; + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => Box::pin(async move { + Ok(http::Response::builder() + .status(200) + .header("grpc-status", "12") + .body(tonic::body::BoxBody::empty()) + .unwrap()) + }), + } + } + } + impl Clone for RegistrationServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { inner } + } + } + impl Clone for _Inner { + fn clone(&self) -> Self { + Self(self.0.clone(), self.1.clone()) + } + } + impl std::fmt::Debug for _Inner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.0) + } + } + impl tonic::transport::NamedService for RegistrationServer { + const NAME: &'static str = "v1beta1.Registration"; + } +} +#[doc = r" Generated server implementations."] +pub mod device_plugin_server { + #![allow(unused_variables, dead_code, missing_docs)] + use tonic::codegen::*; + #[doc = "Generated trait containing gRPC methods that should be implemented for use with DevicePluginServer."] + #[async_trait] + pub trait DevicePlugin: Send + Sync + 'static { + #[doc = " GetDevicePluginOptions returns options to be communicated with Device"] + #[doc = " Manager"] + async fn get_device_plugin_options( + &self, + request: tonic::Request, + ) -> Result, tonic::Status>; + #[doc = "Server streaming response type for the ListAndWatch method."] + type ListAndWatchStream: Stream> + + Send + + Sync + + 'static; + #[doc = " ListAndWatch returns a stream of List of Devices"] + #[doc = " Whenever a Device state change or a Device disapears, ListAndWatch"] + #[doc = " returns the new list"] + async fn list_and_watch( + &self, + request: tonic::Request, + ) -> Result, tonic::Status>; + #[doc = " Allocate is called during container creation so that the Device"] + #[doc = " Plugin can run device specific operations and instruct Kubelet"] + #[doc = " of the steps to make the Device available in the container"] + async fn allocate( + &self, + request: tonic::Request, + ) -> Result, tonic::Status>; + #[doc = " PreStartContainer is called, if indicated by Device Plugin during registeration phase,"] + #[doc = " before each container start. Device plugin can run device specific operations"] + #[doc = " such as reseting the device before making devices available to the container"] + async fn pre_start_container( + &self, + request: tonic::Request, + ) -> Result, tonic::Status>; + } + #[doc = " DevicePlugin is the service advertised by Device Plugins"] + #[derive(Debug)] + #[doc(hidden)] + pub struct DevicePluginServer { + inner: _Inner, + } + struct _Inner(Arc, Option); + impl DevicePluginServer { + pub fn new(inner: T) -> Self { + let inner = Arc::new(inner); + let inner = _Inner(inner, None); + Self { inner } + } + pub fn with_interceptor(inner: T, interceptor: impl Into) -> Self { + let inner = Arc::new(inner); + let inner = _Inner(inner, Some(interceptor.into())); + Self { inner } + } + } + impl Service> for DevicePluginServer { + type Response = http::Response; + type Error = Never; + type Future = BoxFuture; + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + let inner = self.inner.clone(); + match req.uri().path() { + "/v1beta1.DevicePlugin/GetDevicePluginOptions" => { + struct GetDevicePluginOptionsSvc(pub Arc); + impl tonic::server::UnaryService for GetDevicePluginOptionsSvc { + type Response = super::DevicePluginOptions; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = self.0.clone(); + let fut = async move { inner.get_device_plugin_options(request).await }; + Box::pin(fut) + } + } + let inner = self.inner.clone(); + let fut = async move { + let interceptor = inner.1.clone(); + let inner = inner.0; + let method = GetDevicePluginOptionsSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = if let Some(interceptor) = interceptor { + tonic::server::Grpc::with_interceptor(codec, interceptor) + } else { + tonic::server::Grpc::new(codec) + }; + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/v1beta1.DevicePlugin/ListAndWatch" => { + struct ListAndWatchSvc(pub Arc); + impl tonic::server::ServerStreamingService for ListAndWatchSvc { + type Response = super::ListAndWatchResponse; + type ResponseStream = T::ListAndWatchStream; + type Future = + BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = self.0.clone(); + let fut = async move { inner.list_and_watch(request).await }; + Box::pin(fut) + } + } + let inner = self.inner.clone(); + let fut = async move { + let interceptor = inner.1; + let inner = inner.0; + let method = ListAndWatchSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = if let Some(interceptor) = interceptor { + tonic::server::Grpc::with_interceptor(codec, interceptor) + } else { + tonic::server::Grpc::new(codec) + }; + let res = grpc.server_streaming(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/v1beta1.DevicePlugin/Allocate" => { + struct AllocateSvc(pub Arc); + impl tonic::server::UnaryService for AllocateSvc { + type Response = super::AllocateResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = self.0.clone(); + let fut = async move { inner.allocate(request).await }; + Box::pin(fut) + } + } + let inner = self.inner.clone(); + let fut = async move { + let interceptor = inner.1.clone(); + let inner = inner.0; + let method = AllocateSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = if let Some(interceptor) = interceptor { + tonic::server::Grpc::with_interceptor(codec, interceptor) + } else { + tonic::server::Grpc::new(codec) + }; + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/v1beta1.DevicePlugin/PreStartContainer" => { + struct PreStartContainerSvc(pub Arc); + impl + tonic::server::UnaryService + for PreStartContainerSvc + { + type Response = super::PreStartContainerResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = self.0.clone(); + let fut = async move { inner.pre_start_container(request).await }; + Box::pin(fut) + } + } + let inner = self.inner.clone(); + let fut = async move { + let interceptor = inner.1.clone(); + let inner = inner.0; + let method = PreStartContainerSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = if let Some(interceptor) = interceptor { + tonic::server::Grpc::with_interceptor(codec, interceptor) + } else { + tonic::server::Grpc::new(codec) + }; + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => Box::pin(async move { + Ok(http::Response::builder() + .status(200) + .header("grpc-status", "12") + .body(tonic::body::BoxBody::empty()) + .unwrap()) + }), + } + } + } + impl Clone for DevicePluginServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { inner } + } + } + impl Clone for _Inner { + fn clone(&self) -> Self { + Self(self.0.clone(), self.1.clone()) + } + } + impl std::fmt::Debug for _Inner { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.0) + } + } + impl tonic::transport::NamedService for DevicePluginServer { + const NAME: &'static str = "v1beta1.DevicePlugin"; + } +} diff --git a/build/akri-containers.mk b/build/akri-containers.mk new file mode 100644 index 0000000..e12574c --- /dev/null +++ b/build/akri-containers.mk @@ -0,0 +1,265 @@ +# +# +# INSTALL-CROSS: install cargo cross building tool: +# +# `make install-cross` +# +# +.PHONY: install-cross +install-cross: + cargo install cross + + +# +# +# AKRI: make and push the images for akri: +# +# To make all platforms: `make akri` +# To make specific platforms: `BUILD_AMD64=1 BUILD_ARM32=0 BUILD_ARM64=1 make akri` +# To make single component: `make akri-[controller|agent|udev|onvif|streaming]` +# To make specific platforms: `BUILD_AMD64=1 BUILD_ARM32=0 BUILD_ARM64=1 make akri-[controller|agent|udev|onvif|streaming]` +# +# +.PHONY: akri +akri: akri-build akri-docker +akri-controller: akri-build akri-docker-controller +akri-agent: akri-build akri-docker-agent +akri-udev: akri-build akri-docker-udev +akri-onvif: akri-build akri-docker-onvif +akri-streaming: akri-build akri-docker-streaming + +akri-build: install-cross akri-cross-build +akri-docker: akri-docker-build akri-docker-push-per-arch akri-docker-push-multi-arch-create akri-docker-push-multi-arch-push +akri-docker-controller: controller-build controller-docker-per-arch controller-docker-multi-arch-create controller-docker-multi-arch-push +akri-docker-agent: agent-build agent-docker-per-arch agent-docker-multi-arch-create agent-docker-multi-arch-push +akri-docker-udev: udev-build udev-docker-per-arch udev-docker-multi-arch-create udev-docker-multi-arch-push +akri-docker-onvif: onvif-build onvif-docker-per-arch onvif-docker-multi-arch-create onvif-docker-multi-arch-push +akri-docker-streaming: streaming-build streaming-docker-per-arch streaming-docker-multi-arch-create streaming-docker-multi-arch-push + +akri-cross-build: akri-cross-build-amd64 akri-cross-build-arm32 akri-cross-build-arm64 +akri-cross-build-amd64: +ifeq (1, $(BUILD_AMD64)) + PKG_CONFIG_ALLOW_CROSS=1 cross build --release --target=$(AMD64_TARGET) +endif +akri-cross-build-arm32: +ifeq (1, ${BUILD_ARM32}) + PKG_CONFIG_ALLOW_CROSS=1 cross build --release --target=$(ARM32V7_TARGET) +endif +akri-cross-build-arm64: +ifeq (1, ${BUILD_ARM64}) + PKG_CONFIG_ALLOW_CROSS=1 cross build --release --target=$(ARM64V8_TARGET) +endif + +akri-docker-build: controller-build agent-build udev-build onvif-build streaming-build +controller-build: controller-build-amd64 controller-build-arm32 controller-build-arm64 +controller-build-amd64: +ifeq (1, ${BUILD_AMD64}) + docker build $(CACHE_OPTION) -f $(DOCKERFILE_DIR)/Dockerfile.controller . -t $(PREFIX)/controller:$(LABEL_PREFIX)-$(AMD64_SUFFIX) --build-arg PLATFORM=$(AMD64_SUFFIX) --build-arg CROSS_BUILD_TARGET=$(AMD64_TARGET) +endif +controller-build-arm32: +ifeq (1, ${BUILD_ARM32}) + docker build $(CACHE_OPTION) -f $(DOCKERFILE_DIR)/Dockerfile.controller . -t $(PREFIX)/controller:$(LABEL_PREFIX)-$(ARM32V7_SUFFIX) --build-arg PLATFORM=$(ARM32V7_SUFFIX) --build-arg CROSS_BUILD_TARGET=$(ARM32V7_TARGET) +endif +controller-build-arm64: +ifeq (1, ${BUILD_ARM64}) + docker build $(CACHE_OPTION) -f $(DOCKERFILE_DIR)/Dockerfile.controller . -t $(PREFIX)/controller:$(LABEL_PREFIX)-$(ARM64V8_SUFFIX) --build-arg PLATFORM=$(ARM64V8_SUFFIX) --build-arg CROSS_BUILD_TARGET=$(ARM64V8_TARGET) +endif + +agent-build: agent-build-amd64 agent-build-arm32 agent-build-arm64 +agent-build-amd64: +ifeq (1, ${BUILD_AMD64}) + docker build $(CACHE_OPTION) -f $(DOCKERFILE_DIR)/Dockerfile.agent . -t $(PREFIX)/agent:$(LABEL_PREFIX)-$(AMD64_SUFFIX) --build-arg PLATFORM=$(AMD64_SUFFIX) --build-arg CROSS_BUILD_TARGET=$(AMD64_TARGET) +endif +agent-build-arm32: +ifeq (1, ${BUILD_ARM32}) + docker build $(CACHE_OPTION) -f $(DOCKERFILE_DIR)/Dockerfile.agent . -t $(PREFIX)/agent:$(LABEL_PREFIX)-$(ARM32V7_SUFFIX) --build-arg PLATFORM=$(ARM32V7_SUFFIX) --build-arg CROSS_BUILD_TARGET=$(ARM32V7_TARGET) +endif +agent-build-arm64: +ifeq (1, ${BUILD_ARM64}) + docker build $(CACHE_OPTION) -f $(DOCKERFILE_DIR)/Dockerfile.agent . -t $(PREFIX)/agent:$(LABEL_PREFIX)-$(ARM64V8_SUFFIX) --build-arg PLATFORM=$(ARM64V8_SUFFIX) --build-arg CROSS_BUILD_TARGET=$(ARM64V8_TARGET) +endif + +udev-build: udev-build-amd64 udev-build-arm32 udev-build-arm64 +udev-build-amd64: +ifeq (1, ${BUILD_AMD64}) + docker build $(CACHE_OPTION) -f $(DOCKERFILE_DIR)/Dockerfile.udev-video-broker . -t $(PREFIX)/udev-video-broker:$(LABEL_PREFIX)-$(AMD64_SUFFIX) --build-arg PLATFORM=$(AMD64_SUFFIX) --build-arg CROSS_BUILD_TARGET=$(AMD64_TARGET) +endif +udev-build-arm32: +ifeq (1, ${BUILD_ARM32}) + docker build $(CACHE_OPTION) -f $(DOCKERFILE_DIR)/Dockerfile.udev-video-broker . -t $(PREFIX)/udev-video-broker:$(LABEL_PREFIX)-$(ARM32V7_SUFFIX) --build-arg PLATFORM=$(ARM32V7_SUFFIX) --build-arg CROSS_BUILD_TARGET=$(ARM32V7_TARGET) +endif +udev-build-arm64: +ifeq (1, ${BUILD_ARM64}) + docker build $(CACHE_OPTION) -f $(DOCKERFILE_DIR)/Dockerfile.udev-video-broker . -t $(PREFIX)/udev-video-broker:$(LABEL_PREFIX)-$(ARM64V8_SUFFIX) --build-arg PLATFORM=$(ARM64V8_SUFFIX) --build-arg CROSS_BUILD_TARGET=$(ARM64V8_TARGET) +endif + +onvif-build: onvif-build-amd64 onvif-build-arm32 onvif-build-arm64 +onvif-build-amd64: +ifeq (1, ${BUILD_AMD64}) + docker build $(CACHE_OPTION) -f $(DOCKERFILE_DIR)/Dockerfile.onvif-video-broker . -t $(PREFIX)/onvif-video-broker:$(LABEL_PREFIX)-$(AMD64_SUFFIX) --build-arg OUTPUT_PLATFORM_TAG=$(USE_OPENCV_BASE_VERSION)-$(AMD64_SUFFIX) --build-arg DOTNET_PUBLISH_RUNTIME=linux-x64 +endif +onvif-build-arm32: +ifeq (1, ${BUILD_ARM32}) + docker build $(CACHE_OPTION) -f $(DOCKERFILE_DIR)/Dockerfile.onvif-video-broker . -t $(PREFIX)/onvif-video-broker:$(LABEL_PREFIX)-$(ARM32V7_SUFFIX) --build-arg OUTPUT_PLATFORM_TAG=$(USE_OPENCV_BASE_VERSION)-$(ARM32V7_SUFFIX) --build-arg DOTNET_PUBLISH_RUNTIME=linux-arm +endif +onvif-build-arm64: +ifeq (1, ${BUILD_ARM64}) + docker build $(CACHE_OPTION) -f $(DOCKERFILE_DIR)/Dockerfile.onvif-video-broker . -t $(PREFIX)/onvif-video-broker:$(LABEL_PREFIX)-$(ARM64V8_SUFFIX) --build-arg OUTPUT_PLATFORM_TAG=$(USE_OPENCV_BASE_VERSION)-$(ARM64V8_SUFFIX) --build-arg DOTNET_PUBLISH_RUNTIME=linux-arm64 +endif + +streaming-build: streaming-build-amd64 streaming-build-arm32 streaming-build-arm64 +streaming-build-amd64: +ifeq (1, ${BUILD_AMD64}) + docker build $(CACHE_OPTION) -f $(DOCKERFILE_DIR)/Dockerfile.video-streaming-app . -t $(PREFIX)/video-streaming-app:$(LABEL_PREFIX)-$(AMD64_SUFFIX) --build-arg PLATFORM=$(AMD64_SUFFIX) +endif +streaming-build-arm32: +ifeq (1, ${BUILD_ARM32}) + docker build $(CACHE_OPTION) -f $(DOCKERFILE_DIR)/Dockerfile.video-streaming-app . -t $(PREFIX)/video-streaming-app:$(LABEL_PREFIX)-$(ARM32V7_SUFFIX) --build-arg PLATFORM=$(ARM32V7_SUFFIX) +endif +streaming-build-arm64: +ifeq (1, ${BUILD_ARM64}) + docker build $(CACHE_OPTION) -f $(DOCKERFILE_DIR)/Dockerfile.video-streaming-app . -t $(PREFIX)/video-streaming-app:$(LABEL_PREFIX)-$(ARM64V8_SUFFIX) --build-arg PLATFORM=$(ARM64V8_SUFFIX) +endif + +akri-docker-push-per-arch: controller-docker-per-arch agent-docker-per-arch udev-docker-per-arch onvif-docker-per-arch streaming-docker-per-arch + +controller-docker-per-arch: controller-docker-per-arch-amd64 controller-docker-per-arch-arm32 controller-docker-per-arch-arm64 +controller-docker-per-arch-amd64: +ifeq (1, ${BUILD_AMD64}) + docker push $(PREFIX)/controller:$(LABEL_PREFIX)-$(AMD64_SUFFIX) +endif +controller-docker-per-arch-arm32: +ifeq (1, ${BUILD_ARM32}) + docker push $(PREFIX)/controller:$(LABEL_PREFIX)-$(ARM32V7_SUFFIX) +endif +controller-docker-per-arch-arm64: +ifeq (1, ${BUILD_ARM64}) + docker push $(PREFIX)/controller:$(LABEL_PREFIX)-$(ARM64V8_SUFFIX) +endif + +agent-docker-per-arch: agent-docker-per-arch-amd64 agent-docker-per-arch-arm32 agent-docker-per-arch-arm64 +agent-docker-per-arch-amd64: +ifeq (1, ${BUILD_AMD64}) + docker push $(PREFIX)/agent:$(LABEL_PREFIX)-$(AMD64_SUFFIX) +endif +agent-docker-per-arch-arm32: +ifeq (1, ${BUILD_ARM32}) + docker push $(PREFIX)/agent:$(LABEL_PREFIX)-$(ARM32V7_SUFFIX) +endif +agent-docker-per-arch-arm64: +ifeq (1, ${BUILD_ARM64}) + docker push $(PREFIX)/agent:$(LABEL_PREFIX)-$(ARM64V8_SUFFIX) +endif + +onvif-docker-per-arch: onvif-docker-per-arch-amd64 onvif-docker-per-arch-arm32 onvif-docker-per-arch-arm64 +onvif-docker-per-arch-amd64: +ifeq (1, ${BUILD_AMD64}) + docker push $(PREFIX)/onvif-video-broker:$(LABEL_PREFIX)-$(AMD64_SUFFIX) +endif +onvif-docker-per-arch-arm32: +ifeq (1, ${BUILD_ARM32}) + docker push $(PREFIX)/onvif-video-broker:$(LABEL_PREFIX)-$(ARM32V7_SUFFIX) +endif +onvif-docker-per-arch-arm64: +ifeq (1, ${BUILD_ARM64}) + docker push $(PREFIX)/onvif-video-broker:$(LABEL_PREFIX)-$(ARM64V8_SUFFIX) +endif + +udev-docker-per-arch: udev-docker-per-arch-amd64 udev-docker-per-arch-arm32 udev-docker-per-arch-arm64 +udev-docker-per-arch-amd64: +ifeq (1, ${BUILD_AMD64}) + docker push $(PREFIX)/udev-video-broker:$(LABEL_PREFIX)-$(AMD64_SUFFIX) +endif +udev-docker-per-arch-arm32: +ifeq (1, ${BUILD_ARM32}) + docker push $(PREFIX)/udev-video-broker:$(LABEL_PREFIX)-$(ARM32V7_SUFFIX) +endif +udev-docker-per-arch-arm64: +ifeq (1, ${BUILD_ARM64}) + docker push $(PREFIX)/udev-video-broker:$(LABEL_PREFIX)-$(ARM64V8_SUFFIX) +endif + +streaming-docker-per-arch: streaming-docker-per-arch-amd64 streaming-docker-per-arch-arm32 streaming-docker-per-arch-arm64 +streaming-docker-per-arch-amd64: +ifeq (1, ${BUILD_AMD64}) + docker push $(PREFIX)/video-streaming-app:$(LABEL_PREFIX)-$(AMD64_SUFFIX) +endif +streaming-docker-per-arch-arm32: +ifeq (1, ${BUILD_ARM32}) + docker push $(PREFIX)/video-streaming-app:$(LABEL_PREFIX)-$(ARM32V7_SUFFIX) +endif +streaming-docker-per-arch-arm64: +ifeq (1, ${BUILD_ARM64}) + docker push $(PREFIX)/video-streaming-app:$(LABEL_PREFIX)-$(ARM64V8_SUFFIX) +endif + +akri-docker-push-multi-arch-create: controller-docker-multi-arch-create agent-docker-multi-arch-create udev-docker-multi-arch-create onvif-docker-multi-arch-create streaming-docker-multi-arch-create + +controller-docker-multi-arch-create: +ifeq (1, ${BUILD_AMD64}) + $(ENABLE_DOCKER_MANIFEST) docker manifest create --amend $(PREFIX)/controller:$(LABEL_PREFIX) $(PREFIX)/controller:$(LABEL_PREFIX)-$(AMD64_SUFFIX) +endif +ifeq (1, ${BUILD_ARM32}) + $(ENABLE_DOCKER_MANIFEST) docker manifest create --amend $(PREFIX)/controller:$(LABEL_PREFIX) $(PREFIX)/controller:$(LABEL_PREFIX)-$(ARM32V7_SUFFIX) +endif +ifeq (1, ${BUILD_ARM64}) + $(ENABLE_DOCKER_MANIFEST) docker manifest create --amend $(PREFIX)/controller:$(LABEL_PREFIX) $(PREFIX)/controller:$(LABEL_PREFIX)-$(ARM64V8_SUFFIX) +endif + +agent-docker-multi-arch-create: +ifeq (1, ${BUILD_AMD64}) + $(ENABLE_DOCKER_MANIFEST) docker manifest create --amend $(PREFIX)/agent:$(LABEL_PREFIX) $(PREFIX)/agent:$(LABEL_PREFIX)-$(AMD64_SUFFIX) +endif +ifeq (1, ${BUILD_ARM32}) + $(ENABLE_DOCKER_MANIFEST) docker manifest create --amend $(PREFIX)/agent:$(LABEL_PREFIX) $(PREFIX)/agent:$(LABEL_PREFIX)-$(ARM32V7_SUFFIX) +endif +ifeq (1, ${BUILD_ARM64}) + $(ENABLE_DOCKER_MANIFEST) docker manifest create --amend $(PREFIX)/agent:$(LABEL_PREFIX) $(PREFIX)/agent:$(LABEL_PREFIX)-$(ARM64V8_SUFFIX) +endif + +udev-docker-multi-arch-create: +ifeq (1, ${BUILD_AMD64}) + $(ENABLE_DOCKER_MANIFEST) docker manifest create --amend $(PREFIX)/udev-video-broker:$(LABEL_PREFIX) $(PREFIX)/udev-video-broker:$(LABEL_PREFIX)-$(AMD64_SUFFIX) +endif +ifeq (1, ${BUILD_ARM32}) + $(ENABLE_DOCKER_MANIFEST) docker manifest create --amend $(PREFIX)/udev-video-broker:$(LABEL_PREFIX) $(PREFIX)/udev-video-broker:$(LABEL_PREFIX)-$(ARM32V7_SUFFIX) +endif +ifeq (1, ${BUILD_ARM64}) + $(ENABLE_DOCKER_MANIFEST) docker manifest create --amend $(PREFIX)/udev-video-broker:$(LABEL_PREFIX) $(PREFIX)/udev-video-broker:$(LABEL_PREFIX)-$(ARM64V8_SUFFIX) +endif + +onvif-docker-multi-arch-create: +ifeq (1, ${BUILD_AMD64}) + $(ENABLE_DOCKER_MANIFEST) docker manifest create --amend $(PREFIX)/onvif-video-broker:$(LABEL_PREFIX) $(PREFIX)/onvif-video-broker:$(LABEL_PREFIX)-$(AMD64_SUFFIX) +endif +ifeq (1, ${BUILD_ARM32}) + $(ENABLE_DOCKER_MANIFEST) docker manifest create --amend $(PREFIX)/onvif-video-broker:$(LABEL_PREFIX) $(PREFIX)/onvif-video-broker:$(LABEL_PREFIX)-$(ARM32V7_SUFFIX) +endif +ifeq (1, ${BUILD_ARM64}) + $(ENABLE_DOCKER_MANIFEST) docker manifest create --amend $(PREFIX)/onvif-video-broker:$(LABEL_PREFIX) $(PREFIX)/onvif-video-broker:$(LABEL_PREFIX)-$(ARM64V8_SUFFIX) +endif + +streaming-docker-multi-arch-create: +ifeq (1, ${BUILD_AMD64}) + $(ENABLE_DOCKER_MANIFEST) docker manifest create --amend $(PREFIX)/video-streaming-app:$(LABEL_PREFIX) $(PREFIX)/video-streaming-app:$(LABEL_PREFIX)-$(AMD64_SUFFIX) +endif +ifeq (1, ${BUILD_ARM32}) + $(ENABLE_DOCKER_MANIFEST) docker manifest create --amend $(PREFIX)/video-streaming-app:$(LABEL_PREFIX) $(PREFIX)/video-streaming-app:$(LABEL_PREFIX)-$(ARM32V7_SUFFIX) +endif +ifeq (1, ${BUILD_ARM64}) + $(ENABLE_DOCKER_MANIFEST) docker manifest create --amend $(PREFIX)/video-streaming-app:$(LABEL_PREFIX) $(PREFIX)/video-streaming-app:$(LABEL_PREFIX)-$(ARM64V8_SUFFIX) +endif + +akri-docker-push-multi-arch-push: controller-docker-multi-arch-push agent-docker-multi-arch-push udev-docker-multi-arch-push onvif-docker-multi-arch-push streaming-docker-multi-arch-push + +controller-docker-multi-arch-push: + $(ENABLE_DOCKER_MANIFEST) docker manifest push $(PREFIX)/controller:$(LABEL_PREFIX) +agent-docker-multi-arch-push: + $(ENABLE_DOCKER_MANIFEST) docker manifest push $(PREFIX)/agent:$(LABEL_PREFIX) +udev-docker-multi-arch-push: + $(ENABLE_DOCKER_MANIFEST) docker manifest push $(PREFIX)/udev-video-broker:$(LABEL_PREFIX) +onvif-docker-multi-arch-push: + $(ENABLE_DOCKER_MANIFEST) docker manifest push $(PREFIX)/onvif-video-broker:$(LABEL_PREFIX) +streaming-docker-multi-arch-push: + $(ENABLE_DOCKER_MANIFEST) docker manifest push $(PREFIX)/video-streaming-app:$(LABEL_PREFIX) + diff --git a/build/container-images-legal-notice.md b/build/container-images-legal-notice.md new file mode 100644 index 0000000..0cd6f30 --- /dev/null +++ b/build/container-images-legal-notice.md @@ -0,0 +1,3 @@ +**Legal Notice:** Microsoft containers are licensed under various licenses depending upon the container’s content. If the container has a Debian, Ubuntu or Alpine Linux base image and you want to retrieve the licenses, copyright notices or source code information for the base image and its official packages, please review the [Linux Legal Metadata](https://aka.ms/mcr/osslinuxmetadata) instructions file. If you are using a Windows based container the governing licenses are referenced in this Docker Hub “License” section or in the container itself, usually as a NOTICE or LICENSE file. All other components (whether open or closed source) are licensed as indicated either in this Docker Hub “License” section or in the container itself. + +This container may incorporate material from third parties. Microsoft makes certain open source code available at [https://3rdpartysource.microsoft.com](https://3rdpartysource.microsoft.com), or if you want a copy of the source code in this container you may send a check or money order for US $5.00, including the product name, the open source component name, and version number, to: Source Code Compliance Team, Microsoft Corporation, One Microsoft Way, Redmond, WA 98052 USA \ No newline at end of file diff --git a/build/containers/Dockerfile.agent b/build/containers/Dockerfile.agent new file mode 100644 index 0000000..54fdcca --- /dev/null +++ b/build/containers/Dockerfile.agent @@ -0,0 +1,17 @@ +ARG PLATFORM=amd64 +ARG CROSS_BUILD_TARGET=x86_64-unknown-linux-gnu +FROM ${PLATFORM}/debian:buster-slim +ARG CROSS_BUILD_TARGET +RUN echo "Creating container based on ${PLATFORM}/debian:buster-slim" +RUN echo "Using Rust binaries from ${CROSS_BUILD_TARGET}" + +# Link the container to the Akri repository +LABEL org.opencontainers.image.source https://github.com/deislabs/akri + +# Copy over container legal notice +COPY ./build/container-images-legal-notice.md . + +RUN apt-get update && apt-get install -y --no-install-recommends libssl-dev openssl && apt-get clean +COPY ./target/${CROSS_BUILD_TARGET}/release/agent /agent +ENV RUST_LOG agent,akri_shared +CMD ["./agent"] diff --git a/build/containers/Dockerfile.controller b/build/containers/Dockerfile.controller new file mode 100644 index 0000000..fd0ea05 --- /dev/null +++ b/build/containers/Dockerfile.controller @@ -0,0 +1,17 @@ +ARG PLATFORM=amd64 +ARG CROSS_BUILD_TARGET=x86_64-unknown-linux-gnu +FROM ${PLATFORM}/debian:buster-slim +ARG CROSS_BUILD_TARGET +RUN echo "Creating container based on ${PLATFORM}/debian:buster-slim" +RUN echo "Using Rust binaries from ${CROSS_BUILD_TARGET}" + +# Link the container to the Akri repository +LABEL org.opencontainers.image.source https://github.com/deislabs/akri + +# Copy over container legal notice +COPY ./build/container-images-legal-notice.md . + +RUN apt-get update && apt-get install -y --no-install-recommends libssl-dev openssl && apt-get clean +COPY ./target/${CROSS_BUILD_TARGET}/release/controller /controller +ENV RUST_LOG controller,akri_shared +CMD ["./controller"] diff --git a/build/containers/Dockerfile.onvif-video-broker b/build/containers/Dockerfile.onvif-video-broker new file mode 100644 index 0000000..f1e2739 --- /dev/null +++ b/build/containers/Dockerfile.onvif-video-broker @@ -0,0 +1,36 @@ +ARG BUILD_PLATFORM_TAG=3.1-buster +ARG OUTPUT_PLATFORM_TAG=3.1-buster-slim +ARG DOTNET_PUBLISH_RUNTIME=linux-x64 + +FROM mcr.microsoft.com/dotnet/core/sdk:${BUILD_PLATFORM_TAG} AS build +ARG BUILD_PLATFORM_TAG +RUN echo "Build base image: mcr.microsoft.com/dotnet/core/sdk:${BUILD_PLATFORM_TAG}" + +WORKDIR /src +COPY ["samples/brokers/onvif-video-broker/onvif-video-broker.csproj", "."] +RUN find . && dotnet restore "onvif-video-broker.csproj" +COPY ["samples/brokers/onvif-video-broker", "."] +RUN find . && dotnet build "onvif-video-broker.csproj" -c Release -o /app/build + +FROM build AS publish +ARG DOTNET_PUBLISH_RUNTIME +RUN echo "Publishing to: ${DOTNET_PUBLISH_RUNTIME}" +RUN dotnet publish -r ${DOTNET_PUBLISH_RUNTIME} "onvif-video-broker.csproj" -c Release -o /app/publish + +FROM ghcr.io/deislabs/akri/opencvsharp-build:${OUTPUT_PLATFORM_TAG} AS final +ARG OUTPUT_PLATFORM_TAG +RUN echo "Output base image: ghcr.io/deislabs/akri/opencvsharp-build:${OUTPUT_PLATFORM_TAG}" + +WORKDIR /app +COPY --from=publish /app/publish . + +# Link the container to the Akri repository +LABEL org.opencontainers.image.source https://github.com/deislabs/akri + +# Copy over container legal notice +COPY ./build/container-images-legal-notice.md . + +EXPOSE 8083 +ENV ASPNETCORE_URLS=http://*:8083 + +CMD dotnet onvif-video-broker.dll \ No newline at end of file diff --git a/build/containers/Dockerfile.udev-video-broker b/build/containers/Dockerfile.udev-video-broker new file mode 100644 index 0000000..bb6ae7a --- /dev/null +++ b/build/containers/Dockerfile.udev-video-broker @@ -0,0 +1,23 @@ +ARG PLATFORM=amd64 +ARG CROSS_BUILD_TARGET=x86_64-unknown-linux-gnu +FROM ${PLATFORM}/debian:buster-slim +ARG CROSS_BUILD_TARGET +RUN echo "Creating container based on ${PLATFORM}/debian:buster-slim" +RUN echo "Using Rust binaries from ${CROSS_BUILD_TARGET}" + +# Link the container to the Akri repository +LABEL org.opencontainers.image.source https://github.com/deislabs/akri + +# Copy over container legal notice +COPY ./build/container-images-legal-notice.md . + +RUN apt-get update && apt-get install -y --no-install-recommends libssl-dev openssl \ + libv4l-dev && \ + apt-get clean +COPY ./target/${CROSS_BUILD_TARGET}/release/udev-video-broker /udev-video-broker + +# Expose port used by broker service +EXPOSE 8083 + +ENV RUST_LOG udev_video_broker +CMD ["./udev-video-broker"] diff --git a/build/containers/Dockerfile.video-streaming-app b/build/containers/Dockerfile.video-streaming-app new file mode 100644 index 0000000..f5821e4 --- /dev/null +++ b/build/containers/Dockerfile.video-streaming-app @@ -0,0 +1,24 @@ +ARG PLATFORM=amd64 +FROM ${PLATFORM}/debian:buster-slim + +# Avoid tzdata prompt +ARG DEBIAN_FRONTEND=noninteractive + +RUN echo "Creating container based on ${PLATFORM}/debian:buster-slim" && \ + apt update && \ + apt install -y protobuf-compiler libprotoc-dev python3-pip python3-grpcio && \ + apt clean + +RUN pip3 install --upgrade pip && \ + pip3 install protobuf flask + +WORKDIR /app +COPY ./samples/apps/video-streaming-app . + +# Link the container to the Akri repository +LABEL org.opencontainers.image.source https://github.com/deislabs/akri + +# Copy over container legal notice +COPY ./build/container-images-legal-notice.md . + +CMD python3 ./streaming.py \ No newline at end of file diff --git a/build/containers/intermediate/Dockerfile.opencvsharp-build b/build/containers/intermediate/Dockerfile.opencvsharp-build new file mode 100644 index 0000000..4780219 --- /dev/null +++ b/build/containers/intermediate/Dockerfile.opencvsharp-build @@ -0,0 +1,74 @@ +# +# This Dockerfile is used to create intermediate containers used +# by the Akri build. +# +# Any changes to this file need to be made in its own Pull Request +# and need include a change to Makefile's BUILD_OPENCV_BASE_VERSION. +# +# A second Pull Request should subsequently be made to update +# Makefile's USE_OPENCV_BASE_VERSION. +# + +ARG PLATFORM_TAG=3.1-buster-slim +FROM mcr.microsoft.com/dotnet/core/aspnet:${PLATFORM_TAG} AS base +WORKDIR /app + +# Copy over container legal notice +COPY ./build/container-images-legal-notice.md . + +# based on https://xaviergeerinck.com/opencv-in-dotnet-core +ENV OPENCV_VERSION="4.1.1" +ENV OPENCV_INSTALLATION_DIR="/opt/opencv/" +ENV OPENCV_SHARP_VERSION="4.1.0.20190417" + +RUN whoami +RUN pwd + +RUN apt update && \ + apt -y install build-essential cmake pkg-config yasm git gfortran \ + libjpeg-dev libpng-dev libtiff-dev libavcodec-dev \ + libavformat-dev libswscale-dev libdc1394-22-dev libxine2-dev libv4l-dev \ + libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev libgtk2.0-dev libtbb-dev \ + qt5-default libatlas-base-dev libmp3lame-dev libtheora-dev \ + libvorbis-dev libxvidcore-dev libopencore-amrnb-dev libopencore-amrwb-dev \ + libavresample-dev x264 v4l-utils libprotobuf-dev protobuf-compiler \ + libgoogle-glog-dev libgflags-dev libgphoto2-dev libeigen3-dev libhdf5-dev doxygen \ + libtbb2 libdc1394-22-dev unzip wget && \ + apt clean + +RUN mkdir ${OPENCV_INSTALLATION_DIR} && \ + cd ${OPENCV_INSTALLATION_DIR} && \ + wget https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip -Oopencv-${OPENCV_VERSION}.zip && \ + unzip opencv-${OPENCV_VERSION}.zip && \ + wget https://github.com/opencv/opencv_contrib/archive/${OPENCV_VERSION}.zip -Oopencv_contrib-${OPENCV_VERSION}.zip && \ + unzip opencv_contrib-${OPENCV_VERSION}.zip && \ + rm ${OPENCV_INSTALLATION_DIR}/*.zip && \ + cd ${OPENCV_INSTALLATION_DIR}/opencv-${OPENCV_VERSION} && \ + ln -s /usr/include/eigen3/Eigen /usr/include/Eigen + +RUN cd ${OPENCV_INSTALLATION_DIR}/opencv-${OPENCV_VERSION} && \ + mkdir build && cd build && \ + cmake .. -DCMAKE_BUILD_TYPE=Release -DOPENCV_EXTRA_MODULES_PATH=../../opencv_contrib-${OPENCV_VERSION}/modules && \ + make -j$(grep -c ^processor /proc/cpuinfo) && \ + make install -j8 && \ + ldconfig && \ + cd && \ + rm -r ${OPENCV_INSTALLATION_DIR}/opencv-${OPENCV_VERSION} && \ + rm -r ${OPENCV_INSTALLATION_DIR}/opencv_contrib-${OPENCV_VERSION} + +RUN cd ${OPENCV_INSTALLATION_DIR} && \ + git clone https://github.com/shimat/opencvsharp.git opencvsharp && \ + cd opencvsharp && \ + git fetch --all --tags --prune && git checkout ${OPENCV_SHARP_VERSION} && \ + cd src && \ + mkdir build && \ + cd build && \ + cmake -D CMAKE_INSTALL_PREFIX=${OPENCV_INSTALLATION_DIR} .. && \ + make -j$(grep -c ^processor /proc/cpuinfo) && \ + make install && \ + ldconfig && \ + cp OpenCvSharpExtern/libOpenCvSharpExtern.so /usr/lib && \ + cd && \ + rm -r ${OPENCV_INSTALLATION_DIR}/opencvsharp && \ + apt remove -y git unzip wget build-essential cmake && \ + apt autoremove -y diff --git a/build/containers/intermediate/Dockerfile.rust-crossbuild-amd64 b/build/containers/intermediate/Dockerfile.rust-crossbuild-amd64 new file mode 100644 index 0000000..744ff1e --- /dev/null +++ b/build/containers/intermediate/Dockerfile.rust-crossbuild-amd64 @@ -0,0 +1,20 @@ +# +# This Dockerfile is used to create intermediate containers used +# by the Akri build. +# +# Any changes to this file need to be made in its own Pull Request +# and need include a change to Makefile's BUILD_RUST_CROSSBUILD_VERSION. +# +# A second Pull Request should subsequently be made to update +# Cross.toml to point at the new versions. +# + +FROM rustembedded/cross:x86_64-unknown-linux-gnu-0.1.16 + +RUN apt-get update && \ + apt-get install -y --no-install-recommends pkg-config \ + g++ ca-certificates curl libssl-dev \ + libv4l-dev libudev-dev + +# Copy over container legal notice +COPY ./build/container-images-legal-notice.md . \ No newline at end of file diff --git a/build/containers/intermediate/Dockerfile.rust-crossbuild-arm32v7 b/build/containers/intermediate/Dockerfile.rust-crossbuild-arm32v7 new file mode 100644 index 0000000..06c0fd1 --- /dev/null +++ b/build/containers/intermediate/Dockerfile.rust-crossbuild-arm32v7 @@ -0,0 +1,25 @@ +# +# This Dockerfile is used to create intermediate containers used +# by the Akri build. +# +# Any changes to this file need to be made in its own Pull Request +# and need include a change to Makefile's BUILD_RUST_CROSSBUILD_VERSION. +# +# A second Pull Request should subsequently be made to update +# Cross.toml to point at the new versions. +# + +FROM rustembedded/cross:arm-unknown-linux-gnueabihf-0.1.16 + +RUN sed -i 's/^deb h'/'deb [arch=amd64,i386] h/' /etc/apt/sources.list && \ + dpkg --add-architecture armhf && \ + echo deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports $(lsb_release -c -s) main restricted universe multiverse >> /etc/apt/sources.list && \ + echo deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports $(lsb_release -c -s)-updates main restricted universe multiverse >> /etc/apt/sources.list && \ + echo deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports $(lsb_release -c -s)-backports main restricted universe multiverse >> /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -y --no-install-recommends pkg-config:armhf \ + g++ ca-certificates curl libssl-dev:armhf \ + libv4l-dev:armhf libudev-dev:armhf + +# Copy over container legal notice +COPY ./build/container-images-legal-notice.md . diff --git a/build/containers/intermediate/Dockerfile.rust-crossbuild-arm64v8 b/build/containers/intermediate/Dockerfile.rust-crossbuild-arm64v8 new file mode 100644 index 0000000..9bb3013 --- /dev/null +++ b/build/containers/intermediate/Dockerfile.rust-crossbuild-arm64v8 @@ -0,0 +1,25 @@ +# +# This Dockerfile is used to create intermediate containers used +# by the Akri build. +# +# Any changes to this file need to be made in its own Pull Request +# and need include a change to Makefile's BUILD_RUST_CROSSBUILD_VERSION. +# +# A second Pull Request should subsequently be made to update +# Cross.toml to point at the new versions. +# + +FROM rustembedded/cross:aarch64-unknown-linux-gnu-0.1.16 + +RUN sed -i 's/^deb h'/'deb [arch=amd64,i386] h/' /etc/apt/sources.list && \ + dpkg --add-architecture arm64 && \ + echo deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports $(lsb_release -c -s) main restricted universe multiverse >> /etc/apt/sources.list && \ + echo deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports $(lsb_release -c -s)-updates main restricted universe multiverse >> /etc/apt/sources.list && \ + echo deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports $(lsb_release -c -s)-backports main restricted universe multiverse >> /etc/apt/sources.list && \ + apt-get update && \ + apt-get install -y --no-install-recommends pkg-config:arm64 \ + g++ ca-certificates curl libssl-dev:arm64 \ + libv4l-dev:arm64 libudev-dev:arm64 + +# Copy over container legal notice +COPY ./build/container-images-legal-notice.md . \ No newline at end of file diff --git a/build/intermediate-containers.mk b/build/intermediate-containers.mk new file mode 100644 index 0000000..42ad2bf --- /dev/null +++ b/build/intermediate-containers.mk @@ -0,0 +1,82 @@ + +BUILD_RUST_CROSSBUILD_VERSION = 0.0.4 + +BUILD_OPENCV_BASE_VERSION = 0.0.4 +USE_OPENCV_BASE_VERSION = 0.0.4 + +CROSS_VERSION = 0.1.16 + +# +# +# OPENCV: make and push the open cv intermediate images: +# +# To make all platforms: `make opencv-base` +# To make specific platforms: `BUILD_AMD64=1 BUILD_ARM32=0 BUILD_ARM64=1 make opencv` +# +# +.PHONY: opencv-base +opencv-base: opencv-base-build opencv-base-docker-per-arch +opencv-base-build: opencv-base-build-amd64 opencv-base-build-arm32 opencv-base-build-arm64 +opencv-base-build-amd64: +ifeq (1, ${BUILD_AMD64}) + docker build $(CACHE_OPTION) -f $(INTERMEDIATE_DOCKERFILE_DIR)/Dockerfile.opencvsharp-build . -t $(PREFIX)/opencvsharp-build:$(BUILD_OPENCV_BASE_VERSION)-$(AMD64_SUFFIX) --build-arg PLATFORM_TAG=3.1-buster-slim +endif +opencv-base-build-arm32: +ifeq (1, ${BUILD_ARM32}) + docker build $(CACHE_OPTION) -f $(INTERMEDIATE_DOCKERFILE_DIR)/Dockerfile.opencvsharp-build . -t $(PREFIX)/opencvsharp-build:$(BUILD_OPENCV_BASE_VERSION)-$(ARM32V7_SUFFIX) --build-arg PLATFORM_TAG=3.1-buster-slim-$(ARM32V7_SUFFIX) +endif +opencv-base-build-arm64: +ifeq (1, ${BUILD_ARM64}) + docker build $(CACHE_OPTION) -f $(INTERMEDIATE_DOCKERFILE_DIR)/Dockerfile.opencvsharp-build . -t $(PREFIX)/opencvsharp-build:$(BUILD_OPENCV_BASE_VERSION)-$(ARM64V8_SUFFIX) --build-arg PLATFORM_TAG=3.1-buster-slim-$(ARM64V8_SUFFIX) +endif +opencv-base-docker-per-arch: opencv-base-docker-per-arch-amd64 opencv-base-docker-per-arch-arm32 opencv-base-docker-per-arch-arm64 +opencv-base-docker-per-arch-amd64: +ifeq (1, ${BUILD_AMD64}) + docker push $(PREFIX)/opencvsharp-build:$(BUILD_OPENCV_BASE_VERSION)-$(AMD64_SUFFIX) +endif +opencv-base-docker-per-arch-arm32: +ifeq (1, ${BUILD_ARM32}) + docker push $(PREFIX)/opencvsharp-build:$(BUILD_OPENCV_BASE_VERSION)-$(ARM32V7_SUFFIX) +endif +opencv-base-docker-per-arch-arm64: +ifeq (1, ${BUILD_ARM64}) + docker push $(PREFIX)/opencvsharp-build:$(BUILD_OPENCV_BASE_VERSION)-$(ARM64V8_SUFFIX) +endif + +# +# +# CROSS: make and push the intermediate images for the cross building Rust: +# +# To make all platforms: `make rust-crossbuild` +# To make specific platforms: `BUILD_AMD64=1 BUILD_ARM32=0 BUILD_ARM64=1 make rust-crossbuild` +# +# +.PHONY: rust-crossbuild +rust-crossbuild: rust-crossbuild-build rust-crossbuild-docker-per-arch +rust-crossbuild-build: rust-crossbuild-build-amd64 rust-crossbuild-build-arm32 rust-crossbuild-build-arm64 +rust-crossbuild-build-amd64: +ifeq (1, $(BUILD_AMD64)) + docker build $(CACHE_OPTION) -f $(INTERMEDIATE_DOCKERFILE_DIR)/Dockerfile.rust-crossbuild-$(AMD64_SUFFIX) . -t $(PREFIX)/rust-crossbuild:$(AMD64_TARGET)-$(CROSS_VERSION)-$(BUILD_RUST_CROSSBUILD_VERSION) +endif +rust-crossbuild-build-arm32: +ifeq (1, ${BUILD_ARM32}) + docker build $(CACHE_OPTION) -f $(INTERMEDIATE_DOCKERFILE_DIR)/Dockerfile.rust-crossbuild-$(ARM32V7_SUFFIX) . -t $(PREFIX)/rust-crossbuild:$(ARM32V7_TARGET)-$(CROSS_VERSION)-$(BUILD_RUST_CROSSBUILD_VERSION) +endif +rust-crossbuild-build-arm64: +ifeq (1, ${BUILD_ARM64}) + docker build $(CACHE_OPTION) -f $(INTERMEDIATE_DOCKERFILE_DIR)/Dockerfile.rust-crossbuild-$(ARM64V8_SUFFIX) . -t $(PREFIX)/rust-crossbuild:$(ARM64V8_TARGET)-$(CROSS_VERSION)-$(BUILD_RUST_CROSSBUILD_VERSION) +endif + +rust-crossbuild-docker-per-arch: rust-crossbuild-docker-per-arch-amd64 rust-crossbuild-docker-per-arch-arm32 rust-crossbuild-docker-per-arch-arm64 +rust-crossbuild-docker-per-arch-amd64: +ifeq (1, $(BUILD_AMD64)) + docker push $(PREFIX)/rust-crossbuild:$(AMD64_TARGET)-$(CROSS_VERSION)-$(BUILD_RUST_CROSSBUILD_VERSION) +endif +rust-crossbuild-docker-per-arch-arm32: +ifeq (1, ${BUILD_ARM32}) + docker push $(PREFIX)/rust-crossbuild:$(ARM32V7_TARGET)-$(CROSS_VERSION)-$(BUILD_RUST_CROSSBUILD_VERSION) +endif +rust-crossbuild-docker-per-arch-arm64: +ifeq (1, ${BUILD_ARM64}) + docker push $(PREFIX)/rust-crossbuild:$(ARM64V8_TARGET)-$(CROSS_VERSION)-$(BUILD_RUST_CROSSBUILD_VERSION) +endif diff --git a/build/setup.sh b/build/setup.sh new file mode 100755 index 0000000..b77dda5 --- /dev/null +++ b/build/setup.sh @@ -0,0 +1,21 @@ + +apt_dependencies="git curl libssl-dev pkg-config libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev gstreamer1.0-plugins-base gstreamer1.0-plugins-good gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly gstreamer1.0-libav libgstrtspserver-1.0-dev libges-1.0-dev libudev-dev libv4l-dev" + +echo "User: $(whoami)" + +echo "Install rustfmt" +rustup component add rustfmt + +echo "Install dependencies: $apt_dependencies" +which sudo > /dev/null 2>&1 +if [ "$?" -eq "0" ]; +then + echo "Run sudo apt install ..." + sudo apt update + sudo apt install -y $apt_dependencies +else + echo "Run apt update and apt install without sudo" + apt update + apt install -y $apt_dependencies +fi + diff --git a/controller/Cargo.toml b/controller/Cargo.toml new file mode 100644 index 0000000..7cb6d16 --- /dev/null +++ b/controller/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "controller" +version = "0.0.33" +authors = [""] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +async-std = "1.5.0" +async-trait = "0.1.0" +chrono = "0.4.10" +env_logger = "0.6.1" +futures = "0.3.1" +failure = "0.1.5" +kube = { version = "0.23.0", features = ["openapi"] } +k8s-openapi = { version = "0.6.0", features = ["v1_16"] } +log = "0.4" +mockall = "0.6.0" +serde = "1.0.104" +serde_derive = "1.0.104" +serde_json = "1.0.45" +serde_yaml = "0.8.11" +akri-shared = { path = "../shared" } +tokio = { version = "0.2", features = ["full"] } diff --git a/controller/src/main.rs b/controller/src/main.rs new file mode 100644 index 0000000..60eec06 --- /dev/null +++ b/controller/src/main.rs @@ -0,0 +1,66 @@ +mod util; + +use akri_shared::akri::API_NAMESPACE; +use async_std::sync::Mutex; +use env_logger; +use std::sync::Arc; +use util::{instance_action, node_watcher, pod_watcher}; + +/// Length of time to sleep between controller system validation checks +pub const SYSTEM_CHECK_DELAY_SECS: u64 = 30; + +/// This is the entry point for the controller. +#[tokio::main] +async fn main() -> Result<(), Box> { + println!("{} Controller start", API_NAMESPACE); + + println!( + "{} KUBERNETES_PORT found ... env_logger::init", + API_NAMESPACE + ); + env_logger::try_init()?; + println!( + "{} KUBERNETES_PORT found ... env_logger::init finished", + API_NAMESPACE + ); + + log::info!("{} Controller logging started", API_NAMESPACE); + + let synchronization = Arc::new(Mutex::new(())); + let instance_watch_synchronization = synchronization.clone(); + let mut tasks = Vec::new(); + + // Handle existing instances + tasks.push(tokio::spawn({ + async move { + instance_action::handle_existing_instances().await.unwrap(); + } + })); + // Handle instance changes + tasks.push(tokio::spawn({ + async move { + instance_action::do_instance_watch(instance_watch_synchronization) + .await + .unwrap(); + } + })); + // Watch for node disappearance + tasks.push(tokio::spawn({ + async move { + let mut node_watcher = node_watcher::NodeWatcher::new(); + node_watcher.watch().await.unwrap(); + } + })); + // Watch for broker Pod state changes + tasks.push(tokio::spawn({ + async move { + let mut broker_pod_watcher = pod_watcher::BrokerPodWatcher::new(); + broker_pod_watcher.watch().await.unwrap(); + } + })); + + futures::future::try_join_all(tasks).await?; + + log::info!("{} Controller end", API_NAMESPACE); + Ok(()) +} diff --git a/controller/src/util/instance_action.rs b/controller/src/util/instance_action.rs new file mode 100644 index 0000000..1c06045 --- /dev/null +++ b/controller/src/util/instance_action.rs @@ -0,0 +1,979 @@ +use super::{pod_action::PodAction, pod_action::PodActionInfo}; +use akri_shared::{ + akri::{ + configuration::KubeAkriConfig, instance::KubeAkriInstance, AKRI_PREFIX, API_INSTANCES, + API_NAMESPACE, API_VERSION, + }, + k8s, + k8s::{ + pod, + pod::{AKRI_INSTANCE_LABEL_NAME, AKRI_TARGET_NODE_LABEL_NAME}, + KubeInterface, OwnershipInfo, OwnershipType, + }, +}; +use async_std::sync::Mutex; +use futures::StreamExt; +use k8s_openapi::api::core::v1::{PodSpec, PodStatus}; +use kube::api::{Informer, Object, RawApi, WatchEvent}; +use log::{error, info, trace}; +use std::collections::HashMap; +use std::sync::Arc; + +/// Length of time a Pod can be pending before we give up and retry +pub const PENDING_POD_GRACE_PERIOD_MINUTES: i64 = 5; +/// Length of time a Pod can be in an error state before we retry +pub const FAILED_POD_GRACE_PERIOD_MINUTES: i64 = 0; + +/// Instance action types +/// +/// Instance actions describe the types of actions the controller can +/// react to for DeviceCapabiltiyInstances. +/// +#[derive(Clone, Debug, PartialEq)] +pub enum InstanceAction { + /// An Instance is added + Add, + /// An Instance is removed + Remove, + /// An Instance is updated + Update, +} + +/// This invokes an internal method that watches for Instance events +pub async fn handle_existing_instances( +) -> Result<(), Box> { + internal_handle_existing_instances(&k8s::create_kube_interface()).await +} + +/// This invokes an internal method that watches for Instance events +pub async fn do_instance_watch( + synchronization: Arc>, +) -> Result<(), Box> { + // Watch for instance changes + internal_do_instance_watch(&synchronization, &k8s::create_kube_interface()).await +} + +/// This invokes an internal method that watches for Instance events +async fn internal_handle_existing_instances( + kube_interface: &impl KubeInterface, +) -> Result<(), Box> { + let mut tasks = Vec::new(); + + // Handle existing instances + let pre_existing_instances = kube_interface.get_instances().await?; + for instance in pre_existing_instances { + tasks.push(tokio::spawn(async move { + let inner_kube_interface = k8s::create_kube_interface(); + handle_instance_change(&instance, &InstanceAction::Update, &inner_kube_interface) + .await + .unwrap(); + })); + } + futures::future::try_join_all(tasks).await?; + Ok(()) +} + +/// This watches for Instance events +async fn internal_do_instance_watch( + synchronization: &Arc>, + kube_interface: &impl KubeInterface, +) -> Result<(), Box> { + trace!("internal_do_instance_watch - enter"); + let akri_instance_type = RawApi::customResource(API_INSTANCES) + .group(API_NAMESPACE) + .version(API_VERSION); + + let informer = Informer::raw(kube_interface.get_kube_client(), akri_instance_type) + .init() + .await?; + loop { + let mut instances = informer.poll().await?.boxed(); + + // Currently, this does not handle None except to break the + // while. + while let Some(event) = instances.next().await { + // Aquire lock to ensure cleanup_instance_and_configuration_svcs and the + // inner loop handle_instance call in internal_do_instance_watch + // cannot execute at the same time. + let _lock = synchronization.lock().await; + trace!("internal_do_instance_watch - aquired sync lock"); + handle_instance(event?, kube_interface).await?; + } + } +} + +/// This takes an event off the Instance stream and delegates it to the +/// correct function based on the event type. +async fn handle_instance( + event: WatchEvent, + kube_interface: &impl KubeInterface, +) -> Result<(), Box> { + trace!("handle_instance - enter"); + match event { + WatchEvent::Added(instance) => { + info!( + "handle_instance - added Akri Instance {}: {:?}", + instance.metadata.name, instance.spec + ); + handle_instance_change(&instance, &InstanceAction::Add, kube_interface).await?; + Ok(()) + } + WatchEvent::Deleted(instance) => { + info!( + "handle_instance - deleted Akri Instance {}: {:?}", + instance.metadata.name, instance.spec + ); + handle_instance_change(&instance, &InstanceAction::Remove, kube_interface).await?; + Ok(()) + } + WatchEvent::Modified(instance) => { + info!( + "handle_instance - modified Akri Instance {}: {:?}", + instance.metadata.name, instance.spec + ); + handle_instance_change(&instance, &InstanceAction::Update, kube_interface).await?; + Ok(()) + } + WatchEvent::Error(ref e) => { + trace!("handle_instance - error for Akri Instance: {}", e); + Ok(()) + } + } +} + +/// PodContext stores a set of details required to track/create/delete broker +/// Pods. +/// +/// The PodContext stores what is required to determine how to handle a +/// specific Node's protocol broker Pod. +/// +/// * the node is described by node_name +/// * the protocol (or capability) is described by instance_name and namespace +/// * what to do with the broker Pod is described by action +#[derive(Clone, Debug, PartialEq)] +struct PodContext { + node_name: Option, + namespace: Option, + action: PodAction, +} + +/// This finds what to do with a given broker Pod based on its current state and +/// the Instance event action. If this method has enough information, +/// it will update the nodes_to_act_on map with the required action. +fn determine_action_for_pod( + k8s_pod: &Object, + action: &InstanceAction, + nodes_to_act_on: &mut HashMap, +) { + if k8s_pod.status.is_none() { + error!( + "determine_action_for_pod - no pod status found for {}", + &k8s_pod.metadata.name + ); + return; + } + + if k8s_pod.status.as_ref().unwrap().phase.is_none() { + error!( + "determine_action_for_pod - no pod phase found for {}", + &k8s_pod.metadata.name + ); + return; + } + + // Early exits above ensure unwrap will not panic + let pod_phase = k8s_pod.status.as_ref().unwrap().phase.as_ref().unwrap(); + + if k8s_pod + .metadata + .labels + .get(AKRI_TARGET_NODE_LABEL_NAME) + .is_none() + { + error!( + "determine_action_for_pod - no {} label found for {}", + AKRI_TARGET_NODE_LABEL_NAME, &k8s_pod.metadata.name + ); + return; + } + + // Early exits above ensure unwrap will not panic + let node_to_run_pod_on = k8s_pod + .metadata + .labels + .get(AKRI_TARGET_NODE_LABEL_NAME) + .unwrap(); + + if k8s_pod + .metadata + .labels + .get(AKRI_INSTANCE_LABEL_NAME) + .is_none() + { + error!( + "determine_action_for_pod - no {} label found for {}", + AKRI_INSTANCE_LABEL_NAME, &k8s_pod.metadata.name + ); + return; + } + + let mut update_pod_context = PodContext { + node_name: Some(node_to_run_pod_on.to_string()), + namespace: k8s_pod.metadata.namespace.clone(), + action: PodAction::NoAction, + }; + + // Early exits above ensure unwrap will not panic + let pod_start_time = k8s_pod.status.as_ref().unwrap().start_time.clone(); + + let pod_action_info = PodActionInfo { + pending_grace_time_in_minutes: PENDING_POD_GRACE_PERIOD_MINUTES, + ended_grace_time_in_minutes: FAILED_POD_GRACE_PERIOD_MINUTES, + phase: pod_phase.to_string(), + instance_action: action.clone(), + status_start_time: pod_start_time, + unknown_node: !nodes_to_act_on.contains_key(node_to_run_pod_on), + trace_node_name: k8s_pod.metadata.name.clone(), + }; + update_pod_context.action = match pod_action_info.select_pod_action() { + Ok(action) => action, + Err(e) => { + error!( + "determine_action_for_pod - failed ({}) to get pod action for {:?}", + e, update_pod_context + ); + return; + } + }; + nodes_to_act_on.insert(node_to_run_pod_on.to_string(), update_pod_context); +} + +/// This handles Instance deletion event by deleting the +/// broker Pod, the broker Service (if there are no remaining broker Pods), +/// and the capability Service (if there are no remaining capability Pods). +async fn handle_deletion_work( + instance_name: &str, + instance_shared: bool, + node_to_delete_pod: &str, + context: &PodContext, + kube_interface: &impl KubeInterface, +) -> Result<(), Box> { + let context_node_name = context.node_name.as_ref().ok_or(format!( + "handle_deletion_work - Context node_name is missing for {}: {:?}", + node_to_delete_pod, context + ))?; + let context_namespace = context.namespace.as_ref().ok_or(format!( + "handle_deletion_work - Context namespace is missing for {}: {:?}", + node_to_delete_pod, context + ))?; + + trace!( + "handle_deletion_work - pod::create_pod_app_name({:?}, {:?}, {:?}, {:?})", + &instance_name, + context_node_name, + instance_shared, + &"pod".to_string() + ); + let pod_app_name = pod::create_pod_app_name( + &instance_name, + context_node_name, + instance_shared, + &"pod".to_string(), + ); + trace!( + "handle_deletion_work - pod::remove_pod name={:?}, namespace={:?}", + &pod_app_name, + &context_namespace + ); + kube_interface + .remove_pod(&pod_app_name, &context_namespace) + .await?; + trace!("handle_deletion_work - pod::remove_pod succeeded",); + Ok(()) +} + +#[cfg(test)] +mod handle_deletion_work_tests { + use super::*; + use akri_shared::k8s::test_kube::MockKubeImpl; + + #[tokio::test] + async fn test_handle_deletion_work_with_no_node_name() { + let _ = env_logger::builder().is_test(true).try_init(); + + let context = PodContext { + node_name: None, + namespace: Some("namespace".into()), + action: PodAction::NoAction, + }; + + assert!(handle_deletion_work( + "instance_name", + true, + "node_to_delete_pod", + &context, + &MockKubeImpl::new(), + ) + .await + .is_err()); + } + + #[tokio::test] + async fn test_handle_deletion_work_with_no_namespace() { + let _ = env_logger::builder().is_test(true).try_init(); + + let context = PodContext { + node_name: Some("node-a".into()), + namespace: None, + action: PodAction::NoAction, + }; + + assert!(handle_deletion_work( + "instance_name", + true, + "node_to_delete_pod", + &context, + &MockKubeImpl::new(), + ) + .await + .is_err()); + } +} + +/// This handles Instance addition event by creating the +/// broker Pod, the broker Service, and the capability Service. +async fn handle_addition_work( + instance_name: &str, + instance_uid: &str, + instance_namespace: &str, + instance_class_name: &str, + instance_shared: bool, + new_node: &str, + instance_configuration: &KubeAkriConfig, + kube_interface: &impl KubeInterface, +) -> Result<(), Box> { + trace!( + "handle_addition_work - Create new Pod for Node={:?}", + new_node + ); + + if let Some(broker_pod_spec) = &instance_configuration.spec.broker_pod_spec { + let capability_id = format!("{}/{}", AKRI_PREFIX, instance_name); + let new_pod = pod::create_new_pod_from_spec( + &instance_namespace, + &instance_name, + &instance_class_name, + OwnershipInfo::new( + OwnershipType::Instance, + instance_name.to_string(), + instance_uid.to_string(), + ), + &capability_id, + &new_node.to_string(), + instance_shared, + &broker_pod_spec, + )?; + + trace!("handle_addition_work - New pod spec={:?}", new_pod); + + kube_interface + .create_pod(&new_pod, &instance_namespace) + .await?; + trace!("handle_addition_work - pod::create_pod succeeded",); + } + trace!("handle_addition_work - POST nodeInfo.SetNode \n"); + Ok(()) +} + +/// Handle Instance change by watching for node +/// disappearances, starting broker Pods/Services that are missing, +/// and stopping Pods/Services that are no longer needed. +pub async fn handle_instance_change( + instance: &KubeAkriInstance, + action: &InstanceAction, + kube_interface: &impl KubeInterface, +) -> Result<(), Box> { + trace!("handle_instance_change - enter {:?}", action); + + let instance_name = instance.metadata.name.clone(); + let instance_namespace = instance.metadata.namespace.as_ref().ok_or(format!( + "Namespace not found for instance: {}", + &instance_name + ))?; + let instance_uid = instance + .metadata + .uid + .as_ref() + .ok_or(format!("UID not found for instance: {}", &instance_name))?; + + // If InstanceAction::Remove, assume all nodes require PodAction::NoAction (reflect that there is no running Pod unless we find one) + // Otherwise, assume all nodes require PodAction::Add (reflect that there is no running Pod, unless we find one) + let default_action = match action { + InstanceAction::Remove => PodAction::NoAction, + _ => PodAction::Add, + }; + let mut nodes_to_act_on: HashMap = instance + .spec + .nodes + .iter() + .map(|node| { + ( + node.to_string(), + PodContext { + node_name: None, + namespace: None, + action: default_action, + }, + ) + }) + .collect(); + trace!( + "handle_instance_change - nodes tracked from instance={:?}", + nodes_to_act_on + ); + + trace!( + "handle_instance_change - find all pods that have {}={}", + AKRI_INSTANCE_LABEL_NAME, + instance_name + ); + let instance_pods = kube_interface + .find_pods_with_label(&format!("{}={}", AKRI_INSTANCE_LABEL_NAME, instance_name)) + .await?; + trace!( + "handle_instance_change - found {} pods", + instance_pods.items.len() + ); + + trace!("handle_instance_change - update actions based on the existing pods"); + // By default, assume any pod tracked by the instance need to be added. + // Query the existing pods to see if some of these are already added, or + // need to be removed + instance_pods + .items + .iter() + .for_each(|x| determine_action_for_pod(x, action, &mut nodes_to_act_on)); + trace!( + "handle_instance_change - nodes tracked after querying existing pods={:?}", + nodes_to_act_on + ); + + // Iterate over nodes_to_act_on where value == (PodAction::Remove | PodAction::RemoveAndAdd) + for (node_to_delete_pod, context) in nodes_to_act_on.iter().filter(|&(_, v)| { + ((v.action) == PodAction::Remove) | ((v.action) == PodAction::RemoveAndAdd) + }) { + handle_deletion_work( + &instance_name, + instance.spec.shared, + node_to_delete_pod, + context, + kube_interface, + ) + .await? + } + + let nodes_to_add = nodes_to_act_on + .iter() + .filter_map(|(node, context)| { + if ((context.action) == PodAction::Add) | ((context.action) == PodAction::RemoveAndAdd) + { + Some(node.to_string()) + } else { + None + } + }) + .collect::>(); + + let instance_configuration_option = if !nodes_to_add.is_empty() { + // Only retrieve Config if needed + trace!( + "handle_instance_change - find configuration for {:?}", + &instance.spec.configuration_name + ); + let instance_configuration = match kube_interface + .find_configuration(&instance.spec.configuration_name, &instance_namespace) + .await + { + Ok(config) => config, + _ => { + // In this scenario, a configuration has been deleted without a Akri Agent deleting the associated Instances. + // Furthermore, Akri Agent is still modifying the Instances. This should not happen beacuse Agent + // is designed to shutdown when it's Configuration watcher fails. + error!( + "handle_instance_change - no configuration found for {} yet instance {} exists - check that device plugin is running propertly", + &instance.spec.configuration_name, &instance.metadata.name + ); + return Ok(()); + } + }; + trace!( + "handle_instance_change - found configuration for {:?}", + &instance_configuration.metadata.name + ); + Some(instance_configuration) + } else { + None + }; + + // Iterate over nodes_to_act_on where value == (PodAction::Add | PodAction::RemoveAndAdd) + for new_node in nodes_to_add { + handle_addition_work( + &instance_name, + &instance_uid, + &instance_namespace, + &instance.spec.configuration_name, + instance.spec.shared, + &new_node, + &instance_configuration_option.as_ref().unwrap(), + kube_interface, + ) + .await?; + } + + trace!("handle_instance_change - exit"); + + Ok(()) +} + +#[cfg(test)] +mod handle_instance_tests { + use super::super::shared_test_utils::config_for_tests; + use super::super::shared_test_utils::config_for_tests::PodList; + use super::*; + use akri_shared::{ + akri::instance::KubeAkriInstance, + k8s::{pod::AKRI_INSTANCE_LABEL_NAME, test_kube::MockKubeImpl}, + os::file, + }; + use chrono::prelude::*; + use chrono::Utc; + use mockall::predicate::*; + + fn configure_find_pods_with_phase( + mock: &mut MockKubeImpl, + pod_selector: &'static str, + result_file: &'static str, + specified_phase: &'static str, + ) { + trace!( + "mock.expect_find_pods_with_label pod_selector:{}", + pod_selector + ); + mock.expect_find_pods_with_label() + .times(1) + .withf(move |selector| selector == pod_selector) + .returning(move |_| { + let pods_json = file::read_file_to_string(result_file); + let phase_adjusted_json = pods_json.replace( + "\"phase\": \"Running\"", + &format!("\"phase\": \"{}\"", specified_phase), + ); + let pods: PodList = serde_json::from_str(&phase_adjusted_json).unwrap(); + Ok(pods) + }); + } + + fn configure_find_pods_with_phase_and_start_time( + mock: &mut MockKubeImpl, + pod_selector: &'static str, + result_file: &'static str, + specified_phase: &'static str, + start_time: DateTime, + ) { + trace!( + "mock.expect_find_pods_with_label pod_selector:{}", + pod_selector + ); + mock.expect_find_pods_with_label() + .times(1) + .withf(move |selector| selector == pod_selector) + .returning(move |_| { + let pods_json = file::read_file_to_string(result_file); + let phase_adjusted_json = pods_json.replace( + "\"phase\": \"Running\"", + &format!("\"phase\": \"{}\"", specified_phase), + ); + let start_time_adjusted_json = phase_adjusted_json.replace( + "\"startTime\": \"2020-02-25T20:48:03Z\"", + &format!( + "\"startTime\": \"{}\"", + start_time.format("%Y-%m-%dT%H:%M:%SZ").to_string() + ), + ); + let pods: PodList = serde_json::from_str(&start_time_adjusted_json).unwrap(); + Ok(pods) + }); + } + + fn configure_find_pods_with_phase_and_no_start_time( + mock: &mut MockKubeImpl, + pod_selector: &'static str, + result_file: &'static str, + specified_phase: &'static str, + ) { + trace!( + "mock.expect_find_pods_with_label pod_selector:{}", + pod_selector + ); + mock.expect_find_pods_with_label() + .times(1) + .withf(move |selector| selector == pod_selector) + .returning(move |_| { + let pods_json = file::read_file_to_string(result_file); + let phase_adjusted_json = pods_json.replace( + "\"phase\": \"Running\"", + &format!("\"phase\": \"{}\"", specified_phase), + ); + let start_time_adjusted_json = + phase_adjusted_json.replace("\"startTime\": \"2020-02-25T20:48:03Z\",", ""); + let pods: PodList = serde_json::from_str(&start_time_adjusted_json).unwrap(); + Ok(pods) + }); + } + + #[derive(Clone)] + struct HandleInstanceWork { + find_pods_selector: &'static str, + find_pods_result: &'static str, + find_pods_phase: Option<&'static str>, + find_pods_start_time: Option>, + find_pods_delete_start_time: bool, + deletion_work: Option, + addition_work: Option, + } + + fn configure_for_handle_instance_change(mock: &mut MockKubeImpl, work: &HandleInstanceWork) { + if let Some(phase) = work.find_pods_phase { + if let Some(start_time) = work.find_pods_start_time { + configure_find_pods_with_phase_and_start_time( + mock, + work.find_pods_selector, + work.find_pods_result, + phase, + start_time, + ); + } else if work.find_pods_delete_start_time { + configure_find_pods_with_phase_and_no_start_time( + mock, + work.find_pods_selector, + work.find_pods_result, + phase, + ); + } else { + configure_find_pods_with_phase( + mock, + work.find_pods_selector, + work.find_pods_result, + phase, + ); + } + } else { + config_for_tests::configure_find_pods( + mock, + work.find_pods_selector, + work.find_pods_result, + false, + ); + } + + if let Some(deletion_work) = &work.deletion_work { + configure_for_handle_deletion_work(mock, deletion_work); + } + + if let Some(addition_work) = &work.addition_work { + config_for_tests::configure_find_config( + mock, + addition_work.find_config_name, + addition_work.find_config_namespace, + addition_work.find_config_result, + false, + ); + configure_for_handle_addition_work(mock, addition_work); + } + } + + #[derive(Clone)] + struct HandleDeletionWork { + broker_pod_names: Vec<&'static str>, + // instance_svc_names: Vec<&'static str>, + cleanup_namespaces: Vec<&'static str>, + } + + fn configure_deletion_work_for_config_a_359973() -> HandleDeletionWork { + HandleDeletionWork { + broker_pod_names: vec!["node-a-config-a-359973-pod"], + // instance_svc_names: vec!["config-a-359973-svc"], + cleanup_namespaces: vec!["config-a-namespace"], + } + } + + fn configure_deletion_work_for_config_a_b494b6() -> HandleDeletionWork { + HandleDeletionWork { + broker_pod_names: vec!["config-a-b494b6-pod"], + // instance_svc_names: vec!["config-a-b494b6-svc"], + cleanup_namespaces: vec!["config-a-namespace"], + } + } + + fn configure_for_handle_deletion_work(mock: &mut MockKubeImpl, work: &HandleDeletionWork) { + for i in 0..work.broker_pod_names.len() { + let broker_pod_name = work.broker_pod_names[i]; + let cleanup_namespace = work.cleanup_namespaces[i]; + + config_for_tests::configure_remove_pod(mock, broker_pod_name, cleanup_namespace); + } + } + + #[derive(Clone)] + struct HandleAdditionWork { + find_config_name: &'static str, + find_config_namespace: &'static str, + find_config_result: &'static str, + new_pod_names: Vec<&'static str>, + new_pod_instance_names: Vec<&'static str>, + new_pod_namespaces: Vec<&'static str>, + } + + fn configure_add_shared_config_a_359973(pod_name: &'static str) -> HandleAdditionWork { + HandleAdditionWork { + find_config_name: "config-a", + find_config_namespace: "config-a-namespace", + find_config_result: "../test/json/config-a.json", + new_pod_names: vec![pod_name], + new_pod_instance_names: vec!["config-a-359973"], + new_pod_namespaces: vec!["config-a-namespace"], + } + } + + fn configure_add_local_config_a_b494b6() -> HandleAdditionWork { + HandleAdditionWork { + find_config_name: "config-a", + find_config_namespace: "config-a-namespace", + find_config_result: "../test/json/config-a.json", + new_pod_names: vec!["config-a-b494b6-pod"], + new_pod_instance_names: vec!["config-a-b494b6"], + new_pod_namespaces: vec!["config-a-namespace"], + } + } + + fn configure_for_handle_addition_work(mock: &mut MockKubeImpl, work: &HandleAdditionWork) { + for i in 0..work.new_pod_names.len() { + config_for_tests::configure_add_pod( + mock, + work.new_pod_names[i], + work.new_pod_namespaces[i], + AKRI_INSTANCE_LABEL_NAME, + work.new_pod_instance_names[i], + ); + } + } + + async fn run_handle_instance_change_test( + mock: &mut MockKubeImpl, + instance_file: &'static str, + action: &'static InstanceAction, + ) { + trace!("run_handle_instance_change_test enter"); + let instance_json = file::read_file_to_string(instance_file); + let instance: KubeAkriInstance = serde_json::from_str(&instance_json).unwrap(); + handle_instance( + match action { + InstanceAction::Add => WatchEvent::Added(instance), + InstanceAction::Update => WatchEvent::Modified(instance), + InstanceAction::Remove => WatchEvent::Deleted(instance), + }, + mock, + ) + .await + .unwrap(); + trace!("run_handle_instance_change_test exit"); + } + + #[tokio::test] + async fn test_internal_handle_existing_instances_no_instances() { + let _ = env_logger::builder().is_test(true).try_init(); + + let mut mock = MockKubeImpl::new(); + config_for_tests::configure_get_instances(&mut mock, "../test/json/empty-list.json", false); + internal_handle_existing_instances(&mock).await.unwrap(); + } + + #[tokio::test] + async fn test_handle_instance_change_for_add_new_local_instance() { + let _ = env_logger::builder().is_test(true).try_init(); + + let mut mock = MockKubeImpl::new(); + configure_for_handle_instance_change( + &mut mock, + &HandleInstanceWork { + find_pods_selector: "akri.sh/instance=config-a-b494b6", + find_pods_result: "../test/json/empty-list.json", + find_pods_phase: None, + find_pods_start_time: None, + find_pods_delete_start_time: false, + deletion_work: None, + addition_work: Some(configure_add_local_config_a_b494b6()), + }, + ); + run_handle_instance_change_test( + &mut mock, + "../test/json/local-instance.json", + &InstanceAction::Add, + ) + .await; + } + + #[tokio::test] + async fn test_handle_instance_change_for_remove_running_local_instance() { + let _ = env_logger::builder().is_test(true).try_init(); + + let mut mock = MockKubeImpl::new(); + configure_for_handle_instance_change( + &mut mock, + &HandleInstanceWork { + find_pods_selector: "akri.sh/instance=config-a-b494b6", + find_pods_result: "../test/json/running-pod-list-for-config-a-local.json", + find_pods_phase: None, + find_pods_start_time: None, + find_pods_delete_start_time: false, + deletion_work: Some(configure_deletion_work_for_config_a_b494b6()), + addition_work: None, + }, + ); + run_handle_instance_change_test( + &mut mock, + "../test/json/local-instance.json", + &InstanceAction::Remove, + ) + .await; + } + + #[tokio::test] + async fn test_handle_instance_change_for_add_new_shared_instance() { + let _ = env_logger::builder().is_test(true).try_init(); + + let mut mock = MockKubeImpl::new(); + configure_for_handle_instance_change( + &mut mock, + &HandleInstanceWork { + find_pods_selector: "akri.sh/instance=config-a-359973", + find_pods_result: "../test/json/empty-list.json", + find_pods_phase: None, + find_pods_start_time: None, + find_pods_delete_start_time: false, + deletion_work: None, + addition_work: Some(configure_add_shared_config_a_359973( + "node-a-config-a-359973-pod", + )), + }, + ); + run_handle_instance_change_test( + &mut mock, + "../test/json/shared-instance.json", + &InstanceAction::Add, + ) + .await; + } + + #[tokio::test] + async fn test_handle_instance_change_for_remove_running_shared_instance() { + let _ = env_logger::builder().is_test(true).try_init(); + + let mut mock = MockKubeImpl::new(); + configure_for_handle_instance_change( + &mut mock, + &HandleInstanceWork { + find_pods_selector: "akri.sh/instance=config-a-359973", + find_pods_result: "../test/json/running-pod-list-for-config-a-shared.json", + find_pods_phase: None, + find_pods_start_time: None, + find_pods_delete_start_time: false, + deletion_work: Some(configure_deletion_work_for_config_a_359973()), + addition_work: None, + }, + ); + run_handle_instance_change_test( + &mut mock, + "../test/json/shared-instance.json", + &InstanceAction::Remove, + ) + .await; + } + + #[tokio::test] + async fn test_handle_instance_change_for_update_active_shared_instance() { + let _ = env_logger::builder().is_test(true).try_init(); + + let mut mock = MockKubeImpl::new(); + configure_for_handle_instance_change( + &mut mock, + &HandleInstanceWork { + find_pods_selector: "akri.sh/instance=config-a-359973", + find_pods_result: "../test/json/running-pod-list-for-config-a-shared.json", + find_pods_phase: None, + find_pods_start_time: None, + find_pods_delete_start_time: false, + deletion_work: Some(configure_deletion_work_for_config_a_359973()), + addition_work: Some(configure_add_shared_config_a_359973( + "node-b-config-a-359973-pod", + )), + }, + ); + run_handle_instance_change_test( + &mut mock, + "../test/json/shared-instance-update.json", + &InstanceAction::Update, + ) + .await; + } + + #[tokio::test] + async fn test_handle_instance_change_when_node_disappears_shared() { + let _ = env_logger::builder().is_test(true).try_init(); + + let deleted_node = "node-b"; + let instance_file = "../test/json/shared-instance-update.json"; + let instance_json = file::read_file_to_string(instance_file); + let kube_object_instance: KubeAkriInstance = serde_json::from_str(&instance_json).unwrap(); + let mut instance = kube_object_instance.spec; + instance.nodes = instance + .nodes + .iter() + .filter_map(|n| { + if n != deleted_node { + Some(n.to_string()) + } else { + None + } + }) + .collect(); + instance.device_usage = instance + .device_usage + .iter() + .map(|(k, v)| { + if v != deleted_node { + (k.to_string(), v.to_string()) + } else { + (k.to_string(), "".to_string()) + } + }) + .collect::>(); + + let mut mock = MockKubeImpl::new(); + configure_for_handle_instance_change( + &mut mock, + &HandleInstanceWork { + find_pods_selector: "akri.sh/instance=config-a-359973", + find_pods_result: "../test/json/running-pod-list-for-config-a-shared.json", + find_pods_phase: None, + find_pods_start_time: None, + find_pods_delete_start_time: false, + deletion_work: Some(configure_deletion_work_for_config_a_359973()), + addition_work: Some(configure_add_shared_config_a_359973( + "node-b-config-a-359973-pod", + )), + }, + ); + run_handle_instance_change_test(&mut mock, &instance_file, &InstanceAction::Update).await; + } +} diff --git a/controller/src/util/mod.rs b/controller/src/util/mod.rs new file mode 100644 index 0000000..cc51d6a --- /dev/null +++ b/controller/src/util/mod.rs @@ -0,0 +1,6 @@ +mod pod_action; +mod shared_test_utils; + +pub mod instance_action; +pub mod node_watcher; +pub mod pod_watcher; diff --git a/controller/src/util/node_watcher.rs b/controller/src/util/node_watcher.rs new file mode 100644 index 0000000..c3e9752 --- /dev/null +++ b/controller/src/util/node_watcher.rs @@ -0,0 +1,635 @@ +use akri_shared::{ + akri::{ + instance::{Instance, KubeAkriInstance}, + retry::{random_delay, MAX_INSTANCE_UPDATE_TRIES}, + }, + k8s, + k8s::KubeInterface, +}; +use futures::StreamExt; +use k8s_openapi::api::core::v1::{NodeSpec, NodeStatus}; +use kube::api::{Api, Informer, Object, WatchEvent}; +use log::trace; +use std::collections::HashMap; + +type NodeObject = Object; + +/// Node states that NodeWatcher is interested in +/// +/// NodeState describes the various states that the controller can +/// react to for Nodes. +#[derive(Clone, Debug, PartialEq)] +enum NodeState { + /// Node has been seen, but not Running yet + Known, + /// Node has been seen Running + Running, + /// A previously Running Node has been seen as not Running + /// and the Instances have been cleaned of references to that + /// vanished Node + InstancesCleaned, +} + +/// This is used to handle Nodes disappearing. +/// +/// When a Node disapears, make sure that any Instance that +/// references the Node is cleaned. This means that the +/// Instance.nodes property no longer contains the node and +/// that the Instance.deviceUsage property no longer contains +/// slots that are occupied by the node. +pub struct NodeWatcher { + known_nodes: HashMap, +} + +impl NodeWatcher { + /// Create new instance of BrokerPodWatcher + pub fn new() -> Self { + NodeWatcher { + known_nodes: HashMap::new(), + } + } + + /// This watches for Node events + pub async fn watch( + &mut self, + ) -> Result<(), Box> { + trace!("watch - enter"); + let kube_interface = k8s::create_kube_interface(); + let resource = Api::v1Node(kube_interface.get_kube_client()); + let inf = Informer::new(resource.clone()).init().await?; + + loop { + let mut nodes = inf.poll().await?.boxed(); + + // Currently, this does not handle None except to break the + // while. + while let Some(event) = nodes.next().await { + self.handle_node(event?, &kube_interface).await?; + } + } + } + + /// This takes an event off the Node stream and if a Node is no longer + /// available, it calls handle_node_disappearance. + /// + /// Nodes are constantly updated. Cleanup work for our services only + /// needs to be called once. + /// + /// To achieve this, store each Node's state as either Known (Node has + /// been seen, but not Running), Running (Node has been seen as Running), + /// and InstanceCleaned (previously Running Node has been seen as not + /// Running). + /// + /// When a Node is in the Known state, it is not Running. If it has + /// never been seen as Running, it is likely being created and there is + /// no need to clean any Instance. + /// + /// Once a Node moves through the Running state into a non Running + /// state, it becomes important to clean Instances referencing the + /// non-Running Node. + async fn handle_node( + &mut self, + event: WatchEvent, + kube_interface: &impl KubeInterface, + ) -> Result<(), Box> { + trace!("handle_node - enter"); + match event { + WatchEvent::Added(node) => { + trace!("handle_node - Added: {:?}", &node.metadata.name); + if self.is_node_ready(&node) { + self.known_nodes + .insert(node.metadata.name, NodeState::Running); + } else { + self.known_nodes + .insert(node.metadata.name, NodeState::Known); + } + } + WatchEvent::Modified(node) => { + trace!("handle_node - Modified: {:?}", &node.metadata.name); + if self.is_node_ready(&node) { + self.known_nodes + .insert(node.metadata.name.clone(), NodeState::Running); + } else { + self.call_handle_node_disappearance_if_needed(&node, kube_interface) + .await?; + } + } + WatchEvent::Deleted(node) => { + trace!("handle_node - Deleted: {:?}", &node.metadata.name); + self.call_handle_node_disappearance_if_needed(&node, kube_interface) + .await?; + } + WatchEvent::Error(e) => { + trace!("handle_node - error for Node: {}", e); + } + }; + Ok(()) + } + + /// This should be called for Nodes that are either !Ready or Deleted. + /// This function ensures that handle_node_disappearance is called + /// only once for any Node as it disappears. + async fn call_handle_node_disappearance_if_needed( + &mut self, + node: &NodeObject, + kube_interface: &impl KubeInterface, + ) -> Result<(), Box> { + trace!( + "call_handle_node_disappearance_if_needed - enter: {:?}", + &node.metadata.name + ); + let last_known_state = self + .known_nodes + .get(&node.metadata.name) + .unwrap_or(&NodeState::Running); + trace!( + "call_handle_node_disappearance_if_needed - last_known_state: {:?}", + &last_known_state + ); + // Nodes are updated roughly once a minute ... try to only call + // handle_node_disappearance once for a node that disappears. + // + // Also, there is no need to call handle_node_disappearance if a + // Node has never been in the Running state. + if last_known_state == &NodeState::Running { + trace!( + "call_handle_node_disappearance_if_needed - call handle_node_disappearance: {:?}", + &node.metadata.name + ); + self.handle_node_disappearance(&node.metadata.name, kube_interface) + .await?; + self.known_nodes + .insert(node.metadata.name.clone(), NodeState::InstancesCleaned); + } + Ok(()) + } + + /// This determines if a node is in the Ready state. + fn is_node_ready(&self, k8s_node: &NodeObject) -> bool { + k8s_node + .status + .as_ref() + .unwrap_or(&NodeStatus::default()) + .conditions + .as_ref() + .unwrap_or(&Vec::new()) + .iter() + .filter_map(|condition| { + if condition.type_ == "Ready" { + Some(condition.status == "True") + } else { + None + } + }) + .collect::>() + .last() + .unwrap_or(&false) + == &true + } + + /// This handles when a node disappears by clearing nodes from + /// the nodes list and deviceUsage map and then trying 5 times to + /// update the Instance. + async fn handle_node_disappearance( + &self, + vanished_node_name: &str, + kube_interface: &impl KubeInterface, + ) -> Result<(), Box> { + trace!( + "handle_node_disappearance - enter vanished_node_name={:?}", + vanished_node_name, + ); + + let instances = kube_interface.get_instances().await?; + trace!( + "handle_node_disappearance - found {:?} instances", + instances.items.len() + ); + for instance in instances.items { + let instance_name = instance.metadata.name.clone(); + let instance_namespace = instance.metadata.namespace.as_ref().ok_or(format!( + "Namespace not found for instance: {}", + instance_name + ))?; + + trace!( + "handle_node_disappearance - make sure node is not referenced here: {:?}", + &instance_name + ); + + // Try up to MAX_INSTANCE_UPDATE_TRIES times to update/create/get instance + for x in 0..MAX_INSTANCE_UPDATE_TRIES { + match if x == 0 { + self.try_remove_nodes_from_instance( + &vanished_node_name, + &instance_name, + &instance_namespace, + &instance, + kube_interface, + ) + .await + } else { + let retry_instance = kube_interface + .find_instance(&instance_name, &instance_namespace) + .await?; + self.try_remove_nodes_from_instance( + &vanished_node_name, + &instance_name, + &instance_namespace, + &retry_instance, + kube_interface, + ) + .await + } { + Ok(_) => break, + Err(e) => { + if x == (MAX_INSTANCE_UPDATE_TRIES - 1) { + return Err(e); + } + random_delay().await; + } + } + } + } + + trace!("handle_node_disappearance - exit"); + Ok(()) + } + + /// This attempts to remove nodes from the nodes list and deviceUsage + /// map in an Instance. An attempt is made to update + /// the instance in etcd, any failure is returned. + async fn try_remove_nodes_from_instance( + &self, + vanished_node_name: &str, + instance_name: &str, + instance_namespace: &str, + instance: &KubeAkriInstance, + kube_interface: &impl KubeInterface, + ) -> Result<(), Box> { + trace!( + "try_remove_nodes_from_instance - vanished_node_name: {:?}", + &vanished_node_name + ); + let modified_nodes = instance + .spec + .nodes + .iter() + .filter(|node| &vanished_node_name != node) + .map(|node| node.into()) + .collect::>(); + // Remove nodes from instance.deviceusage + let modified_device_usage = instance + .spec + .device_usage + .iter() + .map(|(slot, node)| { + ( + slot.to_string(), + if vanished_node_name == node { + "".into() + } else { + node.into() + }, + ) + }) + .collect::>(); + + // Save the instance + let modified_instance = Instance { + configuration_name: instance.spec.configuration_name.clone(), + metadata: instance.spec.metadata.clone(), + rbac: instance.spec.rbac.clone(), + shared: instance.spec.shared, + device_usage: modified_device_usage, + nodes: modified_nodes, + }; + + trace!( + "handle_node_disappearance - kube_interface.update_instance name: {}, namespace: {}, {:?}", + &instance_name, + &instance_namespace, + &modified_instance + ); + + kube_interface + .update_instance(&modified_instance, &instance_name, &instance_namespace) + .await + } +} + +#[cfg(test)] +mod tests { + use super::super::shared_test_utils::config_for_tests; + use super::*; + use akri_shared::{ + akri::instance::KubeAkriInstanceList, k8s::test_kube::MockKubeImpl, os::file, + }; + + #[derive(Clone)] + struct UpdateInstance { + instance_to_update: Instance, + instance_name: &'static str, + instance_namespace: &'static str, + } + + #[derive(Clone)] + struct HandleNodeDisappearance { + get_instances_result_file: &'static str, + get_instances_result_listify: bool, + update_instance: Option, + } + + fn configure_for_handle_node_disappearance( + mock: &mut MockKubeImpl, + work: &HandleNodeDisappearance, + ) { + config_for_tests::configure_get_instances( + mock, + work.get_instances_result_file, + work.get_instances_result_listify, + ); + + if let Some(update_instance) = &work.update_instance { + config_for_tests::configure_update_instance( + mock, + update_instance.instance_to_update.clone(), + update_instance.instance_name, + update_instance.instance_namespace, + false, + ); + } + } + + #[tokio::test] + async fn test_handle_node_added_unready() { + let _ = env_logger::builder().is_test(true).try_init(); + let node_json = file::read_file_to_string("../test/json/node-a-not-ready.json"); + let node: NodeObject = serde_json::from_str(&node_json).unwrap(); + let mut node_watcher = NodeWatcher::new(); + node_watcher + .handle_node(WatchEvent::Added(node), &MockKubeImpl::new()) + .await + .unwrap(); + + assert_eq!(1, node_watcher.known_nodes.len()); + + assert_eq!( + &NodeState::Known, + node_watcher.known_nodes.get(&"node-a".to_string()).unwrap() + ) + } + + #[tokio::test] + async fn test_handle_node_added_ready() { + let _ = env_logger::builder().is_test(true).try_init(); + + let node_json = file::read_file_to_string("../test/json/node-a.json"); + let node: NodeObject = serde_json::from_str(&node_json).unwrap(); + let mut node_watcher = NodeWatcher::new(); + node_watcher + .handle_node(WatchEvent::Added(node), &MockKubeImpl::new()) + .await + .unwrap(); + + assert_eq!(1, node_watcher.known_nodes.len()); + + assert_eq!( + &NodeState::Running, + node_watcher.known_nodes.get(&"node-a".to_string()).unwrap() + ) + } + + #[tokio::test] + async fn test_handle_node_modified_unready_unknown() { + let _ = env_logger::builder().is_test(true).try_init(); + + let node_json = file::read_file_to_string("../test/json/node-b-not-ready.json"); + let node: NodeObject = serde_json::from_str(&node_json).unwrap(); + let mut node_watcher = NodeWatcher::new(); + + let instance_file = "../test/json/shared-instance-update.json"; + let instance_json = file::read_file_to_string(instance_file); + let kube_object_instance: KubeAkriInstance = serde_json::from_str(&instance_json).unwrap(); + let mut instance = kube_object_instance.spec; + instance.nodes.clear(); + instance + .device_usage + .insert("config-a-359973-2".to_string(), "".to_string()); + + let mut mock = MockKubeImpl::new(); + configure_for_handle_node_disappearance( + &mut mock, + &HandleNodeDisappearance { + get_instances_result_file: "../test/json/shared-instance-update.json", + get_instances_result_listify: true, + update_instance: Some(UpdateInstance { + instance_to_update: instance, + instance_name: "config-a-359973", + instance_namespace: "config-a-namespace", + }), + }, + ); + + node_watcher + .handle_node(WatchEvent::Modified(node), &mock) + .await + .unwrap(); + + assert_eq!(1, node_watcher.known_nodes.len()); + + assert_eq!( + &NodeState::InstancesCleaned, + node_watcher.known_nodes.get(&"node-b".to_string()).unwrap() + ) + } + + #[tokio::test] + async fn test_handle_node_modified_ready_unknown() { + let _ = env_logger::builder().is_test(true).try_init(); + + let node_json = file::read_file_to_string("../test/json/node-b.json"); + let node: NodeObject = serde_json::from_str(&node_json).unwrap(); + let mut node_watcher = NodeWatcher::new(); + + let mock = MockKubeImpl::new(); + node_watcher + .handle_node(WatchEvent::Modified(node), &mock) + .await + .unwrap(); + + assert_eq!(1, node_watcher.known_nodes.len()); + + assert_eq!( + &NodeState::Running, + node_watcher.known_nodes.get(&"node-b".to_string()).unwrap() + ) + } + + #[tokio::test] + async fn test_handle_node_deleted_unready_unknown() { + let _ = env_logger::builder().is_test(true).try_init(); + + let node_json = file::read_file_to_string("../test/json/node-b-not-ready.json"); + let node: NodeObject = serde_json::from_str(&node_json).unwrap(); + let mut node_watcher = NodeWatcher::new(); + + let instance_file = "../test/json/shared-instance-update.json"; + let instance_json = file::read_file_to_string(instance_file); + let kube_object_instance: KubeAkriInstance = serde_json::from_str(&instance_json).unwrap(); + let mut instance = kube_object_instance.spec; + instance.nodes.clear(); + instance + .device_usage + .insert("config-a-359973-2".to_string(), "".to_string()); + + let mut mock = MockKubeImpl::new(); + configure_for_handle_node_disappearance( + &mut mock, + &HandleNodeDisappearance { + get_instances_result_file: "../test/json/shared-instance-update.json", + get_instances_result_listify: true, + update_instance: Some(UpdateInstance { + instance_to_update: instance, + instance_name: "config-a-359973", + instance_namespace: "config-a-namespace", + }), + }, + ); + + node_watcher + .handle_node(WatchEvent::Deleted(node), &mock) + .await + .unwrap(); + + assert_eq!(1, node_watcher.known_nodes.len()); + + assert_eq!( + &NodeState::InstancesCleaned, + node_watcher.known_nodes.get(&"node-b".to_string()).unwrap() + ) + } + + const LIST_PREFIX: &str = r#" +{ + "apiVersion": "v1", + "items": ["#; + const LIST_SUFFIX: &str = r#" + ], + "kind": "List", + "metadata": { + "resourceVersion": "", + "selfLink": "" + } +}"#; + fn listify_node(node_json: &String) -> String { + format!("{}\n{}\n{}", LIST_PREFIX, node_json, LIST_SUFFIX) + } + + #[tokio::test] + async fn test_handle_node_disappearance_update_failure_retries() { + let _ = env_logger::builder().is_test(true).try_init(); + + let mut mock = MockKubeImpl::new(); + mock.expect_get_instances().times(1).returning(move || { + let instance_file = "../test/json/shared-instance-update.json"; + let instance_json = file::read_file_to_string(instance_file); + let instance_list_json = listify_node(&instance_json); + let list: KubeAkriInstanceList = serde_json::from_str(&instance_list_json).unwrap(); + Ok(list) + }); + mock.expect_update_instance() + .times(MAX_INSTANCE_UPDATE_TRIES as usize) + .withf(move |_instance, n, ns| n == "config-a-359973" && ns == "config-a-namespace") + .returning(move |_, _, _| Err(None.ok_or("failure")?)); + mock.expect_find_instance() + .times((MAX_INSTANCE_UPDATE_TRIES - 1) as usize) + .withf(move |n, ns| n == "config-a-359973" && ns == "config-a-namespace") + .returning(move |_, _| { + let instance_file = "../test/json/shared-instance-update.json"; + let instance_json = file::read_file_to_string(instance_file); + let instance: KubeAkriInstance = serde_json::from_str(&instance_json).unwrap(); + Ok(instance) + }); + + let node_watcher = NodeWatcher::new(); + assert!(node_watcher + .handle_node_disappearance(&"foo-a", &mock,) + .await + .is_err()); + } + + #[tokio::test] + async fn test_try_remove_nodes_from_instance() { + let _ = env_logger::builder().is_test(true).try_init(); + + let instance_file = "../test/json/shared-instance-update.json"; + let instance_json = file::read_file_to_string(instance_file); + let kube_object_instance: KubeAkriInstance = serde_json::from_str(&instance_json).unwrap(); + + let mut mock = MockKubeImpl::new(); + mock.expect_update_instance() + .times(1) + .withf(move |ins, n, ns| { + n == "config-a" + && ns == "config-a-namespace" + && !ins.nodes.contains(&"node-b".to_string()) + && ins + .device_usage + .iter() + .filter_map(|(_slot, value)| { + if value == &"node-b".to_string() { + Some(value.to_string()) + } else { + None + } + }) + .collect::>() + .first() + .is_none() + }) + .returning(move |_, _, _| Ok(())); + + let node_watcher = NodeWatcher::new(); + assert!(node_watcher + .try_remove_nodes_from_instance( + "node-b", + "config-a", + "config-a-namespace", + &kube_object_instance, + &mock, + ) + .await + .is_ok()); + } + + #[test] + fn test_is_node_ready_ready() { + let _ = env_logger::builder().is_test(true).try_init(); + + let tests = [ + ("../test/json/node-a.json", true), + ("../test/json/node-a-not-ready.json", false), + ("../test/json/node-a-no-conditions.json", false), + ("../test/json/node-a-no-ready-condition.json", false), + ]; + + for (node_file, result) in tests.iter() { + trace!( + "Testing {} should reflect node is ready={}", + node_file, + result + ); + + let node_json = file::read_file_to_string(node_file); + let kube_object_node: Object = + serde_json::from_str(&node_json).unwrap(); + + let node_watcher = NodeWatcher::new(); + assert_eq!( + result.clone(), + node_watcher.is_node_ready(&kube_object_node) + ); + } + } +} diff --git a/controller/src/util/pod_action.rs b/controller/src/util/pod_action.rs new file mode 100644 index 0000000..dd92008 --- /dev/null +++ b/controller/src/util/pod_action.rs @@ -0,0 +1,539 @@ +use super::instance_action::InstanceAction; +use chrono::Utc; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time; + +/// Pod action types +/// +/// Pod actions describe the types of actions the controller can +/// take for broker Pods. +/// +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum PodAction { + /// The broker Pod must be added + Add, + /// The broker Pod must be removed + Remove, + /// The broker Pod must be removed and added + RemoveAndAdd, + /// No action should be taken for the broker Pod + NoAction, +} + +/// This is used to determine what action to take for +/// a broker Pod. +/// +/// The action to take is based on several factors: +/// 1. what the InstanceAction is (Add, Delete, Modify) +/// 1. what phase the Pod is in (Running, Pending, etc) +/// 1. when the Pod started +/// 1. the relevant grace time +/// +pub struct PodActionInfo { + pub pending_grace_time_in_minutes: i64, + pub ended_grace_time_in_minutes: i64, + pub phase: String, + pub instance_action: InstanceAction, + pub status_start_time: Option