From 01180697a80555afbca6126bb64f68475f397137 Mon Sep 17 00:00:00 2001 From: Olivier Lemasle Date: Tue, 12 Jan 2021 18:54:08 +0100 Subject: [PATCH] Initial commit - release project --- .github/workflows/pr-check.yaml | 44 ++ .github/workflows/release.yaml | 131 ++++++ .gitignore | 4 + LICENSE | 201 ++++++++ Makefile | 58 +++ README.md | 130 ++++++ cmd/cloudstack-csi-driver/Dockerfile | 10 + cmd/cloudstack-csi-driver/main.go | 85 ++++ cmd/cloudstack-csi-sc-syncer/Dockerfile | 10 + cmd/cloudstack-csi-sc-syncer/README.md | 103 +++++ cmd/cloudstack-csi-sc-syncer/main.go | 55 +++ deploy/k8s/controller-deployment.yaml | 81 ++++ deploy/k8s/csidriver.yaml | 8 + deploy/k8s/node-daemonset.yaml | 102 +++++ deploy/k8s/rbac.yaml | 51 +++ examples/k8s/0-storageclass.yaml | 10 + examples/k8s/pod-block.yaml | 18 + examples/k8s/pod.yaml | 18 + examples/k8s/pvc-block.yaml | 12 + examples/k8s/pvc.yaml | 11 + go.mod | 23 + go.sum | 581 ++++++++++++++++++++++++ pkg/cloud/cloud.go | 63 +++ pkg/cloud/config.go | 46 ++ pkg/cloud/fake/fake.go | 102 +++++ pkg/cloud/metadata.go | 83 ++++ pkg/cloud/node.go | 17 + pkg/cloud/vms.go | 53 +++ pkg/cloud/volumes.go | 122 +++++ pkg/cloud/zones.go | 24 + pkg/driver/constants.go | 17 + pkg/driver/controller.go | 370 +++++++++++++++ pkg/driver/controller_test.go | 42 ++ pkg/driver/driver.go | 46 ++ pkg/driver/identity.go | 57 +++ pkg/driver/node.go | 339 ++++++++++++++ pkg/driver/server.go | 85 ++++ pkg/driver/topology.go | 40 ++ pkg/mount/fake.go | 53 +++ pkg/mount/mount.go | 172 +++++++ pkg/syncer/error.go | 13 + pkg/syncer/name.go | 50 ++ pkg/syncer/name_test.go | 44 ++ pkg/syncer/run.go | 201 ++++++++ pkg/syncer/syncer.go | 104 +++++ pkg/util/doc.go | 2 + pkg/util/gb.go | 13 + pkg/util/gb_test.go | 41 ++ test/e2e/README.md | 21 + test/e2e/run.sh | 37 ++ test/e2e/storageclass.yaml | 10 + test/e2e/testdriver.yaml | 10 + test/sanity/sanity_test.go | 47 ++ 53 files changed, 4070 insertions(+) create mode 100644 .github/workflows/pr-check.yaml create mode 100644 .github/workflows/release.yaml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 cmd/cloudstack-csi-driver/Dockerfile create mode 100644 cmd/cloudstack-csi-driver/main.go create mode 100644 cmd/cloudstack-csi-sc-syncer/Dockerfile create mode 100644 cmd/cloudstack-csi-sc-syncer/README.md create mode 100644 cmd/cloudstack-csi-sc-syncer/main.go create mode 100644 deploy/k8s/controller-deployment.yaml create mode 100644 deploy/k8s/csidriver.yaml create mode 100644 deploy/k8s/node-daemonset.yaml create mode 100644 deploy/k8s/rbac.yaml create mode 100644 examples/k8s/0-storageclass.yaml create mode 100644 examples/k8s/pod-block.yaml create mode 100644 examples/k8s/pod.yaml create mode 100644 examples/k8s/pvc-block.yaml create mode 100644 examples/k8s/pvc.yaml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 pkg/cloud/cloud.go create mode 100644 pkg/cloud/config.go create mode 100644 pkg/cloud/fake/fake.go create mode 100644 pkg/cloud/metadata.go create mode 100644 pkg/cloud/node.go create mode 100644 pkg/cloud/vms.go create mode 100644 pkg/cloud/volumes.go create mode 100644 pkg/cloud/zones.go create mode 100644 pkg/driver/constants.go create mode 100644 pkg/driver/controller.go create mode 100644 pkg/driver/controller_test.go create mode 100644 pkg/driver/driver.go create mode 100644 pkg/driver/identity.go create mode 100644 pkg/driver/node.go create mode 100644 pkg/driver/server.go create mode 100644 pkg/driver/topology.go create mode 100644 pkg/mount/fake.go create mode 100644 pkg/mount/mount.go create mode 100644 pkg/syncer/error.go create mode 100644 pkg/syncer/name.go create mode 100644 pkg/syncer/name_test.go create mode 100644 pkg/syncer/run.go create mode 100644 pkg/syncer/syncer.go create mode 100644 pkg/util/doc.go create mode 100644 pkg/util/gb.go create mode 100644 pkg/util/gb_test.go create mode 100644 test/e2e/README.md create mode 100755 test/e2e/run.sh create mode 100644 test/e2e/storageclass.yaml create mode 100644 test/e2e/testdriver.yaml create mode 100644 test/sanity/sanity_test.go diff --git a/.github/workflows/pr-check.yaml b/.github/workflows/pr-check.yaml new file mode 100644 index 0000000..7909c84 --- /dev/null +++ b/.github/workflows/pr-check.yaml @@ -0,0 +1,44 @@ +name: PR Check + +on: + pull_request: {} + +jobs: + lint: + name: Lint + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - name: golangci-lint + uses: golangci/golangci-lint-action@v2 + with: + version: v1.34 + + build: + name: Test & Build + runs-on: ubuntu-18.04 + steps: + - name: Setup up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: "^1.15" + + - name: Check out code + uses: actions/checkout@v2 + + - name: Cache + uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Run unit tests + run: make test + + - name: Run sanity tests + run: make test-sanity + + - name: Build + run: make diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..250b027 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,131 @@ +name: Release + +on: + push: + branches: + - master + tags: + - v* + +env: + REGISTRY_NAME: quay.io/apalia + IMAGES: "cloudstack-csi-driver cloudstack-csi-sc-syncer" + +jobs: + push: + name: Push images + runs-on: ubuntu-18.04 + + steps: + - name: Check out code + uses: actions/checkout@v2 + + - name: Cache + uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Build container images + run: make container + + - name: Log into registry + uses: docker/login-action@v1 + with: + registry: quay.io + username: apalia+github_cloudstack_csi_driver + password: ${{ secrets.QUAY_TOKEN }} + + - name: Push master + if: github.ref == 'refs/heads/master' + run: | + for img in $IMAGES; do + docker tag ${img} ${REGISTRY_NAME}/${img}:master + docker push ${REGISTRY_NAME}/${img}:master + done + + - name: Push tagged release + if: startsWith(github.ref, 'refs/tags/v') + run: | + # Strip prefix from version + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,' | sed -e 's/^v//') + + for img in $IMAGES; do + docker tag ${img} ${REGISTRY_NAME}/${img}:${VERSION} + docker push ${REGISTRY_NAME}/${img}:${VERSION} + done + + - name: Upload cloudstack-csi-sc-syncer artifact + if: startsWith(github.ref, 'refs/tags/v') + uses: actions/upload-artifact@v2 + with: + name: bin + path: bin/cloudstack-csi-sc-syncer + retention-days: 1 + + release: + name: Release + runs-on: ubuntu-18.04 + + # Run only if previous job has succeeded + needs: [push] + + # Create a release only for tags v* + if: startsWith(github.ref, 'refs/tags/v') + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Create manifest + run: | + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,' | sed -e 's/^v//') + echo "---" >> manifest.yaml + cat deploy/k8s/rbac.yaml >> manifest.yaml + echo "---" >> manifest.yaml + cat deploy/k8s/csidriver.yaml >> manifest.yaml + echo "---" >> manifest.yaml + sed -E "s|image: +cloudstack-csi-driver|image: ${REGISTRY_NAME}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/controller-deployment.yaml >> manifest.yaml + echo "---" >> manifest.yaml + sed -E "s|image: +cloudstack-csi-driver|image: ${REGISTRY_NAME}/cloudstack-csi-driver:${VERSION}|" deploy/k8s/node-daemonset.yaml >> manifest.yaml + + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: ${{ github.ref }} + draft: false + prerelease: false + + - name: Upload Release Asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: manifest.yaml + asset_name: manifest.yaml + asset_content_type: application/x-yaml + + - name: Download cloudstack-csi-sc-syncer artifact + uses: actions/download-artifact@v2 + with: + name: bin + path: bin + + - run: ls -l + + - name: Upload cloudstack-csi-sc-syncer asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.create_release.outputs.upload_url }} + asset_path: bin/cloudstack-csi-sc-syncer + asset_name: cloudstack-csi-sc-syncer + asset_content_type: application/x-executable diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..68ff0d9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/bin +/test/e2e/e2e.test +/test/e2e/ginkgo +/cloudstack.ini diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..9bb1e58 --- /dev/null +++ b/Makefile @@ -0,0 +1,58 @@ +CMDS=cloudstack-csi-driver cloudstack-csi-sc-syncer + +# Revision that gets built into each binary via the main.version +# string. Uses the `git describe` output based on the most recent +# version tag with a short revision suffix or, if nothing has been +# tagged yet, just the revision. +# +# Beware that tags may also be missing in shallow clones as done by +# some CI systems (like TravisCI, which pulls only 50 commits). +REV=$(shell git describe --long --tags --match='v*' --dirty 2>/dev/null || git rev-list -n1 HEAD) + +DOCKER?=docker + +IMPORTPATH_LDFLAGS = -X main.version=$(REV) +LDFLAGS = -s -w +FULL_LDFLAGS = $(LDFLAGS) $(IMPORTPATH_LDFLAGS) + +.PHONY: all +all: build + +.PHONY: build +build: $(CMDS:%=build-%) + +.PHONY: container +container: $(CMDS:%=container-%) + +.PHONY: clean +clean: + rm -rf bin test/e2e/e2e.test test/e2e/ginkgo + +.PHONY: build-% +$(CMDS:%=build-%): build-%: + mkdir -p bin + CGO_ENABLED=0 go build -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*" ./cmd/$* + +.PHONY: container-% +$(CMDS:%=container-%): container-%: build-% + $(DOCKER) build -f ./cmd/$*/Dockerfile -t $*:latest \ + --label org.opencontainers.image.revision=$(REV) . + +.PHONY: test +test: + go test ./... + +.PHONY: test-sanity +test-sanity: + go test --tags=sanity ./test/sanity + +.PHONY: setup-external-e2e +setup-external-e2e: test/e2e/e2e.test test/e2e/ginkgo + +test/e2e/e2e.test test/e2e/ginkgo: + curl --location https://dl.k8s.io/v1.19.5/kubernetes-test-linux-amd64.tar.gz | \ + tar --strip-components=3 -C test/e2e -zxf - kubernetes/test/bin/e2e.test kubernetes/test/bin/ginkgo + +.PHONY: test-e2e +test-e2e: setup-external-e2e + bash ./test/e2e/run.sh \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..14f029d --- /dev/null +++ b/README.md @@ -0,0 +1,130 @@ +# CloudStack CSI Driver + +[![Quay.io](https://img.shields.io/badge/Quay.io-container_image-informational)](https://quay.io/repository/apalia/cloudstack-csi-driver) +[![Go Reference](https://pkg.go.dev/badge/github.com/apalia/cloudstack-csi-driver.svg)](https://pkg.go.dev/github.com/apalia/cloudstack-csi-driver) +[![Go Report Card](https://goreportcard.com/badge/github.com/apalia/cloudstack-csi-driver)](https://goreportcard.com/report/github.com/apalia/cloudstack-csi-driver) +[![Release](https://github.com/apalia/cloudstack-csi-driver/workflows/Release/badge.svg?branch=master)](https://github.com/apalia/cloudstack-csi-driver/actions) + +This repository provides a [Container Storage Interface (CSI)](https://github.com/container-storage-interface/spec) +plugin for [Apache CloudStack](https://cloudstack.apache.org/). + +## Usage with Kubernetes + +### Requirements + +- Minimal Kubernetes version: v1.17 + +- The Kubernetes cluster must run in CloudStack. Tested only in a KVM zone. + +- A disk offering with custom size must be available, with type "shared". + +- In order to match the Kubernetes node and the CloudStack instance, + they should both have the same name. If not, it is also possible to use + [cloud-init instance metadata](https://cloudinit.readthedocs.io/en/latest/topics/instancedata.html) + to get the instance name: if the node has cloud-init enabled, metadata will + be available in `/run/cloud-init/instance-data.json`; you should then make + sure that `/run/cloud-init/` is mounted from the node. + +- Kubernetes nodes must be in the Root domain, and be created by the CloudStack + account whose credentials are used in [configuration](#configuration). + +### Configuration + +Create the CloudStack configuration file `cloudstack.ini`. + +It should have the following format, defined for the [CloudStack Kubernetes Provider](https://github.com/apache/cloudstack-kubernetes-provider): + +```ini +[Global] +api-url = +api-key = +secret-key = +ssl-no-verify = +``` + +Create a secret named `cloudstack-secret` in namespace `kube-system`: + +``` +kubectl create secret generic \ + --namespace kube-system \ + --from-file ./cloudstack.ini \ + cloudstack-secret +``` + +If you have also deployed the [CloudStack Kubernetes Provider](https://github.com/apache/cloudstack-kubernetes-provider), +you may use the same secret for both tools. + +### Deployment + +``` +kubectl apply -f https://github.com/apalia/cloudstack-csi-driver/releases/latest/download/manifest.yaml +``` + +### Creation of Storage classes + +#### Manually + +A storage class can be created manually: see [example](./examples/k8s/0-storageclass.yaml). + +The `provisioner` value must be `csi.cloudstack.apache.org`. + +The `volumeBindingMode` must be `WaitForFirstConsumer`, in order to delay the +binding and provisioning of a PersistentVolume until a Pod using the +PersistentVolumeClaim is created. It enables the provisioning of volumes +in respect to topology constraints (e.g. volume in the right zone). + +The storage class must also have a parameter named +`csi.cloudstack.apache.org/disk-offering-id` whose value is the CloudStack disk +offering ID. + +#### Using cloudstack-csi-sc-syncer + +The tool `cloudstack-csi-sc-syncer` may also be used to synchronize CloudStack +disk offerings to Kubernetes storage classes. + +[More info...](./cmd/cloudstack-csi-sc-syncer/README.md) + +### Usage + +Example: + +``` +kubectl apply -f ./examples/k8s/pvc.yaml +kubectl apply -f ./examples/k8s/pod.yaml +``` + +## Building + +To build the driver binary: + +``` +make build-cloudstack-csi-driver +``` + +To build the container images: + +``` +make container +``` + +## See also + +- [CloudStack Kubernetes Provider](https://github.com/apache/cloudstack-kubernetes-provider) - Kubernetes Cloud Controller Manager for Apache CloudStack +- [CloudStack documentation on storage](http://docs.cloudstack.apache.org/en/latest/adminguide/storage.html) +- [CSI (Container Storage Interface) specification](https://github.com/container-storage-interface/spec) + +--- + + Copyright 2021 Apalia SAS + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/cmd/cloudstack-csi-driver/Dockerfile b/cmd/cloudstack-csi-driver/Dockerfile new file mode 100644 index 0000000..b100368 --- /dev/null +++ b/cmd/cloudstack-csi-driver/Dockerfile @@ -0,0 +1,10 @@ +FROM alpine:3.12.3 + +LABEL \ + org.opencontainers.image.description="CloudStack CSI driver" \ + org.opencontainers.image.source="https://github.com/apalia/cloudstack-csi-driver/" + +RUN apk add --no-cache xfsprogs e2fsprogs blkid ca-certificates + +COPY ./bin/cloudstack-csi-driver /cloudstack-csi-driver +ENTRYPOINT ["/cloudstack-csi-driver"] \ No newline at end of file diff --git a/cmd/cloudstack-csi-driver/main.go b/cmd/cloudstack-csi-driver/main.go new file mode 100644 index 0000000..2fe9aa1 --- /dev/null +++ b/cmd/cloudstack-csi-driver/main.go @@ -0,0 +1,85 @@ +// cloudstack-csi-driver binary. +// +// To get usage information: +// +// cloudstack-csi-driver -h +// +package main + +import ( + "flag" + "fmt" + "os" + "path" + + "github.com/apalia/cloudstack-csi-driver/pkg/cloud" + "github.com/apalia/cloudstack-csi-driver/pkg/driver" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +var ( + endpoint = flag.String("endpoint", "unix:///tmp/csi.sock", "CSI endpoint") + cloudstackconfig = flag.String("cloudstackconfig", "./cloudstack.ini", "CloudStack configuration file") + nodeName = flag.String("nodeName", "", "Node name") + debug = flag.Bool("debug", false, "Enable debug logging") + showVersion = flag.Bool("version", false, "Show version") + + // Version is set by the build proces + version = "" + isDevEnv = false +) + +func main() { + flag.Parse() + + if *showVersion { + baseName := path.Base(os.Args[0]) + fmt.Println(baseName, version) + return + } + + if version == "" { + isDevEnv = true + } + + run() + os.Exit(0) +} + +func run() { + // Setup logging + var logConfig zap.Config + if isDevEnv { + logConfig = zap.NewDevelopmentConfig() + } else { + logConfig = zap.NewProductionConfig() + } + if *debug { + logConfig.Level.SetLevel(zapcore.DebugLevel) + } + logger, _ := logConfig.Build() + defer func() { _ = logger.Sync() }() + undo := zap.ReplaceGlobals(logger) + defer undo() + + // Setup cloud connector + config, err := cloud.ReadConfig(*cloudstackconfig) + if err != nil { + logger.Sugar().Errorw("Cannot read CloudStack configuration", "error", err) + os.Exit(1) + } + logger.Sugar().Debugf("Successfully read CloudStack configuration %v", *cloudstackconfig) + csConnector := cloud.New(config) + + d, err := driver.New(*endpoint, csConnector, nil, *nodeName, version, logger) + if err != nil { + logger.Sugar().Errorw("Failed to initialize driver", "error", err) + os.Exit(1) + } + + if err = d.Run(); err != nil { + logger.Sugar().Errorw("Server error", "error", err) + os.Exit(1) + } +} diff --git a/cmd/cloudstack-csi-sc-syncer/Dockerfile b/cmd/cloudstack-csi-sc-syncer/Dockerfile new file mode 100644 index 0000000..15bffc1 --- /dev/null +++ b/cmd/cloudstack-csi-sc-syncer/Dockerfile @@ -0,0 +1,10 @@ +FROM alpine:3.12.3 + +LABEL \ + org.opencontainers.image.description="CloudStack disk offering to Kubernetes storage class syncer" \ + org.opencontainers.image.source="https://github.com/apalia/cloudstack-csi-driver/" + +RUN apk add --no-cache ca-certificates + +COPY ./bin/cloudstack-csi-sc-syncer /cloudstack-csi-sc-syncer +ENTRYPOINT ["/cloudstack-csi-sc-syncer"] \ No newline at end of file diff --git a/cmd/cloudstack-csi-sc-syncer/README.md b/cmd/cloudstack-csi-sc-syncer/README.md new file mode 100644 index 0000000..2a431eb --- /dev/null +++ b/cmd/cloudstack-csi-sc-syncer/README.md @@ -0,0 +1,103 @@ +# cloudstack-csi-sc-syncer + +`cloudstack-csi-sc-syncer` connects to CloudStack (using the same CloudStack +configuration file as `cloudstack-csi-driver`), lists all disk offerings +suitable for usage in Kubernetes (currently: checks they have a custom size), +and creates corresponding Storage Classes in Kubernetes if needed. + +It also adds a label to the Storage Classes it creates. + +If option `-delete=true` is passed, it may also delete Kubernetes Storage +Classes, when they have its label and their corresponding CloudStack disk +offering has been deleted. + +## Usage + +You may use it locally or as a Kubernetes Job. + +### Locally + +You must have a CloudStack configuration file and a Kubernetes `kubeconfig` +file. + +1. Download `cloudstack-csi-sc-syncer` from [latest release](https://github.com/apalia/cloudstack-csi-driver/releases/latest/); + +1. Set the execution permission: + + ``` + chmod +x ./cloudstack-csi-sc-syncer + ``` + +1. Then simply execute the tool: + + ``` + ./cloudstack-csi-sc-syncer + ``` + +Run `./cloudstack-csi-sc-syncer -h` to get the complete list of options and their default values. + +### As a Kubernetes Job + +You may run `cloudstack-csi-sc-syncer` as a Kubernetes Job. In that case, it +re-uses the CloudStack configuration file in Secret `cloudstack-secret`, and use +in-cluster Kubernetes authentification, using a ServiceAccount. + +```sh +export version=... + +kubectl apply -f - < diff --git a/examples/k8s/pod-block.yaml b/examples/k8s/pod-block.yaml new file mode 100644 index 0000000..5016e66 --- /dev/null +++ b/examples/k8s/pod-block.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: example-pod-block +spec: + containers: + - name: example + image: ubuntu + volumeDevices: + - devicePath: "/dev/example-block" + name: example-volume + stdin: true + stdinOnce: true + tty: true + volumes: + - name: example-volume + persistentVolumeClaim: + claimName: example-pvc-block diff --git a/examples/k8s/pod.yaml b/examples/k8s/pod.yaml new file mode 100644 index 0000000..afcc288 --- /dev/null +++ b/examples/k8s/pod.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: example-pod +spec: + containers: + - name: example + image: busybox + volumeMounts: + - mountPath: "/data" + name: example-volume + stdin: true + stdinOnce: true + tty: true + volumes: + - name: example-volume + persistentVolumeClaim: + claimName: example-pvc diff --git a/examples/k8s/pvc-block.yaml b/examples/k8s/pvc-block.yaml new file mode 100644 index 0000000..b689a86 --- /dev/null +++ b/examples/k8s/pvc-block.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: example-pvc-block +spec: + storageClassName: cloudstack-custom + volumeMode: Block + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/examples/k8s/pvc.yaml b/examples/k8s/pvc.yaml new file mode 100644 index 0000000..a050773 --- /dev/null +++ b/examples/k8s/pvc.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: example-pvc +spec: + storageClassName: cloudstack-custom + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..9487ee0 --- /dev/null +++ b/go.mod @@ -0,0 +1,23 @@ +module github.com/apalia/cloudstack-csi-driver + +go 1.15 + +require ( + github.com/container-storage-interface/spec v1.3.0 + github.com/golang/protobuf v1.4.3 + github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 + github.com/hashicorp/go-uuid v1.0.2 + github.com/kubernetes-csi/csi-lib-utils v0.9.0 + github.com/kubernetes-csi/csi-test/v4 v4.0.2 + github.com/xanzy/go-cloudstack/v2 v2.9.0 + go.uber.org/zap v1.16.0 + golang.org/x/text v0.3.4 + google.golang.org/grpc v1.34.0 + gopkg.in/gcfg.v1 v1.2.3 + gopkg.in/warnings.v0 v0.1.2 // indirect + k8s.io/api v0.20.1 + k8s.io/apimachinery v0.20.1 + k8s.io/client-go v0.20.1 + k8s.io/mount-utils v0.20.1 + k8s.io/utils v0.0.0-20201110183641-67b214c5f920 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..9da5ede --- /dev/null +++ b/go.sum @@ -0,0 +1,581 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= +github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= +github.com/container-storage-interface/spec v1.3.0 h1:wMH4UIoWnK/TXYw8mbcIHgZmB6kHOeIsYsiaTJwa6bc= +github.com/container-storage-interface/spec v1.3.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.2 h1:FlFbCRLd5Jr4iYXZufAvgWN6Ao0JrI5chLINnUXDDr0= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= +github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kubernetes-csi/csi-lib-utils v0.9.0 h1:TbuDmxoVqM+fvVkzG/7sShyX/8jUln0ElLHuETcsQJI= +github.com/kubernetes-csi/csi-lib-utils v0.9.0/go.mod h1:8E2jVUX9j3QgspwHXa6LwyN7IHQDjW9jX3kwoWnSC+M= +github.com/kubernetes-csi/csi-test/v4 v4.0.2 h1:MNj94SFHOGK6lOy+yDgxI+zlFWaPcgByqBH3JZZGyZI= +github.com/kubernetes-csi/csi-test/v4 v4.0.2/go.mod h1:z3FYigjLFAuzmFzKdHQr8gUPm5Xr4Du2twKcxfys0eI= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/robertkrimen/otto v0.0.0-20191219234010-c382bd3c16ff/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/xanzy/go-cloudstack/v2 v2.9.0 h1:Vb4uMqQe+QCyb7wpH0yhx7gJQdEzmJEg4trexCdDv4Q= +github.com/xanzy/go-cloudstack/v2 v2.9.0/go.mod h1:+SiI2stR3n/P6IKCjrlD2e2EWzk+rQqK4SxC4V9QhnY= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPjc0178IU44m4EjOO5IY= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb h1:iKlO7ROJc6SttHKlxzwGytRtBUqX4VARrNTgP2YLX5M= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191114150713-6bbd007550de/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.0/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.34.0 h1:raiipEjMOIC/TO2AvyTxP25XFdLxNIBwzDh3FM3XztI= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= +gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.19.0/go.mod h1:I1K45XlvTrDjmj5LoM5LuP/KYrhWbjUKT/SoPG0qTjw= +k8s.io/api v0.20.1 h1:ud1c3W3YNzGd6ABJlbFfKXBKXO+1KdGfcgGGNgFR03E= +k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= +k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= +k8s.io/apimachinery v0.20.1 h1:LAhz8pKbgR8tUwn7boK+b2HZdt7MiTu2mkYtFMUjTRQ= +k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/client-go v0.19.0/go.mod h1:H9E/VT95blcFQnlyShFgnFT9ZnJOAceiUHM3MlRC+mU= +k8s.io/client-go v0.20.1 h1:Qquik0xNFbK9aUG92pxHYsyfea5/RPO9o9bSywNor+M= +k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= +k8s.io/component-base v0.19.0/go.mod h1:dKsY8BxkA+9dZIAh2aWJLL/UdASFDNtGYTCItL4LM7Y= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/mount-utils v0.20.1 h1:PI8Qo/3+8Z/1hIEqe3u8PomanMo+6O0ZrO4/+wj7Fbs= +k8s.io/mount-utils v0.20.1/go.mod h1:Jv9NRZ5L2LF87A17GaGlArD+r3JAJdZFvo4XD1cG4Kc= +k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go new file mode 100644 index 0000000..03af562 --- /dev/null +++ b/pkg/cloud/cloud.go @@ -0,0 +1,63 @@ +// Package cloud contains CloudStack related +// functions. +package cloud + +import ( + "context" + "errors" + + "github.com/xanzy/go-cloudstack/v2/cloudstack" +) + +// Interface is the CloudStack client interface. +type Interface interface { + GetNodeInfo(ctx context.Context, vmName string) (*VM, error) + GetVMByID(ctx context.Context, vmID string) (*VM, error) + + ListZonesID(ctx context.Context) ([]string, error) + + GetVolumeByID(ctx context.Context, volumeID string) (*Volume, error) + GetVolumeByName(ctx context.Context, name string) (*Volume, error) + CreateVolume(ctx context.Context, diskOfferingID, zoneID, name string, sizeInGB int64) (string, error) + DeleteVolume(ctx context.Context, id string) error + AttachVolume(ctx context.Context, volumeID, vmID string) (string, error) + DetachVolume(ctx context.Context, volumeID string) error +} + +// Volume represents a CloudStack volume. +type Volume struct { + ID string + Name string + + // Size in Bytes + Size int64 + + DiskOfferingID string + ZoneID string + + VirtualMachineID string + DeviceID string +} + +// VM represents a CloudStack Virtual Machine. +type VM struct { + ID string + ZoneID string +} + +// Specific errors +var ( + ErrNotFound = errors.New("Not found") + ErrTooManyResults = errors.New("Too many results") +) + +// client is the implementation of Interface. +type client struct { + *cloudstack.CloudStackClient +} + +// New creates a new cloud connector, given its configuration. +func New(config *Config) Interface { + csClient := cloudstack.NewAsyncClient(config.APIURL, config.APIKey, config.SecretKey, config.VerifySSL) + return &client{csClient} +} diff --git a/pkg/cloud/config.go b/pkg/cloud/config.go new file mode 100644 index 0000000..cb851bd --- /dev/null +++ b/pkg/cloud/config.go @@ -0,0 +1,46 @@ +package cloud + +import ( + "fmt" + + "gopkg.in/gcfg.v1" +) + +// Config holds CloudStack connection configuration. +type Config struct { + APIURL string + APIKey string + SecretKey string + VerifySSL bool +} + +// csConfig wraps the config for the CloudStack cloud provider. +// It is taken from https://github.com/apache/cloudstack-kubernetes-provider +// in order to have the same config in cloudstack-kubernetes-provider +// and in this cloudstack-csi-driver +type csConfig struct { + Global struct { + APIURL string `gcfg:"api-url"` + APIKey string `gcfg:"api-key"` + SecretKey string `gcfg:"secret-key"` + SSLNoVerify bool `gcfg:"ssl-no-verify"` + ProjectID string `gcfg:"project-id"` + Zone string `gcfg:"zone"` + } +} + +// ReadConfig reads a config file with a format defined by CloudStack +// Cloud Controller Manager, and returns a CloudStackConfig. +func ReadConfig(configFilePath string) (*Config, error) { + cfg := &csConfig{} + if err := gcfg.ReadFileInto(cfg, configFilePath); err != nil { + return nil, fmt.Errorf("Could not parse CloudStack config: %w", err) + } + + return &Config{ + APIURL: cfg.Global.APIURL, + APIKey: cfg.Global.APIKey, + SecretKey: cfg.Global.SecretKey, + VerifySSL: cfg.Global.SSLNoVerify, + }, nil +} diff --git a/pkg/cloud/fake/fake.go b/pkg/cloud/fake/fake.go new file mode 100644 index 0000000..23c7e12 --- /dev/null +++ b/pkg/cloud/fake/fake.go @@ -0,0 +1,102 @@ +// Package fake provides a fake implementation of the cloud +// connector interface, to be used in tests. +package fake + +import ( + "context" + + "github.com/apalia/cloudstack-csi-driver/pkg/cloud" + "github.com/apalia/cloudstack-csi-driver/pkg/util" + "github.com/hashicorp/go-uuid" +) + +const zoneID = "a1887604-237c-4212-a9cd-94620b7880fa" + +type fakeConnector struct { + node *cloud.VM + volumesByID map[string]cloud.Volume + volumesByName map[string]cloud.Volume +} + +// New returns a new fake implementation of the +// CloudStack connector. +func New() cloud.Interface { + volume := cloud.Volume{ + ID: "ace9f28b-3081-40c1-8353-4cc3e3014072", + Name: "vol-1", + Size: 10, + DiskOfferingID: "9743fd77-0f5d-4ef9-b2f8-f194235c769c", + ZoneID: zoneID, + VirtualMachineID: "", + DeviceID: "", + } + node := &cloud.VM{ + ID: "0d7107a3-94d2-44e7-89b8-8930881309a5", + ZoneID: zoneID, + } + return &fakeConnector{ + node: node, + volumesByID: map[string]cloud.Volume{volume.ID: volume}, + volumesByName: map[string]cloud.Volume{volume.Name: volume}, + } +} + +func (f *fakeConnector) GetVMByID(ctx context.Context, vmID string) (*cloud.VM, error) { + if vmID == f.node.ID { + return f.node, nil + } + return nil, cloud.ErrNotFound +} + +func (f *fakeConnector) GetNodeInfo(ctx context.Context, vmName string) (*cloud.VM, error) { + return f.node, nil +} + +func (f *fakeConnector) ListZonesID(ctx context.Context) ([]string, error) { + return []string{zoneID}, nil +} + +func (f *fakeConnector) GetVolumeByID(ctx context.Context, volumeID string) (*cloud.Volume, error) { + vol, ok := f.volumesByID[volumeID] + if ok { + return &vol, nil + } + return nil, cloud.ErrNotFound +} + +func (f *fakeConnector) GetVolumeByName(ctx context.Context, name string) (*cloud.Volume, error) { + vol, ok := f.volumesByName[name] + if ok { + return &vol, nil + } + return nil, cloud.ErrNotFound +} + +func (f *fakeConnector) CreateVolume(ctx context.Context, diskOfferingID, zoneID, name string, sizeInGB int64) (string, error) { + id, _ := uuid.GenerateUUID() + vol := cloud.Volume{ + ID: id, + Name: name, + Size: util.GigaBytesToBytes(sizeInGB), + DiskOfferingID: diskOfferingID, + ZoneID: zoneID, + } + f.volumesByID[vol.ID] = vol + f.volumesByName[vol.Name] = vol + return vol.ID, nil +} + +func (f *fakeConnector) DeleteVolume(ctx context.Context, id string) error { + if vol, ok := f.volumesByID[id]; ok { + name := vol.Name + delete(f.volumesByName, name) + } + delete(f.volumesByID, id) + return nil +} + +func (f *fakeConnector) AttachVolume(ctx context.Context, volumeID, vmID string) (string, error) { + return "1", nil +} + +func (f *fakeConnector) DetachVolume(ctx context.Context, volumeID string) error { return nil } diff --git a/pkg/cloud/metadata.go b/pkg/cloud/metadata.go new file mode 100644 index 0000000..b2b5638 --- /dev/null +++ b/pkg/cloud/metadata.go @@ -0,0 +1,83 @@ +package cloud + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" +) + +const ( + cloudInitInstanceFilePath = "/run/cloud-init/instance-data.json" + cloudStackCloudName = "cloudstack" +) + +func (c *client) metadataInstanceID(ctx context.Context) string { + slog := ctxzap.Extract(ctx).Sugar() + + // Try a NODE_ID environment variable + if envNodeID := os.Getenv("NODE_ID"); envNodeID != "" { + slog.Debugf("Found CloudStack VM ID from environment variable NODE_ID: %s", envNodeID) + return envNodeID + } + + // Try cloud-init + slog.Debug("Try with cloud-init") + if _, err := os.Stat(cloudInitInstanceFilePath); err == nil { + slog.Debugf("File %s exists", cloudInitInstanceFilePath) + ciData, err := c.readCloudInit(ctx, cloudInitInstanceFilePath) + if err != nil { + slog.Errorf("Cannot read cloud-init instance data: %v", err) + } else { + if ciData.V1.InstanceID != "" { + slog.Debugf("Found CloudStack VM ID from cloud-init: %s", ciData.V1.InstanceID) + return ciData.V1.InstanceID + } + } + slog.Error("cloud-init instance ID is not provided") + } else if os.IsNotExist(err) { + slog.Debugf("File %s does not exist", cloudInitInstanceFilePath) + } else { + slog.Errorf("Cannot read %s: %v", cloudInitInstanceFilePath, err) + } + + slog.Debug("CloudStack VM ID not found in meta-data.") + return "" +} + +type cloudInitInstanceData struct { + V1 cloudInitV1 `json:"v1"` +} + +type cloudInitV1 struct { + CloudName string `json:"cloud_name"` + InstanceID string `json:"instance_id"` + Zone string `json:"availability_zone"` +} + +func (c *client) readCloudInit(ctx context.Context, instanceFilePath string) (*cloudInitInstanceData, error) { + slog := ctxzap.Extract(ctx).Sugar() + + b, err := ioutil.ReadFile(instanceFilePath) + if err != nil { + slog.Errorf("Cannot read %s", instanceFilePath) + return nil, err + } + + var data cloudInitInstanceData + if err := json.Unmarshal(b, &data); err != nil { + slog.Errorf("Cannot parse JSON file %s", instanceFilePath) + return nil, err + } + + if strings.ToLower(data.V1.CloudName) != cloudStackCloudName { + slog.Errorf("Cloud-Init cloud name is %s, only %s is supported", data.V1.CloudName, cloudStackCloudName) + return nil, fmt.Errorf("Cloud-Init cloud name is %s, only %s is supported", data.V1.CloudName, cloudStackCloudName) + } + + return &data, nil +} diff --git a/pkg/cloud/node.go b/pkg/cloud/node.go new file mode 100644 index 0000000..29c11bf --- /dev/null +++ b/pkg/cloud/node.go @@ -0,0 +1,17 @@ +package cloud + +import "context" + +func (c *client) GetNodeInfo(ctx context.Context, vmName string) (*VM, error) { + // First, try to read the instance ID from meta-data (cloud-init) + if id := c.metadataInstanceID(ctx); id != "" { + // Instance ID found using metadata + + // Use CloudStack API to get VM info + return c.GetVMByID(ctx, id) + } + + // VM ID was not found using metadata. + // Use VM name instead + return c.getVMByName(ctx, vmName) +} diff --git a/pkg/cloud/vms.go b/pkg/cloud/vms.go new file mode 100644 index 0000000..29124c3 --- /dev/null +++ b/pkg/cloud/vms.go @@ -0,0 +1,53 @@ +package cloud + +import ( + "context" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" +) + +func (c *client) GetVMByID(ctx context.Context, vmID string) (*VM, error) { + p := c.VirtualMachine.NewListVirtualMachinesParams() + p.SetId(vmID) + ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "ListVirtualMachines", "params", map[string]string{ + "id": vmID, + }) + l, err := c.VirtualMachine.ListVirtualMachines(p) + if err != nil { + return nil, err + } + if l.Count == 0 { + return nil, ErrNotFound + } + if l.Count > 1 { + return nil, ErrTooManyResults + } + vm := l.VirtualMachines[0] + return &VM{ + ID: vm.Id, + ZoneID: vm.Zoneid, + }, nil +} + +func (c *client) getVMByName(ctx context.Context, name string) (*VM, error) { + p := c.VirtualMachine.NewListVirtualMachinesParams() + p.SetName(name) + ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "ListVirtualMachines", "params", map[string]string{ + "name": name, + }) + l, err := c.VirtualMachine.ListVirtualMachines(p) + if err != nil { + return nil, err + } + if l.Count == 0 { + return nil, ErrNotFound + } + if l.Count > 1 { + return nil, ErrTooManyResults + } + vm := l.VirtualMachines[0] + return &VM{ + ID: vm.Id, + ZoneID: vm.Zoneid, + }, nil +} diff --git a/pkg/cloud/volumes.go b/pkg/cloud/volumes.go new file mode 100644 index 0000000..394be39 --- /dev/null +++ b/pkg/cloud/volumes.go @@ -0,0 +1,122 @@ +package cloud + +import ( + "context" + "strconv" + "strings" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" +) + +func (c *client) GetVolumeByID(ctx context.Context, volumeID string) (*Volume, error) { + p := c.Volume.NewListVolumesParams() + p.SetId(volumeID) + ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "ListVolumes", "params", map[string]string{ + "id": volumeID, + }) + l, err := c.Volume.ListVolumes(p) + if err != nil { + return nil, err + } + if l.Count == 0 { + return nil, ErrNotFound + } + if l.Count > 1 { + return nil, ErrTooManyResults + } + vol := l.Volumes[0] + v := Volume{ + ID: vol.Id, + Name: vol.Name, + Size: vol.Size, + DiskOfferingID: vol.Diskofferingid, + ZoneID: vol.Zoneid, + VirtualMachineID: vol.Virtualmachineid, + DeviceID: strconv.FormatInt(vol.Deviceid, 10), + } + return &v, nil +} + +func (c *client) GetVolumeByName(ctx context.Context, name string) (*Volume, error) { + p := c.Volume.NewListVolumesParams() + p.SetName(name) + ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "ListVolumes", "params", map[string]string{ + "name": name, + }) + l, err := c.Volume.ListVolumes(p) + if err != nil { + return nil, err + } + if l.Count == 0 { + return nil, ErrNotFound + } + if l.Count > 1 { + return nil, ErrTooManyResults + } + vol := l.Volumes[0] + v := Volume{ + ID: vol.Id, + Name: vol.Name, + Size: vol.Size, + DiskOfferingID: vol.Diskofferingid, + ZoneID: vol.Zoneid, + VirtualMachineID: vol.Virtualmachineid, + DeviceID: strconv.FormatInt(vol.Deviceid, 10), + } + return &v, nil +} + +func (c *client) CreateVolume(ctx context.Context, diskOfferingID, zoneID, name string, sizeInGB int64) (string, error) { + p := c.Volume.NewCreateVolumeParams() + p.SetDiskofferingid(diskOfferingID) + p.SetZoneid(zoneID) + p.SetName(name) + p.SetSize(sizeInGB) + ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "CreateVolume", "params", map[string]string{ + "diskofferingid": diskOfferingID, + "zoneid": zoneID, + "name": name, + "size": strconv.FormatInt(sizeInGB, 10), + }) + vol, err := c.Volume.CreateVolume(p) + if err != nil { + return "", err + } + return vol.Id, nil +} + +func (c *client) DeleteVolume(ctx context.Context, id string) error { + p := c.Volume.NewDeleteVolumeParams(id) + ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "DeleteVolume", "params", map[string]string{ + "id": id, + }) + _, err := c.Volume.DeleteVolume(p) + if err != nil && strings.Contains(err.Error(), "4350") { + // CloudStack error InvalidParameterValueException + return ErrNotFound + } + return err +} + +func (c *client) AttachVolume(ctx context.Context, volumeID, vmID string) (string, error) { + p := c.Volume.NewAttachVolumeParams(volumeID, vmID) + ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "AttachVolume", "params", map[string]string{ + "id": volumeID, + "virtualmachineid": vmID, + }) + r, err := c.Volume.AttachVolume(p) + if err != nil { + return "", err + } + return strconv.FormatInt(r.Deviceid, 10), nil +} + +func (c *client) DetachVolume(ctx context.Context, volumeID string) error { + p := c.Volume.NewDetachVolumeParams() + p.SetId(volumeID) + ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "DetachVolume", "params", map[string]string{ + "id": volumeID, + }) + _, err := c.Volume.DetachVolume(p) + return err +} diff --git a/pkg/cloud/zones.go b/pkg/cloud/zones.go new file mode 100644 index 0000000..7eee620 --- /dev/null +++ b/pkg/cloud/zones.go @@ -0,0 +1,24 @@ +package cloud + +import ( + "context" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" +) + +func (c *client) ListZonesID(ctx context.Context) ([]string, error) { + result := []string{} + p := c.Zone.NewListZonesParams() + p.SetAvailable(true) + ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "ListZones", "params", map[string]string{ + "available": "true", + }) + r, err := c.Zone.ListZones(p) + if err != nil { + return result, err + } + for _, zone := range r.Zones { + result = append(result, zone.Id) + } + return result, nil +} diff --git a/pkg/driver/constants.go b/pkg/driver/constants.go new file mode 100644 index 0000000..2f548e5 --- /dev/null +++ b/pkg/driver/constants.go @@ -0,0 +1,17 @@ +package driver + +// DriverName is the name of the CSI plugin +const DriverName = "csi.cloudstack.apache.org" + +// Topology keys +const ( + ZoneKey = "topology." + DriverName + "/zone" + HostKey = "topology." + DriverName + "/host" +) + +// Volume parameters keys +const ( + DiskOfferingKey = DriverName + "/disk-offering-id" +) + +const deviceIDContextKey = "deviceID" diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go new file mode 100644 index 0000000..4cc3f14 --- /dev/null +++ b/pkg/driver/controller.go @@ -0,0 +1,370 @@ +package driver + +import ( + "context" + "fmt" + "math/rand" + + "github.com/apalia/cloudstack-csi-driver/pkg/cloud" + "github.com/apalia/cloudstack-csi-driver/pkg/util" + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// onlyVolumeCapAccessMode is the only volume capability access +// mode possible for CloudStack: SINGLE_NODE_WRITER, since a +// CloudStack volume can only be attached to a single node at +// any given time. +var onlyVolumeCapAccessMode = csi.VolumeCapability_AccessMode{ + Mode: csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, +} + +type controllerServer struct { + csi.UnimplementedControllerServer + connector cloud.Interface +} + +// NewControllerServer creates a new Controller gRPC server. +func NewControllerServer(connector cloud.Interface) csi.ControllerServer { + return &controllerServer{ + connector: connector, + } +} + +func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { + + // Check arguments + + if req.GetName() == "" { + return nil, status.Error(codes.InvalidArgument, "Volume name missing in request") + } + name := req.GetName() + + volCaps := req.GetVolumeCapabilities() + if len(volCaps) == 0 { + return nil, status.Error(codes.InvalidArgument, "Volume capabilities missing in request") + } + if !isValidVolumeCapabilities(volCaps) { + return nil, status.Error(codes.InvalidArgument, "Volume capabilities not supported. Only SINGLE_NODE_WRITER supported.") + } + + if req.GetParameters() == nil { + return nil, status.Error(codes.InvalidArgument, "Volume parameters missing in request") + } + diskOfferingID := req.GetParameters()[DiskOfferingKey] + if diskOfferingID == "" { + return nil, status.Errorf(codes.InvalidArgument, "Missing parameter %v", DiskOfferingKey) + } + + // Check if a volume with that name already exists + if vol, err := cs.connector.GetVolumeByName(ctx, name); err == cloud.ErrNotFound { + // The volume does not exist + } else if err != nil { + // Error with CloudStack + return nil, status.Errorf(codes.Internal, "CloudStack error: %v", err) + } else { + // The volume exists. Check if it suits the request. + if ok, message := checkVolumeSuitable(vol, diskOfferingID, req.GetCapacityRange(), req.GetAccessibilityRequirements()); !ok { + return nil, status.Errorf(codes.AlreadyExists, "Volume %v already exists but does not satisfy request: %s", name, message) + } + // Existing volume is ok + return &csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + VolumeId: vol.ID, + CapacityBytes: vol.Size, + AccessibleTopology: []*csi.Topology{ + Topology{ZoneID: vol.ZoneID}.ToCSI(), + }, + }, + }, nil + } + + // We have to create the volume + + // Determine volume size using requested capacity range + sizeInGB, err := determineSize(req) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + + // Determine zone using topology constraints + var zoneID string + topologyRequirement := req.GetAccessibilityRequirements() + if topologyRequirement == nil || topologyRequirement.GetRequisite() == nil { + // No topology requirement. Use random zone + zones, err := cs.connector.ListZonesID(ctx) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + n := len(zones) + if n == 0 { + return nil, status.Error(codes.Internal, "No zone available") + } + zoneID = zones[rand.Intn(n)] + } else { + reqTopology := topologyRequirement.GetRequisite() + if len(reqTopology) > 1 { + return nil, status.Error(codes.InvalidArgument, "Too many topology requirements") + } + t, err := NewTopology(reqTopology[0]) + if err != nil { + return nil, status.Error(codes.InvalidArgument, "Cannot parse topology requirements") + } + zoneID = t.ZoneID + } + + volID, err := cs.connector.CreateVolume(ctx, diskOfferingID, zoneID, name, sizeInGB) + if err != nil { + return nil, status.Errorf(codes.Internal, "Cannot create volume %s: %v", name, err.Error()) + } + + return &csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + VolumeId: volID, + CapacityBytes: util.GigaBytesToBytes(sizeInGB), + AccessibleTopology: []*csi.Topology{ + Topology{ZoneID: zoneID}.ToCSI(), + }, + }, + }, nil +} + +func checkVolumeSuitable(vol *cloud.Volume, + diskOfferingID string, capRange *csi.CapacityRange, topologyRequirement *csi.TopologyRequirement) (bool, string) { + + if vol.DiskOfferingID != diskOfferingID { + return false, fmt.Sprintf("Disk offering %s; requested disk offering %s", vol.DiskOfferingID, diskOfferingID) + } + + if capRange != nil { + if capRange.GetLimitBytes() > 0 && vol.Size > capRange.GetLimitBytes() { + return false, fmt.Sprintf("Disk size %v bytes > requested limit size %v bytes", vol.Size, capRange.GetLimitBytes()) + } + if capRange.GetRequiredBytes() > 0 && vol.Size < capRange.GetRequiredBytes() { + return false, fmt.Sprintf("Disk size %v bytes < requested required size %v bytes", vol.Size, capRange.GetRequiredBytes()) + } + } + + if topologyRequirement != nil && topologyRequirement.GetRequisite() != nil { + reqTopology := topologyRequirement.GetRequisite() + if len(reqTopology) > 1 { + return false, "Too many topology requirements" + } + t, err := NewTopology(reqTopology[0]) + if err != nil { + return false, "Cannot parse topology requirements" + } + if t.ZoneID != vol.ZoneID { + return false, fmt.Sprintf("Volume in zone %s, requested zone is %s", vol.ZoneID, t.ZoneID) + } + } + + return true, "" +} + +func determineSize(req *csi.CreateVolumeRequest) (int64, error) { + var sizeInGB int64 + + if req.GetCapacityRange() != nil { + capRange := req.GetCapacityRange() + + required := capRange.GetRequiredBytes() + sizeInGB = util.RoundUpBytesToGB(required) + if sizeInGB == 0 { + sizeInGB = 1 + } + + if limit := capRange.GetLimitBytes(); limit > 0 { + if util.GigaBytesToBytes(sizeInGB) > limit { + return 0, fmt.Errorf("After round-up, volume size %v GB exceeds the limit specified of %v bytes", sizeInGB, limit) + } + } + } + + if sizeInGB == 0 { + sizeInGB = 1 + } + return sizeInGB, nil +} + +func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { + if req.GetVolumeId() == "" { + return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") + } + + volumeID := req.GetVolumeId() + err := cs.connector.DeleteVolume(ctx, volumeID) + if err != nil && err != cloud.ErrNotFound { + return nil, status.Errorf(codes.Internal, "Cannot delete volume %s: %s", volumeID, err.Error()) + } + return &csi.DeleteVolumeResponse{}, nil +} + +func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { + // Check arguments + + if req.GetVolumeId() == "" { + return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") + } + volumeID := req.GetVolumeId() + + if req.GetNodeId() == "" { + return nil, status.Error(codes.InvalidArgument, "Node ID missing in request") + } + nodeID := req.GetNodeId() + + if req.GetReadonly() { + return nil, status.Error(codes.InvalidArgument, "Readonly not possible") + } + + if req.GetVolumeCapability() == nil { + return nil, status.Error(codes.InvalidArgument, "Volume capability missing in request") + } + if req.GetVolumeCapability().AccessMode.Mode != onlyVolumeCapAccessMode.GetMode() { + return nil, status.Error(codes.InvalidArgument, "Access mode not accepted") + } + + // Check volume + vol, err := cs.connector.GetVolumeByID(ctx, volumeID) + if err == cloud.ErrNotFound { + return nil, status.Errorf(codes.NotFound, "Volume %v not found", volumeID) + } else if err != nil { + // Error with CloudStack + return nil, status.Errorf(codes.Internal, "Error %v", err) + } + + if vol.VirtualMachineID != "" && vol.VirtualMachineID != nodeID { + return nil, status.Error(codes.AlreadyExists, "Volume already assigned") + } + + if _, err := cs.connector.GetVMByID(ctx, nodeID); err == cloud.ErrNotFound { + return nil, status.Errorf(codes.NotFound, "VM %v not found", volumeID) + } else if err != nil { + // Error with CloudStack + return nil, status.Errorf(codes.Internal, "Error %v", err) + } + + if vol.VirtualMachineID == nodeID { + // volume already attached + + publishContext := map[string]string{ + deviceIDContextKey: vol.DeviceID, + } + return &csi.ControllerPublishVolumeResponse{PublishContext: publishContext}, nil + } + + deviceID, err := cs.connector.AttachVolume(ctx, volumeID, nodeID) + if err != nil { + return nil, status.Errorf(codes.Internal, "Cannot attach volume %s: %s", volumeID, err.Error()) + } + + publishContext := map[string]string{ + deviceIDContextKey: deviceID, + } + return &csi.ControllerPublishVolumeResponse{PublishContext: publishContext}, nil +} + +func (cs *controllerServer) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { + // Check arguments + + if req.GetVolumeId() == "" { + return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") + } + volumeID := req.GetVolumeId() + + if req.GetNodeId() == "" { + return nil, status.Error(codes.InvalidArgument, "Node ID missing in request") + } + nodeID := req.GetNodeId() + + // Check volume + if vol, err := cs.connector.GetVolumeByID(ctx, volumeID); err == cloud.ErrNotFound { + return nil, status.Errorf(codes.NotFound, "Volume %v not found", volumeID) + } else if err != nil { + // Error with CloudStack + return nil, status.Errorf(codes.Internal, "Error %v", err) + } else if vol.VirtualMachineID != nodeID { + // Nothing to do + return &csi.ControllerUnpublishVolumeResponse{}, nil + } + + // Check VM existence + if _, err := cs.connector.GetVolumeByID(ctx, volumeID); err == cloud.ErrNotFound { + return nil, status.Errorf(codes.NotFound, "Volume %v not found", volumeID) + } else if err != nil { + // Error with CloudStack + return nil, status.Errorf(codes.Internal, "Error %v", err) + } + + if _, err := cs.connector.GetVMByID(ctx, nodeID); err == cloud.ErrNotFound { + return nil, status.Errorf(codes.NotFound, "VM %v not found", volumeID) + } else if err != nil { + // Error with CloudStack + return nil, status.Errorf(codes.Internal, "Error %v", err) + } + + err := cs.connector.DetachVolume(ctx, volumeID) + if err != nil { + return nil, status.Errorf(codes.Internal, "Cannot detach volume %s: %s", volumeID, err.Error()) + } + + return &csi.ControllerUnpublishVolumeResponse{}, nil +} + +func (cs *controllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { + volumeID := req.GetVolumeId() + if len(volumeID) == 0 { + return nil, status.Error(codes.InvalidArgument, "Volume ID not provided") + } + + volCaps := req.GetVolumeCapabilities() + if len(volCaps) == 0 { + return nil, status.Error(codes.InvalidArgument, "Volume capabilities not provided") + } + + if _, err := cs.connector.GetVolumeByID(ctx, volumeID); err == cloud.ErrNotFound { + return nil, status.Errorf(codes.NotFound, "Volume %v not found", volumeID) + } else if err != nil { + // Error with CloudStack + return nil, status.Errorf(codes.Internal, "Error %v", err) + } + + var confirmed *csi.ValidateVolumeCapabilitiesResponse_Confirmed + if isValidVolumeCapabilities(volCaps) { + confirmed = &csi.ValidateVolumeCapabilitiesResponse_Confirmed{VolumeCapabilities: volCaps} + } + return &csi.ValidateVolumeCapabilitiesResponse{ + Confirmed: confirmed, + }, nil +} + +func isValidVolumeCapabilities(volCaps []*csi.VolumeCapability) bool { + for _, c := range volCaps { + if c.GetAccessMode() != nil && c.GetAccessMode().GetMode() != onlyVolumeCapAccessMode.GetMode() { + return false + } + } + return true +} + +func (cs *controllerServer) ControllerGetCapabilities(ctx context.Context, req *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { + return &csi.ControllerGetCapabilitiesResponse{ + Capabilities: []*csi.ControllerServiceCapability{ + { + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, + }, + }, + }, + { + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME, + }, + }, + }, + }, + }, nil +} diff --git a/pkg/driver/controller_test.go b/pkg/driver/controller_test.go new file mode 100644 index 0000000..bf9f3d1 --- /dev/null +++ b/pkg/driver/controller_test.go @@ -0,0 +1,42 @@ +package driver + +import ( + "testing" + + "github.com/container-storage-interface/spec/lib/go/csi" +) + +func TestDetermineSize(t *testing.T) { + cases := []struct { + name string + capacityRange *csi.CapacityRange + expectedSize int64 + expectError bool + }{ + {"no range", nil, 1, false}, + {"only limit", &csi.CapacityRange{LimitBytes: 100 * 1024 * 1024 * 1024}, 1, false}, + {"only limit (too small)", &csi.CapacityRange{LimitBytes: 1024 * 1024}, 0, true}, + {"only required", &csi.CapacityRange{RequiredBytes: 50 * 1024 * 1024 * 1024}, 50, false}, + {"required and limit", &csi.CapacityRange{RequiredBytes: 25 * 1024 * 1024 * 1024, LimitBytes: 100 * 1024 * 1024 * 1024}, 25, false}, + {"required = limit", &csi.CapacityRange{RequiredBytes: 30 * 1024 * 1024 * 1024, LimitBytes: 30 * 1024 * 1024 * 1024}, 30, false}, + {"required = limit (not GB int)", &csi.CapacityRange{RequiredBytes: 3_000_000_000, LimitBytes: 3_000_000_000}, 0, true}, + {"no int GB int possible", &csi.CapacityRange{RequiredBytes: 4_000_000_000, LimitBytes: 1_000_001_000}, 0, true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + req := &csi.CreateVolumeRequest{ + CapacityRange: c.capacityRange, + } + size, err := determineSize(req) + if err != nil && !c.expectError { + t.Errorf("Unexepcted error: %v", err.Error()) + } + if err == nil && c.expectError { + t.Error("Expected an error") + } + if size != c.expectedSize { + t.Errorf("Expected size %v, got %v", c.expectedSize, size) + } + }) + } +} diff --git a/pkg/driver/driver.go b/pkg/driver/driver.go new file mode 100644 index 0000000..55db71e --- /dev/null +++ b/pkg/driver/driver.go @@ -0,0 +1,46 @@ +// Package driver provides the implementation of the CSI plugin. +// +// It contains the gRPC server implementation of CSI specification. +package driver + +import ( + "github.com/apalia/cloudstack-csi-driver/pkg/cloud" + "github.com/apalia/cloudstack-csi-driver/pkg/mount" + "go.uber.org/zap" +) + +// Interface is the CloudStack CSI driver interface. +type Interface interface { + // Run the CSI driver gRPC server + Run() error +} + +type cloudstackDriver struct { + endpoint string + nodeName string + version string + + connector cloud.Interface + mounter mount.Interface + logger *zap.Logger +} + +// New instantiates a new CloudStack CSI driver +func New(endpoint string, csConnector cloud.Interface, mounter mount.Interface, nodeName string, version string, logger *zap.Logger) (Interface, error) { + return &cloudstackDriver{ + endpoint: endpoint, + nodeName: nodeName, + version: version, + connector: csConnector, + mounter: mounter, + logger: logger, + }, nil +} + +func (cs *cloudstackDriver) Run() error { + ids := NewIdentityServer(cs.version) + ctrls := NewControllerServer(cs.connector) + ns := NewNodeServer(cs.connector, cs.mounter, cs.nodeName) + + return cs.serve(ids, ctrls, ns) +} diff --git a/pkg/driver/identity.go b/pkg/driver/identity.go new file mode 100644 index 0000000..52186f1 --- /dev/null +++ b/pkg/driver/identity.go @@ -0,0 +1,57 @@ +package driver + +import ( + "context" + + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type identityServer struct { + csi.UnimplementedIdentityServer + version string +} + +// NewIdentityServer creates a new Identity gRPC server. +func NewIdentityServer(version string) csi.IdentityServer { + return &identityServer{ + version: version, + } +} + +func (ids *identityServer) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { + if ids.version == "" { + return nil, status.Error(codes.Unavailable, "Driver is missing version") + } + + return &csi.GetPluginInfoResponse{ + Name: DriverName, + VendorVersion: ids.version, + }, nil +} + +func (ids *identityServer) Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeResponse, error) { + return &csi.ProbeResponse{}, nil +} + +func (ids *identityServer) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { + return &csi.GetPluginCapabilitiesResponse{ + Capabilities: []*csi.PluginCapability{ + { + Type: &csi.PluginCapability_Service_{ + Service: &csi.PluginCapability_Service{ + Type: csi.PluginCapability_Service_CONTROLLER_SERVICE, + }, + }, + }, + { + Type: &csi.PluginCapability_Service_{ + Service: &csi.PluginCapability_Service{ + Type: csi.PluginCapability_Service_VOLUME_ACCESSIBILITY_CONSTRAINTS, + }, + }, + }, + }, + }, nil +} diff --git a/pkg/driver/node.go b/pkg/driver/node.go new file mode 100644 index 0000000..dcb96e3 --- /dev/null +++ b/pkg/driver/node.go @@ -0,0 +1,339 @@ +package driver + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/apalia/cloudstack-csi-driver/pkg/cloud" + "github.com/apalia/cloudstack-csi-driver/pkg/mount" + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const ( + // default file system type to be used when it is not provided + defaultFsType = "ext4" +) + +type nodeServer struct { + csi.UnimplementedNodeServer + connector cloud.Interface + mounter mount.Interface + nodeName string +} + +// NewNodeServer creates a new Node gRPC server. +func NewNodeServer(connector cloud.Interface, mounter mount.Interface, nodeName string) csi.NodeServer { + if mounter == nil { + mounter = mount.New() + } + return &nodeServer{ + connector: connector, + mounter: mounter, + nodeName: nodeName, + } +} + +func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { + + // Check parameters + + volumeID := req.GetVolumeId() + if volumeID == "" { + return nil, status.Error(codes.InvalidArgument, "Volume ID not provided") + } + + target := req.GetStagingTargetPath() + if target == "" { + return nil, status.Error(codes.InvalidArgument, "Staging target not provided") + } + + volCap := req.GetVolumeCapability() + if volCap == nil { + return nil, status.Error(codes.InvalidArgument, "Volume capability not provided") + } + if !isValidVolumeCapabilities([]*csi.VolumeCapability{volCap}) { + return nil, status.Error(codes.InvalidArgument, "Volume capability not supported") + } + + // Now, find the device path + + deviceID := req.PublishContext[deviceIDContextKey] + + devicePath, err := ns.mounter.GetDevicePath(ctx, volumeID) + if err != nil { + return nil, status.Errorf(codes.Internal, "Cannot find device path for volume %s: %s", volumeID, err.Error()) + } + + ctxzap.Extract(ctx).Sugar().Infow("Device found", + "devicePath", devicePath, + "deviceID", deviceID, + ) + + // If the access type is block, do nothing for stage + switch volCap.GetAccessType().(type) { + case *csi.VolumeCapability_Block: + return &csi.NodeStageVolumeResponse{}, nil + } + + // The access type should now be "Mount". + // We have to format the partition. + + mnt := volCap.GetMount() + if mnt == nil { + return nil, status.Error(codes.InvalidArgument, "Neither block nor mount volume capability") + } + + // Verify whether mounted + notMnt, err := ns.mounter.IsLikelyNotMountPoint(target) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + + fsType := mnt.GetFsType() + if fsType == "" { + fsType = defaultFsType + } + + var mountOptions []string + for _, f := range mnt.GetMountFlags() { + if !hasMountOption(mountOptions, f) { + mountOptions = append(mountOptions, f) + } + } + + // Volume Mount + if notMnt { + err = ns.mounter.FormatAndMount(devicePath, target, fsType, mountOptions) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + } + return &csi.NodeStageVolumeResponse{}, nil +} + +// hasMountOption returns a boolean indicating whether the given +// slice already contains a mount option. This is used to prevent +// passing duplicate option to the mount command. +func hasMountOption(options []string, opt string) bool { + for _, o := range options { + if o == opt { + return true + } + } + return false +} + +func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { + // Check parameters + + volumeID := req.GetVolumeId() + if volumeID == "" { + return nil, status.Error(codes.InvalidArgument, "Volume ID not provided") + } + + target := req.GetStagingTargetPath() + if target == "" { + return nil, status.Error(codes.InvalidArgument, "Staging target not provided") + } + + // Check if target directory is a mount point. GetDeviceNameFromMount + // given a mnt point, finds the device from /proc/mounts + // returns the device name, reference count, and error code + dev, refCount, err := ns.mounter.GetDeviceName(target) + if err != nil { + msg := fmt.Sprintf("failed to check if volume is mounted: %v", err) + return nil, status.Error(codes.Internal, msg) + } + + // From the spec: If the volume corresponding to the volume_id + // is not staged to the staging_target_path, the Plugin MUST + // reply 0 OK. + if refCount == 0 { + return &csi.NodeUnstageVolumeResponse{}, nil + } + + if refCount > 1 { + ctxzap.Extract(ctx).Sugar().Warnf("NodeUnstageVolume: found %d references to device %s mounted at target path %s", refCount, dev, target) + } + + err = ns.mounter.Unmount(target) + if err != nil { + return nil, status.Errorf(codes.Internal, "Could not unmount target %q: %v", target, err) + } + + return &csi.NodeUnstageVolumeResponse{}, nil +} + +func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { + // Check arguments + if req.GetVolumeCapability() == nil { + return nil, status.Error(codes.InvalidArgument, "Volume capability missing in request") + } + if req.GetVolumeId() == "" { + return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") + } + volumeID := req.GetVolumeId() + + if req.GetTargetPath() == "" { + return nil, status.Error(codes.InvalidArgument, "Target path missing in request") + } + targetPath := req.GetTargetPath() + + if req.GetVolumeCapability().GetBlock() != nil && + req.GetVolumeCapability().GetMount() != nil { + return nil, status.Error(codes.InvalidArgument, "Cannot have both block and mount access type") + } + if req.GetStagingTargetPath() == "" { + return nil, status.Error(codes.InvalidArgument, "Staging target path missing in request") + } + + readOnly := req.GetReadonly() + options := []string{"bind"} + if readOnly { + options = append(options, "ro") + } + + deviceID := "" + if req.GetPublishContext() != nil { + deviceID = req.GetPublishContext()[deviceIDContextKey] + } + + if req.GetVolumeCapability().GetMount() != nil { + source := req.GetStagingTargetPath() + + notMnt, err := ns.mounter.IsLikelyNotMountPoint(targetPath) + if err != nil { + if os.IsNotExist(err) { + if err := ns.mounter.MakeDir(targetPath); err != nil { + return nil, status.Errorf(codes.Internal, "Could not create dir %q: %v", targetPath, err) + } + } else { + return nil, status.Error(codes.Internal, err.Error()) + } + } + if !notMnt { + return &csi.NodePublishVolumeResponse{}, nil + } + + fsType := req.GetVolumeCapability().GetMount().GetFsType() + + mountFlags := req.GetVolumeCapability().GetMount().GetMountFlags() + + ctxzap.Extract(ctx).Sugar().Infow("Mounting device", + "targetPath", targetPath, + "fsType", fsType, + "deviceID", deviceID, + "readOnly", readOnly, + "volumeID", volumeID, + "mountFlags", mountFlags, + ) + + if err := ns.mounter.Mount(source, targetPath, fsType, options); err != nil { + return nil, status.Errorf(codes.Internal, "failed to mount %s at %s: %s", source, targetPath, err.Error()) + } + } + + if req.GetVolumeCapability().GetBlock() != nil { + volumeID := req.GetVolumeId() + + devicePath, err := ns.mounter.GetDevicePath(ctx, volumeID) + if err != nil { + return nil, status.Errorf(codes.Internal, "Cannot find device path for volume %s: %s", volumeID, err.Error()) + } + + globalMountPath := filepath.Dir(targetPath) + exists, err := ns.mounter.ExistsPath(globalMountPath) + if err != nil { + return nil, status.Errorf(codes.Internal, "Could not check if path exists %q: %v", globalMountPath, err) + } + if !exists { + if err = ns.mounter.MakeDir(globalMountPath); err != nil { + return nil, status.Errorf(codes.Internal, "Could not create dir %q: %v", globalMountPath, err) + } + } + + err = ns.mounter.MakeFile(targetPath) + if err != nil { + if removeErr := os.Remove(targetPath); removeErr != nil { + return nil, status.Errorf(codes.Internal, "Could not remove mount target %q: %v", targetPath, removeErr) + } + return nil, status.Errorf(codes.Internal, "Could not create file %q: %v", targetPath, err) + } + + if err := ns.mounter.Mount(devicePath, targetPath, "", options); err != nil { + return nil, status.Errorf(codes.Internal, "failed to mount %s at %s: %s", devicePath, targetPath, err.Error()) + } + } + + return &csi.NodePublishVolumeResponse{}, nil +} + +func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { + if req.GetVolumeId() == "" { + return nil, status.Error(codes.InvalidArgument, "Volume ID missing in request") + } + if req.GetTargetPath() == "" { + return nil, status.Error(codes.InvalidArgument, "Target path missing in request") + } + targetPath := req.GetTargetPath() + + volumeID := req.GetVolumeId() + if _, err := ns.connector.GetVolumeByID(ctx, volumeID); err == cloud.ErrNotFound { + return nil, status.Errorf(codes.NotFound, "Volume %v not found", volumeID) + } else if err != nil { + // Error with CloudStack + return nil, status.Errorf(codes.Internal, "Error %v", err) + } + + err := ns.mounter.Unmount(targetPath) + if err != nil { + return nil, status.Errorf(codes.Internal, "Unmount of targetpath %s failed with error %v", targetPath, err) + } + err = os.Remove(targetPath) + if err != nil && !os.IsNotExist(err) { + return nil, status.Errorf(codes.Internal, "Deleting %s failed with error %v", targetPath, err) + } + return &csi.NodeUnpublishVolumeResponse{}, nil +} + +func (ns *nodeServer) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { + if ns.nodeName == "" { + return nil, status.Error(codes.Internal, "Missing node name") + } + + vm, err := ns.connector.GetNodeInfo(ctx, ns.nodeName) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + if vm.ID == "" { + return nil, status.Error(codes.Internal, "Node with no ID") + } + if vm.ZoneID == "" { + return nil, status.Error(codes.Internal, "Node zone ID not found") + } + + topology := Topology{ZoneID: vm.ZoneID} + return &csi.NodeGetInfoResponse{ + NodeId: vm.ID, + AccessibleTopology: topology.ToCSI(), + }, nil +} + +func (ns *nodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { + return &csi.NodeGetCapabilitiesResponse{ + Capabilities: []*csi.NodeServiceCapability{ + { + Type: &csi.NodeServiceCapability_Rpc{ + Rpc: &csi.NodeServiceCapability_RPC{ + Type: csi.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, + }, + }, + }, + }, + }, nil +} diff --git a/pkg/driver/server.go b/pkg/driver/server.go new file mode 100644 index 0000000..d3e5fe3 --- /dev/null +++ b/pkg/driver/server.go @@ -0,0 +1,85 @@ +package driver + +import ( + "context" + "fmt" + "io" + "net" + "os" + "strings" + + //nolint:staticcheck // deprecated but still required by gRPC + "github.com/golang/protobuf/proto" + + "github.com/container-storage-interface/spec/lib/go/csi" + grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" + grpc_zap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap" + "github.com/kubernetes-csi/csi-lib-utils/protosanitizer" + "google.golang.org/grpc" +) + +func (cs *cloudstackDriver) serve(ids csi.IdentityServer, ctrls csi.ControllerServer, ns csi.NodeServer) error { + proto, addr, err := parseEndpoint(cs.endpoint) + if err != nil { + return err + } + + if proto == "unix" { + if !strings.HasPrefix(addr, "/") { + addr = "/" + addr + } + if err := os.Remove(addr); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("Failed to remove %s, error: %s", addr, err.Error()) + } + } + + listener, err := net.Listen(proto, addr) + if err != nil { + return fmt.Errorf("Failed to listen: %w", err) + } + + // Sanitize payload before logging + grpc_zap.JsonPbMarshaller = sanitizer{} + + // Log every request and payloads (request + response) + opts := []grpc.ServerOption{ + grpc_middleware.WithUnaryServerChain( + grpc_zap.UnaryServerInterceptor(cs.logger), + grpc_zap.PayloadUnaryServerInterceptor(cs.logger, func(context.Context, string, interface{}) bool { return true }), + ), + } + // Make sure that log statements internal to gRPC library are logged using the zapLogger as well. + grpc_zap.ReplaceGrpcLoggerV2(cs.logger) + + server := grpc.NewServer(opts...) + + if ids != nil { + csi.RegisterIdentityServer(server, ids) + } + if ctrls != nil { + csi.RegisterControllerServer(server, ctrls) + } + if ns != nil { + csi.RegisterNodeServer(server, ns) + } + + cs.logger.Sugar().Infow("Listening for connections", "address", listener.Addr()) + return server.Serve(listener) +} + +func parseEndpoint(ep string) (string, string, error) { + if strings.HasPrefix(strings.ToLower(ep), "unix://") || strings.HasPrefix(strings.ToLower(ep), "tcp://") { + s := strings.SplitN(ep, "://", 2) + if s[1] != "" { + return s[0], s[1], nil + } + } + return "", "", fmt.Errorf("Invalid endpoint: %v", ep) +} + +type sanitizer struct{} + +func (sanitizer) Marshal(out io.Writer, pb proto.Message) error { + _, err := io.WriteString(out, protosanitizer.StripSecrets(pb).String()) + return err +} diff --git a/pkg/driver/topology.go b/pkg/driver/topology.go new file mode 100644 index 0000000..513c794 --- /dev/null +++ b/pkg/driver/topology.go @@ -0,0 +1,40 @@ +package driver + +import ( + "errors" + + "github.com/container-storage-interface/spec/lib/go/csi" +) + +// Topology represents CloudStack storage topology. +type Topology struct { + ZoneID string + HostID string +} + +// NewTopology converts a *csi.Topology to Topology. +func NewTopology(t *csi.Topology) (Topology, error) { + segments := t.GetSegments() + if segments == nil { + return Topology{}, errors.New("Nil segment in topology") + } + + zoneID, ok := segments[ZoneKey] + if !ok { + return Topology{}, errors.New("No zone in topology") + } + hostID := segments[HostKey] + return Topology{zoneID, hostID}, nil +} + +// ToCSI converts a Topology to a *csi.Topology. +func (t Topology) ToCSI() *csi.Topology { + segments := make(map[string]string) + segments[ZoneKey] = t.ZoneID + if t.HostID != "" { + segments[ZoneKey] = t.ZoneID + } + return &csi.Topology{ + Segments: segments, + } +} diff --git a/pkg/mount/fake.go b/pkg/mount/fake.go new file mode 100644 index 0000000..985b920 --- /dev/null +++ b/pkg/mount/fake.go @@ -0,0 +1,53 @@ +package mount + +import ( + "context" + "os" + + "k8s.io/mount-utils" + utilsexec "k8s.io/utils/exec" + exec "k8s.io/utils/exec/testing" +) + +type fakeMounter struct { + mount.SafeFormatAndMount + utilsexec.Interface +} + +// NewFake creates an fake implementation of the +// mount.Interface, to be used in tests. +func NewFake() Interface { + return &fakeMounter{ + mount.SafeFormatAndMount{ + Interface: mount.NewFakeMounter([]mount.MountPoint{}), + Exec: &exec.FakeExec{DisableScripts: true}, + }, + utilsexec.New(), + } +} + +func (m *fakeMounter) GetDevicePath(ctx context.Context, volumeID string) (string, error) { + return "/dev/sdb", nil +} + +func (m *fakeMounter) GetDeviceName(mountPath string) (string, int, error) { + return mount.GetDeviceNameFromMount(m, mountPath) +} + +func (*fakeMounter) ExistsPath(filename string) (bool, error) { + return true, nil +} + +func (*fakeMounter) MakeDir(pathname string) error { + err := os.MkdirAll(pathname, os.FileMode(0755)) + if err != nil { + if !os.IsExist(err) { + return err + } + } + return nil +} + +func (*fakeMounter) MakeFile(pathname string) error { + return nil +} diff --git a/pkg/mount/mount.go b/pkg/mount/mount.go new file mode 100644 index 0000000..7a9f79b --- /dev/null +++ b/pkg/mount/mount.go @@ -0,0 +1,172 @@ +// Package mount provides utilities to detect, +// format and mount storage devices. +package mount + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/mount-utils" + "k8s.io/utils/exec" +) + +const ( + diskIDPath = "/dev/disk/by-id" +) + +// Interface defines the set of methods to allow for +// mount operations on a system. +type Interface interface { + mount.Interface + exec.Interface + + FormatAndMount(source string, target string, fstype string, options []string) error + + GetDevicePath(ctx context.Context, volumeID string) (string, error) + GetDeviceName(mountPath string) (string, int, error) + ExistsPath(filename string) (bool, error) + MakeDir(pathname string) error + MakeFile(pathname string) error +} + +type mounter struct { + mount.SafeFormatAndMount + exec.Interface +} + +// New creates an implementation of the mount.Interface. +func New() Interface { + return &mounter{ + mount.SafeFormatAndMount{ + Interface: mount.New(""), + Exec: exec.New(), + }, + exec.New(), + } +} + +func (m *mounter) GetDevicePath(ctx context.Context, volumeID string) (string, error) { + backoff := wait.Backoff{ + Duration: 1 * time.Second, + Factor: 1.1, + Steps: 15, + } + + var devicePath string + err := wait.ExponentialBackoffWithContext(ctx, backoff, func() (bool, error) { + path, err := m.getDevicePathBySerialID(volumeID) + if err != nil { + return false, err + } + if path != "" { + devicePath = path + return true, nil + } + m.probeVolume(ctx) + return false, nil + }) + + if err == wait.ErrWaitTimeout { + return "", fmt.Errorf("Failed to find device for the volumeID: %q within the alloted time", volumeID) + } else if devicePath == "" { + return "", fmt.Errorf("Device path was empty for volumeID: %q", volumeID) + } + return devicePath, nil +} + +func (m *mounter) getDevicePathBySerialID(volumeID string) (string, error) { + sourcePathPrefixes := []string{"virtio-", "scsi-", "scsi-0QEMU_QEMU_HARDDISK_"} + serial := diskUUIDToSerial(volumeID) + for _, prefix := range sourcePathPrefixes { + source := filepath.Join(diskIDPath, prefix+serial) + _, err := os.Stat(source) + if err == nil { + return source, nil + } + if !os.IsNotExist(err) { + return "", err + } + } + return "", nil +} + +func (m *mounter) probeVolume(ctx context.Context) { + log := ctxzap.Extract(ctx).Sugar() + log.Debug("Scaning SCSI host...") + + scsiPath := "/sys/class/scsi_host/" + if dirs, err := ioutil.ReadDir(scsiPath); err == nil { + for _, f := range dirs { + name := scsiPath + f.Name() + "/scan" + data := []byte("- - -") + if err = ioutil.WriteFile(name, data, 0666); err != nil { + log.Warnf("Failed to rescan scsi host %s", name) + } + } + } else { + log.Warnf("Failed to read %s, err %v", scsiPath, err) + } + + args := []string{"trigger"} + cmd := m.Exec.Command("udevadm", args...) + _, err := cmd.CombinedOutput() + if err != nil { + log.Warnf("Error running udevadm trigger %v\n", err) + } +} + +func (m *mounter) GetDeviceName(mountPath string) (string, int, error) { + return mount.GetDeviceNameFromMount(m, mountPath) +} + +// diskUUIDToSerial reproduces CloudStack function diskUuidToSerial +// from https://github.com/apache/cloudstack/blob/0f3f2a0937/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java#L3000 +// +// This is what CloudStack do *with KVM hypervisor* to translate +// a CloudStack volume UUID to libvirt disk serial. +func diskUUIDToSerial(uuid string) string { + uuidWithoutHyphen := strings.ReplaceAll(uuid, "-", "") + if len(uuidWithoutHyphen) < 20 { + return uuidWithoutHyphen + } + return uuidWithoutHyphen[:20] +} + +func (*mounter) ExistsPath(filename string) (bool, error) { + if _, err := os.Stat(filename); os.IsNotExist(err) { + return false, nil + } else if err != nil { + return false, err + } + return true, nil +} + +func (*mounter) MakeDir(pathname string) error { + err := os.MkdirAll(pathname, os.FileMode(0755)) + if err != nil { + if !os.IsExist(err) { + return err + } + } + return nil +} + +func (*mounter) MakeFile(pathname string) error { + f, err := os.OpenFile(pathname, os.O_CREATE, os.FileMode(0644)) + if err != nil { + if !os.IsExist(err) { + return err + } + } + if err = f.Close(); err != nil { + return err + } + return nil +} diff --git a/pkg/syncer/error.go b/pkg/syncer/error.go new file mode 100644 index 0000000..1c23c37 --- /dev/null +++ b/pkg/syncer/error.go @@ -0,0 +1,13 @@ +package syncer + +import "fmt" + +type combinedError []error + +func (errs combinedError) Error() string { + err := "Collected errors:\n" + for i, e := range errs { + err += fmt.Sprintf("\tError %d: %s\n", i, e.Error()) + } + return err +} diff --git a/pkg/syncer/name.go b/pkg/syncer/name.go new file mode 100644 index 0000000..b82dcd0 --- /dev/null +++ b/pkg/syncer/name.go @@ -0,0 +1,50 @@ +package syncer + +import ( + "fmt" + "regexp" + "strings" + "unicode" + + "golang.org/x/text/runes" + "golang.org/x/text/transform" + "golang.org/x/text/unicode/norm" +) + +func createStorageClassName(origName string) (string, error) { + // Remove accents / diacritics + nonSpacingMarks := runes.In(unicode.Mn) + t := transform.Chain(norm.NFD, runes.Remove(nonSpacingMarks), norm.NFC) + name, _, err := transform.String(t, origName) + if err != nil { + return "", err + } + + // Replace non alphanumeric characters (except .) by a space + nonAlpha := regexp.MustCompile("[^a-zA-Z0-9.]+") + name = nonAlpha.ReplaceAllString(name, " ") + + // Use lowercase + name = strings.ToLower(name) + + // Trim whitespaces + name = strings.TrimSpace(name) + + // Replace whitespaces by a single dash + name = regexp.MustCompile(`\s+`).ReplaceAllString(name, "-") + + // Truncate + if len(name) > 253 { + name = name[:253] + } + + // Remove trailing and leading "." and "-" + name = strings.TrimFunc(name, func(r rune) bool { return r == '.' || r == '-' }) + + // Return an error if the resulting name is empty + if len(name) == 0 { + return "", fmt.Errorf("%s transformed to an empty name", origName) + } + + return name, nil +} diff --git a/pkg/syncer/name_test.go b/pkg/syncer/name_test.go new file mode 100644 index 0000000..89e623b --- /dev/null +++ b/pkg/syncer/name_test.go @@ -0,0 +1,44 @@ +package syncer + +import ( + "testing" +) + +func TestCreateStorageClassName(t *testing.T) { + cases := []struct { + OrigName string + ExpectedName string + ShouldErr bool + }{ + {OrigName: "cloudstack-gold", ExpectedName: "cloudstack-gold"}, + {OrigName: "cloudstack-Silver", ExpectedName: "cloudstack-silver"}, + {OrigName: "cloudstack-copper-1.2", ExpectedName: "cloudstack-copper-1.2"}, + {OrigName: "cloudstack-Custom Storage 1.2 - experimental", ExpectedName: "cloudstack-custom-storage-1.2-experimental"}, + {OrigName: "étendu", ExpectedName: "etendu"}, + {OrigName: "stockage NFS", ExpectedName: "stockage-nfs"}, + {OrigName: "Disque 123", ExpectedName: "disque-123"}, + {OrigName: "123", ExpectedName: "123"}, + {OrigName: "Platinium +", ExpectedName: "platinium"}, + {OrigName: " Platinium Plus ", ExpectedName: "platinium-plus"}, + {OrigName: "cloudstack-Ruthénium", ExpectedName: "cloudstack-ruthenium"}, + {OrigName: "--- gold ---", ExpectedName: "gold"}, + {OrigName: ".silver.", ExpectedName: "silver"}, + {OrigName: "Don't use me!", ExpectedName: "don-t-use-me"}, + {OrigName: "cloudstack-東京", ExpectedName: "cloudstack"}, + {OrigName: "こんにちは世界", ShouldErr: true}, + {OrigName: "", ShouldErr: true}, + } + + for _, c := range cases { + t.Run(c.OrigName, func(t *testing.T) { + name, err := createStorageClassName(c.OrigName) + if err != nil && !c.ShouldErr { + t.Error(err) + } else if err == nil && c.ShouldErr { + t.Error("Expected a non-nil error; error was nil") + } else if err == nil && name != c.ExpectedName { + t.Errorf("Expected name %s; got %s", c.ExpectedName, name) + } + }) + } +} diff --git a/pkg/syncer/run.go b/pkg/syncer/run.go new file mode 100644 index 0000000..f6f698a --- /dev/null +++ b/pkg/syncer/run.go @@ -0,0 +1,201 @@ +package syncer + +import ( + "context" + "errors" + "fmt" + "log" + + "github.com/apalia/cloudstack-csi-driver/pkg/driver" + "github.com/xanzy/go-cloudstack/v2/cloudstack" + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" +) + +var ( + volBindingMode = storagev1.VolumeBindingWaitForFirstConsumer + reclaimPolicy = corev1.PersistentVolumeReclaimDelete + allowVolumeExpansion = false +) + +func (s syncer) Run(ctx context.Context) error { + oldSc := make([]string, 0) + newSc := make([]string, 0) + errs := make([]error, 0) + + // List existing K8s storage classes + + labelSelector := s.labelsSet.String() + log.Printf("Listing Storage classes with label selector \"%s\"...", labelSelector) + scList, err := s.k8sClient.StorageV1().StorageClasses().List(ctx, metav1.ListOptions{ + LabelSelector: labelSelector, + }) + if err != nil { + return fmt.Errorf("Cannot list existing storage classes: %w", err) + } + for _, sc := range scList.Items { + oldSc = append(oldSc, sc.Name) + } + log.Printf("Found %v: %v\n", len(oldSc), oldSc) + + // List CloudStack disk offerings + + log.Println("Listing CloudStack disk offerings...") + p := s.csClient.DiskOffering.NewListDiskOfferingsParams() + diskOfferings, err := s.csClient.DiskOffering.ListDiskOfferings(p) + if err != nil { + return fmt.Errorf("Cannot list CloudStack disk offerings: %w", err) + } + + // Iterate over CloudStack disk offerings to synchronize them + + for _, offering := range diskOfferings.DiskOfferings { + name, err := s.syncOffering(ctx, offering) + if err != nil { + err = fmt.Errorf("Error with offering %s: %w", offering.Name, err) + log.Println(err.Error()) + errs = append(errs, err) + } + if name != "" { + newSc = append(newSc, name) + } + } + log.Println("No more CloudStack disk offerings") + + // If enabled, delete unused labelled storage classes + + if s.delete { + del := toDelete(oldSc, newSc) + if len(del) == 0 { + log.Println("No storage class to delete") + } else { + for _, sc := range del { + log.Printf("Deleting storage class %s", sc) + err = s.k8sClient.StorageV1().StorageClasses().Delete(ctx, sc, metav1.DeleteOptions{}) + if err != nil { + err = fmt.Errorf("Error deleting storage class %s: %w", sc, err) + log.Println(err.Error()) + errs = append(errs, err) + } + } + } + } + + if len(errs) == 0 { + return nil + } + return combinedError(errs) +} + +func (s syncer) syncOffering(ctx context.Context, offering *cloudstack.DiskOffering) (string, error) { + offeringName := offering.Name + custom := offering.Iscustomized + if !custom { + log.Printf("Disk offering \"%s\" has a fixed size: ignoring\n", offeringName) + return "", nil + } + + log.Printf("Syncing disk offering %s...", offeringName) + name, err := createStorageClassName(s.namePrefix + offeringName) + if err != nil { + log.Printf("Cannot transform name: %s", err.Error()) + name = offering.Id + } + log.Printf("Storage class name: %s", name) + + sc, err := s.k8sClient.StorageV1().StorageClasses().Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if k8serrors.IsNotFound(err) { + + // Storage class does not exist; creating it + + log.Printf("Creating storage class %s", name) + + newSc := &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: s.labelsSet, + }, + Provisioner: driver.DriverName, + VolumeBindingMode: &volBindingMode, + ReclaimPolicy: &reclaimPolicy, + AllowVolumeExpansion: &allowVolumeExpansion, + Parameters: map[string]string{ + driver.DiskOfferingKey: offering.Id, + }, + } + _, err = s.k8sClient.StorageV1().StorageClasses().Create(ctx, newSc, metav1.CreateOptions{}) + return name, err + } + return "", err + } + + // Storage class already exists + + err = checkStorageClass(sc, offering.Id) + if err != nil { + // Updates to provisioner, reclaimpolicy, volumeBindingMode and parameters are forbidden + log.Printf("Storage class %s exists but it not compatible.", name) + return name, err + } + + // Update labels if needed + + existingLabels := labels.Set(sc.Labels) + if !s.labelsSet.AsSelector().Matches(existingLabels) { + log.Printf("Storage class %s misses labels %s: updating...", sc.Name, s.labelsSet.String()) + + sc.Labels = labels.Merge(existingLabels, s.labelsSet) + _, err = s.k8sClient.StorageV1().StorageClasses().Update(ctx, sc, metav1.UpdateOptions{}) + return name, err + } + + log.Printf("Storage class %s already ok", sc.Name) + + return name, nil +} + +func checkStorageClass(sc *storagev1.StorageClass, expectedOfferingID string) error { + errs := make([]error, 0) + diskOfferingID, ok := sc.Parameters[driver.DiskOfferingKey] + if !ok { + errs = append(errs, fmt.Errorf("Missing parameter %s", driver.DiskOfferingKey)) + } else if diskOfferingID != expectedOfferingID { + errs = append(errs, fmt.Errorf("Storage class %s has parameter %s=%s, should be %s", sc.Name, driver.DiskOfferingKey, diskOfferingID, expectedOfferingID)) + } + + if sc.ReclaimPolicy == nil || *sc.ReclaimPolicy != reclaimPolicy { + errs = append(errs, errors.New("Wrong ReclaimPolicy")) + } + if sc.VolumeBindingMode == nil || *sc.VolumeBindingMode != volBindingMode { + errs = append(errs, errors.New("Wrong VolumeBindingMode")) + } + if sc.AllowVolumeExpansion == nil || *sc.AllowVolumeExpansion != allowVolumeExpansion { + errs = append(errs, errors.New("Wrong AllowVolumeExpansion")) + } + + if len(errs) > 0 { + return combinedError(errs) + } + return nil +} + +func toDelete(oldSc, newSc []string) []string { + del := make([]string, 0) + for _, old := range oldSc { + var found bool + for _, new := range newSc { + if new == old { + found = true + break + } + } + if !found { + del = append(del, old) + } + } + return del +} diff --git a/pkg/syncer/syncer.go b/pkg/syncer/syncer.go new file mode 100644 index 0000000..ccd0f4f --- /dev/null +++ b/pkg/syncer/syncer.go @@ -0,0 +1,104 @@ +// Package syncer provides the logic used by command line tool cloudstack-csi-sc-syncer. +// +// It provides functions to synchronize CloudStack disk offerings +// to Kubernetes storage classes. +package syncer + +import ( + "context" + "fmt" + "strings" + + "github.com/apalia/cloudstack-csi-driver/pkg/cloud" + "github.com/xanzy/go-cloudstack/v2/cloudstack" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +// Config holds the syncer tool configuration. +type Config struct { + Agent string + CloudStackConfig string + KubeConfig string + Label string + NamePrefix string + Delete bool +} + +// Syncer has a function Run which synchronises CloudStack +// disk offerings to Kubernetes Storage classes. +type Syncer interface { + Run(context.Context) error +} + +// syncer is Syncer implementation. +type syncer struct { + k8sClient *kubernetes.Clientset + csClient *cloudstack.CloudStackClient + labelsSet labels.Set + namePrefix string + delete bool +} + +func createK8sClient(kubeconfig, agent string) (*kubernetes.Clientset, error) { + var config *rest.Config + var err error + if kubeconfig == "-" { + config, err = rest.InClusterConfig() + if err != nil { + return nil, err + } + } else { + config, err = clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + return nil, err + } + } + config.UserAgent = agent + return kubernetes.NewForConfig(config) +} + +func createCloudStackClient(cloudstackconfig string) (*cloudstack.CloudStackClient, error) { + config, err := cloud.ReadConfig(cloudstackconfig) + if err != nil { + return nil, err + } + client := cloudstack.NewAsyncClient(config.APIURL, config.APIKey, config.SecretKey, config.VerifySSL) + return client, nil +} + +func createLabelsSet(label string) labels.Set { + m := make(map[string]string) + if len(label) > 0 { + parts := strings.SplitN(label, "=", 2) + key := parts[0] + value := "" + if len(parts) > 1 { + value = parts[1] + } + m[key] = value + } + return labels.Set(m) +} + +// New creates a new Syncer instance. +func New(config Config) (Syncer, error) { + k8sClient, err := createK8sClient(config.KubeConfig, config.Agent) + if err != nil { + return nil, fmt.Errorf("Cannot create Kubernetes client: %w", err) + } + csClient, err := createCloudStackClient(config.CloudStackConfig) + if err != nil { + return nil, fmt.Errorf("Cannot create CloudStack client: %w", err) + } + + return syncer{ + k8sClient: k8sClient, + csClient: csClient, + labelsSet: createLabelsSet(config.Label), + namePrefix: config.NamePrefix, + delete: config.Delete, + }, nil +} diff --git a/pkg/util/doc.go b/pkg/util/doc.go new file mode 100644 index 0000000..e1beef7 --- /dev/null +++ b/pkg/util/doc.go @@ -0,0 +1,2 @@ +// Package util provides shared utility functions. +package util diff --git a/pkg/util/gb.go b/pkg/util/gb.go new file mode 100644 index 0000000..3454857 --- /dev/null +++ b/pkg/util/gb.go @@ -0,0 +1,13 @@ +package util + +// RoundUpBytesToGB converts a size given in bytes to GB with +// an upper rounding (it gives the smallest amount in GB +// which is greater than the original amount) +func RoundUpBytesToGB(n int64) int64 { + return (((n+1023)/1024+1023)/1024 + 1023) / 1024 +} + +// GigaBytesToBytes gives an exact conversion from GigaBytes to Bytes +func GigaBytesToBytes(gb int64) int64 { + return gb * 1024 * 1024 * 1024 +} diff --git a/pkg/util/gb_test.go b/pkg/util/gb_test.go new file mode 100644 index 0000000..bde5ad8 --- /dev/null +++ b/pkg/util/gb_test.go @@ -0,0 +1,41 @@ +package util + +import ( + "strconv" + "testing" +) + +func TestRoundUpBytesToGB(t *testing.T) { + cases := []struct { + b int64 + expectedGb int64 + }{ + {100, 1}, + {3221225472, 3}, + {3000000000, 3}, + {50 * 1024 * 1024 * 1024, 50}, + {50*1024*1024*1024 - 1, 50}, + {50*1024*1024*1024 + 1, 51}, + } + for _, c := range cases { + t.Run(strconv.FormatInt(c.b, 10), func(t *testing.T) { + gb := RoundUpBytesToGB(c.b) + if gb != c.expectedGb { + t.Errorf("%v bytes: expecting %v, got %v", c.b, c.expectedGb, gb) + } + }) + } +} + +func TestGigaBytesToBytes(t *testing.T) { + var gb int64 = 5 + b := GigaBytesToBytes(gb) + var expectedBytes int64 = 5368709120 + if b != expectedBytes { + t.Errorf("Expected %v, got %v", expectedBytes, b) + } + back := RoundUpBytesToGB(b) + if back != gb { + t.Errorf("Expected %v, got %v", gb, back) + } +} diff --git a/test/e2e/README.md b/test/e2e/README.md new file mode 100644 index 0000000..5221d2d --- /dev/null +++ b/test/e2e/README.md @@ -0,0 +1,21 @@ +# e2e tests + +1. Deploy a Kubernetes cluster on Apache CloudStack; + +1. Deploy the cloudstack-csi-driver; + +1. Set environment variables: + + | Name | Description | Required / Default behaviour | + | ------------------ | -------------------------------------------------------------------------------------------------------- | --------------------------------------------- | + | `DISK_OFFERING_ID` | ID of the CloudStack disk offering to be used in e2e tests. Must accept custom sizes. | **REQUIRED** | + | `KUBECONFIG` | Path to your `kubeconfig` file | Optional - Defaults to `${HOME}/.kube/config` | + | `KUBE_SSH_USER` | Username to use to connect to cluster nodes via SSH | Optional - Defaults to `${USER}`. | + | `KUBE_SSH_KEY` | Path of the SSH key to use to connect to cluster nodes via SSH - may be absolute or relative to `~/.ssh` | Optional - Defaults to `id_rsa` | + | `KUBE_SSH_BASTION` | Address (`host:port`) of a bastion host to use to connect to cluster nodes via SSH | Optional - Direct connection if not set | + +1. From the root of this repository, execute tests with: + + ``` + make test-e2e + ``` diff --git a/test/e2e/run.sh b/test/e2e/run.sh new file mode 100755 index 0000000..28fc3b1 --- /dev/null +++ b/test/e2e/run.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +cd "$(dirname "$0")" || exit + +KUBECONFIG=${KUBECONFIG:=${HOME}/.kube/config} + +if [ ! -f "$KUBECONFIG" ]; then + echo "Kubeconfig $KUBECONFIG not found!" + exit 1 +fi + +if [ -z "$DISK_OFFERING_ID" ]; then + echo "Variable DISK_OFFERING_ID not set!" + exit 1 +fi + +# Create storage class "cloudstack-csi-driver-e2e" +scName="cloudstack-csi-driver-e2e" +sed "s//${DISK_OFFERING_ID}/" storageclass.yaml | kubectl apply -f - + +# Run in parallel when possible (exclude [Feature:.*], [Disruptive] and [Serial]): +./ginkgo -p -progress -v \ + -focus='External.Storage.*csi-cloudstack' \ + -skip='\[Feature:|\[Disruptive\]|\[Serial\]' \ + e2e.test -- \ + -storage.testdriver=testdriver.yaml \ + --kubeconfig="$KUBECONFIG" + +# Then run the remaining tests, sequentially: +./ginkgo -progress -v \ + -focus='External.Storage.*csi-cloudstack.*(\[Feature:|\[Disruptive\]|\[Serial\])' \ + e2e.test -- \ + -storage.testdriver=testdriver.yaml \ + --kubeconfig="$KUBECONFIG" + +# Delete storage class +kubectl delete storageclasses.storage.k8s.io "${scName}" || echo "No storage class named ${scName}" \ No newline at end of file diff --git a/test/e2e/storageclass.yaml b/test/e2e/storageclass.yaml new file mode 100644 index 0000000..116d77f --- /dev/null +++ b/test/e2e/storageclass.yaml @@ -0,0 +1,10 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: cloudstack-csi-driver-e2e +provisioner: csi.cloudstack.apache.org +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: false +parameters: + csi.cloudstack.apache.org/disk-offering-id: diff --git a/test/e2e/testdriver.yaml b/test/e2e/testdriver.yaml new file mode 100644 index 0000000..701563a --- /dev/null +++ b/test/e2e/testdriver.yaml @@ -0,0 +1,10 @@ +StorageClass: + FromExistingClassName: cloudstack-csi-driver-e2e +SnapshotClass: + FromName: true +DriverInfo: + Name: csi-cloudstack + Capabilities: + persistence: true + block: true + exec: true diff --git a/test/sanity/sanity_test.go b/test/sanity/sanity_test.go new file mode 100644 index 0000000..382e86d --- /dev/null +++ b/test/sanity/sanity_test.go @@ -0,0 +1,47 @@ +// +build sanity + +package sanity + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/apalia/cloudstack-csi-driver/pkg/cloud/fake" + "github.com/apalia/cloudstack-csi-driver/pkg/driver" + "github.com/apalia/cloudstack-csi-driver/pkg/mount" + "github.com/kubernetes-csi/csi-test/v4/pkg/sanity" + "go.uber.org/zap" +) + +func TestSanity(t *testing.T) { + // Setup driver + dir, err := ioutil.TempDir("", "sanity-cloudstack-csi") + if err != nil { + t.Fatalf("error creating directory: %v", err) + } + defer os.RemoveAll(dir) + + targetPath := filepath.Join(dir, "target") + stagingPath := filepath.Join(dir, "staging") + endpoint := "unix://" + filepath.Join(dir, "csi.sock") + + config := sanity.NewTestConfig() + config.TargetPath = targetPath + config.StagingPath = stagingPath + config.Address = endpoint + config.TestVolumeParameters = map[string]string{ + driver.DiskOfferingKey: "9743fd77-0f5d-4ef9-b2f8-f194235c769c", + } + + csiDriver, err := driver.New(endpoint, fake.New(), mount.NewFake(), "node", "v0", zap.NewNop()) + if err != nil { + t.Fatalf("error creating driver: %v", err) + } + go func() { + csiDriver.Run() + }() + + sanity.Test(t, config) +}