From b615173c77b3d1ac3d1250dc41b1cc9cea7a00b6 Mon Sep 17 00:00:00 2001 From: Michal Fiedorowicz Date: Thu, 3 Oct 2024 09:25:55 +0100 Subject: [PATCH 1/4] chore: remove redundant environment variable (#180) Signed-off-by: Michal Fiedorowicz --- diode-server/docker/docker-compose.yaml | 1 - diode-server/docker/sample.env | 1 - 2 files changed, 2 deletions(-) diff --git a/diode-server/docker/docker-compose.yaml b/diode-server/docker/docker-compose.yaml index e659e5c8..8abf878c 100644 --- a/diode-server/docker/docker-compose.yaml +++ b/diode-server/docker/docker-compose.yaml @@ -36,7 +36,6 @@ services: diode-ingester: image: netboxlabs/diode-ingester:${DIODE_TAG:-latest} environment: - - API_KEY=${RECONCILER_API_KEY} - REDIS_PASSWORD=${REDIS_PASSWORD} - REDIS_HOST=${REDIS_HOST} - REDIS_PORT=${REDIS_PORT} diff --git a/diode-server/docker/sample.env b/diode-server/docker/sample.env index 3e0b65f9..410be79c 100644 --- a/diode-server/docker/sample.env +++ b/diode-server/docker/sample.env @@ -1,5 +1,4 @@ DIODE_NGINX_PORT=8080 -RECONCILER_API_KEY=CHANGE_.ME REDIS_PASSWORD=@FmnLoA*VnebyVnZoL.!-.6z REDIS_HOST=diode-redis REDIS_PORT=6378 From e7c7313b68a8a2dbb4afb2c1f142da18ba68c1ef Mon Sep 17 00:00:00 2001 From: Michal Fiedorowicz Date: Thu, 3 Oct 2024 11:16:00 +0100 Subject: [PATCH 2/4] chore: GHA go lint and tests on diode-server/** changes only (#181) Signed-off-by: Michal Fiedorowicz --- .github/workflows/go-test.yaml | 4 ++++ .github/workflows/golangci-lint.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/go-test.yaml b/.github/workflows/go-test.yaml index 58b11be3..36e72a30 100644 --- a/.github/workflows/go-test.yaml +++ b/.github/workflows/go-test.yaml @@ -3,7 +3,11 @@ on: push: branches: - "!release" + paths: + - "diode-server/**" pull_request: + paths: + - "diode-server/**" concurrency: group: ${{ github.workflow }} diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml index 11a80268..01218d15 100644 --- a/.github/workflows/golangci-lint.yaml +++ b/.github/workflows/golangci-lint.yaml @@ -3,7 +3,11 @@ on: push: branches: - "!release" + paths: + - "diode-server/**" pull_request: + paths: + - "diode-server/**" permissions: contents: read From 216f3a6d556cf19c7d828473b5369b01ed2286de Mon Sep 17 00:00:00 2001 From: Michal Fiedorowicz Date: Tue, 15 Oct 2024 18:56:12 +0100 Subject: [PATCH 3/4] feat: diode helm chart (#184) Signed-off-by: Michal Fiedorowicz --- .github/pull_request_labeler.yaml | 14 +- .github/workflows/helm-lint.yaml | 35 +++ .github/workflows/helm-release.yaml | 45 ++++ charts/diode/.helmignore | 23 ++ charts/diode/Chart.lock | 12 ++ charts/diode/Chart.yaml | 26 +++ charts/diode/README.md | 118 ++++++++++ charts/diode/README.md.gotmpl | 41 ++++ charts/diode/templates/NOTES.txt | 4 + charts/diode/templates/_helpers.tpl | 43 ++++ .../templates/diode-ingester-configmap.yaml | 11 + .../templates/diode-ingester-deployment.yaml | 85 ++++++++ .../templates/diode-ingester-secret.yaml | 10 + .../templates/diode-ingester-service.yaml | 16 ++ .../diode-ingester-serviceaccount.yaml | 9 + charts/diode/templates/diode-ingress.yaml | 47 ++++ charts/diode/templates/diode-issuer.yaml | 22 ++ .../templates/diode-reconciler-configmap.yaml | 13 ++ .../diode-reconciler-deployment.yaml | 78 +++++++ .../templates/diode-reconciler-secret.yaml | 13 ++ .../templates/diode-reconciler-service.yaml | 16 ++ .../diode-reconciler-serviceaccount.yaml | 9 + charts/diode/values.yaml | 202 ++++++++++++++++++ 23 files changed, 884 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/helm-lint.yaml create mode 100644 .github/workflows/helm-release.yaml create mode 100644 charts/diode/.helmignore create mode 100644 charts/diode/Chart.lock create mode 100644 charts/diode/Chart.yaml create mode 100644 charts/diode/README.md create mode 100644 charts/diode/README.md.gotmpl create mode 100644 charts/diode/templates/NOTES.txt create mode 100644 charts/diode/templates/_helpers.tpl create mode 100644 charts/diode/templates/diode-ingester-configmap.yaml create mode 100644 charts/diode/templates/diode-ingester-deployment.yaml create mode 100644 charts/diode/templates/diode-ingester-secret.yaml create mode 100644 charts/diode/templates/diode-ingester-service.yaml create mode 100644 charts/diode/templates/diode-ingester-serviceaccount.yaml create mode 100644 charts/diode/templates/diode-ingress.yaml create mode 100644 charts/diode/templates/diode-issuer.yaml create mode 100644 charts/diode/templates/diode-reconciler-configmap.yaml create mode 100644 charts/diode/templates/diode-reconciler-deployment.yaml create mode 100644 charts/diode/templates/diode-reconciler-secret.yaml create mode 100644 charts/diode/templates/diode-reconciler-service.yaml create mode 100644 charts/diode/templates/diode-reconciler-serviceaccount.yaml create mode 100644 charts/diode/values.yaml diff --git a/.github/pull_request_labeler.yaml b/.github/pull_request_labeler.yaml index 466c1c4c..88e7c3be 100644 --- a/.github/pull_request_labeler.yaml +++ b/.github/pull_request_labeler.yaml @@ -74,14 +74,6 @@ diode-server: - 'diode-server/*' - 'diode-server/**/*' -diode-distributor: - - changed-files: - - any-glob-to-any-file: - - 'diode-server/cmd/distributor/*' - - 'diode-server/cmd/distributor/**/*' - - 'diode-server/distributor/*' - - 'diode-server/distributor/**/*' - diode-ingester: - changed-files: - any-glob-to-any-file: @@ -97,3 +89,9 @@ diode-reconciler: - 'diode-server/cmd/reconciler/**/*' - 'diode-server/reconciler/*' - 'diode-server/reconciler/**/*' + +diode-chart: + - changed-files: + - any-glob-to-any-file: + - 'charts/*' + - 'charts/**/*' diff --git a/.github/workflows/helm-lint.yaml b/.github/workflows/helm-lint.yaml new file mode 100644 index 00000000..44b19f05 --- /dev/null +++ b/.github/workflows/helm-lint.yaml @@ -0,0 +1,35 @@ +name: Helm - lint +on: + push: + branches: + - "!release" + paths: + - "charts/**" + pull_request: + paths: + - "charts/**" + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +permissions: + contents: write + pull-requests: write + +jobs: + helm-lint: + runs-on: ubuntu-latest + timeout-minutes: 10 + defaults: + run: + working-directory: charts + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Helm + uses: azure/setup-helm@v4.2.0 + - name: Update helm dependencies + run: helm dependency update diode + - name: Run helm lint + run: helm lint diode diff --git a/.github/workflows/helm-release.yaml b/.github/workflows/helm-release.yaml new file mode 100644 index 00000000..76515cf2 --- /dev/null +++ b/.github/workflows/helm-release.yaml @@ -0,0 +1,45 @@ +name: Helm - release +on: + workflow_dispatch: + push: + branches: [ release ] + paths: + - "charts/**" + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +permissions: + contents: write + +jobs: + helm-release: + runs-on: ubuntu-latest + timeout-minutes: 10 + defaults: + run: + working-directory: charts + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Configure Git + run: | + git config user.name "$GITHUB_ACTOR" + git config user.email "$GITHUB_ACTOR@users.noreply.github.com" + - name: Setup Helm + uses: azure/setup-helm@v4.2.0 + - name: Update helm dependencies + run: | + helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx + helm repo add jetstack https://charts.jetstack.io + helm repo add bitnami https://charts.bitnami.com/bitnami + - name: Run chart-releaser + uses: helm/chart-releaser-action@v1.6.0 + env: + CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + CR_RELEASE_NAME_TEMPLATE: helm-chart-{{ .Name }}-{{ .Version }} + CR_INDEX_PATH: charts/index.yaml + CR_PAGES_INDEX_PATH: charts/index.yaml diff --git a/charts/diode/.helmignore b/charts/diode/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/charts/diode/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/diode/Chart.lock b/charts/diode/Chart.lock new file mode 100644 index 00000000..672b5843 --- /dev/null +++ b/charts/diode/Chart.lock @@ -0,0 +1,12 @@ +dependencies: +- name: ingress-nginx + repository: https://kubernetes.github.io/ingress-nginx + version: 4.11.2 +- name: cert-manager + repository: https://charts.jetstack.io + version: v1.16.1 +- name: redis + repository: oci://registry-1.docker.io/bitnamicharts + version: 20.1.4 +digest: sha256:f89ee5fc93ebfc48d7566073c20cbe8ab7b632e73f2fbd860b84d1b7a01ecf48 +generated: "2024-10-14T19:29:43.398885+01:00" diff --git a/charts/diode/Chart.yaml b/charts/diode/Chart.yaml new file mode 100644 index 00000000..e4d5210a --- /dev/null +++ b/charts/diode/Chart.yaml @@ -0,0 +1,26 @@ +apiVersion: v2 +name: diode +description: A Helm chart for Diode +type: application +version: 0.1.0 +appVersion: "0.6.0" +home: https://github.com/netboxlabs/diode +sources: + - https://github.com/netboxlabs/diode +maintainers: + - name: NetBox Labs + email: support@netboxlabs.com + url: https://github.com/netboxlabs +dependencies: + - name: ingress-nginx + version: 4.11.2 + repository: https://kubernetes.github.io/ingress-nginx + condition: ingress-nginx.enabled + - name: cert-manager + version: 1.16.1 + repository: https://charts.jetstack.io + condition: cert-manager.enabled + - name: redis + version: 20.1.4 + repository: oci://registry-1.docker.io/bitnamicharts + condition: redis.enabled diff --git a/charts/diode/README.md b/charts/diode/README.md new file mode 100644 index 00000000..3b3962c6 --- /dev/null +++ b/charts/diode/README.md @@ -0,0 +1,118 @@ +# diode + +A Helm chart for Diode + +![Version: 0.1.0](https://img.shields.io/badge/Version-0.1.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.0](https://img.shields.io/badge/AppVersion-0.6.0-informational?style=flat-square) + +## Installing the Chart + +Install custom resource definitions for cert-manager (if enabled): + +```console +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.16.1/cert-manager.crds.yaml +``` + +Create namespaces for ingress-nginx and cert-manager: + +```console +kubectl create namespace diode-ingress +kubectl create namespace diode-cert-manager +``` + +Install the chart with the release name `my-release`: + +```console +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo add jetstack https://charts.jetstack.io +helm repo add bitnami https://charts.bitnami.com/bitnami +helm repo add diode https://netboxlabs.github.io/diode/charts +helm install my-release diode/diode --namespace my-namespace --create-namespace +``` + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.jetstack.io | cert-manager | 1.16.1 | +| https://kubernetes.github.io/ingress-nginx | ingress-nginx | 4.11.2 | +| oci://registry-1.docker.io/bitnamicharts | redis | 20.1.4 | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| cert-manager | object | `{"enabled":false,"namespace":"diode-cert-manager"}` | ref: https://github.com/cert-manager/cert-manager/blob/master/deploy/charts/cert-manager/values.yaml | +| cert-manager.enabled | bool | `false` | cert-manager enabled | +| cert-manager.namespace | string | `"diode-cert-manager"` | cert-manager namespace | +| certIssuer.email | string | `""` | email address for ACME registration | +| certIssuer.enabled | bool | `false` | enable certificate issuer creation | +| certIssuer.kind | string | `"Issuer"` | issuer kind (Issuer or ClusterIssuer) ref: https://cert-manager.io/docs/configuration/acme/ | +| certIssuer.name | string | `""` | issuer name | +| certIssuer.prod | bool | `false` | determines whether to use Let's Encrypt production or staging environment | +| certIssuer.solvers | list | `[{"http01":{"ingress":{"ingressClassName":"nginx"}}}]` | solvers for the issuer | +| diodeIngester.affinity | object | `{}` | custom affinity rules for the pod | +| diodeIngester.config.reconcilerGrpcHost | string | `"diode-reconciler"` | diode-reconciler gRPC host | +| diodeIngester.config.reconcilerGrpcPort | int | `8081` | diode-reconciler gRPC port | +| diodeIngester.config.sentryDsn | string | `""` | sentry DSN | +| diodeIngester.containerPort | int | `8081` | port to listen on | +| diodeIngester.existingSecret | string | `""` | existing secret for diode-ingester | +| diodeIngester.image.pullPolicy | string | `"IfNotPresent"` | image pull policy | +| diodeIngester.image.repository | string | `"netboxlabs/diode-ingester"` | image repository | +| diodeIngester.image.securityContext | object | `{}` | security context for the container | +| diodeIngester.image.tag | string | `"v0.6.0"` | image tag | +| diodeIngester.nodeSelector | object | `{}` | node selector for the pod | +| diodeIngester.podAnnotations | object | `{}` | additional pod annotations | +| diodeIngester.podLabels | object | `{}` | additional pod labels | +| diodeIngester.podSecurityContext | object | `{}` | additional pod security context | +| diodeIngester.replicas | int | `1` | number of replicas | +| diodeIngester.resources | object | `{}` | resources to allocate for the container | +| diodeIngester.secrets.ingesterToReconcilerAPIKey | string | `""` | API key for authentication between diode-ingester and diode-reconciler | +| diodeIngester.secrets.redisPassword | string | `""` | redis password, must match the password in the redis chart or external redis | +| diodeIngester.serviceAccount.create | bool | `true` | create service account | +| diodeIngester.serviceAccount.name | string | `"diode-ingester"` | service account name | +| diodeIngester.serviceName | string | `"diode-ingester"` | service name | +| diodeIngester.tolerations | list | `[]` | tolerations to use with node taints | +| diodeReconciler.affinity | object | `{}` | custom affinity rules for the pod | +| diodeReconciler.config.loggingLevel | string | `"DEBUG"` | logging level | +| diodeReconciler.config.migrationEnabled | bool | `true` | migration enabled | +| diodeReconciler.config.netboxDiodePluginAPIBaseURL | string | `"https:///api/plugins/diode"` | NetBox plugin API base URL | +| diodeReconciler.config.netboxDiodePluginSkipTLSVerify | bool | `false` | NetBox plugin skip TLS verify | +| diodeReconciler.config.sentryDsn | string | `""` | sentry DSN | +| diodeReconciler.containerPort | int | `8081` | port to listen on | +| diodeReconciler.existingSecret | string | `""` | existing secret for diode-ingester | +| diodeReconciler.image.pullPolicy | string | `"IfNotPresent"` | image pull policy | +| diodeReconciler.image.repository | string | `"netboxlabs/diode-reconciler"` | image repository | +| diodeReconciler.image.securityContext | object | `{}` | security context for the container | +| diodeReconciler.image.tag | string | `"v0.6.0"` | image tag | +| diodeReconciler.nodeSelector | object | `{}` | node selector for the pod | +| diodeReconciler.podAnnotations | object | `{}` | additional pod annotations | +| diodeReconciler.podLabels | object | `{}` | additional pod labels | +| diodeReconciler.podSecurityContext | object | `{}` | additional pod security context | +| diodeReconciler.replicas | int | `1` | number of replicas | +| diodeReconciler.resources | object | `{}` | | +| diodeReconciler.secrets.diodeAPIKey | string | `""` | API key for authentication of diode ingestion requests | +| diodeReconciler.secrets.diodeToNetboxAPIKey | string | `""` | API key for authentication between diode and NetBox API | +| diodeReconciler.secrets.ingesterToReconcilerAPIKey | string | `""` | API key for authentication between diode-ingester and diode-reconciler | +| diodeReconciler.secrets.netboxToDiodeAPIKey | string | `""` | API key for authentication between NetBox API and diode | +| diodeReconciler.secrets.redisPassword | string | `""` | redis password, must match the password in the redis chart or external redis | +| diodeReconciler.serviceAccount.create | bool | `true` | create service account | +| diodeReconciler.serviceAccount.name | string | `"diode-reconciler"` | service account name | +| diodeReconciler.serviceName | string | `"diode-reconciler"` | service name | +| diodeReconciler.tolerations | list | `[]` | tolerations to use with node taints | +| externalRedis.host | string | `""` | external redis host | +| externalRedis.port | int | `6379` | external redis port | +| ingress-nginx | object | `{"controller":{"allowSnippetAnnotations":true},"enabled":true,"hostname":"","ingressClass":"nginx","namespaceOverride":"diode-ingress"}` | ref: https://github.com/kubernetes/ingress-nginx/blob/main/charts/ingress-nginx/values.yaml | +| ingress-nginx.controller.allowSnippetAnnotations | bool | `true` | allow snippet annotations | +| ingress-nginx.enabled | bool | `true` | ingress-nginx enabled | +| ingress-nginx.hostname | string | `""` | hostname | +| ingress-nginx.ingressClass | string | `"nginx"` | ingress class | +| ingress-nginx.namespaceOverride | string | `"diode-ingress"` | override ingress-nginx namespace | +| redis | object | `{"auth":{"existingSecret":"diode-ingester-secret","existingSecretPasswordKey":"REDIS_PASSWORD"},"commonConfiguration":"appendonly yes\nsave 60 1\nloadmodule /opt/redis-stack/lib/rejson.so\nloadmodule /opt/redis-stack/lib/redisearch.so","enabled":true,"image":{"pullPolicy":"IfNotPresent","repository":"redis/redis-stack-server","tag":"latest"},"replica":{"replicaCount":1}}` | ref: https://github.com/bitnami/charts/blob/main/bitnami/redis/values.yaml | +| redis.auth.existingSecret | string | `"diode-ingester-secret"` | existing secret for redis password, either diodeIngester.existingSecret, diode-ingester-secret (created from diodeIngester.secrets) or your custom secret | +| redis.auth.existingSecretPasswordKey | string | `"REDIS_PASSWORD"` | existing secret key for redis password | +| redis.commonConfiguration | string | `"appendonly yes\nsave 60 1\nloadmodule /opt/redis-stack/lib/rejson.so\nloadmodule /opt/redis-stack/lib/redisearch.so"` | redis configuration | +| redis.enabled | bool | `true` | redis enabled | +| redis.image.pullPolicy | string | `"IfNotPresent"` | redis image pull policy | +| redis.image.repository | string | `"redis/redis-stack-server"` | redis image repository | +| redis.image.tag | string | `"latest"` | redis image tag | +| redis.replica.replicaCount | int | `1` | number of redis replicas | \ No newline at end of file diff --git a/charts/diode/README.md.gotmpl b/charts/diode/README.md.gotmpl new file mode 100644 index 00000000..1288fa4a --- /dev/null +++ b/charts/diode/README.md.gotmpl @@ -0,0 +1,41 @@ +{{ template "chart.header" . }} +{{ template "chart.description" . }} + +{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }} + +## Installing the Chart + +{{- define "cert-manager-version" }} +{{- range .Dependencies }} +{{- if eq .Name "cert-manager" }} +{{- .Version }} +{{- end }} +{{- end }} +{{- end }} + +Install custom resource definitions for cert-manager (if enabled): + +```console +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v{{ template "cert-manager-version" . }}/cert-manager.crds.yaml +``` + +Create namespaces for ingress-nginx and cert-manager: + +```console +kubectl create namespace diode-ingress +kubectl create namespace diode-cert-manager +``` + +Install the chart with the release name `my-release`: + +```console +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo add jetstack https://charts.jetstack.io +helm repo add bitnami https://charts.bitnami.com/bitnami +helm repo add diode https://netboxlabs.github.io/diode/charts +helm install my-release diode/{{ template "chart.name" . }} --namespace my-namespace --create-namespace +``` + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} \ No newline at end of file diff --git a/charts/diode/templates/NOTES.txt b/charts/diode/templates/NOTES.txt new file mode 100644 index 00000000..de728069 --- /dev/null +++ b/charts/diode/templates/NOTES.txt @@ -0,0 +1,4 @@ +CHART NAME: {{ .Chart.Name }} +CHART VERSION: {{ .Chart.Version }} +APP VERSION: {{ .Chart.AppVersion }} +DESCRIPTION: {{ .Chart.Description }} diff --git a/charts/diode/templates/_helpers.tpl b/charts/diode/templates/_helpers.tpl new file mode 100644 index 00000000..e56fb3b6 --- /dev/null +++ b/charts/diode/templates/_helpers.tpl @@ -0,0 +1,43 @@ +{{/* +Define redis host +*/}} +{{- define "diode.redis.host" -}} +{{- if .Values.redis.enabled -}} +{{- printf "%s-redis-master.%s.svc.cluster.local" .Release.Name .Release.Namespace -}} +{{- else -}} +{{- .Values.externalRedis.host -}} +{{- end -}} +{{- end -}} + +{{/* +Define redis port +*/}} +{{- define "diode.redis.port" -}} +{{- if .Values.redis.enabled -}} +{{- .Values.redis.master.containerPorts.redis -}} +{{- else -}} +{{- .Values.externalRedis.port -}} +{{- end -}} +{{- end -}} + +{{/* +Define diode-ingester-secret +*/}} +{{- define "diode-ingester.secret" -}} +{{- if .Values.diodeIngester.existingSecret -}} +{{- .Values.diodeIngester.existingSecret -}} +{{- else -}} +{{- printf "%s-secret" .Values.diodeIngester.serviceName -}} +{{- end -}} +{{- end -}} + +{{/* +Define diode-reconciler-secret +*/}} +{{- define "diode-reconciler.secret" -}} +{{- if .Values.diodeReconciler.existingSecret -}} +{{- .Values.diodeReconciler.existingSecret -}} +{{- else -}} +{{- printf "%s-secret" .Values.diodeReconciler.serviceName -}} +{{- end -}} +{{- end -}} diff --git a/charts/diode/templates/diode-ingester-configmap.yaml b/charts/diode/templates/diode-ingester-configmap.yaml new file mode 100644 index 00000000..7d800ab2 --- /dev/null +++ b/charts/diode/templates/diode-ingester-configmap.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.diodeIngester.serviceName }}-config + namespace: {{ .Release.Namespace }} +data: + RECONCILER_GRPC_HOST: {{ .Values.diodeIngester.config.reconcilerGrpcHost | quote }} + RECONCILER_GRPC_PORT: {{ .Values.diodeIngester.config.reconcilerGrpcPort | quote }} + REDIS_HOST: {{ include "diode.redis.host" . | quote }} + REDIS_PORT: {{ include "diode.redis.port" . | quote }} + SENTRY_DSN: {{ .Values.diodeIngester.config.sentryDsn | quote }} diff --git a/charts/diode/templates/diode-ingester-deployment.yaml b/charts/diode/templates/diode-ingester-deployment.yaml new file mode 100644 index 00000000..c17850c7 --- /dev/null +++ b/charts/diode/templates/diode-ingester-deployment.yaml @@ -0,0 +1,85 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.diodeIngester.serviceName }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Values.diodeIngester.serviceName }} +spec: + replicas: {{ .Values.diodeIngester.replicas }} + selector: + matchLabels: + app: {{ .Values.diodeIngester.serviceName }} + template: + metadata: + annotations: + checksum/config-ingester: {{ include (printf "%s/%s-configmap.yaml" $.Template.BasePath .Values.diodeIngester.serviceName) . | sha256sum }} + checksum/config-reconciler: {{ include (printf "%s/%s-configmap.yaml" $.Template.BasePath .Values.diodeReconciler.serviceName) . | sha256sum }} + {{- if not .Values.diodeIngester.existingSecret }} + checksum/secret-ingester: {{ include (printf "%s/%s-secret.yaml" $.Template.BasePath .Values.diodeIngester.serviceName ) . | sha256sum }} + {{- end }} + {{- if not .Values.diodeReconciler.existingSecret }} + checksum/secret-reconciler: {{ include (printf "%s/%s-secret.yaml" $.Template.BasePath .Values.diodeReconciler.serviceName ) . | sha256sum }} + {{- end }} + {{- with .Values.diodeIngester.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app: {{ .Values.diodeIngester.serviceName }} + {{- with .Values.diodeIngester.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.diodeIngester.serviceAccount.name }} + {{- with .Values.diodeIngester.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.diodeIngester.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.diodeIngester.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.diodeIngester.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: {{ include "diode-ingester.secret" . }} + secret: + secretName: {{ include "diode-ingester.secret" . }} + initContainers: + {{- if .Values.redis.enabled }} + - name: wait-for-redis + image: busybox:latest + command: [ 'sh', '-c', 'until nc -z {{ include "diode.redis.host" . }} {{ include "diode.redis.port" . }}; do echo "Waiting for Redis"; sleep 3; done; echo "Redis is up and running";' ] + {{- end }} + - name: wait-for-diode-reconciler + image: busybox:latest + command: [ 'sh', '-c', 'until nc -z {{ .Values.diodeReconciler.serviceName }} {{ .Values.diodeReconciler.containerPort | default 8081 }}; do echo "Waiting for Diode Reconciler"; sleep 3; done; echo "Diode Reconciler is up and running";' ] + containers: + - name: diode-ingester + image: "{{ .Values.diodeIngester.image.repository }}:{{ .Values.diodeIngester.image.tag }}" + imagePullPolicy: {{ .Values.diodeIngester.image.pullPolicy }} + ports: + - containerPort: {{ .Values.diodeIngester.containerPort | default 8081 }} + {{- with .Values.diodeIngester.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.diodeIngester.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - mountPath: /{{ include "diode-ingester.secret" . }} + name: {{ include "diode-ingester.secret" . }} + readOnly: true + envFrom: + - configMapRef: + name: {{ .Values.diodeIngester.serviceName }}-config + - secretRef: + name: {{ include "diode-ingester.secret" . }} diff --git a/charts/diode/templates/diode-ingester-secret.yaml b/charts/diode/templates/diode-ingester-secret.yaml new file mode 100644 index 00000000..9f448f5f --- /dev/null +++ b/charts/diode/templates/diode-ingester-secret.yaml @@ -0,0 +1,10 @@ +{{ if not .Values.diodeIngester.existingSecret -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.diodeIngester.serviceName }}-secret + namespace: {{ .Release.Namespace }} +stringData: + INGESTER_TO_RECONCILER_API_KEY: {{ .Values.diodeIngester.secrets.ingesterToReconcilerAPIKey | quote }} + REDIS_PASSWORD: {{ .Values.diodeIngester.secrets.redisPassword | quote }} +{{- end -}} diff --git a/charts/diode/templates/diode-ingester-service.yaml b/charts/diode/templates/diode-ingester-service.yaml new file mode 100644 index 00000000..811fe884 --- /dev/null +++ b/charts/diode/templates/diode-ingester-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.diodeIngester.serviceName }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Values.diodeIngester.serviceName }} +spec: + type: ClusterIP + selector: + app: {{ .Values.diodeIngester.serviceName }} + ports: + - name: grpc + port: {{ .Values.diodeIngester.containerPort }} + targetPort: {{ .Values.diodeIngester.containerPort }} + protocol: TCP diff --git a/charts/diode/templates/diode-ingester-serviceaccount.yaml b/charts/diode/templates/diode-ingester-serviceaccount.yaml new file mode 100644 index 00000000..c4ae9c64 --- /dev/null +++ b/charts/diode/templates/diode-ingester-serviceaccount.yaml @@ -0,0 +1,9 @@ +{{- if .Values.diodeIngester.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.diodeIngester.serviceAccount.name }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Values.diodeIngester.serviceName }} +{{- end -}} diff --git a/charts/diode/templates/diode-ingress.yaml b/charts/diode/templates/diode-ingress.yaml new file mode 100644 index 00000000..01e1541f --- /dev/null +++ b/charts/diode/templates/diode-ingress.yaml @@ -0,0 +1,47 @@ +{{- $ingressNginx := index .Values "ingress-nginx" -}} +{{- $certManager := index .Values "cert-manager" -}} +{{- if $ingressNginx.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ .Release.Name}}-ingress-nginx + namespace: {{ .Release.Namespace }} + annotations: + nginx.ingress.kubernetes.io/backend-protocol: "GRPC" + nginx.ingress.kubernetes.io/rewrite-target: /$1 + {{- if and $certManager.enabled .Values.certIssuer.enabled }} + {{ if eq .Values.certIssuer.kind "Issuer" }} + cert-manager.io/issuer: {{ .Values.certIssuer.name }} + {{- else }} + cert-manager.io/cluster-issuer: {{ .Values.certIssuer.name }} + {{- end }} + {{- end }} + {{- if $ingressNginx.controller.allowSnippetAnnotations }} + {{- end }} +spec: + ingressClassName: {{ $ingressNginx.ingressClass }} + {{- if and $certManager.enabled $ingressNginx.hostname }} + tls: + - hosts: + - {{ $ingressNginx.hostname }} + secretName: {{ .Release.Name }}-ingress-tls + {{- end }} + rules: + - host: {{ $ingressNginx.hostname }} + http: + paths: + - path: /diode/(diode.v1.IngesterService.*) + pathType: Prefix + backend: + service: + name: {{ .Values.diodeIngester.serviceName }} + port: + number: {{ .Values.diodeIngester.containerPort }} + - path: /diode/(diode.v1.ReconcilerService.*) + pathType: Prefix + backend: + service: + name: {{ .Values.diodeReconciler.serviceName }} + port: + number: {{ .Values.diodeReconciler.containerPort }} +{{- end }} diff --git a/charts/diode/templates/diode-issuer.yaml b/charts/diode/templates/diode-issuer.yaml new file mode 100644 index 00000000..217564fa --- /dev/null +++ b/charts/diode/templates/diode-issuer.yaml @@ -0,0 +1,22 @@ +{{- $certManager := index .Values "cert-manager" -}} +{{- if and $certManager.enabled .Values.certIssuer.enabled }} +apiVersion: cert-manager.io/v1 +kind: {{ .Values.certIssuer.kind }} +metadata: + name: {{ .Values.certIssuer.name }} + namespace: {{ .Release.Namespace }} +spec: + acme: + # The ACME server URL + server: "https://acme-{{- if not .Values.certIssuer.prod }}staging-{{- end }}v02.api.letsencrypt.org/directory" + # Email address used for ACME registration + email: {{ .Values.certIssuer.email }} + # Name of a secret used to store the ACME account private key + privateKeySecretRef: + name: {{ .Values.certIssuer.name }} + # Solvers that will be used to obtain the certificate + {{- with .Values.certIssuer.solvers }} + solvers: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/charts/diode/templates/diode-reconciler-configmap.yaml b/charts/diode/templates/diode-reconciler-configmap.yaml new file mode 100644 index 00000000..6c55dc95 --- /dev/null +++ b/charts/diode/templates/diode-reconciler-configmap.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.diodeReconciler.serviceName }}-config + namespace: {{ .Release.Namespace }} +data: + REDIS_HOST: {{ include "diode.redis.host" . | quote }} + REDIS_PORT: {{ include "diode.redis.port" . | quote }} + NETBOX_DIODE_PLUGIN_API_BASE_URL: {{ .Values.diodeReconciler.config.netboxDiodePluginAPIBaseURL | quote }} + NETBOX_DIODE_PLUGIN_SKIP_TLS_VERIFY: {{ .Values.diodeReconciler.config.netboxDiodePluginSkipTLSVerify | quote }} + LOGGING_LEVEL: {{ .Values.diodeReconciler.config.loggingLevel | quote }} + MIGRATION_ENABLED: {{ .Values.diodeReconciler.config.migrationEnabled | quote }} + SENTRY_DSN: {{ .Values.diodeReconciler.config.sentryDsn | quote }} diff --git a/charts/diode/templates/diode-reconciler-deployment.yaml b/charts/diode/templates/diode-reconciler-deployment.yaml new file mode 100644 index 00000000..757228dd --- /dev/null +++ b/charts/diode/templates/diode-reconciler-deployment.yaml @@ -0,0 +1,78 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Values.diodeReconciler.serviceName }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Values.diodeReconciler.serviceName }} +spec: + replicas: {{ .Values.diodeReconciler.replicas }} + selector: + matchLabels: + app: {{ .Values.diodeReconciler.serviceName }} + template: + metadata: + annotations: + checksum/config: {{ include (printf "%s/%s-configmap.yaml" $.Template.BasePath .Values.diodeReconciler.serviceName) . | sha256sum }} + {{- if not .Values.diodeReconciler.existingSecret }} + checksum/secret: {{ include (printf "%s/%s-secret.yaml" $.Template.BasePath .Values.diodeReconciler.serviceName ) . | sha256sum }} + {{- end }} + {{- with .Values.diodeReconciler.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + app: {{ .Values.diodeReconciler.serviceName }} + {{- with .Values.diodeReconciler.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.diodeReconciler.serviceAccount.name }} + {{- with .Values.diodeReconciler.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.diodeReconciler.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.diodeReconciler.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.diodeReconciler.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: {{ include "diode-reconciler.secret" . }} + secret: + secretName: {{ include "diode-reconciler.secret" . }} + initContainers: + {{- if .Values.redis.enabled }} + - name: wait-for-redis + image: busybox:latest + command: [ 'sh', '-c', 'until nc -z {{ include "diode.redis.host" . }} {{ include "diode.redis.port" . }}; do echo "Waiting for Redis"; sleep 3; done; echo "Redis is up and running";' ] + {{- end }} + containers: + - name: diode-ingester + image: "{{ .Values.diodeReconciler.image.repository }}:{{ .Values.diodeReconciler.image.tag }}" + imagePullPolicy: {{ .Values.diodeReconciler.image.pullPolicy }} + ports: + - containerPort: {{ .Values.diodeReconciler.containerPort | default 8081 }} + {{- with .Values.diodeReconciler.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.diodeReconciler.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - mountPath: /{{ include "diode-reconciler.secret" . }} + name: {{ include "diode-reconciler.secret" . }} + readOnly: true + envFrom: + - configMapRef: + name: {{ .Values.diodeReconciler.serviceName }}-config + - secretRef: + name: {{ include "diode-reconciler.secret" . }} diff --git a/charts/diode/templates/diode-reconciler-secret.yaml b/charts/diode/templates/diode-reconciler-secret.yaml new file mode 100644 index 00000000..9e8fc9c3 --- /dev/null +++ b/charts/diode/templates/diode-reconciler-secret.yaml @@ -0,0 +1,13 @@ +{{ if not .Values.diodeReconciler.existingSecret -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.diodeReconciler.serviceName }}-secret + namespace: {{ .Release.Namespace }} +stringData: + DIODE_TO_NETBOX_API_KEY: {{ .Values.diodeReconciler.secrets.diodeToNetboxAPIKey | quote }} + NETBOX_TO_DIODE_API_KEY: {{ .Values.diodeReconciler.secrets.netboxToDiodeAPIKey | quote }} + DIODE_API_KEY: {{ .Values.diodeReconciler.secrets.diodeAPIKey | quote }} + INGESTER_TO_RECONCILER_API_KEY: {{ .Values.diodeReconciler.secrets.ingesterToReconcilerAPIKey | quote }} + REDIS_PASSWORD: {{ .Values.diodeReconciler.secrets.redisPassword | quote }} +{{- end -}} diff --git a/charts/diode/templates/diode-reconciler-service.yaml b/charts/diode/templates/diode-reconciler-service.yaml new file mode 100644 index 00000000..09ef4aae --- /dev/null +++ b/charts/diode/templates/diode-reconciler-service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Values.diodeReconciler.serviceName }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Values.diodeReconciler.serviceName }} +spec: + type: ClusterIP + selector: + app: {{ .Values.diodeReconciler.serviceName }} + ports: + - name: grpc + port: {{ .Values.diodeReconciler.containerPort }} + targetPort: {{ .Values.diodeReconciler.containerPort }} + protocol: TCP diff --git a/charts/diode/templates/diode-reconciler-serviceaccount.yaml b/charts/diode/templates/diode-reconciler-serviceaccount.yaml new file mode 100644 index 00000000..22e6234e --- /dev/null +++ b/charts/diode/templates/diode-reconciler-serviceaccount.yaml @@ -0,0 +1,9 @@ +{{- if .Values.diodeReconciler.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.diodeReconciler.serviceAccount.name }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ .Values.diodeReconciler.serviceName }} +{{- end -}} diff --git a/charts/diode/values.yaml b/charts/diode/values.yaml new file mode 100644 index 00000000..ff07638f --- /dev/null +++ b/charts/diode/values.yaml @@ -0,0 +1,202 @@ +# Default values for diode. + +# diode-ingester service configuration +diodeIngester: + image: + # -- image repository + repository: netboxlabs/diode-ingester + # -- image tag + tag: v0.6.0 + # -- image pull policy + pullPolicy: IfNotPresent + # -- security context for the container + securityContext: { } + # -- number of replicas + replicas: 1 + # -- service name + serviceName: diode-ingester + serviceAccount: + # -- create service account + create: true + # -- service account name + name: diode-ingester + # -- custom affinity rules for the pod + affinity: { } + # -- tolerations to use with node taints + tolerations: [ ] + # -- node selector for the pod + nodeSelector: { } + # -- additional pod annotations + podAnnotations: { } + # -- additional pod labels + podLabels: { } + # -- additional pod security context + podSecurityContext: { } + # -- port to listen on + containerPort: 8081 + # -- resources to allocate for the container + resources: { } + #resources: + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + # -- existing secret for diode-ingester + existingSecret: "" + secrets: + # -- API key for authentication between diode-ingester and diode-reconciler + ingesterToReconcilerAPIKey: "" + # -- redis password, must match the password in the redis chart or external redis + redisPassword: "" + config: + # -- diode-reconciler gRPC host + reconcilerGrpcHost: diode-reconciler + # -- diode-reconciler gRPC port + reconcilerGrpcPort: 8081 + # -- sentry DSN + sentryDsn: "" + +# diode-reconciler service configuration +diodeReconciler: + image: + # -- image repository + repository: netboxlabs/diode-reconciler + # -- image tag + tag: v0.6.0 + # -- image pull policy + pullPolicy: IfNotPresent + # -- security context for the container + securityContext: { } + # -- number of replicas + replicas: 1 + # -- service name + serviceName: diode-reconciler + serviceAccount: + # -- create service account + create: true + # -- service account name + name: diode-reconciler + # -- custom affinity rules for the pod + affinity: { } + # -- tolerations to use with node taints + tolerations: [ ] + # -- node selector for the pod + nodeSelector: { } + # -- additional pod annotations + podAnnotations: { } + # -- additional pod labels + podLabels: { } + # -- additional pod security context + podSecurityContext: { } + # -- port to listen on + containerPort: 8081 + resources: { } + #resources: + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + # -- existing secret for diode-ingester + existingSecret: "" + secrets: + # -- API key for authentication between diode and NetBox API + diodeToNetboxAPIKey: "" + # -- API key for authentication between NetBox API and diode + netboxToDiodeAPIKey: "" + # -- API key for authentication of diode ingestion requests + diodeAPIKey: "" + # -- API key for authentication between diode-ingester and diode-reconciler + ingesterToReconcilerAPIKey: "" + # -- redis password, must match the password in the redis chart or external redis + redisPassword: "" + config: + # -- NetBox plugin API base URL + netboxDiodePluginAPIBaseURL: https:///api/plugins/diode + # -- NetBox plugin skip TLS verify + netboxDiodePluginSkipTLSVerify: false + # -- logging level + loggingLevel: DEBUG + # -- migration enabled + migrationEnabled: true + # -- sentry DSN + sentryDsn: "" + +# ingress-nginx configuration +# -- ref: https://github.com/kubernetes/ingress-nginx/blob/main/charts/ingress-nginx/values.yaml +ingress-nginx: + # -- ingress-nginx enabled + enabled: true + # -- override ingress-nginx namespace + namespaceOverride: diode-ingress + # -- hostname + hostname: "" + # -- ingress class + ingressClass: nginx + controller: + # -- allow snippet annotations + allowSnippetAnnotations: true + +# cert-manager configuration +# -- ref: https://github.com/cert-manager/cert-manager/blob/master/deploy/charts/cert-manager/values.yaml +cert-manager: + # -- cert-manager enabled + enabled: false + # -- cert-manager namespace + namespace: diode-cert-manager + +# Certificate issuer configuration +certIssuer: + # -- enable certificate issuer creation + enabled: false + # -- determines whether to use Let's Encrypt production or staging environment + prod: false + # -- issuer kind (Issuer or ClusterIssuer) ref: https://cert-manager.io/docs/configuration/acme/ + kind: Issuer + # -- issuer name + name: "" + # -- email address for ACME registration + email: "" + # -- solvers for the issuer + solvers: + - http01: + ingress: + ingressClassName: nginx + +# external redis configuration +externalRedis: + # -- external redis host + host: "" + # -- external redis port + port: 6379 + +# redis configuration +# -- ref: https://github.com/bitnami/charts/blob/main/bitnami/redis/values.yaml +redis: + # -- redis enabled + enabled: true + image: + # -- redis image repository + repository: redis/redis-stack-server + # -- redis image tag + tag: latest + # -- redis image pull policy + pullPolicy: IfNotPresent + replica: + # -- number of redis replicas + replicaCount: 1 + auth: + # -- existing secret for redis password, either diodeIngester.existingSecret, diode-ingester-secret (created from + # diodeIngester.secrets) or your custom secret + existingSecret: "diode-ingester-secret" + # -- existing secret key for redis password + existingSecretPasswordKey: "REDIS_PASSWORD" + # -- redis configuration + commonConfiguration: |- + appendonly yes + save 60 1 + loadmodule /opt/redis-stack/lib/rejson.so + loadmodule /opt/redis-stack/lib/redisearch.so From 5a8cf580e65f7f9b3b99a77a961a3ac17c7f3cda Mon Sep 17 00:00:00 2001 From: Michal Fiedorowicz Date: Tue, 15 Oct 2024 21:44:01 +0100 Subject: [PATCH 4/4] gha(server-release): trigger on diode-server/** changes only (#188) Signed-off-by: Michal Fiedorowicz --- .github/workflows/go-test.yaml | 2 ++ .github/workflows/server-release.yaml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/.github/workflows/go-test.yaml b/.github/workflows/go-test.yaml index 36e72a30..81f3adf0 100644 --- a/.github/workflows/go-test.yaml +++ b/.github/workflows/go-test.yaml @@ -5,9 +5,11 @@ on: - "!release" paths: - "diode-server/**" + - "!diode-server/docker/**" pull_request: paths: - "diode-server/**" + - "!diode-server/docker/**" concurrency: group: ${{ github.workflow }} diff --git a/.github/workflows/server-release.yaml b/.github/workflows/server-release.yaml index dc4936dc..6a16ab62 100644 --- a/.github/workflows/server-release.yaml +++ b/.github/workflows/server-release.yaml @@ -3,6 +3,9 @@ on: workflow_dispatch: push: branches: [ release ] + paths: + - "diode-server/**" + - "!diode-server/docker/**" concurrency: group: ${{ github.workflow }}