feat(localenv): local development tooling for PluginDefinition (#579)

* (chore): adds .idea to ignore list

* (chore): adds kustomization for plugindefintions

ability to apply all plugindefinitions with kustomize

* (chore): fix from relative path to content root

* (chore): single kustomize at root

* (chore): adds shell script to generate kustomization

shell script tooling to generate plugindefinitions for local development

* (chore): adds setup dependencies and generate kustomization hook

* (chore): adds bin dir to ignore

* (chore): adds license to script

* Automatic application of license header

* Update Makefile

Co-authored-by: David Gogl <[email protected]>

* chore(release): bump version to 0.1.9 and update service URL format

Signed-off-by: IB Akshay <[email protected]>

* fix(Makefile): update generate-kustomization dependencies to include kustomize

Signed-off-by: IB Akshay <[email protected]>

* (chore): rename to local-plugin-definitions

* (chore): rename cmd to local plugin definition

Co-authored-by: Akshay Iyyadurai Balasundaram <[email protected]>

* (docs): setup guide

---------

Signed-off-by: IB Akshay <[email protected]>
Co-authored-by: License Bot <[email protected]>
Co-authored-by: David Gogl <[email protected]>
Co-authored-by: IB Akshay <[email protected]>

.gitignore

@@ -1,7 +1,12 @@
+# ignore local dependencies
+bin/
+
 # act artifacts needed for testing workflows
 .secrets
 .env
 act_*.json
 
 .DS_Store
 
+# ignore IntelliJ IDEA files
+.idea

Makefile

@@ -1,3 +1,56 @@
+# Detect OS (Linux/macOS)
+OS := $(shell uname -s | tr '[:upper:]' '[:lower:]')
+
+# Detect ARCH (AMD64 or ARM64)
+UNAME_P := $(shell uname -p)
+ARCH :=
+ifeq ($(UNAME_P),x86_64)
+	ARCH =amd64
+endif
+ifneq ($(filter arm%,$(UNAME_P)),)
+	ARCH = arm64
+endif
+
+
+## tools versions
+KUSTOMIZE_VERSION ?= 5.5.0 # Update to the latest version as needed
+YQ_VERSION ?= v4.45.1  # Update to the latest version as needed
+
+## Location to install dependencies to
+LOCALBIN ?= $(shell pwd)/bin
+$(LOCALBIN):
+	mkdir -p $(LOCALBIN)
+KUSTOMIZE ?= $(LOCALBIN)/kustomize
+YQ ?= $(LOCALBIN)/yq
+
+KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"
+YQ_INSTALL_SCRIPT ?= https://github.com/mikefarah/yq/releases/latest/download/yq_$(OS)_$(ARCH)
+
+## Download `kustomize` locally if necessary
+.PHONY: kustomize
+kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. If wrong version is installed, it will be removed before downloading.
+$(KUSTOMIZE): $(LOCALBIN)
+	@if test -x $(LOCALBIN)/kustomize && ! $(LOCALBIN)/kustomize version | grep -q $(KUSTOMIZE_VERSION); then \
+		echo "$(LOCALBIN)/kustomize version is not expected $(KUSTOMIZE_VERSION). Removing it before installing."; \
+		rm -rf $(LOCALBIN)/kustomize; \
+	fi
+	test -s $(LOCALBIN)/kustomize || curl -s $(KUSTOMIZE_INSTALL_SCRIPT) | bash -s -- $(subst v,,$(KUSTOMIZE_VERSION)) $(LOCALBIN)
+
+## Download `yq` locally if necessary
+.PHONY: yq
+yq: $(YQ)
+$(YQ): $(LOCALBIN)
+	@if test -x $(LOCALBIN)/yq && ! $(LOCALBIN)/yq --version | grep -q $(YQ_VERSION); then \
+		echo "$(LOCALBIN)/yq version is not expected $(YQ_VERSION). Removing it before installing."; \
+		rm -rf $(LOCALBIN)/yq; \
+	fi
+	test -s $(LOCALBIN)/yq || curl -L $(YQ_INSTALL_SCRIPT) -o $(LOCALBIN)/yq && chmod +x $(LOCALBIN)/yq
+
+
 .PHONY: generate-documentation
 generate-documentation:
 	hack/generate-catalog-markdown
+
+.PHONY: local-plugin-definitions
+local-plugin-definitions: kustomize yq
+	hack/local-plugin-definitions

README.md

@@ -45,6 +45,10 @@ The *README.md* describes the overall Plugin functionality and highlights config
 the *plugin.yaml* specifies the front- and backend of a Plugin and its configuration options using the Greenhouse Plugin CRD.
 An optional *charts* directory defines the backend for a Plugin as a Helm chart and the *ui* directory the frontend part which can be seen in the Greenhouse UI.
 
+### Local development
+
+Please check [Local development](./docs/local-development.md) for details on how to develop and test Greenhouse extensions locally.
+
 ### Walkthrough
 
 See the [walkthrough guide](./docs/extension.md) and the [local development environment](./dev-env/README.md) for details on how to create new Greenhouse extensions.

docs/local-development.md

@@ -0,0 +1,66 @@
+## Generate local plugin definitions
+
+In order to simplify local development and testing of greenhouse extensions the script in `hack/local-plugin-definitions` 
+can be used to generate local `PluginDefinitions`. 
+
+### Prerequisites
+
+- Please follow the [Setting up a local development environment](https://github.com/cloudoperators/greenhouse/blob/main/dev-env/localenv/README.md#setting-up-a-local-development-environment) guide to setup the local environment
+- Follow the instructions in [Test Plugin / Greenhouse Extension charts locally](https://github.com/cloudoperators/greenhouse/blob/main/dev-env/localenv/README.md#test-plugin--greenhouse-extension-charts-locally) to setup a local extension testing environment
+
+### Usage
+
+```bash
+  make local-plugin-definitions
+```
+
+1. The script will extract each `plugindefintion.yaml` from all the extensions' directories
+2. The script will modify certain fields in the `plugindefintion.yaml` to make it compatible with the local setup
+3. All the modified `plugindefintion.yaml` will be saved in the `bin/.generated` directory
+
+** Example **
+
+Original `plugindefintion.yaml` of `perses` extension
+
+```yaml
+apiVersion: greenhouse.sap/v1alpha1
+kind: PluginDefinition
+metadata:
+  name: perses
+spec:
+  version: 0.3.1
+  displayName: Perses
+  description: "Perses is a dashboard tooling to visualize metrics and traces produced by observability tools such as Prometheus/Thanos/Jaeger"
+  docMarkDownUrl: https://raw.githubusercontent.com/cloudoperators/greenhouse-extensions/main/perses/README.md
+  icon: https://raw.githubusercontent.com/cloudoperators/greenhouse-extensions/main/perses/logo.png
+  helmChart:
+    name: perses
+    repository: oci://ghcr.io/cloudoperators/greenhouse-extensions/charts
+    version: '0.9.1'
+  options: [...]
+```
+
+Generated `plugindefintion.yaml` of `perses` extension
+
+```yaml
+apiVersion: greenhouse.sap/v1alpha1
+kind: PluginDefinition
+metadata:
+  name: perses
+spec:
+  description: Perses is a dashboard tooling to visualize metrics and traces produced by observability tools such as Prometheus/Thanos/Jaeger
+  displayName: Perses
+  docMarkDownUrl: https://raw.githubusercontent.com/cloudoperators/greenhouse-extensions/main/perses/README.md
+  helmChart:
+    name: local/plugins/perses
+    repository: ""
+    version: ""
+  icon: https://raw.githubusercontent.com/cloudoperators/greenhouse-extensions/main/perses/logo.png
+  options: [...]
+```
+
+> [!NOTE]
+> `spec.helmChart.name` references only to extension dir name
+> before applying it to the local cluster you would need to append the remaining path to baseDir that contains the `Chart.yaml` file
+> e.g. `local/plugins/perses` -> `local/plugins/perses/charts`
+

hack/local-plugin-definitions

@@ -0,0 +1,132 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
+# SPDX-License-Identifier: Apache-2.0
+
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Paths
+YQ="./bin/yq"  # Use yq from bin folder
+KUSTOMIZE="./bin/kustomize"  # Use kustomize from bin folder
+GENERATED_DIR="bin/.generated"
+BASE_KUSTOMIZATION="${GENERATED_DIR}/kustomization.yaml"
+SOURCE_KUSTOMIZATION="kustomization.yaml"
+
+# Ensure the output directory exists
+mkdir -p $GENERATED_DIR
+
+# extract plugin names from root kustomization.yaml
+extract_plugin_names() {
+  $YQ -r '.resources[]' "$SOURCE_KUSTOMIZATION" | sed 's#/plugindefinition.yaml$##'
+}
+
+# initialize kustomization.yaml to generate the local catalog
+# note: _all.yaml is a temporary file that will be removed after the local catalog is generated
+initialize_kustomization() {
+  cat <<EOF > "$BASE_KUSTOMIZATION"
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - _all.yaml
+patches: []
+EOF
+}
+
+# generate YAML block for patches
+generate_patch_file() {
+  local plugin="$1"
+  local patch_file
+  patch_file=$(mktemp)
+
+  cat <<EOF > "$patch_file"
+|-
+  - op: replace
+    path: /spec/helmChart/version
+    value: ""
+  - op: replace
+    path: /spec/helmChart/repository
+    value: ""
+  - op: replace
+    path: /spec/helmChart/name
+    value: "local/plugins/$plugin"
+EOF
+
+  echo "$patch_file"
+}
+
+# add patches in kustomization.yaml to replace the helmChart values with local values
+add_patches() {
+  local plugin_names
+  plugin_names=$(extract_plugin_names)
+
+  for plugin in $plugin_names; do
+    patch_file=$(generate_patch_file "$plugin")
+
+    # Use `load()` to correctly insert the YAML patch block
+    $YQ -i ".patches += [{
+      \"target\": {
+        \"kind\": \"PluginDefinition\",
+        \"name\": \"$plugin\"
+      },
+      \"patch\": load(\"$patch_file\")
+    }]" "$BASE_KUSTOMIZATION"
+
+    rm -f "$patch_file"
+  done
+}
+
+# build the local catalog using kustomize and extract each resource into its own file
+build_local_catalog() {
+  # build all resources using kustomize
+  $KUSTOMIZE build . > $GENERATED_DIR/_all.yaml
+  # build all in one local catalog from the initialized bin/.generated/kustomization.yaml
+  $KUSTOMIZE build $GENERATED_DIR > $GENERATED_DIR/_local_catalog.yaml
+
+  # extract all metadata names from _local_catalog.yaml
+  $YQ -N e 'select(.metadata.name) | .metadata.name' $GENERATED_DIR/_local_catalog.yaml | while read -r name; do
+    # extract and write each specific resource using its metadata.name
+    $YQ -N e "select(.metadata.name == \"$name\")" $GENERATED_DIR/_local_catalog.yaml > $GENERATED_DIR/"$name.yaml"
+  done
+
+  # clean up temporary files
+  rm -f $GENERATED_DIR/_all.yaml $GENERATED_DIR/_local_catalog.yaml
+}
+
+# prepare a local kustomization.yaml to easily apply the generated catalog
+prepare_local_kustomization() {
+  cat <<EOF > "$BASE_KUSTOMIZATION"
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources: []
+EOF
+}
+
+# extract name of plugindefinitions from root kustomization.yaml to populate the resources array with "<plugin-name>.yaml"
+# kubectl apply -k bin/.generated
+add_resources() {
+  local plugin_names
+  plugin_names=$(extract_plugin_names)
+
+  for plugin in $plugin_names; do
+    $YQ -i ".resources += [\"$plugin.yaml\"]" "$BASE_KUSTOMIZATION"
+  done
+}
+
+# Main Execution (Ordered execution)
+initialize_kustomization #1
+add_patches #2
+build_local_catalog #3
+prepare_local_kustomization #4
+add_resources #5
+
+echo "✅ Successfully generated $BASE_KUSTOMIZATION!"

kustomization.yaml

@@ -0,0 +1,18 @@
+# SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
+# SPDX-License-Identifier: Apache-2.0
+
+resources:
+  - alerts/plugindefinition.yaml
+  - cert-manager/plugindefinition.yaml
+  - exposed-services/plugindefinition.yaml
+  - external-dns/plugindefinition.yaml
+  - github-guard/plugindefinition.yaml
+  - ingress-nginx/plugindefinition.yaml
+  - kube-monitoring/plugindefinition.yaml
+  - openbao/plugindefinition.yaml
+  - opentelemetry/plugindefinition.yaml
+  - perses/plugindefinition.yaml
+  - plutono/plugindefinition.yaml
+  - service-proxy/plugindefinition.yaml
+  - teams2slack/plugindefinition.yaml
+  - thanos/plugindefinition.yaml