Add tool to update CRDs in a Kubernetes job without kubectl

Signed-off-by: Tobias Giese <[email protected]>

cmd/apply-crds/README.md

@@ -0,0 +1,61 @@
+# CRD Apply Tool
+
+This tool is designed to help deploy and manage Custom Resource Definitions (CRDs) in a Kubernetes cluster.
+It applies all CRDs found in specified directories, providing a solution to some of the limitations of Helm when it comes to managing CRDs.
+
+## Motivation
+
+While Helm is commonly used for managing Kubernetes resources, it has certain restrictions with CRDs:
+
+- CRDs placed in Helm's top-level `crds/` directory are not updated on upgrades or rollbacks.
+- Placing CRDs in Helm’s `templates/` directory is not entirely safe, as deletions and upgrades of CRDs are not always handled properly.
+
+This tool offers a more reliable way to apply CRDs, ensuring they are created or updated as needed.
+
+## Features
+
+- **Apply CRDs from multiple directories**: Allows specifying multiple directories containing CRD YAML manifests.
+- **Recursive directory search**: Walks through each specified directory to find and apply all YAML files.
+- **Safe update mechanism**: Checks if a CRD already exists; if so, it updates it with the latest version.
+- **Handles multiple YAML documents**: Supports files containing multiple CRD documents separated by YAML document delimiters.
+
+## Usage
+
+Compile and run the tool by providing the `-crds-dir` flag with paths to the directories containing the CRD YAML files:
+
+```bash
+go build -o crd-apply-tool
+./crd-apply-tool -crds-dir /path/to/crds1 -crds-dir /path/to/crds2
+```
+
+In a Helm pre-install hook it can look like:
+
+```yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: upgrade-crd
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade
+    "helm.sh/hook-weight": "1"
+    "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation
+spec:
+  template:
+    metadata:
+      name: upgrade-crd
+    spec:
+      containers:
+        - name: upgrade-crd
+          image: path-to-your/crd-apply-image
+          imagePullPolicy: IfNotPresent
+          command:
+            - /apply-crds
+          args:
+              - --crds-dir=/crds/operator
+```
+
+> Note: the image must contain all your CRDs in e.g. the `/crds/operator` directory.
+
+## Flags
+
+- `-crds-dir` (required): Specifies a directory path that contains the CRD manifests in YAML format. This flag can be provided multiple times to apply CRDs from multiple directories.

cmd/apply-crds/main.go

@@ -0,0 +1,181 @@
+/*
+Copyright 2024 NVIDIA CORPORATION & AFFILIATES
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/client-go/util/retry"
+	ctrl "sigs.k8s.io/controller-runtime"
+)
+
+type StringList []string
+
+func (s *StringList) String() string {
+	return strings.Join(*s, ", ")
+}
+
+func (s *StringList) Set(value string) error {
+	*s = append(*s, value)
+	return nil
+}
+
+var (
+	crdsDir StringList
+)
+
+func initFlags() {
+	flag.Var(&crdsDir, "crds-dir", "Path to the directory containing the CRD manifests")
+	flag.Parse()
+
+	if len(crdsDir) == 0 {
+		log.Fatalf("CRDs directory is required")
+	}
+
+	for _, crdDir := range crdsDir {
+		if _, err := os.Stat(crdDir); os.IsNotExist(err) {
+			log.Fatalf("CRDs directory %s does not exist", crdsDir)
+		}
+	}
+}
+
+func main() {
+	Run()
+}
+
+func Run() {
+	ctx := context.Background()
+
+	initFlags()
+
+	config, err := ctrl.GetConfig()
+	if err != nil {
+		log.Fatalf("Failed to get Kubernetes config: %v", err)
+	}
+
+	client, err := clientset.NewForConfig(config)
+	if err != nil {
+		log.Fatalf("Failed to create API extensions client: %v", err)
+	}
+
+	if err := walkCrdsDir(ctx, client); err != nil {
+		log.Fatalf("Failed to apply CRDs: %v", err)
+	}
+}
+
+// walkCrdsDir walks the CRDs directory and applies each YAML file.
+func walkCrdsDir(ctx context.Context, client *clientset.Clientset) error {
+	for _, crdDir := range crdsDir {
+		// Walk the directory recursively and apply each YAML file.
+		err := filepath.Walk(crdDir, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			if info.IsDir() || filepath.Ext(path) != ".yaml" {
+				return nil
+			}
+
+			log.Printf("Apply CRDs from file: %s", path)
+			if err := applyCRDsFromFile(ctx, client, path); err != nil {
+				return fmt.Errorf("apply CRD %s: %v", path, err)
+			}
+			return nil
+		})
+		if err != nil {
+			return fmt.Errorf("walk the path %s: %v", crdsDir, err)
+		}
+	}
+	return nil
+}
+
+// applyCRDsFromFile reads a YAML file, splits it into documents, and applies each CRD to the cluster.
+func applyCRDsFromFile(ctx context.Context, client *clientset.Clientset, filePath string) error {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return fmt.Errorf("open file %q: %v", filePath, err)
+	}
+	defer file.Close()
+
+	// Create a decoder that reads multiple YAML documents.
+	decoder := yaml.NewYAMLOrJSONDecoder(file, 4096)
+	var crdsToApply []*apiextensionsv1.CustomResourceDefinition
+	for {
+		crd := &apiextensionsv1.CustomResourceDefinition{}
+		if err := decoder.Decode(crd); err != nil {
+			if err == io.EOF {
+				break
+			}
+			return fmt.Errorf("decode YAML: %v", err)
+		}
+		if crd.GetObjectKind().GroupVersionKind().Kind != "CustomResourceDefinition" {
+			log.Printf("Skipping non-CRD object %s", crd.GetName())
+			continue
+		}
+		crdsToApply = append(crdsToApply, crd)
+	}
+
+	// Apply each CRD separately.
+	for _, crd := range crdsToApply {
+		err := wait.ExponentialBackoffWithContext(ctx, retry.DefaultBackoff, func(context.Context) (bool, error) {
+			if err := applyCRD(ctx, client, crd); err != nil {
+				return false, nil
+			}
+			return true, nil
+		})
+		if err != nil {
+			return fmt.Errorf("apply CRD %s: %v", crd.Name, err)
+		}
+	}
+	return nil
+}
+
+// applyCRD creates or updates the CRD.
+func applyCRD(ctx context.Context, client *clientset.Clientset, crd *apiextensionsv1.CustomResourceDefinition) error {
+	crdClient := client.ApiextensionsV1().CustomResourceDefinitions()
+
+	// Check if CRD already exists in cluster and create if not found.
+	curCRD, err := crdClient.Get(ctx, crd.Name, metav1.GetOptions{})
+	if apierrors.IsNotFound(err) {
+		log.Printf("Create CRD %s", crd.Name)
+		_, err = crdClient.Create(ctx, crd, metav1.CreateOptions{})
+		if err != nil {
+			return fmt.Errorf("create CRD %s: %v", crd.Name, err)
+		}
+	} else {
+		log.Printf("Update CRD %s", crd.Name)
+		// Set resource version to update an existing CRD.
+		crd.SetResourceVersion(curCRD.GetResourceVersion())
+		_, err = crdClient.Update(ctx, crd, metav1.UpdateOptions{})
+		if err != nil {
+			return fmt.Errorf("update CRD %s: %v", crd.Name, err)
+		}
+	}
+	return nil
+}

cmd/apply-crds/main_test.go

@@ -0,0 +1,80 @@
+/*
+Copyright 2024 NVIDIA CORPORATION & AFFILIATES
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("CRD Application", func() {
+	var (
+		ctx context.Context
+	)
+
+	BeforeEach(func() {
+		ctx = context.Background()
+	})
+
+	AfterEach(func() {
+		Expect(testClient.ApiextensionsV1().CustomResourceDefinitions().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{})).NotTo(HaveOccurred())
+	})
+
+	Describe("applyCRDsFromFile", func() {
+		It("should apply CRDs multiple times from a valid YAML file", func() {
+			By("applying CRDs")
+			Expect(applyCRDsFromFile(ctx, testClient, "test-files/test-crds.yaml")).To(Succeed())
+			Expect(applyCRDsFromFile(ctx, testClient, "test-files/test-crds.yaml")).To(Succeed())
+			Expect(applyCRDsFromFile(ctx, testClient, "test-files/test-crds.yaml")).To(Succeed())
+			Expect(applyCRDsFromFile(ctx, testClient, "test-files/test-crds.yaml")).To(Succeed())
+
+			By("verifying CRDs are applied")
+			crds, err := testClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(crds.Items).To(HaveLen(2))
+		})
+
+		It("should update CRDs", func() {
+			By("applying CRDs")
+			Expect(applyCRDsFromFile(ctx, testClient, "test-files/test-crds.yaml")).To(Succeed())
+
+			By("verifying CRDs do not have spec.foobar")
+			for _, crdName := range []string{"bars.example.com", "foos.example.com"} {
+				crd, err := testClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crdName, metav1.GetOptions{})
+				Expect(err).NotTo(HaveOccurred())
+				props := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties
+				Expect(props).To(HaveKey("spec"))
+				Expect(props["spec"].Properties).NotTo(HaveKey("foobar"))
+			}
+
+			By("updating CRDs")
+			Expect(applyCRDsFromFile(ctx, testClient, "test-files/updated-test-crds.yaml")).To(Succeed())
+
+			By("verifying CRDs are updated")
+			for _, crdName := range []string{"bars.example.com", "foos.example.com"} {
+				crd, err := testClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crdName, metav1.GetOptions{})
+				Expect(err).NotTo(HaveOccurred())
+				props := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties
+				Expect(props["spec"].Properties).To(HaveKey("foobar"))
+			}
+		})
+	})
+})

cmd/apply-crds/suite_test.go

@@ -0,0 +1,65 @@
+/*
+Copyright 2024 NVIDIA CORPORATION & AFFILIATES
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"testing"
+
+	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var (
+	testClient *clientset.Clientset
+	testEnv    *envtest.Environment
+)
+
+func TestApplyCrds(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "ApplyCrds Suite")
+}
+
+var _ = BeforeSuite(func() {
+	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
+
+	By("bootstrapping test environment")
+	testEnv = &envtest.Environment{}
+
+	cfg, err := testEnv.Start()
+	Expect(err).NotTo(HaveOccurred())
+	Expect(cfg).NotTo(BeNil())
+
+	// create clientset with scheme
+	testClient, err = clientset.NewForConfig(cfg)
+	Expect(err).NotTo(HaveOccurred())
+	Expect(testClient).NotTo(BeNil())
+
+	go func() {
+		defer GinkgoRecover()
+	}()
+})
+
+var _ = AfterSuite(func() {
+	By("tearing down the test environment")
+	err := testEnv.Stop()
+	Expect(err).NotTo(HaveOccurred())
+})