From 71022eca8258c67a8f2d804913c1a0c2e1c5386d Mon Sep 17 00:00:00 2001 From: Tobias Giese Date: Tue, 5 Nov 2024 17:49:48 +0100 Subject: [PATCH] Add tool to update CRDs in a Kubernetes job without kubectl --- cmd/apply-crds/README.md | 61 ++++++ cmd/apply-crds/main.go | 181 ++++++++++++++++++ cmd/apply-crds/main_test.go | 80 ++++++++ cmd/apply-crds/suite_test.go | 65 +++++++ cmd/apply-crds/test-files/test-crds.yaml | 47 +++++ .../test-files/updated-test-crds.yaml | 51 +++++ go.mod | 2 +- 7 files changed, 486 insertions(+), 1 deletion(-) create mode 100644 cmd/apply-crds/README.md create mode 100644 cmd/apply-crds/main.go create mode 100644 cmd/apply-crds/main_test.go create mode 100644 cmd/apply-crds/suite_test.go create mode 100644 cmd/apply-crds/test-files/test-crds.yaml create mode 100644 cmd/apply-crds/test-files/updated-test-crds.yaml diff --git a/cmd/apply-crds/README.md b/cmd/apply-crds/README.md new file mode 100644 index 0000000..15aa4bd --- /dev/null +++ b/cmd/apply-crds/README.md @@ -0,0 +1,61 @@ +# CRD Apply Tool + +This tool is designed to help deploy and manage Custom Resource Definitions (CRDs) in a Kubernetes cluster. +It applies all CRDs found in specified directories, providing a solution to some of the limitations of Helm when it comes to managing CRDs. + +## Motivation + +While Helm is commonly used for managing Kubernetes resources, it has certain restrictions with CRDs: + +- CRDs placed in Helm's top-level `crds/` directory are not updated on upgrades or rollbacks. +- Placing CRDs in Helm’s `templates/` directory is not entirely safe, as deletions and upgrades of CRDs are not always handled properly. + +This tool offers a more reliable way to apply CRDs, ensuring they are created or updated as needed. + +## Features + +- **Apply CRDs from multiple directories**: Allows specifying multiple directories containing CRD YAML manifests. +- **Recursive directory search**: Walks through each specified directory to find and apply all YAML files. +- **Safe update mechanism**: Checks if a CRD already exists; if so, it updates it with the latest version. +- **Handles multiple YAML documents**: Supports files containing multiple CRD documents separated by YAML document delimiters. + +## Usage + +Compile and run the tool by providing the `-crds-dir` flag with paths to the directories containing the CRD YAML files: + +```bash +go build -o crd-apply-tool +./crd-apply-tool -crds-dir /path/to/crds1 -crds-dir /path/to/crds2 +``` + +In a Helm pre-install hook it can look like: + +```yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: upgrade-crd + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation +spec: + template: + metadata: + name: upgrade-crd + spec: + containers: + - name: upgrade-crd + image: path-to-your/crd-apply-image + imagePullPolicy: IfNotPresent + command: + - /apply-crds + args: + - --crds-dir=/crds/operator +``` + +> Note: the image must contain all your CRDs in e.g. the `/crds/operator` directory. + +## Flags + +- `-crds-dir` (required): Specifies a directory path that contains the CRD manifests in YAML format. This flag can be provided multiple times to apply CRDs from multiple directories. diff --git a/cmd/apply-crds/main.go b/cmd/apply-crds/main.go new file mode 100644 index 0000000..257b714 --- /dev/null +++ b/cmd/apply-crds/main.go @@ -0,0 +1,181 @@ +/* +Copyright 2024 NVIDIA CORPORATION & AFFILIATES + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/util/retry" + ctrl "sigs.k8s.io/controller-runtime" +) + +type StringList []string + +func (s *StringList) String() string { + return strings.Join(*s, ", ") +} + +func (s *StringList) Set(value string) error { + *s = append(*s, value) + return nil +} + +var ( + crdsDir StringList +) + +func initFlags() { + flag.Var(&crdsDir, "crds-dir", "Path to the directory containing the CRD manifests") + flag.Parse() + + if len(crdsDir) == 0 { + log.Fatalf("CRDs directory is required") + } + + for _, crdDir := range crdsDir { + if _, err := os.Stat(crdDir); os.IsNotExist(err) { + log.Fatalf("CRDs directory %s does not exist", crdsDir) + } + } +} + +func main() { + Run() +} + +func Run() { + ctx := context.Background() + + initFlags() + + config, err := ctrl.GetConfig() + if err != nil { + log.Fatalf("Failed to get Kubernetes config: %v", err) + } + + client, err := clientset.NewForConfig(config) + if err != nil { + log.Fatalf("Failed to create API extensions client: %v", err) + } + + if err := walkCrdsDir(ctx, client); err != nil { + log.Fatalf("Failed to apply CRDs: %v", err) + } +} + +// walkCrdsDir walks the CRDs directory and applies each YAML file. +func walkCrdsDir(ctx context.Context, client *clientset.Clientset) error { + for _, crdDir := range crdsDir { + // Walk the directory recursively and apply each YAML file. + err := filepath.Walk(crdDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || filepath.Ext(path) != ".yaml" { + return nil + } + + log.Printf("Apply CRDs from file: %s", path) + if err := applyCRDsFromFile(ctx, client, path); err != nil { + return fmt.Errorf("apply CRD %s: %v", path, err) + } + return nil + }) + if err != nil { + return fmt.Errorf("walk the path %s: %v", crdsDir, err) + } + } + return nil +} + +// applyCRDsFromFile reads a YAML file, splits it into documents, and applies each CRD to the cluster. +func applyCRDsFromFile(ctx context.Context, client *clientset.Clientset, filePath string) error { + file, err := os.Open(filePath) + if err != nil { + return fmt.Errorf("open file %q: %v", filePath, err) + } + defer file.Close() + + // Create a decoder that reads multiple YAML documents. + decoder := yaml.NewYAMLOrJSONDecoder(file, 4096) + var crdsToApply []*apiextensionsv1.CustomResourceDefinition + for { + crd := &apiextensionsv1.CustomResourceDefinition{} + if err := decoder.Decode(crd); err != nil { + if err == io.EOF { + break + } + return fmt.Errorf("decode YAML: %v", err) + } + if crd.GetObjectKind().GroupVersionKind().Kind != "CustomResourceDefinition" { + log.Printf("Skipping non-CRD object %s", crd.GetName()) + continue + } + crdsToApply = append(crdsToApply, crd) + } + + // Apply each CRD separately. + for _, crd := range crdsToApply { + err := wait.ExponentialBackoffWithContext(ctx, retry.DefaultBackoff, func(context.Context) (bool, error) { + if err := applyCRD(ctx, client, crd); err != nil { + return false, nil + } + return true, nil + }) + if err != nil { + return fmt.Errorf("apply CRD %s: %v", crd.Name, err) + } + } + return nil +} + +// applyCRD creates or updates the CRD. +func applyCRD(ctx context.Context, client *clientset.Clientset, crd *apiextensionsv1.CustomResourceDefinition) error { + crdClient := client.ApiextensionsV1().CustomResourceDefinitions() + + // Check if CRD already exists in cluster and create if not found. + curCRD, err := crdClient.Get(ctx, crd.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + log.Printf("Create CRD %s", crd.Name) + _, err = crdClient.Create(ctx, crd, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("create CRD %s: %v", crd.Name, err) + } + } else { + log.Printf("Update CRD %s", crd.Name) + // Set resource version to update an existing CRD. + crd.SetResourceVersion(curCRD.GetResourceVersion()) + _, err = crdClient.Update(ctx, crd, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("update CRD %s: %v", crd.Name, err) + } + } + return nil +} diff --git a/cmd/apply-crds/main_test.go b/cmd/apply-crds/main_test.go new file mode 100644 index 0000000..3700a14 --- /dev/null +++ b/cmd/apply-crds/main_test.go @@ -0,0 +1,80 @@ +/* +Copyright 2024 NVIDIA CORPORATION & AFFILIATES + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("CRD Application", func() { + var ( + ctx context.Context + ) + + BeforeEach(func() { + ctx = context.Background() + }) + + AfterEach(func() { + Expect(testClient.ApiextensionsV1().CustomResourceDefinitions().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{})).NotTo(HaveOccurred()) + }) + + Describe("applyCRDsFromFile", func() { + It("should apply CRDs multiple times from a valid YAML file", func() { + By("applying CRDs") + Expect(applyCRDsFromFile(ctx, testClient, "test-files/test-crds.yaml")).To(Succeed()) + Expect(applyCRDsFromFile(ctx, testClient, "test-files/test-crds.yaml")).To(Succeed()) + Expect(applyCRDsFromFile(ctx, testClient, "test-files/test-crds.yaml")).To(Succeed()) + Expect(applyCRDsFromFile(ctx, testClient, "test-files/test-crds.yaml")).To(Succeed()) + + By("verifying CRDs are applied") + crds, err := testClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(crds.Items).To(HaveLen(2)) + }) + + It("should update CRDs", func() { + By("applying CRDs") + Expect(applyCRDsFromFile(ctx, testClient, "test-files/test-crds.yaml")).To(Succeed()) + + By("verifying CRDs do not have spec.foobar") + for _, crdName := range []string{"bars.example.com", "foos.example.com"} { + crd, err := testClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crdName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + props := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties + Expect(props).To(HaveKey("spec")) + Expect(props["spec"].Properties).NotTo(HaveKey("foobar")) + } + + By("updating CRDs") + Expect(applyCRDsFromFile(ctx, testClient, "test-files/updated-test-crds.yaml")).To(Succeed()) + + By("verifying CRDs are updated") + for _, crdName := range []string{"bars.example.com", "foos.example.com"} { + crd, err := testClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crdName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + props := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties + Expect(props["spec"].Properties).To(HaveKey("foobar")) + } + }) + }) +}) diff --git a/cmd/apply-crds/suite_test.go b/cmd/apply-crds/suite_test.go new file mode 100644 index 0000000..78729d5 --- /dev/null +++ b/cmd/apply-crds/suite_test.go @@ -0,0 +1,65 @@ +/* +Copyright 2024 NVIDIA CORPORATION & AFFILIATES + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" + + "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var ( + testClient *clientset.Clientset + testEnv *envtest.Environment +) + +func TestApplyCrds(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "ApplyCrds Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{} + + cfg, err := testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + // create clientset with scheme + testClient, err = clientset.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) + Expect(testClient).NotTo(BeNil()) + + go func() { + defer GinkgoRecover() + }() +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/cmd/apply-crds/test-files/test-crds.yaml b/cmd/apply-crds/test-files/test-crds.yaml new file mode 100644 index 0000000..67a97a7 --- /dev/null +++ b/cmd/apply-crds/test-files/test-crds.yaml @@ -0,0 +1,47 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: foos.example.com +spec: + group: example.com + names: + kind: Foo + listKind: FooList + singular: foo + plural: foos + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object +--- +# non CRD yamls should not be handled and skipped +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bars.example.com +spec: + group: example.com + names: + kind: Bar + listKind: BarList + singular: bar + plural: bars + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object diff --git a/cmd/apply-crds/test-files/updated-test-crds.yaml b/cmd/apply-crds/test-files/updated-test-crds.yaml new file mode 100644 index 0000000..db2829c --- /dev/null +++ b/cmd/apply-crds/test-files/updated-test-crds.yaml @@ -0,0 +1,51 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: foos.example.com +spec: + group: example.com + names: + kind: Foo + listKind: FooList + singular: foo + plural: foos + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + foobar: + type: string +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bars.example.com +spec: + group: example.com + names: + kind: Bar + listKind: BarList + singular: bar + plural: bars + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + foobar: + type: string diff --git a/go.mod b/go.mod index 649616f..4549960 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/onsi/gomega v1.34.2 github.com/stretchr/testify v1.9.0 k8s.io/api v0.31.2 + k8s.io/apiextensions-apiserver v0.31.0 k8s.io/apimachinery v0.31.2 k8s.io/client-go v0.31.2 k8s.io/kubectl v0.31.2 @@ -91,7 +92,6 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiextensions-apiserver v0.31.0 // indirect k8s.io/cli-runtime v0.31.2 // indirect k8s.io/component-base v0.31.2 // indirect k8s.io/klog/v2 v2.130.1 // indirect