From 7b3aabf36d2cded20fbc534a8da1327a5793ac9b Mon Sep 17 00:00:00 2001 From: Harsha Narayana Date: Mon, 2 May 2022 18:38:10 +0530 Subject: [PATCH] GIT-109: enable CRD setup helper to ease the testing of operators --- examples/crds/README.md | 25 ++++++ examples/crds/envtest_test.go | 65 ++++++++++++++++ examples/crds/main_test.go | 53 +++++++++++++ .../crds/stable.example.com.crontabs.yaml | 30 ++++++++ examples/crds/testdata/crontabs/register.go | 43 +++++++++++ examples/crds/testdata/crontabs/type.go | 76 +++++++++++++++++++ examples/crds/testdata/crs/sample.yaml | 7 ++ klient/decoder/decoder.go | 15 ++++ pkg/envfuncs/resource_funcs.go | 51 +++++++++++++ 9 files changed, 365 insertions(+) create mode 100644 examples/crds/README.md create mode 100644 examples/crds/envtest_test.go create mode 100644 examples/crds/main_test.go create mode 100644 examples/crds/testdata/crds/stable.example.com.crontabs.yaml create mode 100644 examples/crds/testdata/crontabs/register.go create mode 100644 examples/crds/testdata/crontabs/type.go create mode 100644 examples/crds/testdata/crs/sample.yaml create mode 100644 pkg/envfuncs/resource_funcs.go diff --git a/examples/crds/README.md b/examples/crds/README.md new file mode 100644 index 00000000..86b03d87 --- /dev/null +++ b/examples/crds/README.md @@ -0,0 +1,25 @@ +# Kubernetes Custom Resource Integration Test + +While developing a kubernetes operator for your custom use cases, it is very common practice that you end up creating your own custom resources. + +This example shows how to leverage the helper functions provided by the Framework itself to setup +the CRD resources using the `decoder` package against your test cluster before starting the actual test workflow. + +## How does this work ? + +1. You can leverage the framework's `env.Func` type helper for setting up the CRDs and tearing them down after the tests +2. Register the CRD scheme with the `resources.Resources` to leverage the helpers for interacting with Custom resource objects + +## What does this test do ? + +1. Create a Kind cluster with a random name generated with `crdtest-` as the cluster name prefix +2. Create a custom namespace with `my-ns` as the prefix +3. Register the CRDs listed under `./testdata/crds` using the resource decode helpers +4. Create a new Custom Resource for the CRD created in step #3 +5. Fetch the CR created in Test setup and print the value + +## How to run the tests + +```bash +go test -v . +``` diff --git a/examples/crds/envtest_test.go b/examples/crds/envtest_test.go new file mode 100644 index 00000000..d6857078 --- /dev/null +++ b/examples/crds/envtest_test.go @@ -0,0 +1,65 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package crds + +import ( + "context" + "os" + "testing" + + "k8s.io/klog/v2" + "sigs.k8s.io/e2e-framework/examples/crds/testdata/crontabs" + "sigs.k8s.io/e2e-framework/klient/decoder" + "sigs.k8s.io/e2e-framework/klient/k8s/resources" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/features" +) + +func TestCRDSetup(t *testing.T) { + feature := features.New("Custom Controller"). + Setup(func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + r, err := resources.New(c.Client().RESTConfig()) + if err != nil { + t.Fail() + } + crontabs.AddToScheme(r.GetScheme()) + r.WithNamespace(namespace) + decoder.DecodeEachFile( + ctx, os.DirFS("./testdata/crs"), "*", + decoder.CreateHandler(r), + decoder.MutateNamespace(namespace), + ) + return ctx + }). + Assess("Check If Resource created", func(ctx context.Context, t *testing.T, c *envconf.Config) context.Context { + r, err := resources.New(c.Client().RESTConfig()) + if err != nil { + t.Fail() + } + r.WithNamespace(namespace) + crontabs.AddToScheme(r.GetScheme()) + ct := &crontabs.CronTab{} + err = r.Get(ctx, "my-new-cron-object", namespace, ct) + if err != nil { + t.Fail() + } + klog.InfoS("CR Details", "cr", ct) + return ctx + }).Feature() + + testEnv.Test(t, feature) +} diff --git a/examples/crds/main_test.go b/examples/crds/main_test.go new file mode 100644 index 00000000..a87ea82f --- /dev/null +++ b/examples/crds/main_test.go @@ -0,0 +1,53 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package crds + +import ( + "os" + "testing" + + "sigs.k8s.io/e2e-framework/pkg/env" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/envfuncs" +) + +var ( + testEnv env.Environment + kindClusterName string + namespace string +) + +func TestMain(m *testing.M) { + cfg, _ := envconf.NewFromFlags() + testEnv = env.NewWithConfig(cfg) + kindClusterName = envconf.RandomName("crdtest-", 16) + namespace = envconf.RandomName("my-ns", 10) + + testEnv.Setup( + envfuncs.CreateKindCluster(kindClusterName), + envfuncs.CreateNamespace(namespace), + envfuncs.SetupCRDs("./testdata/crds", "*"), + ) + + testEnv.Finish( + envfuncs.DeleteNamespace(namespace), + envfuncs.TeardownCRDs("./testdata/crds", "*"), + envfuncs.DestroyKindCluster(kindClusterName), + ) + + os.Exit(testEnv.Run(m)) +} diff --git a/examples/crds/testdata/crds/stable.example.com.crontabs.yaml b/examples/crds/testdata/crds/stable.example.com.crontabs.yaml new file mode 100644 index 00000000..301efc05 --- /dev/null +++ b/examples/crds/testdata/crds/stable.example.com.crontabs.yaml @@ -0,0 +1,30 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: crontabs.stable.example.com +spec: + group: stable.example.com + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + cronSpec: + type: string + image: + type: string + replicas: + type: integer + scope: Namespaced + names: + plural: crontabs + singular: crontab + kind: CronTab + shortNames: + - ct diff --git a/examples/crds/testdata/crontabs/register.go b/examples/crds/testdata/crontabs/register.go new file mode 100644 index 00000000..0f0c573b --- /dev/null +++ b/examples/crds/testdata/crontabs/register.go @@ -0,0 +1,43 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package crontabs + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +const GroupName = "stable.example.com" +const GroupVersion = "v1" + +var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: GroupVersion} + +var ( + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + AddToScheme = SchemeBuilder.AddToScheme +) + +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &CronTab{}, + &CronTabList{}, + ) + + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil +} diff --git a/examples/crds/testdata/crontabs/type.go b/examples/crds/testdata/crontabs/type.go new file mode 100644 index 00000000..ac73fc6a --- /dev/null +++ b/examples/crds/testdata/crontabs/type.go @@ -0,0 +1,76 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package crontabs + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +type CronTabSpec struct { + CronSpec string `json:"cronSpec"` + Image string `json:"image"` + Replicas int `json:"replicas,omitempty"` +} + +type CronTab struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CronTabSpec `json:"spec"` +} + +type CronTabList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + + Items []CronTab `json:"items"` +} + +// DeepCopyInto copies all properties of this object into another object of the +// same type that is provided as a pointer. +func (in *CronTab) DeepCopyInto(out *CronTab) { + out.TypeMeta = in.TypeMeta + out.ObjectMeta = in.ObjectMeta + out.Spec = CronTabSpec{ + Replicas: in.Spec.Replicas, + } +} + +// DeepCopyObject returns a generically typed copy of an object +func (in *CronTab) DeepCopyObject() runtime.Object { + out := CronTab{} + in.DeepCopyInto(&out) + + return &out +} + +// DeepCopyObject returns a generically typed copy of an object +func (in *CronTabList) DeepCopyObject() runtime.Object { + out := CronTabList{} + out.TypeMeta = in.TypeMeta + out.ListMeta = in.ListMeta + + if in.Items != nil { + out.Items = make([]CronTab, len(in.Items)) + for i := range in.Items { + in.Items[i].DeepCopyInto(&out.Items[i]) + } + } + + return &out +} diff --git a/examples/crds/testdata/crs/sample.yaml b/examples/crds/testdata/crs/sample.yaml new file mode 100644 index 00000000..e3df22e9 --- /dev/null +++ b/examples/crds/testdata/crs/sample.yaml @@ -0,0 +1,7 @@ +apiVersion: "stable.example.com/v1" +kind: CronTab +metadata: + name: my-new-cron-object +spec: + cronSpec: "* * * * */5" + image: my-awesome-cron-image \ No newline at end of file diff --git a/klient/decoder/decoder.go b/klient/decoder/decoder.go index e517ee8b..744fa870 100644 --- a/klient/decoder/decoder.go +++ b/klient/decoder/decoder.go @@ -24,6 +24,7 @@ import ( "fmt" "io" "io/fs" + "os" "strings" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -92,6 +93,20 @@ func DecodeAllFiles(ctx context.Context, fsys fs.FS, pattern string, options ... return objects, err } +// ApplyWithManifestDir resolves all the files in the Directory dirPath against the globbing pattern and creates a kubernetes +// resource for each of the resources found under the manifest directory. +func ApplyWithManifestDir(ctx context.Context, r *resources.Resources, dirPath, pattern string, createOptions []resources.CreateOption, options ...DecodeOption) error { + err := DecodeEachFile(ctx, os.DirFS(dirPath), pattern, CreateHandler(r, createOptions...), options...) + return err +} + +// DeleteWithManifestDir does the reverse of ApplyUsingManifestDir does. This will resolve all files in the dirPath against the pattern and then +// delete those kubernetes resources found under the manifest directory. +func DeleteWithManifestDir(ctx context.Context, r *resources.Resources, dirPath, pattern string, deleteOptions []resources.DeleteOption, options ...DecodeOption) error { + err := DecodeEachFile(ctx, os.DirFS(dirPath), pattern, DeleteHandler(r, deleteOptions...), options...) + return err +} + // Decode a stream of documents of any Kind using either the innate typing of the scheme. // Falls back to the unstructured.Unstructured type if a matching type cannot be found for the Kind. // diff --git a/pkg/envfuncs/resource_funcs.go b/pkg/envfuncs/resource_funcs.go new file mode 100644 index 00000000..54a5f565 --- /dev/null +++ b/pkg/envfuncs/resource_funcs.go @@ -0,0 +1,51 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package envfuncs + +import ( + "context" + + "sigs.k8s.io/e2e-framework/klient/decoder" + "sigs.k8s.io/e2e-framework/klient/k8s/resources" + "sigs.k8s.io/e2e-framework/pkg/env" + "sigs.k8s.io/e2e-framework/pkg/envconf" +) + +// SetupCRDs is provided as a helper env.Func handler that can be used to setup the CRDs that are required +// to process your controller code for testing. For additional control on resource creation handling, please +// use the decoder.ApplyWithManifestDir directly with suitable arguments to customize the behavior +func SetupCRDs(crdPath, pattern string) env.Func { + return func(ctx context.Context, c *envconf.Config) (context.Context, error) { + r, err := resources.New(c.Client().RESTConfig()) + if err != nil { + return ctx, err + } + return ctx, decoder.ApplyWithManifestDir(ctx, r, crdPath, pattern, []resources.CreateOption{}) + } +} + +// TeardownCRDs is provided as a handler function that can be hooked into your test's teardown sequence to +// make sure that you can cleanup the CRDs that were setup as part of the SetupCRDs hook +func TeardownCRDs(crdPath, pattern string) env.Func { + return func(ctx context.Context, c *envconf.Config) (context.Context, error) { + r, err := resources.New(c.Client().RESTConfig()) + if err != nil { + return ctx, err + } + return ctx, decoder.DeleteWithManifestDir(ctx, r, crdPath, pattern, []resources.DeleteOption{}) + } +}