diff --git a/docs/design/k8s-resource-watcher.md b/docs/design/k8s-resource-watcher.md new file mode 100644 index 00000000..c52eaa0a --- /dev/null +++ b/docs/design/k8s-resource-watcher.md @@ -0,0 +1,164 @@ +# Watcher for K8S Objects + +K8S object watchers are great functionality provided by k8s to get efficient change notifications on resources. + +The events supported by these watchers are +1. ADD +2. MODIFY/UPDATE +3. DELETE + +## Motivation +Users must implement watchers such a way that whenever any events recieved some action/job has to be triggered. Developer has to write lots of code to do this and its sometimes difficult to manage. + +This can be achieved by using the k8s client built in function but understanding which packages to import or which core type needs to be used might be a complex for the developer. + +The below design would make developer life easier. They have to just register their actions for respective events. To stay informed about when these events get triggered just use Watch(), which resides inside klient/k8s/resources package. + +## Proposal +Watch function accepts a `object ObjectList` as a argument. ObjectList type is used to inject the resource type in which Watch has to be applied. + +`klient/k8s/resources/resources.go` +```go= +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + "k8s.io/apimachinery/pkg/watch" +) + +func (r *Resources) Watch(object k8s.ObjectList, opts ...ListOption) *watcher.EventHandlerFuncs { + listOptions := &metav1.ListOptions{} + + for _, fn := range opts { + fn(listOptions) + } + + o := &cr.ListOptions{Raw: listOptions} + + return &watcher.EventHandlerFuncs{ + ListOptions: o, + K8sObject: object, + Cfg: r.GetConfig(), + } +} +``` + +Watch() in resources.go will return the `watcher` type which helps to call `Start()`. InvokeEventHandler accepts `EventHandlerFuncs` which carries the user registerd function sets. + +file : klient/k8s/resources/watch.go + +```go= +// Start triggers the registered methods based on the event recieved for particular k8s resources. +func (watcher watch.Interface)Start(ctx context.Context) { + ... + go func() { + for { + select { + case <-ctx.Done(): + if ctx.Err() != nil { + return + } + case event := <-e.watcher.ResultChan(): + // retrieve the event type + eventType := event.Type + + switch eventType { + case watch.Added: + // calls AddFunc if it's not nil. + if e.addFunc != nil { + e.addFunc(event.Object) + } + case watch.Modified: + // calls UpdateFunc if it's not nil. + if e.updateFunc != nil { + e.updateFunc(event.Object) + } + case watch.Deleted: + // calls DeleteFunc if it's not nil. + if e.deleteFunc != nil { + e.deleteFunc(event.Object) + } + } + } + } + }() + ... +} + +// EventHandlerFuncs is an adaptor to let you easily specify as many or +// as few of functions to invoke while getting notification from watcher +type EventHandlerFuncs struct { + addFunc func(obj interface{}) + updateFunc func(newObj interface{}) + deleteFunc func(obj interface{}) + watcher watch.Interface + ListOptions *cr.ListOptions + K8sObject k8s.ObjectList + Cfg *rest.Config +} + +// EventHandler can handle notifications for events that happen to a resource. +// Start will be waiting for the events notification which is responsible +// for invoking the registered user defined functions. +// Stop used to stop the watcher. +type EventHandler interface { + Start(ctx context.Context) + Stop() +} + +``` + +`Start()` is invoked in a goroutine so that whenever watched resource changes the states it will call the registered user defined functions. +`Stop()` should be explicitly invoked by the user after the watch once the feature is done to ensure no unwanted go routine thread leackage. + +If any error while Start() one can retry it for number of times. + +This example shows how to use klient/resources/resources.go Watch() func and how to register the user defined functions. + +```go= + +import ( + "sigs.k8s.io/e2e-framework/klient/conf" + "sigs.k8s.io/e2e-framework/klient/k8s/resources" +) + +func main() { + ... + cfg, _ := conf.New(conf.ResolveKubeConfigFile()) + cl, err := cfg.NewClient() + if err != nil { + t.Fatal(err) + } + + dep := appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "watch-dep", Namespace: cfg.Namespace()}, + } + + // watch for the deployment and triger action based on the event recieved. + cl.Resources().Watch(&appsv1.DeploymentList{}, resources.WithFieldSelector(labels.FormatLabels(map[string]string{"metadata.name": dep.Name}))). + WithAddFunc(onAdd).WithDeleteFunc(onDelete).Start(ctx) + + ... +} + +// onAdd is the function executed when the kubernetes watch notifies the +// presence of a new kubernetes deployment in the cluster +func onAdd(obj interface{}) { + dep := obj.(*appsv1.Deployment) + _, ok := dep.GetLabels()[K8S_LABEL_AWS_REGION] + if ok { + fmt.Printf("It has the label!") + } +} + +// onDelete is the function executed when the kubernetes watch notifies +// delete event on deployment +func onDelete(obj interface{}) { + dep := obj.(*appsv1.Deployment) + _, ok := dep.GetLabels()[K8S_LABEL_AWS_REGION] + if ok { + fmt.Printf("It has the label!") + } +} + +``` + +The e2e flow of how to use watch is demonsrated in the examples/ folder. \ No newline at end of file diff --git a/docs/design/test-harness-framework.md b/docs/design/test-harness-framework.md index f05753e7..b8388b01 100644 --- a/docs/design/test-harness-framework.md +++ b/docs/design/test-harness-framework.md @@ -469,7 +469,7 @@ The framework should automatically inject these filter values into the environme ### Skipping features The test framework should provide the ability to explicitly exclude features during a test run. This could be done with the following flags: -* `--skip-feature` - a regular expression that skips features with matching names +* `--skip-features` - a regular expression that skips features with matching names * `--skip-assessment` - a regular expression that skips assessment with matching name * `--skip-lables` - a comma-separated list of key/value pairs used to skip features with matching lables diff --git a/examples/watch_resources/README.md b/examples/watch_resources/README.md new file mode 100644 index 00000000..a664a250 --- /dev/null +++ b/examples/watch_resources/README.md @@ -0,0 +1,54 @@ +# Watching for Resource Changes + +The test harness supports several methods for querying Kubernetes object types and watching for resource states. This example shows how to watch particular resource and how to register the functions to act upon based on the events recieved. + + +# Watch for the deployment and triger action based on the event + +Watch has to run as goroutine to get the different events based on the k8s resource state changes. +```go +func TestWatchForResources(t *testing.T) { +... +dep := appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "watch-dep", Namespace: cfg.Namespace()}, + } + + // watch for the deployment and triger action based on the event recieved. + cl.Resources().Watch(&appsv1.DeploymentList{}, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("metadata.name", dep.Name), + Namespace: dep.Namespace}, cl.RESTConfig()).WithAddFunc(onAdd).WithDeleteFunc(onDelete).Start(ctx) +... +} +``` + +# Function/Action definition and registering these actions + +```go +// onAdd is the function executed when the kubernetes watch notifies the +// presence of a new kubernetes deployment in the cluster +func onAdd(obj interface{}) { + dep := obj.(*appsv1.Deployment) + depName := dep.GetName() + fmt.Printf("Dep name recieved is %s", depName) + if depName == "watch-dep" { + fmt.Println("Dep name matches with actual name!") + } +} + +// onDelete is the function executed when the kubernetes watch notifies +// delete event on deployment +func onDelete(obj interface{}) { + dep := obj.(*appsv1.Deployment) + depName := dep.GetName() + if depName == "watch-dep" { + fmt.Println("Deployment deleted successfully!") + } +} +``` + +The above functions can be registered using Register functions(WithAddFunc(), WithDeleteFunc(), WithUpdateFunc()) defined under klient/k8s/watcher/watch.go as shown in the example. + +# How to stop the watcher +Create a global EventHandlerFuncs variable to store the watcher object and call Stop() as shown in example TestWatchForResourcesWithStop() test. + +Note: User should explicitly invoke the Stop() after the watch once the feature is done to ensure no unwanted go routine thread leackage. \ No newline at end of file diff --git a/examples/watch_resources/main_test.go b/examples/watch_resources/main_test.go new file mode 100644 index 00000000..887f88f1 --- /dev/null +++ b/examples/watch_resources/main_test.go @@ -0,0 +1,43 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watch_resources + +import ( + "os" + "testing" + + "sigs.k8s.io/e2e-framework/pkg/env" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/envfuncs" +) + +var testenv env.Environment + +func TestMain(m *testing.M) { + testenv = env.New() + kindClusterName := envconf.RandomName("watch-for-resources", 16) + namespace := envconf.RandomName("watch-ns", 16) + testenv.Setup( + envfuncs.CreateKindCluster(kindClusterName), + envfuncs.CreateNamespace(namespace), + ) + testenv.Finish( + envfuncs.DeleteNamespace(namespace), + envfuncs.DestroyKindCluster(kindClusterName), + ) + os.Exit(testenv.Run(m)) +} diff --git a/examples/watch_resources/watch_test.go b/examples/watch_resources/watch_test.go new file mode 100644 index 00000000..c868887c --- /dev/null +++ b/examples/watch_resources/watch_test.go @@ -0,0 +1,171 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watch_resources + +import ( + "context" + "testing" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/klog/v2" + "sigs.k8s.io/e2e-framework/klient/k8s/resources" + "sigs.k8s.io/e2e-framework/klient/k8s/watcher" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/features" +) + +func TestWatchForResources(t *testing.T) { + watchFeature := features.New("test watcher").WithLabel("env", "dev"). + Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + cl, err := cfg.NewClient() + if err != nil { + t.Fatal(err) + } + + dep := appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "watch-dep", Namespace: cfg.Namespace()}, + } + + // watch for the deployment and triger action based on the event recieved. + cl.Resources().Watch(&appsv1.DeploymentList{}, resources.WithFieldSelector(labels.FormatLabels(map[string]string{"metadata.name": dep.Name}))). + WithAddFunc(onAdd).WithDeleteFunc(onDelete).Start(ctx) + + return ctx + }). + Assess("create watch deployment", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + // create a deployment + deployment := newDeployment(cfg.Namespace(), "watch-dep", 1) + client, err := cfg.NewClient() + if err != nil { + t.Fatal(err) + } + if err := client.Resources().Create(ctx, deployment); err != nil { + t.Fatal(err) + } + return context.WithValue(ctx, "test-dep", deployment) + }). + Teardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + client, err := cfg.NewClient() + if err != nil { + t.Fatal(err) + } + depl := ctx.Value("test-dep").(*appsv1.Deployment) + if err := client.Resources().Delete(ctx, depl); err != nil { + t.Fatal(err) + } + return ctx + }).Feature() + + testenv.Test(t, watchFeature) + +} + +// TestWatchForResourcesWithStop() demonstartes how to start and stop the watcher +var w *watcher.EventHandlerFuncs + +func TestWatchForResourcesWithStop(t *testing.T) { + watchFeature := features.New("test watcher with stop").WithLabel("env", "prod"). + Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + cl, err := cfg.NewClient() + if err != nil { + t.Fatal(err) + } + + dep := appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: "watchnstop-dep", Namespace: cfg.Namespace()}, + } + + // watch for the deployment and triger action based on the event recieved. + w = cl.Resources().Watch(&appsv1.DeploymentList{}, resources.WithFieldSelector(labels.FormatLabels(map[string]string{"metadata.name": dep.Name}))). + WithAddFunc(onAdd).WithDeleteFunc(onDelete) + + err = w.Start(ctx) + if err != nil { + t.Error(err) + } + + return ctx + }). + Assess("create watch deployment", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + // create a deployment + deployment := newDeployment(cfg.Namespace(), "watchnstop-dep", 1) + client, err := cfg.NewClient() + if err != nil { + t.Fatal(err) + } + if err := client.Resources().Create(ctx, deployment); err != nil { + t.Fatal(err) + } + return context.WithValue(ctx, "stop-dep", deployment) + }). + Teardown(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + client, err := cfg.NewClient() + if err != nil { + t.Fatal(err) + } + depl := ctx.Value("stop-dep").(*appsv1.Deployment) + if err := client.Resources().Delete(ctx, depl); err != nil { + t.Fatal(err) + } + + w.Stop() + + return ctx + }).Feature() + + testenv.Test(t, watchFeature) + +} + +func newDeployment(namespace string, name string, replicas int32) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace, Labels: map[string]string{"app": "watch-for-resources"}}, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "watch-for-resources"}, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "watch-for-resources"}}, + Spec: v1.PodSpec{Containers: []v1.Container{{Name: "nginx", Image: "nginx"}}}, + }, + }, + } +} + +// onAdd is the function executed when the kubernetes watch notifies the +// presence of a new kubernetes deployment in the cluster +func onAdd(obj interface{}) { + dep := obj.(*appsv1.Deployment) + depName := dep.GetName() + if depName == "watch-dep" || depName == "watchnstop-dep" { + klog.InfoS("Deployment name matches with actual name!") + } +} + +// onDelete is the function executed when the kubernetes watch notifies +// delete event on deployment +func onDelete(obj interface{}) { + dep := obj.(*appsv1.Deployment) + depName := dep.GetName() + if depName == "watch-dep" || depName == "watchnstop-dep" { + klog.InfoS("Deployment deleted successfully!") + } +} diff --git a/klient/decoder/decoder.go b/klient/decoder/decoder.go index 744fa870..6124054d 100644 --- a/klient/decoder/decoder.go +++ b/klient/decoder/decoder.go @@ -132,7 +132,7 @@ func DecodeEach(ctx context.Context, manifest io.Reader, handlerFn HandlerFunc, return nil } -// Decode a stream of documents of any Kind using either the innate typing of the scheme. +// DecodeAll is a stream of documents of any Kind using either the innate typing of the scheme. // Falls back to the unstructured.Unstructured type if a matching type cannot be found for the Kind. // Options may be provided to configure the behavior of the decoder. func DecodeAll(ctx context.Context, manifest io.Reader, options ...DecodeOption) ([]k8s.Object, error) { @@ -144,7 +144,7 @@ func DecodeAll(ctx context.Context, manifest io.Reader, options ...DecodeOption) return objects, err } -// Decode any single-document YAML or JSON input using either the innate typing of the scheme. +// DecodeAny decodes any single-document YAML or JSON input using either the innate typing of the scheme. // Falls back to the unstructured.Unstructured type if a matching type cannot be found for the Kind. // Options may be provided to configure the behavior of the decoder. func DecodeAny(manifest io.Reader, options ...DecodeOption) (k8s.Object, error) { diff --git a/klient/k8s/resources/resources.go b/klient/k8s/resources/resources.go index 5a4c48ce..6b7ac125 100644 --- a/klient/k8s/resources/resources.go +++ b/klient/k8s/resources/resources.go @@ -27,6 +27,7 @@ import ( "k8s.io/client-go/rest" cr "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/e2e-framework/klient/k8s" + "sigs.k8s.io/e2e-framework/klient/k8s/watcher" ) type Resources struct { @@ -66,6 +67,11 @@ func New(cfg *rest.Config) (*Resources, error) { return res, nil } +// GetConfig hepls to get config type *rest.Config +func (r *Resources) GetConfig() *rest.Config { + return r.config +} + func (r *Resources) WithNamespace(ns string) *Resources { r.namespace = ns return r @@ -182,3 +188,19 @@ func (r *Resources) Label(obj k8s.Object, label map[string]string) { func (r *Resources) GetScheme() *runtime.Scheme { return r.scheme } + +func (r *Resources) Watch(object k8s.ObjectList, opts ...ListOption) *watcher.EventHandlerFuncs { + listOptions := &metav1.ListOptions{} + + for _, fn := range opts { + fn(listOptions) + } + + o := &cr.ListOptions{Raw: listOptions} + + return &watcher.EventHandlerFuncs{ + ListOptions: o, + K8sObject: object, + Cfg: r.GetConfig(), + } +} diff --git a/klient/k8s/watcher/watch.go b/klient/k8s/watcher/watch.go new file mode 100644 index 00000000..96af993e --- /dev/null +++ b/klient/k8s/watcher/watch.go @@ -0,0 +1,126 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watcher + +import ( + "context" + + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/rest" + cr "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/e2e-framework/klient/k8s" +) + +// EventHandlerFuncs is an adaptor to let you easily specify as many or +// as few of functions to invoke while getting notification from watcher +type EventHandlerFuncs struct { + addFunc func(obj interface{}) + updateFunc func(newObj interface{}) + deleteFunc func(obj interface{}) + watcher watch.Interface + ListOptions *cr.ListOptions + K8sObject k8s.ObjectList + Cfg *rest.Config +} + +// EventHandler can handle notifications for events that happen to a resource. +// Start will be waiting for the events notification which is responsible +// for invoking the registered user defined functions. +// Stop used to stop the watcher. +type EventHandler interface { + Start(ctx context.Context) + Stop() +} + +// Start triggers the registered methods based on the event received for +// particular k8s resources +func (e *EventHandlerFuncs) Start(ctx context.Context) error { + // check if context is valid and that it has not been cancelled. + if ctx.Err() != nil { + return ctx.Err() + } + + cl, err := cr.NewWithWatch(e.Cfg, cr.Options{}) + if err != nil { + return err + } + + w, err := cl.Watch(ctx, e.K8sObject, e.ListOptions) + if err != nil { + return err + } + + // set watcher object + e.watcher = w + + go func() { + for { + select { + case <-ctx.Done(): + if ctx.Err() != nil { + return + } + case event := <-e.watcher.ResultChan(): + // retrieve the event type + eventType := event.Type + + switch eventType { + case watch.Added: + // calls AddFunc if it's not nil. + if e.addFunc != nil { + e.addFunc(event.Object) + } + case watch.Modified: + // calls UpdateFunc if it's not nil. + if e.updateFunc != nil { + e.updateFunc(event.Object) + } + case watch.Deleted: + // calls DeleteFunc if it's not nil. + if e.deleteFunc != nil { + e.deleteFunc(event.Object) + } + } + } + } + }() + + return nil +} + +// Stop triggers stopping a particular k8s watch resources +func (e *EventHandlerFuncs) Stop() { + e.watcher.Stop() +} + +// SetAddFunc used to set action on create event +func (e *EventHandlerFuncs) WithAddFunc(addfn func(obj interface{})) *EventHandlerFuncs { + e.addFunc = addfn + return e +} + +// SetUpdateFunc sets action for any update events +func (e *EventHandlerFuncs) WithUpdateFunc(updatefn func(updated interface{})) *EventHandlerFuncs { + e.updateFunc = updatefn + return e +} + +// SetDeleteFunc sets action for delete events +func (e *EventHandlerFuncs) WithDeleteFunc(deletefn func(obj interface{})) *EventHandlerFuncs { + e.deleteFunc = deletefn + return e +}