add panic handling

pluralsh · Sep 23, 2023 · 2278ebe · 2278ebe
1 parent e39a792
commit 2278ebe
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 2 deletions.
diff --git a/agent/main.go b/agent/main.go
@@ -63,12 +63,12 @@ func newCmd(log logr.Logger) *cobra.Command {
 			manifestCache := manifests.NewCache(refresh)
 
 			svcChan := make(chan string)
+			deathChan := make(chan interface{})
 
 			// we should enable SSA if kubernetes version supports it
 			clusterCache := cache.NewClusterCache(config,
 				cache.SetLogr(log),
 				cache.SetPopulateResourceInfoHandler(func(un *unstructured.Unstructured, isRoot bool) (info interface{}, cacheManifest bool) {
-					// store gc mark of every resource
 					svcId := un.GetAnnotations()[deploysync.SyncAnnotation]
 					sha := un.GetAnnotations()[deploysync.SyncShaAnnotation]
 					info = deploysync.NewResource(svcId, sha)
@@ -87,7 +87,14 @@ func newCmd(log logr.Logger) *cobra.Command {
 
 			engine := deploysync.New(gitOpsEngine, clusterCache, consoleClient, svcChan, svcCache, manifestCache)
 			engine.RegisterHandlers()
-			go engine.ControlLoop()
+			engine.AddHealthCheck(deathChan)
+			go func() {
+				for {
+					go engine.ControlLoop()
+					failure := <-deathChan
+					fmt.Printf("recovered from panic %v\n", failure)
+				}
+			}()
 
 			for {
 				svcs, err := consoleClient.GetServices()

diff --git a/agent/pkg/sync/engine.go b/agent/pkg/sync/engine.go
@@ -11,6 +11,7 @@ import (
 type Engine struct {
 	client        *client.Client
 	svcChan       chan string
+	deathChan     chan interface{}
 	svcCache      *client.ServiceCache
 	manifestCache *manifests.ManifestCache
 	engine        engine.GitOpsEngine
@@ -28,6 +29,10 @@ func New(engine engine.GitOpsEngine, cache cache.ClusterCache, client *client.Cl
 	}
 }
 
+func (engine *Engine) AddHealthCheck(health chan interface{}) {
+	engine.deathChan = health
+}
+
 func (engine *Engine) RegisterHandlers() {
 	engine.cache.OnResourceUpdated(func(new *cache.Resource, old *cache.Resource, nrs map[kube.ResourceKey]*cache.Resource) {
 		if id := svcId(new); id != nil {

diff --git a/agent/pkg/sync/loop.go b/agent/pkg/sync/loop.go
@@ -17,6 +17,14 @@ const (
 )
 
 func (engine *Engine) ControlLoop() {
+	if engine.deathChan != nil {
+		defer func() {
+			if r := recover(); r != nil {
+				engine.deathChan <- r
+			}
+		}()
+	}
+
 	log := klogr.New()
 	for {
 		id := <-engine.svcChan