[jobmanager] Recover from job panics

Signed-off-by: Dmitrii Okunev <[email protected]>

Author: xaionaro

pkg/jobmanager/jobmanager.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/facebookincubator/go-belt/beltctx"
+	"github.com/facebookincubator/go-belt/tool/experimental/errmon"
 	"github.com/insomniacslk/xjson"
 
 	"github.com/linuxboot/contest/pkg/api"
@@ -183,6 +184,9 @@ loop:
 			logging.Debugf(ev.Context, "Handling event %+v", ev)
 			handlerWg.Add(1)
 			go func() {
+				defer func() {
+					errmon.ObserveRecoverCtx(ev.Context, recover())
+				}()
 				defer handlerWg.Done()
 				jm.handleEvent(ev)
 			}()

pkg/jobmanager/start.go

@@ -98,7 +98,14 @@ func (jm *JobManager) startJob(ctx context.Context, j *job.Job, resumeState *job
 			logging.Debugf(ctx, "cancelling job context")
 			jobCancel()
 		}}
-	go jm.runJob(jobCtx, j, resumeState)
+
+	go func() {
+		defer func() {
+			errmon.ObserveRecoverCtx(ctx, recover())
+		}()
+
+		jm.runJob(jobCtx, j, resumeState)
+	}()
 }
 
 func (jm *JobManager) runJob(ctx context.Context, j *job.Job, resumeState *job.PauseEventPayload) {