feat(worker-visibility): allow update to worker table with status and…

… build info (#482)

* feat(register-and-operate): add ability for worker to update status to idle and error

* docs(README): showing off commitizen cli

* feat: update worker status and runningBuildIDs

* feat: worker can update runningBuildIDs and status

* chore(exec): clean up logging for worker database updates

* feat(worker): update last_build_ timestamps in the database

* fix(worker-visibility): update error to make logrus happy

* chore(worker_visibility): update deps for types and server

* Update cmd/vela-worker/register.go

Co-authored-by: David May <[email protected]>

* add res nil check

* move res nil above err

* add res nil check

* more accurate logging

* make clean

---------

Co-authored-by: Tim Huynh <[email protected]>
Co-authored-by: David May <[email protected]>

.github/README.md

@@ -38,3 +38,4 @@ Copyright (c) 2022 Target Brands, Inc.
 ```
 
 [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0)
+

cmd/vela-worker/exec.go

@@ -7,11 +7,13 @@ package main
 import (
 	"context"
 	"net/http"
+	"strconv"
 	"sync"
 	"time"
 
 	"github.com/go-vela/types"
 	"github.com/go-vela/types/constants"
+	"github.com/go-vela/types/library"
 	"github.com/go-vela/worker/executor"
 	"github.com/go-vela/worker/runtime"
 	"github.com/go-vela/worker/version"
@@ -22,7 +24,7 @@ import (
 // and execute Vela pipelines for the Worker.
 //
 //nolint:nilerr,funlen // ignore returning nil - don't want to crash worker
-func (w *Worker) exec(index int) error {
+func (w *Worker) exec(index int, config *library.Worker) error {
 	var err error
 
 	// setup the version
@@ -71,6 +73,27 @@ func (w *Worker) exec(index int) error {
 		"version":  v.Semantic(),
 	})
 
+	// lock and append the build to the RunningBuildIDs list
+	w.RunningBuildIDsMutex.Lock()
+
+	w.RunningBuildIDs = append(w.RunningBuildIDs, strconv.Itoa(item.Build.GetNumber()))
+
+	config.SetRunningBuildIDs(w.RunningBuildIDs)
+
+	w.RunningBuildIDsMutex.Unlock()
+
+	// set worker status
+	updateStatus := w.getWorkerStatusFromConfig(config)
+	config.SetStatus(updateStatus)
+	config.SetLastStatusUpdateAt(time.Now().Unix())
+	config.SetLastBuildStartedAt(time.Now().Unix())
+
+	// update worker in the database
+	_, _, err = w.VelaClient.Worker.Update(config.GetHostname(), config)
+	if err != nil {
+		logger.Errorf("unable to update worker: %v", err)
+	}
+
 	// handle stale item queued before a Vela upgrade or downgrade.
 	if item.ItemVersion != types.ItemVersion {
 		// If the ItemVersion is older or newer than what we expect, then it might
@@ -154,6 +177,32 @@ func (w *Worker) exec(index int) error {
 		}
 
 		logger.Info("completed build")
+
+		// lock and remove the build from the RunningBuildIDs list
+		w.RunningBuildIDsMutex.Lock()
+
+		for i, v := range w.RunningBuildIDs {
+			if v == strconv.Itoa(item.Build.GetNumber()) {
+				w.RunningBuildIDs = append(w.RunningBuildIDs[:i], w.RunningBuildIDs[i+1:]...)
+			}
+		}
+
+		config.SetRunningBuildIDs(w.RunningBuildIDs)
+
+		w.RunningBuildIDsMutex.Unlock()
+
+		// set worker status
+		updateStatus := w.getWorkerStatusFromConfig(config)
+		config.SetStatus(updateStatus)
+		config.SetLastStatusUpdateAt(time.Now().Unix())
+		config.SetLastBuildFinishedAt(time.Now().Unix())
+
+		// update worker in the database
+		_, _, err := w.VelaClient.Worker.Update(config.GetHostname(), config)
+		if err != nil {
+			logger.Errorf("unable to update worker: %v", err)
+		}
+
 	}()
 
 	// capture the configured build timeout
@@ -222,3 +271,18 @@ func (w *Worker) exec(index int) error {
 
 	return nil
 }
+
+// getWorkerStatusFromConfig is a helper function
+// to determine the appropriate worker status
+func (w *Worker) getWorkerStatusFromConfig(config *library.Worker) string {
+	switch rb := len(config.GetRunningBuildIDs()); {
+	case rb == 0:
+		return constants.WorkerStatusIdle
+	case rb < w.Config.Build.Limit:
+		return constants.WorkerStatusAvailable
+	case rb == w.Config.Build.Limit:
+		return constants.WorkerStatusBusy
+	default:
+		return constants.WorkerStatusError
+	}
+}

cmd/vela-worker/operate.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/go-vela/server/queue"
+	"github.com/go-vela/types/constants"
 	"github.com/go-vela/types/library"
 
 	"github.com/sirupsen/logrus"
@@ -118,6 +119,18 @@ func (w *Worker) operate(ctx context.Context) error {
 	//nolint:contextcheck // ignore passing context
 	w.Queue, err = queue.New(w.Config.Queue)
 	if err != nil {
+		registryWorker.SetStatus(constants.WorkerStatusError)
+		_, resp, logErr := w.VelaClient.Worker.Update(registryWorker.GetHostname(), registryWorker)
+		if resp == nil {
+			// log the error instead of returning so the operation doesn't block worker deployment
+			logrus.Error("status update response is nil")
+		}
+		if logErr != nil {
+			if resp != nil {
+				// log the error instead of returning so the operation doesn't block worker deployment
+				logrus.Errorf("status code: %v, unable to update worker %s status with the server: %v", resp.StatusCode, registryWorker.GetHostname(), logErr)
+			}
+		}
 		return err
 	}
 
@@ -160,13 +173,24 @@ func (w *Worker) operate(ctx context.Context) error {
 					// (do not pass the context to avoid errors in one
 					// executor+build inadvertently canceling other builds)
 					//nolint:contextcheck // ignore passing context
-					err = w.exec(id)
+					err = w.exec(id, registryWorker)
 					if err != nil {
 						// log the error received from the executor
 						//
 						// https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#Errorf
 						logrus.Errorf("failing worker executor: %v", err)
-
+						registryWorker.SetStatus(constants.WorkerStatusError)
+						_, resp, logErr := w.VelaClient.Worker.Update(registryWorker.GetHostname(), registryWorker)
+						if resp == nil {
+							// log the error instead of returning so the operation doesn't block worker deployment
+							logrus.Error("status update response is nil")
+						}
+						if logErr != nil {
+							if resp != nil {
+								// log the error instead of returning so the operation doesn't block worker deployment
+								logrus.Errorf("status code: %v, unable to update worker %s status with the server: %v", resp.StatusCode, registryWorker.GetHostname(), logErr)
+							}
+						}
 						return err
 					}
 				}

cmd/vela-worker/register.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/go-vela/types/constants"
 	"github.com/go-vela/types/library"
 	"github.com/sirupsen/logrus"
 )
@@ -46,12 +47,16 @@ func (w *Worker) checkIn(config *library.Worker) (bool, string, error) {
 func (w *Worker) register(config *library.Worker) (bool, string, error) {
 	logrus.Infof("worker %s not found, registering it with the server", config.GetHostname())
 
+	config.SetStatus(constants.WorkerStatusIdle)
+
 	tkn, _, err := w.VelaClient.Worker.Add(config)
 	if err != nil {
 		// log the error instead of returning so the operation doesn't block worker deployment
 		return false, "", fmt.Errorf("unable to register worker %s with the server: %w", config.GetHostname(), err)
 	}
 
+	logrus.Infof("worker %q status updated successfully to %s", config.GetHostname(), config.GetStatus())
+
 	// successfully added the worker so return nil
 	return true, tkn.GetToken(), nil
 }

cmd/vela-worker/run.go

@@ -138,6 +138,8 @@ func run(c *cli.Context) error {
 		Executors: make(map[int]executor.Engine),
 
 		RegisterToken: make(chan string, 1),
+
+		RunningBuildIDs: make([]string, 0),
 	}
 
 	// set the worker address if no flag was provided

cmd/vela-worker/worker.go

@@ -6,6 +6,7 @@ package main
 
 import (
 	"net/url"
+	"sync"
 	"time"
 
 	"github.com/go-vela/sdk-go/vela"
@@ -62,12 +63,14 @@ type (
 	// Worker represents all configuration and
 	// system processes for the worker.
 	Worker struct {
-		Config        *Config
-		Executors     map[int]executor.Engine
-		Queue         queue.Service
-		Runtime       runtime.Engine
-		VelaClient    *vela.Client
-		RegisterToken chan string
-		CheckedIn     bool
+		Config               *Config
+		Executors            map[int]executor.Engine
+		Queue                queue.Service
+		Runtime              runtime.Engine
+		VelaClient           *vela.Client
+		RegisterToken        chan string
+		CheckedIn            bool
+		RunningBuildIDs      []string
+		RunningBuildIDsMutex sync.Mutex
 	}
 )

go.mod

@@ -9,8 +9,8 @@ require (
 	github.com/docker/go-units v0.5.0
 	github.com/gin-gonic/gin v1.9.1
 	github.com/go-vela/sdk-go v0.19.3-0.20230609172535-4dfd42c3640a
-	github.com/go-vela/server v0.19.3-0.20230609145007-f8c795f7aa03
-	github.com/go-vela/types v0.19.3-0.20230523200921-35a0d5fc088c
+	github.com/go-vela/server v0.19.3-0.20230616181003-4e5d484be2bd
+	github.com/go-vela/types v0.19.3-0.20230614134928-b1b57c0b34af
 	github.com/golang-jwt/jwt/v5 v5.0.0
 	github.com/google/go-cmp v0.5.9
 	github.com/joho/godotenv v1.5.1
@@ -34,7 +34,7 @@ require (
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect
-	github.com/alicebob/miniredis/v2 v2.30.2 // indirect
+	github.com/alicebob/miniredis/v2 v2.30.3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/buildkite/yaml v0.0.0-20230306222819-0e4e032d4835 // indirect
 	github.com/bytedance/sonic v1.9.1 // indirect
@@ -71,7 +71,7 @@ require (
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.2 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.4 // indirect
 	github.com/huandu/xstrings v1.3.3 // indirect
 	github.com/imdario/mergo v0.3.11 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
@@ -94,7 +94,7 @@ require (
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect
-	github.com/redis/go-redis/v9 v9.0.3 // indirect
+	github.com/redis/go-redis/v9 v9.0.5 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/spf13/afero v1.9.5 // indirect

go.sum

@@ -58,8 +58,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk=
 github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
-github.com/alicebob/miniredis/v2 v2.30.2 h1:lc1UAUT9ZA7h4srlfBmBt2aorm5Yftk9nBjxz7EyY9I=
-github.com/alicebob/miniredis/v2 v2.30.2/go.mod h1:b25qWj4fCEsBeAAR2mlb0ufImGC6uH3VlUfb/HS5zKg=
+github.com/alicebob/miniredis/v2 v2.30.3 h1:hrqDB4cHFSHQf4gO3xu6YKQg8PqJpNjLYsQAFYHstqw=
+github.com/alicebob/miniredis/v2 v2.30.3/go.mod h1:b25qWj4fCEsBeAAR2mlb0ufImGC6uH3VlUfb/HS5zKg=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bsm/ginkgo/v2 v2.7.0 h1:ItPMPH90RbmZJt5GtkcNvIRuGEdwlBItdNVoyzaNQao=
@@ -148,10 +148,10 @@ github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QX
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
 github.com/go-vela/sdk-go v0.19.3-0.20230609172535-4dfd42c3640a h1:8ZpMB3fYAyRGKPBq0Y8NltuDjzaD0k8oISjkP0fBP1o=
 github.com/go-vela/sdk-go v0.19.3-0.20230609172535-4dfd42c3640a/go.mod h1:4vnf8+6RVvWmMLaivX1okkA8s2YJxTcqFjIv+z3r5C0=
-github.com/go-vela/server v0.19.3-0.20230609145007-f8c795f7aa03 h1:Vi1y5faOZ5CaCjCTRfc45z+BpR5JgMpLyIxmy7pSgqE=
-github.com/go-vela/server v0.19.3-0.20230609145007-f8c795f7aa03/go.mod h1:b520o4N7ss4kHATH291Ui1LHwuC0qEgJgg/Jab6yPIQ=
-github.com/go-vela/types v0.19.3-0.20230523200921-35a0d5fc088c h1:eAApIK5e5MxFF8RzZAFsvTSdwq/AzdUrdhJHOGQ0ILc=
-github.com/go-vela/types v0.19.3-0.20230523200921-35a0d5fc088c/go.mod h1:0lsuPfGyVyTWJSi2h3NS6uaEW6DgnFvIzaZu1sXYKrs=
+github.com/go-vela/server v0.19.3-0.20230616181003-4e5d484be2bd h1:cw9iz/4xbq+/lAlHke10RMjfjlPbLk7V8xJ2hqd3dmI=
+github.com/go-vela/server v0.19.3-0.20230616181003-4e5d484be2bd/go.mod h1:Vjxxn+BP9bvr43uiQtqhDNelajsJzEuIIJTdeRlSiDU=
+github.com/go-vela/types v0.19.3-0.20230614134928-b1b57c0b34af h1:Ixsa6Ha0j9Edq4v3IooDgyUoGSp08fk9FgrYKuZSML8=
+github.com/go-vela/types v0.19.3-0.20230614134928-b1b57c0b34af/go.mod h1:1ZSmKWX9MamKogwaIb53mzzRpZMV34mJFKiGfVFadFk=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -243,8 +243,8 @@ github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxC
 github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
-github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0=
-github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
+github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA=
+github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4=
@@ -323,8 +323,8 @@ github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI
 github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
 github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
 github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
-github.com/redis/go-redis/v9 v9.0.3 h1:+7mmR26M0IvyLxGZUHxu4GiBkJkVDid0Un+j4ScYu4k=
-github.com/redis/go-redis/v9 v9.0.3/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk=
+github.com/redis/go-redis/v9 v9.0.5 h1:CuQcn5HIEeK7BgElubPP8CGtE0KakrnbBSTLjathl5o=
+github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=