fixing race condition in error counting

paychex · Jul 23, 2019 · 4ea191d · 4ea191d
1 parent cd95a05
commit 4ea191d
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 9 deletions.
diff --git a/cmd/ecs_exporter.go b/cmd/ecs_exporter.go
@@ -167,7 +167,7 @@ func queryHandler(w http.ResponseWriter, r *http.Request) {
 	c := result.(*ecsclient.EcsClient)
 
 	c.RetrieveNodeInfoV2()
-	log.WithFields(log.Fields{"package": "main", "cluster_version": c.EcsVersion, "node_count": c.RetrieveNodeCount}).Debugf("ECS Cluster Info.")
+	log.WithFields(log.Fields{"package": "main", "cluster_version": c.EcsVersion, "node_count": c.RetrieveNodeCount()}).Debugf("ECS Cluster Info.")
 	ecsClusterInfo.WithLabelValues(c.EcsVersion, strconv.Itoa(c.RetrieveNodeCount())).Set(1)
 
 	if r.URL.Query().Get("metering") == "1" {
@@ -206,7 +206,7 @@ func queryHandler(w http.ResponseWriter, r *http.Request) {
 
 	// Delegate http serving to Promethues client library, which will call collector.Collect.
 	log.WithFields(log.Fields{"package": "main"}).Debugf("incrementing requests errors by %v\n", c.ErrorCount)
-	ecsCollectionRequestErrors.Add(c.ErrorCount)
+	ecsCollectionRequestErrors.Add(float64(c.ErrorCount))
 	// we have recorded this round of errors zero out the errorCount before moving on
 	c.ZeroErrorCount()
 	h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{})

diff --git a/pkg/ecsclient/ecsclient.go b/pkg/ecsclient/ecsclient.go
@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	ecsconfig "github.com/paychex/prometheus-emcecs-exporter/pkg/config"
@@ -24,7 +25,7 @@ type EcsClient struct {
 	nodeListMgmtIP []string
 	nodeListDataIP []string
 	EcsVersion     string
-	ErrorCount     float64
+	ErrorCount     int64
 	Config         *ecsconfig.Config
 	httpClient     *http.Client
 }
@@ -343,7 +344,7 @@ func (c *EcsClient) retrieveNodeState(node string, ch chan<- NodeState) {
 	resp, err := c.httpClient.Get(reqStatusURL)
 	if err != nil {
 		log.WithFields(log.Fields{"package": "ecsclient", "cluster": c.ClusterAddress}).Error("Error connecting to ECS Cluster at: " + reqStatusURL)
-		c.ErrorCount++
+		atomic.AddInt64(&c.ErrorCount, 1)
 		ch <- *parsedOutput
 		return
 	}
@@ -354,7 +355,7 @@ func (c *EcsClient) retrieveNodeState(node string, ch chan<- NodeState) {
 	if err != nil {
 		log.WithFields(log.Fields{"package": "ecsclient", "cluster": c.ClusterAddress}).Error("Error un-marshaling XML from: " + reqStatusURL)
 		log.WithFields(log.Fields{"package": "ecsclient", "cluster": c.ClusterAddress}).Error(err)
-		c.ErrorCount++
+		atomic.AddInt64(&c.ErrorCount, 1)
 		ch <- *parsedOutput
 		return
 	}
@@ -369,7 +370,7 @@ func (c *EcsClient) retrieveNodeState(node string, ch chan<- NodeState) {
 	if err != nil {
 		log.WithFields(log.Fields{"package": "ecsclient", "cluster": c.ClusterAddress}).Error("Error connecting to ECS Cluster.")
 		log.WithFields(log.Fields{"package": "ecsclient", "cluster": c.ClusterAddress}).Error(err)
-		c.ErrorCount++
+		atomic.AddInt64(&c.ErrorCount, 1)
 		ch <- *parsedOutput
 		return
 	}
@@ -380,7 +381,7 @@ func (c *EcsClient) retrieveNodeState(node string, ch chan<- NodeState) {
 	if err != nil {
 		log.WithFields(log.Fields{"package": "ecsclient", "cluster": c.ClusterAddress}).Error("Error un-marshaling XML from: " + reqConnectionsURL)
 		log.WithFields(log.Fields{"package": "ecsclient", "cluster": c.ClusterAddress}).Error(err)
-		c.ErrorCount++
+		atomic.AddInt64(&c.ErrorCount, 1)
 		ch <- *parsedOutput
 		return
 	}
@@ -405,8 +406,8 @@ func (c *EcsClient) RetrieveNodeStateParallel() []NodeState {
 	return NodeStates
 }
 
-// Zero out the error count for this client. This should be called after we have recorded any existing error count
+// ZeroErrorCount resets the error count for this client. This should be called after we have recorded any existing error count
 func (c *EcsClient) ZeroErrorCount() {
 	log.WithFields(log.Fields{"package": "ecsclient", "cluster": c.ClusterAddress}).Debug("Zeroing out client error count")
-	c.ErrorCount = 0
+	atomic.StoreInt64(&c.ErrorCount, 0)
 }