Add tests

nyaruka · Dec 13, 2024 · bade8ef · bade8ef
1 parent aa63ea1
commit bade8ef
Show file tree

Hide file tree

Showing 5 changed files with 76 additions and 27 deletions.
diff --git a/archives/archives.go b/archives/archives.go
@@ -967,21 +967,20 @@ func ArchiveActiveOrgs(rt *runtime.Runtime) error {
 
 	timeTaken := dates.Now().Sub(start)
 	slog.Info("archiving of active orgs complete", "time_taken", timeTaken, "num_orgs", len(orgs))
-	if rt.Config.DeploymentID != "dev" {
-
-		rt.CW.Queue(types.MetricDatum{MetricName: aws.String("ArchiveElapsed"), Value: aws.Float64(float64(len(orgs)))})
-		rt.CW.Queue(types.MetricDatum{MetricName: aws.String("OrgsArchived"), Value: aws.Float64(float64(len(orgs)))})
-		rt.CW.Queue(types.MetricDatum{MetricName: aws.String("MsgsRecordsArchived"), Value: aws.Float64(float64(totalMsgsRecordsArchived))})
-		rt.CW.Queue(types.MetricDatum{MetricName: aws.String("MsgsArchivedsCreated"), Value: aws.Float64(float64(totalMsgsArchivesCreated))})
-		rt.CW.Queue(types.MetricDatum{MetricName: aws.String("MsgsArchivedsFailed"), Value: aws.Float64(float64(totalMsgsArchivesFailed))})
-		rt.CW.Queue(types.MetricDatum{MetricName: aws.String("MsgsRollupsCreated"), Value: aws.Float64(float64(totalMsgsRollupsCreated))})
-		rt.CW.Queue(types.MetricDatum{MetricName: aws.String("MsgsRollupsFailed"), Value: aws.Float64(float64(totalMsgsRollupsFailed))})
-		rt.CW.Queue(types.MetricDatum{MetricName: aws.String("RunsRecordsArchived"), Value: aws.Float64(float64(totalRunsRecordsArchived))})
-		rt.CW.Queue(types.MetricDatum{MetricName: aws.String("RunsArchivedsCreated"), Value: aws.Float64(float64(totalRunsArchivesCreated))})
-		rt.CW.Queue(types.MetricDatum{MetricName: aws.String("RunsArchivedsFailed"), Value: aws.Float64(float64(totalRunsArchivesFailed))})
-		rt.CW.Queue(types.MetricDatum{MetricName: aws.String("RunsRollupsCreated"), Value: aws.Float64(float64(totalRunsRollupsCreated))})
-		rt.CW.Queue(types.MetricDatum{MetricName: aws.String("RunsRollupsFailed"), Value: aws.Float64(float64(totalRunsRollupsFailed))})
-	}
+
+	rt.CW.Queue(types.MetricDatum{MetricName: aws.String("ArchiveElapsed"), Value: aws.Float64(timeTaken.Seconds()), Unit: types.StandardUnitSeconds})
+	rt.CW.Queue(types.MetricDatum{MetricName: aws.String("OrgsArchived"), Value: aws.Float64(float64(len(orgs))), Unit: types.StandardUnitCount})
+	rt.CW.Queue(types.MetricDatum{MetricName: aws.String("MsgsRecordsArchived"), Value: aws.Float64(float64(totalMsgsRecordsArchived)), Unit: types.StandardUnitCount})
+	rt.CW.Queue(types.MetricDatum{MetricName: aws.String("MsgsArchivedsCreated"), Value: aws.Float64(float64(totalMsgsArchivesCreated)), Unit: types.StandardUnitCount})
+	rt.CW.Queue(types.MetricDatum{MetricName: aws.String("MsgsArchivedsFailed"), Value: aws.Float64(float64(totalMsgsArchivesFailed)), Unit: types.StandardUnitCount})
+	rt.CW.Queue(types.MetricDatum{MetricName: aws.String("MsgsRollupsCreated"), Value: aws.Float64(float64(totalMsgsRollupsCreated)), Unit: types.StandardUnitCount})
+	rt.CW.Queue(types.MetricDatum{MetricName: aws.String("MsgsRollupsFailed"), Value: aws.Float64(float64(totalMsgsRollupsFailed)), Unit: types.StandardUnitCount})
+	rt.CW.Queue(types.MetricDatum{MetricName: aws.String("RunsRecordsArchived"), Value: aws.Float64(float64(totalRunsRecordsArchived)), Unit: types.StandardUnitCount})
+	rt.CW.Queue(types.MetricDatum{MetricName: aws.String("RunsArchivedsCreated"), Value: aws.Float64(float64(totalRunsArchivesCreated)), Unit: types.StandardUnitCount})
+	rt.CW.Queue(types.MetricDatum{MetricName: aws.String("RunsArchivedsFailed"), Value: aws.Float64(float64(totalRunsArchivesFailed)), Unit: types.StandardUnitCount})
+	rt.CW.Queue(types.MetricDatum{MetricName: aws.String("RunsRollupsCreated"), Value: aws.Float64(float64(totalRunsRollupsCreated)), Unit: types.StandardUnitCount})
+	rt.CW.Queue(types.MetricDatum{MetricName: aws.String("RunsRollupsFailed"), Value: aws.Float64(float64(totalRunsRollupsFailed)), Unit: types.StandardUnitCount})
+
 	analytics.Gauge("archiver.archive_elapsed", timeTaken.Seconds())
 	analytics.Gauge("archiver.orgs_archived", float64(len(orgs)))
 	analytics.Gauge("archiver.msgs_records_archived", float64(totalMsgsRecordsArchived))

diff --git a/archives/archives_test.go b/archives/archives_test.go
@@ -6,9 +6,12 @@ import (
 	"io"
 	"log/slog"
 	"os"
+	"sync"
 	"testing"
 	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
 	"github.com/jmoiron/sqlx"
 	_ "github.com/lib/pq"
 	"github.com/nyaruka/gocommon/analytics"
@@ -44,7 +47,7 @@ func setup(t *testing.T) (context.Context, *runtime.Runtime) {
 
 	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug})))
 
-	CW, err := cwatch.NewService("root", "key", "us-east-1", "Foo", "testing")
+	CW, err := cwatch.NewService(config.AWSAccessKeyID, config.AWSSecretAccessKey, config.AWSRegion, config.CloudwatchNamespace, config.DeploymentID)
 	require.NoError(t, err)
 
 	return ctx, &runtime.Runtime{Config: config, DB: db, S3: s3Client, CW: CW}
@@ -495,7 +498,24 @@ func TestArchiveOrgRuns(t *testing.T) {
 }
 
 func TestArchiveActiveOrgs(t *testing.T) {
-	_, rt := setup(t)
+	ctx, rt := setup(t)
+	wg := &sync.WaitGroup{}
+
+	rt.CW.StartQueue(wg, time.Millisecond*100)
+	assert.Equal(t, 0, rt.CW.Client.(*cwatch.DevClient).CallCount())
+
+	_, err := rt.CW.Client.PutMetricData(ctx, rt.CW.Prepare([]types.MetricDatum{
+		{MetricName: aws.String("NumGoats"), Value: aws.Float64(10), Unit: types.StandardUnitCount},
+		{MetricName: aws.String("NumSheep"), Dimensions: []types.Dimension{{Name: aws.String("Host"), Value: aws.String("foo1")}}, Value: aws.Float64(20), Unit: types.StandardUnitCount},
+	}))
+
+	assert.NoError(t, err)
+	assert.Equal(t, 1, rt.CW.Client.(*cwatch.DevClient).CallCount())
+
+	// test queuing metrics to be sent by batching process
+	rt.CW.Queue(types.MetricDatum{MetricName: aws.String("SleepTime"), Value: aws.Float64(30), Unit: types.StandardUnitSeconds})
+
+	time.Sleep(time.Millisecond * 300)
 
 	mockAnalytics := analytics.NewMock()
 	analytics.RegisterBackend(mockAnalytics)
@@ -504,7 +524,7 @@ func TestArchiveActiveOrgs(t *testing.T) {
 	dates.SetNowFunc(dates.NewSequentialNow(time.Date(2018, 1, 8, 12, 30, 0, 0, time.UTC), time.Second))
 	defer dates.SetNowFunc(time.Now)
 
-	err := ArchiveActiveOrgs(rt)
+	err = ArchiveActiveOrgs(rt)
 	assert.NoError(t, err)
 
 	assert.Equal(t, map[string][]float64{
@@ -523,5 +543,9 @@ func TestArchiveActiveOrgs(t *testing.T) {
 	}, mockAnalytics.Gauges)
 
 	analytics.Stop()
+	rt.CW.StopQueue()
+
+	// check the queued metric was sent
+	assert.Equal(t, 2, rt.CW.Client.(*cwatch.DevClient).CallCount())
 
 }
diff --git a/cmd/rp-archiver/main.go b/cmd/rp-archiver/main.go
@@ -127,17 +127,15 @@ func main() {
 		analytics.RegisterBackend(analytics.NewLibrato(config.LibratoUsername, config.LibratoToken, config.InstanceName, time.Second, wg))
 	}
 
-	if rt.Config.DeploymentID != "dev" {
-		rt.CW, err = cwatch.NewService(config.AWSAccessKeyID, config.AWSSecretAccessKey, config.AWSRegion, "Temba", config.DeploymentID)
-		if err != nil {
-			logger.Error("unable to create cloudwatch service", "error", err)
-		} else {
-			logger.Info("cloudwatch service ok", "state", "starting")
-		}
+	rt.CW, err = cwatch.NewService(config.AWSAccessKeyID, config.AWSSecretAccessKey, config.AWSRegion, config.CloudwatchNamespace, config.DeploymentID)
+	if err != nil {
+		logger.Error("unable to create cloudwatch service", "error", err)
+	} else {
+		logger.Info("cloudwatch service ok", "state", "starting")
 	}
 
 	analytics.Start()
-	rt.CW.StartQueue(wg)
+	rt.CW.StartQueue(wg, time.Second*3)
 
 	if config.Once {
 		doArchival(rt)

diff --git a/go.mod b/go.mod
@@ -13,7 +13,7 @@ require (
 	github.com/jmoiron/sqlx v1.4.0
 	github.com/lib/pq v1.10.9
 	github.com/nyaruka/ezconf v0.3.0
-	github.com/nyaruka/gocommon v1.60.1
+	github.com/nyaruka/gocommon v1.60.2
 	github.com/samber/slog-multi v1.2.0
 	github.com/samber/slog-sentry v1.2.2
 	github.com/stretchr/testify v1.10.0

diff --git a/go.sum b/go.sum
@@ -84,6 +84,34 @@ github.com/nyaruka/gocommon v1.60.0 h1:VjkimXF1dlJltWPYRFnqEIizoXXqrsmCuRBt/t1h4
 github.com/nyaruka/gocommon v1.60.0/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0=
 github.com/nyaruka/gocommon v1.60.1 h1:m/BXoBQ1KVzbpmTJ5vuQrv084mWyQ6gtuX6cOeva+lM=
 github.com/nyaruka/gocommon v1.60.1/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0=
+github.com/nyaruka/gocommon v1.60.2-0.20241213101040-31d318de7516 h1:W2nGSNyLCdtj+FuJe6gC6wxskQHnc1b4guKJbvWuFJ4=
+github.com/nyaruka/gocommon v1.60.2-0.20241213101040-31d318de7516/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0=
+github.com/nyaruka/gocommon v1.60.2-0.20241213101632-7af3f01d418c h1:MWhp6Or3C/S0CRAAEuUFsNanmeBsml9vhIo1rGMAupw=
+github.com/nyaruka/gocommon v1.60.2-0.20241213101632-7af3f01d418c/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0=
+github.com/nyaruka/gocommon v1.60.2-0.20241213101955-a34cc2b9e7a6 h1:yU1kOmeG4jC11R7gqsP0Akyy0XYjzKFTfmGcS+PsLxU=
+github.com/nyaruka/gocommon v1.60.2-0.20241213101955-a34cc2b9e7a6/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0=
+github.com/nyaruka/gocommon v1.60.2-0.20241213102159-aca359588626 h1:mrtup7cuBEVmIQAVjPkvzw/5+lHnfpAjjWooWRa7c38=
+github.com/nyaruka/gocommon v1.60.2-0.20241213102159-aca359588626/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0=
+github.com/nyaruka/gocommon v1.60.2-0.20241213102535-f6608972de8a h1:9jHgS60UIMleaPcfXS7wfSeE5VZohLapsbZD5wIhQQw=
+github.com/nyaruka/gocommon v1.60.2-0.20241213102535-f6608972de8a/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0=
+github.com/nyaruka/gocommon v1.60.2-0.20241213104214-ab724df76615 h1:xU3bWjxh2AGm7YnvpMpJZ/r880AIeKfnnR5iIBL8L4w=
+github.com/nyaruka/gocommon v1.60.2-0.20241213104214-ab724df76615/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0=
+github.com/nyaruka/gocommon v1.60.2-0.20241213110004-7421f1ac47a6 h1:mz5mynR3qWyHgV2il/z5dN+WA0IZQrpIR+wGngZixvg=
+github.com/nyaruka/gocommon v1.60.2-0.20241213110004-7421f1ac47a6/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0=
+github.com/nyaruka/gocommon v1.60.2-0.20241213110203-0444041bd051 h1:yAH0gdA3l/ZcuHe0uuXygbjkiEkI75F7HhvkdR5Iibk=
+github.com/nyaruka/gocommon v1.60.2-0.20241213110203-0444041bd051/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0=
+github.com/nyaruka/gocommon v1.60.2-0.20241213110951-fb183c8bb4a5 h1:PP8mo3GwgKvKPKW4KAKhD4Fk2nzrM2NR1wNxdLPGePw=
+github.com/nyaruka/gocommon v1.60.2-0.20241213110951-fb183c8bb4a5/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0=
+github.com/nyaruka/gocommon v1.60.2-0.20241213111058-b7ec4abe84e0 h1:yZojcEs0ji+H5xzeSTvdX5+RorbtpvQBpGkPSKStuiU=
+github.com/nyaruka/gocommon v1.60.2-0.20241213111058-b7ec4abe84e0/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0=
+github.com/nyaruka/gocommon v1.60.2-0.20241213112709-d4ad3229b2d2 h1:AJTdPSYJb4bjqn8flO0/NfZzTPkAJBhgTWpG8RjQ01w=
+github.com/nyaruka/gocommon v1.60.2-0.20241213112709-d4ad3229b2d2/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0=
+github.com/nyaruka/gocommon v1.60.2-0.20241213113120-30b1b04a98e8 h1:wK38pZqJ6yJHOVzNRgDI0MxOp1fxEuXozHZIVrTlJCM=
+github.com/nyaruka/gocommon v1.60.2-0.20241213113120-30b1b04a98e8/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0=
+github.com/nyaruka/gocommon v1.60.2-0.20241213113333-112a0bc4c3ee h1:MH7kxmPa53XrbDROcK8vzYYw+Im0NMwGL/knR8TsSqE=
+github.com/nyaruka/gocommon v1.60.2-0.20241213113333-112a0bc4c3ee/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0=
+github.com/nyaruka/gocommon v1.60.2 h1:AvvSSAV70SV49ocNtvjpdb9NlcdiA2OQAL4NYVUcuV0=
+github.com/nyaruka/gocommon v1.60.2/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0=
 github.com/nyaruka/librato v1.1.1 h1:0nTYtJLl3Sn7lX3CuHsLf+nXy1k/tGV0OjVxLy3Et4s=
 github.com/nyaruka/librato v1.1.1/go.mod h1:fme1Fu1PT2qvkaBZyw8WW+SrnFe2qeeCWpvqmAaKAKE=
 github.com/nyaruka/null/v2 v2.0.3 h1:rdmMRQyVzrOF3Jff/gpU/7BDR9mQX0lcLl4yImsA3kw=