From bade8ef0e954eefe62513dbf4df445c6930b3c40 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Fri, 13 Dec 2024 13:12:09 +0200 Subject: [PATCH] Add tests --- archives/archives.go | 29 ++++++++++++++--------------- archives/archives_test.go | 30 +++++++++++++++++++++++++++--- cmd/rp-archiver/main.go | 14 ++++++-------- go.mod | 2 +- go.sum | 28 ++++++++++++++++++++++++++++ 5 files changed, 76 insertions(+), 27 deletions(-) diff --git a/archives/archives.go b/archives/archives.go index b170bb5..da037ef 100644 --- a/archives/archives.go +++ b/archives/archives.go @@ -967,21 +967,20 @@ func ArchiveActiveOrgs(rt *runtime.Runtime) error { timeTaken := dates.Now().Sub(start) slog.Info("archiving of active orgs complete", "time_taken", timeTaken, "num_orgs", len(orgs)) - if rt.Config.DeploymentID != "dev" { - - rt.CW.Queue(types.MetricDatum{MetricName: aws.String("ArchiveElapsed"), Value: aws.Float64(float64(len(orgs)))}) - rt.CW.Queue(types.MetricDatum{MetricName: aws.String("OrgsArchived"), Value: aws.Float64(float64(len(orgs)))}) - rt.CW.Queue(types.MetricDatum{MetricName: aws.String("MsgsRecordsArchived"), Value: aws.Float64(float64(totalMsgsRecordsArchived))}) - rt.CW.Queue(types.MetricDatum{MetricName: aws.String("MsgsArchivedsCreated"), Value: aws.Float64(float64(totalMsgsArchivesCreated))}) - rt.CW.Queue(types.MetricDatum{MetricName: aws.String("MsgsArchivedsFailed"), Value: aws.Float64(float64(totalMsgsArchivesFailed))}) - rt.CW.Queue(types.MetricDatum{MetricName: aws.String("MsgsRollupsCreated"), Value: aws.Float64(float64(totalMsgsRollupsCreated))}) - rt.CW.Queue(types.MetricDatum{MetricName: aws.String("MsgsRollupsFailed"), Value: aws.Float64(float64(totalMsgsRollupsFailed))}) - rt.CW.Queue(types.MetricDatum{MetricName: aws.String("RunsRecordsArchived"), Value: aws.Float64(float64(totalRunsRecordsArchived))}) - rt.CW.Queue(types.MetricDatum{MetricName: aws.String("RunsArchivedsCreated"), Value: aws.Float64(float64(totalRunsArchivesCreated))}) - rt.CW.Queue(types.MetricDatum{MetricName: aws.String("RunsArchivedsFailed"), Value: aws.Float64(float64(totalRunsArchivesFailed))}) - rt.CW.Queue(types.MetricDatum{MetricName: aws.String("RunsRollupsCreated"), Value: aws.Float64(float64(totalRunsRollupsCreated))}) - rt.CW.Queue(types.MetricDatum{MetricName: aws.String("RunsRollupsFailed"), Value: aws.Float64(float64(totalRunsRollupsFailed))}) - } + + rt.CW.Queue(types.MetricDatum{MetricName: aws.String("ArchiveElapsed"), Value: aws.Float64(timeTaken.Seconds()), Unit: types.StandardUnitSeconds}) + rt.CW.Queue(types.MetricDatum{MetricName: aws.String("OrgsArchived"), Value: aws.Float64(float64(len(orgs))), Unit: types.StandardUnitCount}) + rt.CW.Queue(types.MetricDatum{MetricName: aws.String("MsgsRecordsArchived"), Value: aws.Float64(float64(totalMsgsRecordsArchived)), Unit: types.StandardUnitCount}) + rt.CW.Queue(types.MetricDatum{MetricName: aws.String("MsgsArchivedsCreated"), Value: aws.Float64(float64(totalMsgsArchivesCreated)), Unit: types.StandardUnitCount}) + rt.CW.Queue(types.MetricDatum{MetricName: aws.String("MsgsArchivedsFailed"), Value: aws.Float64(float64(totalMsgsArchivesFailed)), Unit: types.StandardUnitCount}) + rt.CW.Queue(types.MetricDatum{MetricName: aws.String("MsgsRollupsCreated"), Value: aws.Float64(float64(totalMsgsRollupsCreated)), Unit: types.StandardUnitCount}) + rt.CW.Queue(types.MetricDatum{MetricName: aws.String("MsgsRollupsFailed"), Value: aws.Float64(float64(totalMsgsRollupsFailed)), Unit: types.StandardUnitCount}) + rt.CW.Queue(types.MetricDatum{MetricName: aws.String("RunsRecordsArchived"), Value: aws.Float64(float64(totalRunsRecordsArchived)), Unit: types.StandardUnitCount}) + rt.CW.Queue(types.MetricDatum{MetricName: aws.String("RunsArchivedsCreated"), Value: aws.Float64(float64(totalRunsArchivesCreated)), Unit: types.StandardUnitCount}) + rt.CW.Queue(types.MetricDatum{MetricName: aws.String("RunsArchivedsFailed"), Value: aws.Float64(float64(totalRunsArchivesFailed)), Unit: types.StandardUnitCount}) + rt.CW.Queue(types.MetricDatum{MetricName: aws.String("RunsRollupsCreated"), Value: aws.Float64(float64(totalRunsRollupsCreated)), Unit: types.StandardUnitCount}) + rt.CW.Queue(types.MetricDatum{MetricName: aws.String("RunsRollupsFailed"), Value: aws.Float64(float64(totalRunsRollupsFailed)), Unit: types.StandardUnitCount}) + analytics.Gauge("archiver.archive_elapsed", timeTaken.Seconds()) analytics.Gauge("archiver.orgs_archived", float64(len(orgs))) analytics.Gauge("archiver.msgs_records_archived", float64(totalMsgsRecordsArchived)) diff --git a/archives/archives_test.go b/archives/archives_test.go index 6d1376a..c6234ba 100644 --- a/archives/archives_test.go +++ b/archives/archives_test.go @@ -6,9 +6,12 @@ import ( "io" "log/slog" "os" + "sync" "testing" "time" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types" "github.com/jmoiron/sqlx" _ "github.com/lib/pq" "github.com/nyaruka/gocommon/analytics" @@ -44,7 +47,7 @@ func setup(t *testing.T) (context.Context, *runtime.Runtime) { slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug}))) - CW, err := cwatch.NewService("root", "key", "us-east-1", "Foo", "testing") + CW, err := cwatch.NewService(config.AWSAccessKeyID, config.AWSSecretAccessKey, config.AWSRegion, config.CloudwatchNamespace, config.DeploymentID) require.NoError(t, err) return ctx, &runtime.Runtime{Config: config, DB: db, S3: s3Client, CW: CW} @@ -495,7 +498,24 @@ func TestArchiveOrgRuns(t *testing.T) { } func TestArchiveActiveOrgs(t *testing.T) { - _, rt := setup(t) + ctx, rt := setup(t) + wg := &sync.WaitGroup{} + + rt.CW.StartQueue(wg, time.Millisecond*100) + assert.Equal(t, 0, rt.CW.Client.(*cwatch.DevClient).CallCount()) + + _, err := rt.CW.Client.PutMetricData(ctx, rt.CW.Prepare([]types.MetricDatum{ + {MetricName: aws.String("NumGoats"), Value: aws.Float64(10), Unit: types.StandardUnitCount}, + {MetricName: aws.String("NumSheep"), Dimensions: []types.Dimension{{Name: aws.String("Host"), Value: aws.String("foo1")}}, Value: aws.Float64(20), Unit: types.StandardUnitCount}, + })) + + assert.NoError(t, err) + assert.Equal(t, 1, rt.CW.Client.(*cwatch.DevClient).CallCount()) + + // test queuing metrics to be sent by batching process + rt.CW.Queue(types.MetricDatum{MetricName: aws.String("SleepTime"), Value: aws.Float64(30), Unit: types.StandardUnitSeconds}) + + time.Sleep(time.Millisecond * 300) mockAnalytics := analytics.NewMock() analytics.RegisterBackend(mockAnalytics) @@ -504,7 +524,7 @@ func TestArchiveActiveOrgs(t *testing.T) { dates.SetNowFunc(dates.NewSequentialNow(time.Date(2018, 1, 8, 12, 30, 0, 0, time.UTC), time.Second)) defer dates.SetNowFunc(time.Now) - err := ArchiveActiveOrgs(rt) + err = ArchiveActiveOrgs(rt) assert.NoError(t, err) assert.Equal(t, map[string][]float64{ @@ -523,5 +543,9 @@ func TestArchiveActiveOrgs(t *testing.T) { }, mockAnalytics.Gauges) analytics.Stop() + rt.CW.StopQueue() + + // check the queued metric was sent + assert.Equal(t, 2, rt.CW.Client.(*cwatch.DevClient).CallCount()) } diff --git a/cmd/rp-archiver/main.go b/cmd/rp-archiver/main.go index 83a6c00..5c6f15f 100644 --- a/cmd/rp-archiver/main.go +++ b/cmd/rp-archiver/main.go @@ -127,17 +127,15 @@ func main() { analytics.RegisterBackend(analytics.NewLibrato(config.LibratoUsername, config.LibratoToken, config.InstanceName, time.Second, wg)) } - if rt.Config.DeploymentID != "dev" { - rt.CW, err = cwatch.NewService(config.AWSAccessKeyID, config.AWSSecretAccessKey, config.AWSRegion, "Temba", config.DeploymentID) - if err != nil { - logger.Error("unable to create cloudwatch service", "error", err) - } else { - logger.Info("cloudwatch service ok", "state", "starting") - } + rt.CW, err = cwatch.NewService(config.AWSAccessKeyID, config.AWSSecretAccessKey, config.AWSRegion, config.CloudwatchNamespace, config.DeploymentID) + if err != nil { + logger.Error("unable to create cloudwatch service", "error", err) + } else { + logger.Info("cloudwatch service ok", "state", "starting") } analytics.Start() - rt.CW.StartQueue(wg) + rt.CW.StartQueue(wg, time.Second*3) if config.Once { doArchival(rt) diff --git a/go.mod b/go.mod index 454cb69..4055286 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/jmoiron/sqlx v1.4.0 github.com/lib/pq v1.10.9 github.com/nyaruka/ezconf v0.3.0 - github.com/nyaruka/gocommon v1.60.1 + github.com/nyaruka/gocommon v1.60.2 github.com/samber/slog-multi v1.2.0 github.com/samber/slog-sentry v1.2.2 github.com/stretchr/testify v1.10.0 diff --git a/go.sum b/go.sum index 36a59c8..2d29935 100644 --- a/go.sum +++ b/go.sum @@ -84,6 +84,34 @@ github.com/nyaruka/gocommon v1.60.0 h1:VjkimXF1dlJltWPYRFnqEIizoXXqrsmCuRBt/t1h4 github.com/nyaruka/gocommon v1.60.0/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0= github.com/nyaruka/gocommon v1.60.1 h1:m/BXoBQ1KVzbpmTJ5vuQrv084mWyQ6gtuX6cOeva+lM= github.com/nyaruka/gocommon v1.60.1/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0= +github.com/nyaruka/gocommon v1.60.2-0.20241213101040-31d318de7516 h1:W2nGSNyLCdtj+FuJe6gC6wxskQHnc1b4guKJbvWuFJ4= +github.com/nyaruka/gocommon v1.60.2-0.20241213101040-31d318de7516/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0= +github.com/nyaruka/gocommon v1.60.2-0.20241213101632-7af3f01d418c h1:MWhp6Or3C/S0CRAAEuUFsNanmeBsml9vhIo1rGMAupw= +github.com/nyaruka/gocommon v1.60.2-0.20241213101632-7af3f01d418c/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0= +github.com/nyaruka/gocommon v1.60.2-0.20241213101955-a34cc2b9e7a6 h1:yU1kOmeG4jC11R7gqsP0Akyy0XYjzKFTfmGcS+PsLxU= +github.com/nyaruka/gocommon v1.60.2-0.20241213101955-a34cc2b9e7a6/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0= +github.com/nyaruka/gocommon v1.60.2-0.20241213102159-aca359588626 h1:mrtup7cuBEVmIQAVjPkvzw/5+lHnfpAjjWooWRa7c38= +github.com/nyaruka/gocommon v1.60.2-0.20241213102159-aca359588626/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0= +github.com/nyaruka/gocommon v1.60.2-0.20241213102535-f6608972de8a h1:9jHgS60UIMleaPcfXS7wfSeE5VZohLapsbZD5wIhQQw= +github.com/nyaruka/gocommon v1.60.2-0.20241213102535-f6608972de8a/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0= +github.com/nyaruka/gocommon v1.60.2-0.20241213104214-ab724df76615 h1:xU3bWjxh2AGm7YnvpMpJZ/r880AIeKfnnR5iIBL8L4w= +github.com/nyaruka/gocommon v1.60.2-0.20241213104214-ab724df76615/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0= +github.com/nyaruka/gocommon v1.60.2-0.20241213110004-7421f1ac47a6 h1:mz5mynR3qWyHgV2il/z5dN+WA0IZQrpIR+wGngZixvg= +github.com/nyaruka/gocommon v1.60.2-0.20241213110004-7421f1ac47a6/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0= +github.com/nyaruka/gocommon v1.60.2-0.20241213110203-0444041bd051 h1:yAH0gdA3l/ZcuHe0uuXygbjkiEkI75F7HhvkdR5Iibk= +github.com/nyaruka/gocommon v1.60.2-0.20241213110203-0444041bd051/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0= +github.com/nyaruka/gocommon v1.60.2-0.20241213110951-fb183c8bb4a5 h1:PP8mo3GwgKvKPKW4KAKhD4Fk2nzrM2NR1wNxdLPGePw= +github.com/nyaruka/gocommon v1.60.2-0.20241213110951-fb183c8bb4a5/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0= +github.com/nyaruka/gocommon v1.60.2-0.20241213111058-b7ec4abe84e0 h1:yZojcEs0ji+H5xzeSTvdX5+RorbtpvQBpGkPSKStuiU= +github.com/nyaruka/gocommon v1.60.2-0.20241213111058-b7ec4abe84e0/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0= +github.com/nyaruka/gocommon v1.60.2-0.20241213112709-d4ad3229b2d2 h1:AJTdPSYJb4bjqn8flO0/NfZzTPkAJBhgTWpG8RjQ01w= +github.com/nyaruka/gocommon v1.60.2-0.20241213112709-d4ad3229b2d2/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0= +github.com/nyaruka/gocommon v1.60.2-0.20241213113120-30b1b04a98e8 h1:wK38pZqJ6yJHOVzNRgDI0MxOp1fxEuXozHZIVrTlJCM= +github.com/nyaruka/gocommon v1.60.2-0.20241213113120-30b1b04a98e8/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0= +github.com/nyaruka/gocommon v1.60.2-0.20241213113333-112a0bc4c3ee h1:MH7kxmPa53XrbDROcK8vzYYw+Im0NMwGL/knR8TsSqE= +github.com/nyaruka/gocommon v1.60.2-0.20241213113333-112a0bc4c3ee/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0= +github.com/nyaruka/gocommon v1.60.2 h1:AvvSSAV70SV49ocNtvjpdb9NlcdiA2OQAL4NYVUcuV0= +github.com/nyaruka/gocommon v1.60.2/go.mod h1:kFJuOq8COneV7ssfK6xgCMJ8gP8fQifLQnNXBnE4YL0= github.com/nyaruka/librato v1.1.1 h1:0nTYtJLl3Sn7lX3CuHsLf+nXy1k/tGV0OjVxLy3Et4s= github.com/nyaruka/librato v1.1.1/go.mod h1:fme1Fu1PT2qvkaBZyw8WW+SrnFe2qeeCWpvqmAaKAKE= github.com/nyaruka/null/v2 v2.0.3 h1:rdmMRQyVzrOF3Jff/gpU/7BDR9mQX0lcLl4yImsA3kw=