From 2931fdd5dfa39ba6b5ba855f6ddb385ccac64038 Mon Sep 17 00:00:00 2001 From: AnieeG Date: Mon, 26 Feb 2024 14:38:06 -0800 Subject: [PATCH] send grafana link as slack notification --- .github/workflows/integration-tests.yml | 4 +- integration-tests/ccip-tests/load/helper.go | 64 +++--- .../ccip-tests/testreporters/ccip.go | 203 +++++++++++++----- .../ccip-tests/testsetups/ccip.go | 3 + 4 files changed, 199 insertions(+), 75 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index fcb8523abd..c2f0627435 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -703,7 +703,9 @@ jobs: cl_image_tag: ${{ github.sha }}${{ matrix.product.tag_suffix }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} artifacts_name: ${{ matrix.product.name }}${{ matrix.product.tag_suffix }}-test-logs - artifacts_location: ./integration-tests/smoke/logs/ + artifacts_location: | + ./integration-tests/smoke/logs/ + ./integration-tests/ccip-tests/smoke/logs/* publish_check_name: ${{ matrix.product.name }} token: ${{ secrets.GITHUB_TOKEN }} go_mod_path: ./integration-tests/go.mod diff --git a/integration-tests/ccip-tests/load/helper.go b/integration-tests/ccip-tests/load/helper.go index 8d4fe5b939..41212d9efb 100644 --- a/integration-tests/ccip-tests/load/helper.go +++ b/integration-tests/ccip-tests/load/helper.go @@ -40,6 +40,21 @@ type LoadArgs struct { TestSetupArgs *testsetups.CCIPTestSetUpOutputs ChaosExps []ChaosConfig LoadgenTearDowns []func() + Labels map[string]string +} + +func (l *LoadArgs) SetReportParams() { + var qParams []string + for k, v := range l.Labels { + qParams = append(qParams, fmt.Sprintf("%s=%s", k, v)) + } + // add one of the source and destination network to the grafana query params + if len(l.TestSetupArgs.Lanes) > 0 { + qParams = append(qParams, fmt.Sprintf("source_chain=%s", l.TestSetupArgs.Lanes[0].ForwardLane.SourceNetworkName) + qParams = append(qParams, fmt.Sprintf("dest_chain=%s", l.TestSetupArgs.Lanes[0].ForwardLane.DestNetworkName)) + } + err := l.TestSetupArgs.Reporter.AddToGrafanaDashboardQueryParams(qParams...) + require.NoError(l.t, err, "failed to set grafana query params") } func (l *LoadArgs) Setup() { @@ -50,6 +65,18 @@ func (l *LoadArgs) Setup() { envName = "ccip-runner" } l.TestSetupArgs = testsetups.CCIPDefaultTestSetUp(l.TestCfg.Test, lggr, envName, nil, l.TestCfg) + namespace := l.TestCfg.TestGroupInput.TestRunName + if l.TestSetupArgs.Env != nil && l.TestSetupArgs.Env.K8Env != nil && l.TestSetupArgs.Env.K8Env.Cfg != nil { + namespace = l.TestSetupArgs.Env.K8Env.Cfg.Namespace + } + l.Labels = map[string]string{ + "test_group": "load", + "cluster": "sdlc", + "test_id": "ccip", + "namespace": namespace, + } + l.TestSetupArgs.Reporter.SetGrafanaURLProvider(l.TestCfg.EnvInput) + l.SetReportParams() } func (l *LoadArgs) setSchedule() { @@ -87,7 +114,6 @@ func (l *LoadArgs) SanityCheck() { func (l *LoadArgs) TriggerLoadByLane() { l.setSchedule() l.TestSetupArgs.Reporter.SetDuration(l.TestCfg.TestGroupInput.TestDuration.Duration()) - namespace := l.TestCfg.TestGroupInput.TestRunName // start load for a lane startLoad := func(lane *actions.CCIPLane) { @@ -98,10 +124,13 @@ func (l *LoadArgs) TriggerLoadByLane() { ccipLoad := NewCCIPLoad(l.TestCfg.Test, lane, l.TestCfg.TestGroupInput.PhaseTimeout.Duration(), 100000) ccipLoad.BeforeAllCall(l.TestCfg.TestGroupInput.MsgType, big.NewInt(*l.TestCfg.TestGroupInput.DestGasLimit)) - if lane.TestEnv != nil && lane.TestEnv.K8Env != nil && lane.TestEnv.K8Env.Cfg != nil { - namespace = lane.TestEnv.K8Env.Cfg.Namespace - } lokiConfig := l.TestCfg.EnvInput.Logging.Loki + labels := make(map[string]string) + for k, v := range l.Labels { + labels[k] = v + } + labels["source_chain"] = lane.SourceNetworkName + labels["dest_chain"] = lane.DestNetworkName loadRunner, err := wasp.NewGenerator(&wasp.Config{ T: l.TestCfg.Test, GenName: fmt.Sprintf("lane %s-> %s", lane.SourceNetworkName, lane.DestNetworkName), @@ -114,21 +143,15 @@ func (l *LoadArgs) TriggerLoadByLane() { Logger: ccipLoad.Lane.Logger, SharedData: l.TestCfg.TestGroupInput.MsgType, LokiConfig: wasp.NewLokiConfig(lokiConfig.Endpoint, lokiConfig.TenantId, nil, nil), - Labels: map[string]string{ - "test_group": "load", - "cluster": "sdlc", - "namespace": namespace, - "test_id": "ccip", - "source_chain": lane.SourceNetworkName, - "dest_chain": lane.DestNetworkName, - }, - FailOnErr: true, + Labels: labels, + FailOnErr: true, }) require.NoError(l.TestCfg.Test, err, "initiating loadgen for lane %s --> %s", lane.SourceNetworkName, lane.DestNetworkName) loadRunner.Run(false) l.AddToRunnerGroup(loadRunner) } + for _, lane := range l.TestSetupArgs.Lanes { lane := lane l.LoadStarterWg.Add(1) @@ -213,8 +236,6 @@ func (l *LoadArgs) TriggerLoadBySource() { require.NotNil(l.t, l.TestCfg.TestGroupInput.TestDuration, "test duration input is nil") require.GreaterOrEqual(l.t, 1, len(l.TestCfg.TestGroupInput.RequestPerUnitTime), "time unit input must be specified") l.TestSetupArgs.Reporter.SetDuration(l.TestCfg.TestGroupInput.TestDuration.Duration()) - namespace := l.TestCfg.TestGroupInput.TestRunName - var laneBySource = make(map[string][]*actions.CCIPLane) for _, lane := range l.TestSetupArgs.Lanes { laneBySource[lane.ForwardLane.SourceNetworkName] = append(laneBySource[lane.ForwardLane.SourceNetworkName], lane.ForwardLane) @@ -231,16 +252,11 @@ func (l *LoadArgs) TriggerLoadBySource() { l.lggr.Info(). Str("Source Network", source). Msg("Starting load for source") - if lanes[0].TestEnv != nil && lanes[0].TestEnv.K8Env != nil && lanes[0].TestEnv.K8Env.Cfg != nil { - namespace = lanes[0].TestEnv.K8Env.Cfg.Namespace - } - allLabels := map[string]string{ - "test_group": "load", - "cluster": "sdlc", - "namespace": namespace, - "test_id": "ccip", - "source_chain": source, + allLabels := make(map[string]string) + for k, v := range l.Labels { + allLabels[k] = v } + allLabels["source_chain"] = source multiCallGen, err := NewMultiCallLoadGenerator(l.TestCfg, lanes, l.TestCfg.TestGroupInput.RequestPerUnitTime[0], allLabels) require.NoError(l.t, err) lokiConfig := l.TestCfg.EnvInput.Logging.Loki diff --git a/integration-tests/ccip-tests/testreporters/ccip.go b/integration-tests/ccip-tests/testreporters/ccip.go index e7dc522e35..4c7718fb37 100644 --- a/integration-tests/ccip-tests/testreporters/ccip.go +++ b/integration-tests/ccip-tests/testreporters/ccip.go @@ -5,12 +5,14 @@ import ( "fmt" "os" "path/filepath" + "strings" "sync" "testing" "time" "github.com/rs/zerolog" "github.com/slack-go/slack" + "github.com/smartcontractkit/chainlink-testing-framework/k8s/config" "github.com/smartcontractkit/chainlink-testing-framework/testreporters" ) @@ -198,21 +200,91 @@ func (testStats *CCIPLaneStats) Finalize(lane string) { } type CCIPTestReporter struct { - t *testing.T - logger zerolog.Logger - namespace string - reportFilePath string - duration time.Duration // duration is the duration of the test - FailedLanes map[string]Phase `json:"failed_lanes_and_phases,omitempty"` // FailedLanes is the list of lanes that failed and the phase at which it failed - LaneStats map[string]*CCIPLaneStats `json:"lane_stats"` // LaneStats is the statistics for each lane - mu *sync.Mutex - sendSlackReport bool + t *testing.T + logger zerolog.Logger + startTime int64 + endTime int64 + grafanaURLProvider testreporters.GrafanaURLProvider + grafanaURL string + grafanaQueryParams []string + namespace string + reportFilePath string + duration time.Duration // duration is the duration of the test + FailedLanes map[string]Phase `json:"failed_lanes_and_phases,omitempty"` // FailedLanes is the list of lanes that failed and the phase at which it failed + LaneStats map[string]*CCIPLaneStats `json:"lane_stats"` // LaneStats is the statistics for each lane + mu *sync.Mutex + sendSlackReport bool } func (r *CCIPTestReporter) SetSendSlackReport(sendSlackReport bool) { r.sendSlackReport = sendSlackReport } +func (r *CCIPTestReporter) CompleteGrafanaDashboardURL() error { + if r.grafanaURLProvider == nil { + return fmt.Errorf("grafana URL provider is not set") + } + grafanaUrl, err := r.grafanaURLProvider.GetGrafanaBaseURL() + if err != nil { + return err + } + + dashboardUrl, err := r.grafanaURLProvider.GetGrafanaDashboardURL() + if err != nil { + return err + } + r.grafanaURL = fmt.Sprintf("%s%s", grafanaUrl, dashboardUrl) + err = r.AddToGrafanaDashboardQueryParams( + fmt.Sprintf("from=%d", r.startTime), + fmt.Sprintf("to=%d", r.endTime), + fmt.Sprintf("namespace=%s", r.namespace)) + if err != nil { + return err + } + + err = r.FormatGrafanaURLWithQueryParameters() + if err != nil { + return fmt.Errorf("error formatting grafana URL: %w", err) + } + r.logger.Info().Str("Dashboard", r.grafanaURL).Msg("Dashboard URL") + return nil +} + +// FormatGrafanaURLWithQueryParameters adds query params to the grafana URL +// The query params are added in the format ?key=value if the grafana URL does not have any query params +// If the grafana URL already has query params, the query params are added in the format &key=value +// The function parameter qParam should be in the format key=value +// If the function parameter qParam does not contain an =, an error is returned +func (r *CCIPTestReporter) FormatGrafanaURLWithQueryParameters() error { + for _, v := range r.grafanaQueryParams { + if !strings.Contains(v, "=") { + return fmt.Errorf("invalid query param %s", v) + } + if strings.Contains(r.grafanaURL, "?") { + r.grafanaURL = fmt.Sprintf("%s&%s", r.grafanaURL, v) + continue + } + r.grafanaURL = fmt.Sprintf("%s?%s", r.grafanaURL, v) + } + return nil +} + +// AddToGrafanaDashboardQueryParams adds query params to the QueryParams slice +// The function parameter qParam should be in the format key=value +// If the function parameter qParam does not contain an =, an error is returned +func (r *CCIPTestReporter) AddToGrafanaDashboardQueryParams(qParams ...string) error { + for _, qParam := range qParams { + if !strings.Contains(qParam, "=") { + return fmt.Errorf("invalid query param %s", qParam) + } + r.grafanaQueryParams = append(r.grafanaQueryParams, qParam) + } + return nil +} + +// SendSlackNotification sends a slack notification to the specified channel set in the environment variable "SLACK_CHANNEL" +// notifying the user set in the environment variable "SLACK_USER" +// The function returns an error if the slack notification fails func (r *CCIPTestReporter) SendSlackNotification(t *testing.T, slackClient *slack.Client, _ testreporters.GrafanaURLProvider) error { if r.sendSlackReport { r.logger.Info().Msg("Sending Slack notification") @@ -233,58 +305,56 @@ func (r *CCIPTestReporter) SendSlackNotification(t *testing.T, slackClient *slac if t.Failed() { headerText = ":x: CCIP Test FAILED :x:" } - for name, lane := range r.LaneStats { - if lane.FailedCountsByPhase[E2E] > 0 { - msgTexts = append(msgTexts, - fmt.Sprintf("lane %s :x:", name), - fmt.Sprintf( - "Run Duration = %.0fm "+ + // If grafanaURLProvider is not set, form the message notifying about the failed lanes with the report file path + if r.grafanaURLProvider == nil { + for name, lane := range r.LaneStats { + if lane.FailedCountsByPhase[E2E] > 0 { + msgTexts = append(msgTexts, + fmt.Sprintf("lane %s :x:", name), + fmt.Sprintf( "\nNumber of ccip-send= %d"+ - "\nNo of failed requests = %d", r.duration.Minutes(), lane.TotalRequests, lane.FailedCountsByPhase[E2E])) + "\nNo of failed requests = %d", lane.TotalRequests, lane.FailedCountsByPhase[E2E])) + } } - } - msgTexts = append(msgTexts, fmt.Sprintf( - "\nTest Run Summary created on _remote-test-runner_ at _%s_\nNotifying <@%s>", - r.reportFilePath, testreporters.SlackUserID)) - if r.namespace == "" { - r.SetNamespace("ccip") + msgTexts = append(msgTexts, fmt.Sprintf( + "\nTest Run Summary created on _remote-test-runner_ at _%s_\nNotifying <@%s>", + r.reportFilePath, testreporters.SlackUserID)) + } else { + // If grafanaURLProvider is set, form the message with the grafana dashboard link + err := r.CompleteGrafanaDashboardURL() + if err != nil { + return fmt.Errorf("error formatting grafana dashboard URL: %w", err) + } + msgTexts = append(msgTexts, fmt.Sprintf( + "\nTest Run Completed \nNotifying <@%s>\nGrafana Dashboard: %s", + testreporters.SlackUserID, r.grafanaURL)) } + messageBlocks := testreporters.SlackNotifyBlocks(headerText, r.namespace, msgTexts) ts, err := testreporters.SendSlackMessage(slackClient, slack.MsgOptionBlocks(messageBlocks...)) if err != nil { - return fmt.Errorf("failed to send slack message: %w messageBlocks = %v", err, messageBlocks) + fmt.Println(messageBlocks) + return fmt.Errorf("failed to send slack message: %w", err) } - - return testreporters.UploadSlackFile(slackClient, slack.FileUploadParameters{ - Title: fmt.Sprintf("CCIP Test Report %s", r.namespace), - Filetype: "json", - Filename: fmt.Sprintf("ccip_report_%s.csv", r.namespace), - File: r.reportFilePath, - InitialComment: fmt.Sprintf("CCIP Test Report %s.", r.namespace), - Channels: []string{testreporters.SlackChannel}, - ThreadTimestamp: ts, - }) + // if grafanaURLProvider is set, we don't want to write the report in a file + // the report will be shared in terms of grafana dashboard link + if r.grafanaURLProvider == nil { + return testreporters.UploadSlackFile(slackClient, slack.FileUploadParameters{ + Title: fmt.Sprintf("CCIP Test Report %s", r.namespace), + Filetype: "json", + Filename: fmt.Sprintf("ccip_report_%s.csv", r.namespace), + File: r.reportFilePath, + InitialComment: fmt.Sprintf("CCIP Test Report %s.", r.namespace), + Channels: []string{testreporters.SlackChannel}, + ThreadTimestamp: ts, + }) + } + return nil } func (r *CCIPTestReporter) WriteReport(folderPath string) error { l := r.logger - l.Debug().Str("Folder Path", folderPath).Msg("Writing CCIP Test Report") - if err := testreporters.MkdirIfNotExists(folderPath); err != nil { - return err - } - reportLocation := filepath.Join(folderPath, slackFile) - r.reportFilePath = reportLocation - slackFile, err := os.Create(reportLocation) - defer func() { - err = slackFile.Close() - if err != nil { - l.Error().Err(err).Msg("Error closing slack file") - } - }() - if err != nil { - return err - } for k := range r.LaneStats { r.LaneStats[k].Finalize(k) // if E2E for the lane has failed @@ -303,6 +373,27 @@ func (r *CCIPTestReporter) WriteReport(folderPath string) error { } else { r.logger.Info().Msg("All Lanes Passed") } + // if grafanaURLProvider is set, we don't want to write the report in a file + // the report will be shared in terms of grafana dashboard link + if r.grafanaURLProvider != nil { + return nil + } + l.Debug().Str("Folder Path", folderPath).Msg("Writing CCIP Test Report") + if err := testreporters.MkdirIfNotExists(folderPath); err != nil { + return err + } + reportLocation := filepath.Join(folderPath, slackFile) + r.reportFilePath = reportLocation + slackFile, err := os.Create(reportLocation) + defer func() { + err = slackFile.Close() + if err != nil { + l.Error().Err(err).Msg("Error closing slack file") + } + }() + if err != nil { + return err + } stats, err := json.MarshalIndent(r, "", " ") if err != nil { return err @@ -316,6 +407,12 @@ func (r *CCIPTestReporter) WriteReport(folderPath string) error { // SetNamespace sets the namespace of the report for clean reports func (r *CCIPTestReporter) SetNamespace(namespace string) { + // if the test is run in remote runner, the namespace will be set to the remote runner's namespace + if value, set := os.LookupEnv(config.EnvVarNamespace); set && value != "" { + r.namespace = value + return + } + // if the namespace is not set, set it to the namespace provided r.namespace = namespace } @@ -324,6 +421,10 @@ func (r *CCIPTestReporter) SetDuration(d time.Duration) { r.duration = d } +func (r *CCIPTestReporter) SetGrafanaURLProvider(provider testreporters.GrafanaURLProvider) { + r.grafanaURLProvider = provider +} + func (r *CCIPTestReporter) AddNewLane(name string, lggr zerolog.Logger) *CCIPLaneStats { r.mu.Lock() defer r.mu.Unlock() @@ -341,13 +442,15 @@ func (r *CCIPTestReporter) AddNewLane(name string, lggr zerolog.Logger) *CCIPLan func (r *CCIPTestReporter) SendReport(t *testing.T, namespace string, slackSend bool) error { logsPath := filepath.Join("logs", fmt.Sprintf("%s-%s-%d", t.Name(), namespace, time.Now().Unix())) r.SetNamespace(namespace) - r.SetSendSlackReport(namespace != "" && slackSend) + r.endTime = time.Now().UTC().UnixMilli() + r.SetSendSlackReport(r.namespace != "" && slackSend) return testreporters.SendReport(t, namespace, logsPath, r, nil) } func NewCCIPTestReporter(t *testing.T, lggr zerolog.Logger) *CCIPTestReporter { return &CCIPTestReporter{ LaneStats: make(map[string]*CCIPLaneStats), + startTime: time.Now().UTC().UnixMilli(), logger: lggr, t: t, mu: &sync.Mutex{}, diff --git a/integration-tests/ccip-tests/testsetups/ccip.go b/integration-tests/ccip-tests/testsetups/ccip.go index 971e977e4c..69316b4b00 100644 --- a/integration-tests/ccip-tests/testsetups/ccip.go +++ b/integration-tests/ccip-tests/testsetups/ccip.go @@ -253,6 +253,9 @@ func NewCCIPTestConfig(t *testing.T, lggr zerolog.Logger, tType string) *CCIPTes if testCfg.CCIP.Env.Logging == nil || testCfg.CCIP.Env.Logging.Loki == nil { t.Fatal("loki config is required to be set for load test") } + if testCfg.CCIP.Env.Logging == nil || testCfg.CCIP.Env.Logging.Grafana == nil { + t.Fatal("grafana config is required to be set for load test") + } } ccipTestConfig := &CCIPTestConfig{ Test: t,