feat: add alert silencing (#169)

.github/workflows/release.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       postgres:
-        image: postgres:12
+        image: postgres:13
         ports:
           - 5432:5432
         env:

.github/workflows/test.yaml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       postgres:
-        image: postgres:12
+        image: postgres:13
         ports:
           - 5432:5432
         env:

cli/deps.go

@@ -6,14 +6,16 @@ import (
 
 	"github.com/newrelic/go-agent/v3/newrelic"
 	"github.com/odpf/salt/db"
-	"github.com/odpf/salt/log"
+	saltlog "github.com/odpf/salt/log"
 	"github.com/odpf/siren/config"
 	"github.com/odpf/siren/core/alert"
+	"github.com/odpf/siren/core/log"
 	"github.com/odpf/siren/core/namespace"
 	"github.com/odpf/siren/core/notification"
 	"github.com/odpf/siren/core/provider"
 	"github.com/odpf/siren/core/receiver"
 	"github.com/odpf/siren/core/rule"
+	"github.com/odpf/siren/core/silence"
 	"github.com/odpf/siren/core/subscription"
 	"github.com/odpf/siren/core/template"
 	"github.com/odpf/siren/internal/api"
@@ -30,18 +32,19 @@ import (
 
 func InitDeps(
 	ctx context.Context,
-	logger log.Logger,
+	logger saltlog.Logger,
 	cfg config.Config,
 	queue notification.Queuer,
 ) (*api.Deps, *newrelic.Application, *pgc.Client, map[string]notification.Notifier, error) {
 
 	telemetry.Init(ctx, cfg.Telemetry, logger)
+
 	nrApp, err := newrelic.NewApplication(
-		newrelic.ConfigAppName(cfg.Telemetry.ServiceName),
+		newrelic.ConfigAppName(cfg.Telemetry.NewRelicAppName),
 		newrelic.ConfigLicense(cfg.Telemetry.NewRelicAPIKey),
 	)
 	if err != nil {
-		return nil, nil, nil, nil, err
+		logger.Warn("failed to init newrelic", "err", err)
 	}
 
 	dbClient, err := db.New(cfg.DB)
@@ -59,22 +62,26 @@ func InitDeps(
 		return nil, nil, nil, nil, fmt.Errorf("cannot initialize encryptor: %w", err)
 	}
 
-	idempotencyRepository := postgres.NewIdempotencyRepository(pgClient)
 	templateRepository := postgres.NewTemplateRepository(pgClient)
 	templateService := template.NewService(templateRepository)
 
-	alertRepository := postgres.NewAlertRepository(pgClient)
-
 	providerRepository := postgres.NewProviderRepository(pgClient)
 	providerService := provider.NewService(providerRepository)
 
-	namespaceRepository := postgres.NewNamespaceRepository(pgClient)
+	logRepository := postgres.NewLogRepository(pgClient)
+	logService := log.NewService(logRepository)
 
 	cortexPluginService := cortex.NewPluginService(logger, cfg.Providers.Cortex)
+	alertRepository := postgres.NewAlertRepository(pgClient)
+	alertService := alert.NewService(
+		alertRepository,
+		logService,
+		map[string]alert.AlertTransformer{
+			provider.TypeCortex: cortexPluginService,
+		},
+	)
 
-	alertHistoryService := alert.NewService(alertRepository, map[string]alert.AlertTransformer{
-		provider.TypeCortex: cortexPluginService,
-	})
+	namespaceRepository := postgres.NewNamespaceRepository(pgClient)
 	namespaceService := namespace.NewService(encryptor, namespaceRepository, providerService, map[string]namespace.ConfigSyncer{
 		provider.TypeCortex: cortexPluginService,
 	})
@@ -89,6 +96,9 @@ func InitDeps(
 		},
 	)
 
+	silenceRepository := postgres.NewSilenceRepository(pgClient)
+	silenceService := silence.NewService(silenceRepository)
+
 	// plugin receiver services
 	slackPluginService := slack.NewPluginService(cfg.Receivers.Slack, encryptor)
 	pagerDutyPluginService := pagerduty.NewPluginService(cfg.Receivers.Pagerduty)
@@ -109,6 +119,7 @@ func InitDeps(
 	subscriptionRepository := postgres.NewSubscriptionRepository(pgClient)
 	subscriptionService := subscription.NewService(
 		subscriptionRepository,
+		logService,
 		namespaceService,
 		receiverService,
 	)
@@ -121,17 +132,33 @@ func InitDeps(
 		receiver.TypeFile:      filePluginService,
 	}
 
-	notificationService := notification.NewService(logger, queue, idempotencyRepository, receiverService, subscriptionService, notifierRegistry)
+	idempotencyRepository := postgres.NewIdempotencyRepository(pgClient)
+	notificationRepository := postgres.NewNotificationRepository(pgClient)
+	notificationService := notification.NewService(
+		logger,
+		notificationRepository,
+		queue,
+		notifierRegistry,
+		notification.Deps{
+			LogService:            logService,
+			IdempotencyRepository: idempotencyRepository,
+			ReceiverService:       receiverService,
+			SubscriptionService:   subscriptionService,
+			SilenceService:        silenceService,
+			AlertService:          alertService,
+		},
+	)
 
 	return &api.Deps{
 			TemplateService:     templateService,
 			RuleService:         ruleService,
-			AlertService:        alertHistoryService,
+			AlertService:        alertService,
 			ProviderService:     providerService,
 			NamespaceService:    namespaceService,
 			ReceiverService:     receiverService,
 			SubscriptionService: subscriptionService,
 			NotificationService: notificationService,
+			SilenceService:      silenceService,
 		}, nrApp, pgClient, notifierRegistry,
 		nil
 }

config/init.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/mcuadros/go-defaults"
 	"github.com/odpf/siren/core/receiver"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 func Init(configFile string) error {

core/alert/alert.go

@@ -9,20 +9,22 @@ import (
 type Repository interface {
 	Create(context.Context, Alert) (Alert, error)
 	List(context.Context, Filter) ([]Alert, error)
+	BulkUpdateSilence(context.Context, []int64, string) error
 }
 
 type Alert struct {
-	ID           uint64    `json:"id"`
-	ProviderID   uint64    `json:"provider_id"`
-	NamespaceID  uint64    `json:"namespace_id"`
-	ResourceName string    `json:"resource_name"`
-	MetricName   string    `json:"metric_name"`
-	MetricValue  string    `json:"metric_value"`
-	Severity     string    `json:"severity"`
-	Rule         string    `json:"rule"`
-	TriggeredAt  time.Time `json:"triggered_at"`
-	CreatedAt    time.Time `json:"created_at"`
-	UpdatedAt    time.Time `json:"updated_at"`
+	ID            uint64    `json:"id"`
+	ProviderID    uint64    `json:"provider_id"`
+	NamespaceID   uint64    `json:"namespace_id"`
+	ResourceName  string    `json:"resource_name"`
+	MetricName    string    `json:"metric_name"`
+	MetricValue   string    `json:"metric_value"`
+	Severity      string    `json:"severity"`
+	Rule          string    `json:"rule"`
+	TriggeredAt   time.Time `json:"triggered_at"`
+	CreatedAt     time.Time `json:"created_at"`
+	UpdatedAt     time.Time `json:"updated_at"`
+	SilenceStatus string    `json:"silence_status"`
 
 	// These fields won't be stored in the DB
 	// these are additional information for notification purposes

core/alert/filter.go

@@ -6,4 +6,6 @@ type Filter struct {
 	NamespaceID  uint64
 	StartTime    int64
 	EndTime      int64
+	SilenceID    string
+	IDs          []int64
 }

core/alert/service.go

@@ -7,15 +7,21 @@ import (
 	"github.com/odpf/siren/pkg/errors"
 )
 
+//go:generate mockery --name=LogService -r --case underscore --with-expecter --structname LogService --filename log_service.go --output=./mocks
+type LogService interface {
+	ListAlertIDsBySilenceID(ctx context.Context, silenceID string) ([]int64, error)
+}
+
 // Service handles business logic
 type Service struct {
 	repository Repository
+	logService LogService
 	registry   map[string]AlertTransformer
 }
 
 // NewService returns repository struct
-func NewService(repository Repository, registry map[string]AlertTransformer) *Service {
-	return &Service{repository, registry}
+func NewService(repository Repository, logService LogService, registry map[string]AlertTransformer) *Service {
+	return &Service{repository, logService, registry}
 }
 
 func (s *Service) CreateAlerts(ctx context.Context, providerType string, providerID uint64, namespaceID uint64, body map[string]interface{}) ([]Alert, int, error) {
@@ -29,15 +35,15 @@ func (s *Service) CreateAlerts(ctx context.Context, providerType string, provide
 		return nil, 0, err
 	}
 
-	for _, alrt := range alerts {
-		createdAlert, err := s.repository.Create(ctx, alrt)
+	for i := 0; i < len(alerts); i++ {
+		createdAlert, err := s.repository.Create(ctx, alerts[i])
 		if err != nil {
 			if errors.Is(err, ErrRelation) {
 				return nil, 0, errors.ErrNotFound.WithMsgf(err.Error())
 			}
 			return nil, 0, err
 		}
-		alrt.ID = createdAlert.ID
+		alerts[i].ID = createdAlert.ID
 	}
 
 	return alerts, firingLen, nil
@@ -48,9 +54,21 @@ func (s *Service) List(ctx context.Context, flt Filter) ([]Alert, error) {
 		flt.EndTime = time.Now().Unix()
 	}
 
+	if flt.SilenceID != "" {
+		alertIDs, err := s.logService.ListAlertIDsBySilenceID(ctx, flt.SilenceID)
+		if err != nil {
+			return nil, err
+		}
+		flt.IDs = alertIDs
+	}
+
 	return s.repository.List(ctx, flt)
 }
 
+func (s *Service) UpdateSilenceStatus(ctx context.Context, alertIDs []int64, hasSilenced bool, hasNonSilenced bool) error {
+	return s.repository.BulkUpdateSilence(ctx, alertIDs, silenceStatus(hasSilenced, hasNonSilenced))
+}
+
 func (s *Service) getProviderPluginService(providerType string) (AlertTransformer, error) {
 	pluginService, exist := s.registry[providerType]
 	if !exist {