Disable prospect dlq alarm via aws (#105)

* chore(temp-disable-alert): disabling DLQ low alert temporarily as it generated false alert * feat(alarms): add ability to disable alarm actions w/o deleting alarm --------- Co-authored-by: Srilekha Kirsh <[email protected]>
Pocket · Feb 16, 2023 · 6656852 · 6656852
1 parent 303bc7c
commit 6656852
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 1 deletion.
diff --git a/.aws/src/event-rules/prospect-events/prospectEventRules.ts b/.aws/src/event-rules/prospect-events/prospectEventRules.ts
@@ -83,11 +83,15 @@ export class ProspectEvents extends Resource {
     // TODO: create a policy function for dev event bridge
     // this.createPolicyForEventBridgeToDevEventBridge();
 
+    //todo: disabling till this ticket is done:
+    //https://getpocket.atlassian.net/browse/INFRA-1048
+    //prospect-alert triggering false alert for 1 message in the DLQ
     createDeadLetterQueueAlarm(
       this,
       pagerDuty,
       this.sqsDlq.name,
-      `${eventConfig.name}-Rule-DLQ-Alarm`
+      `${eventConfig.name}-Rule-DLQ-Alarm`,
+      false // temporarily disabled, see note above
     );
 
     new NullProviders.Resource(this, 'null-resource', {

diff --git a/.aws/src/event-rules/user-api-events/userApiEventRules.ts b/.aws/src/event-rules/user-api-events/userApiEventRules.ts
@@ -45,6 +45,7 @@ export class UserApiEvents extends Resource {
       pagerDuty,
       this.snsTopicDlq.name,
       `${eventConfig.name}-Rule-dlq-alarm`,
+      true,
       4,
       300,
       10

diff --git a/.aws/src/event-rules/utils.ts b/.aws/src/event-rules/utils.ts
@@ -14,13 +14,15 @@ import { PocketPagerDuty } from '@pocket-tools/terraform-modules';
  * @param evaluationPeriods
  * @param periodInSeconds
  * @param threshold
+ * @param enabled if alarm should act on state changes (pass/fail), defaults to true
  * @private
  */
 export function createDeadLetterQueueAlarm(
   stack,
   pagerDuty: PocketPagerDuty,
   queueName: string,
   alarmName: string,
+  enabled: boolean = true,
   evaluationPeriods = 2,
   periodInSeconds = 900,
   threshold = 15
@@ -38,5 +40,6 @@ export function createDeadLetterQueueAlarm(
     statistic: 'Sum',
     alarmActions: config.isDev ? [] : [pagerDuty.snsNonCriticalAlarmTopic.arn],
     okActions: config.isDev ? [] : [pagerDuty.snsNonCriticalAlarmTopic.arn],
+    actionsEnabled: enabled,
   });
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -45,6 +45,7 @@ export class UserApiEvents extends Resource { @@
           pagerDuty,
           this.snsTopicDlq.name,
           `${eventConfig.name}-Rule-dlq-alarm`,
+          true,
 ,
 ,
@@ Expand Down @@