marbot-ec2-instances.yml

---
# Copyright widdix GmbH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
AWSTemplateFormatVersion: '2010-09-09'
Description: 'marbot.io: EC2 instances (up to ten) monitoring (https://github.com/marbot-io/monitoring-jump-start)'
Metadata:
  'AWS::CloudFormation::Interface':
    ParameterGroups:
    - Label:
        default: 'marbot endpoint'
      Parameters:
      - EndpointId
      - Stage
    - Label:
        default: 'EC2'
      Parameters:
      - InstanceIds
      - NumberOfInstanceIds
    - Label:
        default: 'Thresholds'
      Parameters:
      - CPUUtilizationThreshold
      - CPUCreditBalanceThreshold
      - EBSIOCreditBalanceThreshold
      - EBSThroughputCreditBalanceThreshold
      - NetworkUtilizationThreshold
Parameters:
  EndpointId:
    Description: 'Your marbot endpoint ID (to get this value: select a channel where marbot belongs to and send a message like this: "@marbot show me my endpoint id").'
    Type: String
  Stage:
    Description: 'marbot stage (never change this!).'
    Type: String
    Default: v1
    AllowedValues: [v1, dev]
  InstanceIds:
    Description: 'The instance IDs of up to ten EC2 instances that you want to monitor.'
    Type: 'List<AWS::EC2::Instance::Id>'
  NumberOfInstanceIds:
    Description: 'The number of InstanceIds you entered (needs to be provided because CloudFormation can not count).'
    Type: 'Number'
    Default: 1
    MinValue: 1
    MaxValue: 10
  CPUUtilizationThreshold:
    Description: 'The maximum percentage of CPU utilization (set to -1 to disable).'
    Type: Number
    Default: 80
    MinValue: -1
    MaxValue: 100
  CPUCreditBalanceThreshold:
    Description: 'The minimum number of CPU credits available (t* instances only; set to -1 to disable).'
    Type: Number
    Default: 20
    MinValue: -1
  EBSIOCreditBalanceThreshold:
    Description: 'The minimum percentage of I/O credits remaining in the burst bucket (smaller instance only; set to -1 to disable).'
    Type: Number
    Default: 20
    MinValue: -1
    MaxValue: 100
  EBSThroughputCreditBalanceThreshold:
    Description: 'The minimum percentage of throughput credits remaining in the burst bucket (smaller instance only; set to -1 to disable).'
    Type: Number
    Default: 20
    MinValue: -1
    MaxValue: 100
  NetworkUtilizationThreshold:
    Description: 'The maximum percentage of network utilization (set to -1 to disable).'
    Type: Number
    Default: 80
    MinValue: -1
    MaxValue: 100
Conditions:
  Has1: !Equals [!Ref NumberOfInstanceIds, 1]
  Has2: !Equals [!Ref NumberOfInstanceIds, 2]
  Has3: !Equals [!Ref NumberOfInstanceIds, 3]
  Has4: !Equals [!Ref NumberOfInstanceIds, 4]
  Has5: !Equals [!Ref NumberOfInstanceIds, 5]
  Has6: !Equals [!Ref NumberOfInstanceIds, 6]
  Has7: !Equals [!Ref NumberOfInstanceIds, 7]
  Has8: !Equals [!Ref NumberOfInstanceIds, 8]
  Has9: !Equals [!Ref NumberOfInstanceIds, 9]
  Has10: !Equals [!Ref NumberOfInstanceIds, 10]
  HasInstance0Id: !Or [!Condition Has10, !Condition Has9, !Condition Has8, !Condition Has7, !Condition Has6, !Condition Has5, !Condition Has4, !Condition Has3, !Condition Has2, !Condition Has1]
  HasInstance1Id: !Or [!Condition Has10, !Condition Has9, !Condition Has8, !Condition Has7, !Condition Has6, !Condition Has5, !Condition Has4, !Condition Has3, !Condition Has2]
  HasInstance2Id: !Or [!Condition Has10, !Condition Has9, !Condition Has8, !Condition Has7, !Condition Has6, !Condition Has5, !Condition Has4, !Condition Has3]
  HasInstance3Id: !Or [!Condition Has10, !Condition Has9, !Condition Has8, !Condition Has7, !Condition Has6, !Condition Has5, !Condition Has4]
  HasInstance4Id: !Or [!Condition Has10, !Condition Has9, !Condition Has8, !Condition Has7, !Condition Has6, !Condition Has5]
  HasInstance5Id: !Or [!Condition Has10, !Condition Has9, !Condition Has8, !Condition Has7, !Condition Has6]
  HasInstance6Id: !Or [!Condition Has10, !Condition Has9, !Condition Has8, !Condition Has7]
  HasInstance7Id: !Or [!Condition Has10, !Condition Has9, !Condition Has8]
  HasInstance8Id: !Or [!Condition Has10, !Condition Has9]
  HasInstance9Id: !Condition Has10
Resources:
  ##########################################################################
  #                                                                        #
  #                                 TOPIC                                  #
  #                                                                        #
  ##########################################################################
  Topic:
    Type: 'AWS::SNS::Topic'
    Properties: {}
  TopicPolicy:
    Type: 'AWS::SNS::TopicPolicy'
    Properties:
      PolicyDocument:
        Id: Id1
        Version: '2012-10-17'
        Statement:
        - Sid: Sid1
          Effect: Allow
          Principal:
            Service: 'events.amazonaws.com' # Allow EventBridge
          Action: 'sns:Publish'
          Resource: !Ref Topic
        - Sid: Sid2
          Effect: Allow
          Principal:
            AWS: '*' # Allow CloudWatch Alarms
          Action: 'sns:Publish'
          Resource: !Ref Topic
          Condition:
            StringEquals:
              'AWS:SourceOwner': !Ref 'AWS::AccountId'
      Topics:
      - !Ref Topic
  TopicEndpointSubscription:
    DependsOn: TopicPolicy
    Type: 'AWS::SNS::Subscription'
    Properties:
      DeliveryPolicy:
        healthyRetryPolicy:
          minDelayTarget: 1
          maxDelayTarget: 60
          numRetries: 100
          numNoDelayRetries: 0
          backoffFunction: exponential
        throttlePolicy:
          maxReceivesPerSecond: 1
      Endpoint: !Sub 'https://api.marbot.io/${Stage}/endpoint/${EndpointId}'
      Protocol: https
      TopicArn: !Ref Topic
  MonitoringJumpStartEvent:
    DependsOn: TopicEndpointSubscription
    Type: 'AWS::Events::Rule'
    Properties:
      Description: 'Monitoring Jump Start connection. (created by marbot)'
      ScheduleExpression: 'rate(30 days)'
      State: ENABLED
      Targets:
      - Arn: !Ref Topic
        Id: marbot
        Input: !Sub |
          {
            "Type": "monitoring-jump-start-connection",
            "StackTemplate": "marbot-ec2-instances",
            "StackVersion": "2.6.0",
            "Partition": "${AWS::Partition}",
            "AccountId": "${AWS::AccountId}",
            "Region": "${AWS::Region}",
            "StackId": "${AWS::StackId}",
            "StackName": "${AWS::StackName}"
          }
  ##########################################################################
  #                                                                        #
  #                                 ALARMS                                 #
  #                                                                        #
  ##########################################################################
  Instance0NestedStack:
    DependsOn: [TopicEndpointSubscription, LambdaPolicy]
    Condition: HasInstance0Id
    Type: 'AWS::CloudFormation::Stack'
    Properties:
      Parameters:
        Topic: !Ref Topic
        CustomResourceLambda: !GetAtt 'LambdaFunction.Arn'
        InstanceId: !Select [0, !Ref InstanceIds]
        CPUUtilizationThreshold: !Ref CPUUtilizationThreshold
        CPUCreditBalanceThreshold: !Ref CPUCreditBalanceThreshold
        EBSIOCreditBalanceThreshold: !Ref EBSIOCreditBalanceThreshold
        EBSThroughputCreditBalanceThreshold: !Ref EBSThroughputCreditBalanceThreshold
        NetworkUtilizationThreshold: !Ref NetworkUtilizationThreshold
      TemplateURL: 'https://s3-eu-west-1.amazonaws.com/monitoring-jump-start/marbot-ec2-instances-nested.yml'
  Instance1NestedStack:
    DependsOn: [TopicEndpointSubscription, LambdaPolicy]
    Condition: HasInstance1Id
    Type: 'AWS::CloudFormation::Stack'
    Properties:
      Parameters:
        Topic: !Ref Topic
        CustomResourceLambda: !GetAtt 'LambdaFunction.Arn'
        InstanceId: !Select [1, !Ref InstanceIds]
        CPUUtilizationThreshold: !Ref CPUUtilizationThreshold
        CPUCreditBalanceThreshold: !Ref CPUCreditBalanceThreshold
        EBSIOCreditBalanceThreshold: !Ref EBSIOCreditBalanceThreshold
        EBSThroughputCreditBalanceThreshold: !Ref EBSThroughputCreditBalanceThreshold
        NetworkUtilizationThreshold: !Ref NetworkUtilizationThreshold
      TemplateURL: 'https://s3-eu-west-1.amazonaws.com/monitoring-jump-start/marbot-ec2-instances-nested.yml'
  Instance2NestedStack:
    DependsOn: [TopicEndpointSubscription, LambdaPolicy]
    Condition: HasInstance2Id
    Type: 'AWS::CloudFormation::Stack'
    Properties:
      Parameters:
        Topic: !Ref Topic
        CustomResourceLambda: !GetAtt 'LambdaFunction.Arn'
        InstanceId: !Select [2, !Ref InstanceIds]
        CPUUtilizationThreshold: !Ref CPUUtilizationThreshold
        CPUCreditBalanceThreshold: !Ref CPUCreditBalanceThreshold
        EBSIOCreditBalanceThreshold: !Ref EBSIOCreditBalanceThreshold
        EBSThroughputCreditBalanceThreshold: !Ref EBSThroughputCreditBalanceThreshold
        NetworkUtilizationThreshold: !Ref NetworkUtilizationThreshold
      TemplateURL: 'https://s3-eu-west-1.amazonaws.com/monitoring-jump-start/marbot-ec2-instances-nested.yml'
  Instance3NestedStack:
    DependsOn: [TopicEndpointSubscription, LambdaPolicy]
    Condition: HasInstance3Id
    Type: 'AWS::CloudFormation::Stack'
    Properties:
      Parameters:
        Topic: !Ref Topic
        CustomResourceLambda: !GetAtt 'LambdaFunction.Arn'
        InstanceId: !Select [3, !Ref InstanceIds]
        CPUUtilizationThreshold: !Ref CPUUtilizationThreshold
        CPUCreditBalanceThreshold: !Ref CPUCreditBalanceThreshold
        EBSIOCreditBalanceThreshold: !Ref EBSIOCreditBalanceThreshold
        EBSThroughputCreditBalanceThreshold: !Ref EBSThroughputCreditBalanceThreshold
        NetworkUtilizationThreshold: !Ref NetworkUtilizationThreshold
      TemplateURL: 'https://s3-eu-west-1.amazonaws.com/monitoring-jump-start/marbot-ec2-instances-nested.yml'
  Instance4NestedStack:
    DependsOn: [TopicEndpointSubscription, LambdaPolicy]
    Condition: HasInstance4Id
    Type: 'AWS::CloudFormation::Stack'
    Properties:
      Parameters:
        Topic: !Ref Topic
        CustomResourceLambda: !GetAtt 'LambdaFunction.Arn'
        InstanceId: !Select [4, !Ref InstanceIds]
        CPUUtilizationThreshold: !Ref CPUUtilizationThreshold
        CPUCreditBalanceThreshold: !Ref CPUCreditBalanceThreshold
        EBSIOCreditBalanceThreshold: !Ref EBSIOCreditBalanceThreshold
        EBSThroughputCreditBalanceThreshold: !Ref EBSThroughputCreditBalanceThreshold
        NetworkUtilizationThreshold: !Ref NetworkUtilizationThreshold
      TemplateURL: 'https://s3-eu-west-1.amazonaws.com/monitoring-jump-start/marbot-ec2-instances-nested.yml'
  Instance5NestedStack:
    DependsOn: [TopicEndpointSubscription, LambdaPolicy]
    Condition: HasInstance5Id
    Type: 'AWS::CloudFormation::Stack'
    Properties:
      Parameters:
        Topic: !Ref Topic
        CustomResourceLambda: !GetAtt 'LambdaFunction.Arn'
        InstanceId: !Select [5, !Ref InstanceIds]
        CPUUtilizationThreshold: !Ref CPUUtilizationThreshold
        CPUCreditBalanceThreshold: !Ref CPUCreditBalanceThreshold
        EBSIOCreditBalanceThreshold: !Ref EBSIOCreditBalanceThreshold
        EBSThroughputCreditBalanceThreshold: !Ref EBSThroughputCreditBalanceThreshold
        NetworkUtilizationThreshold: !Ref NetworkUtilizationThreshold
      TemplateURL: 'https://s3-eu-west-1.amazonaws.com/monitoring-jump-start/marbot-ec2-instances-nested.yml'
  Instance6NestedStack:
    DependsOn: [TopicEndpointSubscription, LambdaPolicy]
    Condition: HasInstance6Id
    Type: 'AWS::CloudFormation::Stack'
    Properties:
      Parameters:
        Topic: !Ref Topic
        CustomResourceLambda: !GetAtt 'LambdaFunction.Arn'
        InstanceId: !Select [6, !Ref InstanceIds]
        CPUUtilizationThreshold: !Ref CPUUtilizationThreshold
        CPUCreditBalanceThreshold: !Ref CPUCreditBalanceThreshold
        EBSIOCreditBalanceThreshold: !Ref EBSIOCreditBalanceThreshold
        EBSThroughputCreditBalanceThreshold: !Ref EBSThroughputCreditBalanceThreshold
        NetworkUtilizationThreshold: !Ref NetworkUtilizationThreshold
      TemplateURL: 'https://s3-eu-west-1.amazonaws.com/monitoring-jump-start/marbot-ec2-instances-nested.yml'
  Instance7NestedStack:
    DependsOn: [TopicEndpointSubscription, LambdaPolicy]
    Condition: HasInstance7Id
    Type: 'AWS::CloudFormation::Stack'
    Properties:
      Parameters:
        Topic: !Ref Topic
        CustomResourceLambda: !GetAtt 'LambdaFunction.Arn'
        InstanceId: !Select [7, !Ref InstanceIds]
        CPUUtilizationThreshold: !Ref CPUUtilizationThreshold
        CPUCreditBalanceThreshold: !Ref CPUCreditBalanceThreshold
        EBSIOCreditBalanceThreshold: !Ref EBSIOCreditBalanceThreshold
        EBSThroughputCreditBalanceThreshold: !Ref EBSThroughputCreditBalanceThreshold
        NetworkUtilizationThreshold: !Ref NetworkUtilizationThreshold
      TemplateURL: 'https://s3-eu-west-1.amazonaws.com/monitoring-jump-start/marbot-ec2-instances-nested.yml'
  Instance8NestedStack:
    DependsOn: [TopicEndpointSubscription, LambdaPolicy]
    Condition: HasInstance8Id
    Type: 'AWS::CloudFormation::Stack'
    Properties:
      Parameters:
        Topic: !Ref Topic
        CustomResourceLambda: !GetAtt 'LambdaFunction.Arn'
        InstanceId: !Select [8, !Ref InstanceIds]
        CPUUtilizationThreshold: !Ref CPUUtilizationThreshold
        CPUCreditBalanceThreshold: !Ref CPUCreditBalanceThreshold
        EBSIOCreditBalanceThreshold: !Ref EBSIOCreditBalanceThreshold
        EBSThroughputCreditBalanceThreshold: !Ref EBSThroughputCreditBalanceThreshold
        NetworkUtilizationThreshold: !Ref NetworkUtilizationThreshold
      TemplateURL: 'https://s3-eu-west-1.amazonaws.com/monitoring-jump-start/marbot-ec2-instances-nested.yml'
  Instance9NestedStack:
    DependsOn: [TopicEndpointSubscription, LambdaPolicy]
    Condition: HasInstance9Id
    Type: 'AWS::CloudFormation::Stack'
    Properties:
      Parameters:
        Topic: !Ref Topic
        CustomResourceLambda: !GetAtt 'LambdaFunction.Arn'
        InstanceId: !Select [9, !Ref InstanceIds]
        CPUUtilizationThreshold: !Ref CPUUtilizationThreshold
        CPUCreditBalanceThreshold: !Ref CPUCreditBalanceThreshold
        EBSIOCreditBalanceThreshold: !Ref EBSIOCreditBalanceThreshold
        EBSThroughputCreditBalanceThreshold: !Ref EBSThroughputCreditBalanceThreshold
        NetworkUtilizationThreshold: !Ref NetworkUtilizationThreshold
      TemplateURL: 'https://s3-eu-west-1.amazonaws.com/monitoring-jump-start/marbot-ec2-instances-nested.yml'
  ##########################################################################
  #                                                                        #
  #                            CUSTOM RESOURCES                            #
  #                                                                        #
  ##########################################################################
  LambdaRole:
    Type: 'AWS::IAM::Role'
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
        - Effect: Allow
          Principal:
            Service: 'lambda.amazonaws.com'
          Action: 'sts:AssumeRole'
      Policies:
      - PolicyName: ec2
        PolicyDocument:
          Statement:
          - Effect: Allow
            Action: 'ec2:DescribeInstances'
            Resource: '*'
  LambdaPolicy:
    Type: 'AWS::IAM::Policy'
    Properties:
      PolicyDocument:
        Statement:
        - Effect: Allow
          Action:
          - 'logs:CreateLogStream'
          - 'logs:PutLogEvents'
          Resource: !GetAtt 'LambdaLogGroup.Arn'
      PolicyName: lambda
      Roles:
      - !Ref LambdaRole
  LambdaFunction: # needs no monitoring because it is used as a custom resource
    Type: 'AWS::Lambda::Function'
    Properties:
      Code:
        ZipFile: |
          'use strict';
          const https = require('https');
          const response = require('cfn-response');
          const { EC2Client, DescribeInstancesCommand } = require('@aws-sdk/client-ec2');
          const ec2 = new EC2Client({apiVersion: '2016-11-15'});
          function getData(cb) {
            https.get('https://s3-eu-west-1.amazonaws.com/monitoring-jump-start/data/network.json', (res) => {
              if (res.statusCode === 200) {
                const chunks = [];
                res.on('data', (chunk) => chunks.push(chunk));
                res.on('end', () => {
                  const body = Buffer.concat(chunks);
                  const json = body.toString('utf8');
                  const data = JSON.parse(json);
                  cb(null, data);
                });
                } else {
                  cb(new Error(`unexpeczted status code: ${res.statusCode}`));
                }
            }).on('error', (err) => {
              cb(err);
            });
          }
          function format(n, networkUtilizationThreshold) {
            if (n === undefined) {
              return 99;
            }
            return Math.round(n * networkUtilizationThreshold) / 100;
          }
          exports.handler = (event, context) => {
            console.log(`Invoke: ${JSON.stringify(event)}`);
            if (event.RequestType === 'Delete') {
              response.send(event, context, response.SUCCESS, {});
            } else if (event.RequestType === 'Create' || event.RequestType === 'Update') {
              getData((err, networkData) => {
                if (err) {
                  console.log(`Error: ${JSON.stringify(err)}`);
                  response.send(event, context, response.FAILED, {});
                } else {
                  ec2.send(new DescribeInstancesCommand({
                    InstanceIds: [event.ResourceProperties.InstanceId]
                  }), (err, instanceData) => {
                    if (err) {
                      console.log(`Error: ${JSON.stringify(err)}`);
                      response.send(event, context, response.FAILED, {});
                    } else {
                      const instance = instanceData.Reservations[0].Instances[0];
                      console.log(`Instance data: ${JSON.stringify(instance)}`);
                      const network = networkData[instance.InstanceType];
                      let networkMaximum = undefined;
                      let networkBurst = undefined;
                      let networkBaseline = undefined;
                      if (network !== undefined) {
                        if (network.baseline !== undefined && network.burst !== undefined) {
                          networkBaseline = network.baseline;
                          networkBurst = network.burst;
                        } else if (network.baseline !== undefined) {
                          networkMaximum = network.baseline;
                        }
                      } else {
                        console.log(`No network data found for instance #${event.ResourceProperties.InstanceId} of type ${instance.InstanceType}`);
                      }
                      response.send(event, context, response.SUCCESS, {
                        InstanceType: instance.InstanceType,
                        NetworkMaximum: format(networkMaximum, event.ResourceProperties.NetworkUtilizationThreshold), // in Gbit/s
                        NetworkBurst: format(networkBurst, event.ResourceProperties.NetworkUtilizationThreshold), // in Gbit/s
                        NetworkBaseline: format(networkBaseline, event.ResourceProperties.NetworkUtilizationThreshold) // in Gbit/s
                      }, event.ResourceProperties.InstanceId);
                    }
                  });
                }
              });
            } else {
              cb(new Error(`unsupported RequestType: ${event.RequestType}`));
            }
          };
      Handler: 'index.handler'
      MemorySize: 128
      Role: !GetAtt 'LambdaRole.Arn'
      Runtime: 'nodejs18.x'
      Timeout: 60
  LambdaLogGroup:
    Type: 'AWS::Logs::LogGroup'
    Properties:
      LogGroupName: !Sub '/aws/lambda/${LambdaFunction}'
      RetentionInDays: 14
Outputs:
  StackName:
    Description: 'Stack name.'
    Value: !Sub '${AWS::StackName}'
  StackTemplate:
    Description: 'Stack template.'
    Value: 'marbot-ec2-instances'
  StackVersion:
    Description: 'Stack version.'
    Value: '2.6.0'