WIP

content/en/database_monitoring/guide/rds_auto_install.md

@@ -0,0 +1,5 @@
+---
+title: Automatically Monitoring RDS Instances With DBM
+
+---
+

static/resources/yaml/dbm/rds-auto-install/cloudformation.yaml

@@ -0,0 +1,235 @@
+# Automatically setup RDS instances to be ready for the Datadog integration
+# Before deploying, run deploy.sh in the lambda subdirectory to build the Lambda function
+#
+# To deploy run deploy.sh in the same directory as this file
+#
+
+
+Parameters:
+  VPC:
+    Type: "AWS::EC2::VPC::Id"
+    Description: "The VPC to deploy the resources in"
+  SubnetId:
+    Type: "AWS::EC2::Subnet::Id"
+    Description: "The subnet to deploy the Lambda function in"
+  LambdaBucket:
+    Type: String
+    Description: "The S3 bucket containing the Lambda function code"
+  LambdaKey:
+    Type: String
+    Description: "The S3 key for the Lambda function code"
+
+Resources:
+  # The SNS topic is used as the target for RDS instance events
+  SNS:
+    Type: AWS::SNS::Topic
+    Properties:
+      TopicName: rds-db-instance-events
+
+  # This subscription automatically routes messages to the SQS queue
+  # to be processed by the Lambda function
+  SNSSubscription:
+    Type: AWS::SNS::Subscription
+    Properties:
+      TopicArn: !Ref SNS
+      Protocol: sqs
+      RawMessageDelivery: true
+      Endpoint: !GetAtt SQS.Arn
+
+  # The SQS queue holds messages about RDS instance events
+  # There can be a delay between event creation and all permissions
+  # being applied, so we use a queue to retry until the permissions are set
+  SQS:
+    Type: AWS::SQS::Queue
+    Properties:
+      QueueName: rds-db-instance-events
+      VisibilityTimeout: 30
+      MessageRetentionPeriod: 3600 # 1 hour
+
+  # This policy allows the SNS topic to send messages to the SQS queue
+  SQSPolicy:
+    Type: AWS::SQS::QueuePolicy
+    Properties:
+      Queues:
+        - !Ref SQS
+      PolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: sns.amazonaws.com
+            Action: sqs:SendMessage
+            Resource: !GetAtt SQS.Arn
+            Condition:
+              ArnEquals:
+                "aws:SourceArn": !Ref SNS
+
+  # Map SQS queue directly to the Lambda function
+  SQSLambdaMapping:
+    Type: AWS::Lambda::EventSourceMapping
+    Properties:
+      BatchSize: 1
+      EventSourceArn: !GetAtt SQS.Arn
+      FunctionName: !GetAtt LambdaFunction.Arn
+      Enabled: true
+      MaximumBatchingWindowInSeconds: 0
+
+  # Subscribe to all RDS instance modification events
+  # availability and maintence events may include version changes
+  # configuration change events may include changes that enable new Datadog features
+  RDSEventSubscription:
+    Type: "AWS::RDS::EventSubscription"
+    Properties:
+      SnsTopicArn: !GetAtt SNS.TopicArn
+      EventCategories:
+        - "availability"
+        - "configuration change"
+        - "creation"
+        - "maintenance"
+      SourceType: "db-instance"
+      Enabled: true
+
+  # Allow any RDS instance to publish to the SNS topic
+  RDSPermission:
+    Type: AWS::SNS::TopicPolicy
+    Properties:
+      PolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+        - Effect: Allow
+          Principal:
+            Service: events.rds.amazonaws.com
+          Action: sns:Publish
+          Resource: "*"
+          Condition:
+            StringEquals:
+              "aws:SourceAccount": !Ref AWS::AccountId
+      Topics:
+        - !GetAtt SNS.TopicArn
+
+  # Permissions for the Lambda function that processes the SQS messages
+  LambdaExecutionRole:
+    Type: AWS::IAM::Role
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+        - Effect: Allow
+          Principal:
+            Service: lambda.amazonaws.com
+          Action: sts:AssumeRole
+      Policies:
+      - PolicyName: "AllowLogs"
+        PolicyDocument:
+          Version: "2012-10-17"
+          Statement:
+            - Effect: Allow
+              Action:
+                - logs:CreateLogGroup
+                - logs:CreateLogStream
+                - logs:PutLogEvents
+              Resource: "*"
+      # The execution role for Lambdas within a VPC need permissions to manage network interfaces
+      - PolicyName: "AllowEC2"
+        PolicyDocument:
+          Version: "2012-10-17"
+          Statement:
+            - Effect: Allow
+              Action:
+                - ec2:CreateNetworkInterface
+                - ec2:DescribeNetworkInterfaces
+                - ec2:DescribeSubnets
+                - ec2:DeleteNetworkInterface
+                - ec2:AssignPrivateIpAddresses
+                - ec2:UnassignPrivateIpAddresses
+              Resource: "*"
+      # Allow the Lambda to read from the SQS queue
+      - PolicyName: "AllowSQS"
+        PolicyDocument:
+          Version: "2012-10-17"
+          Statement:
+            - Effect: Allow
+              Action:
+                - sqs:ReceiveMessage
+                - sqs:DeleteMessage
+                - sqs:GetQueueAttributes
+              Resource: !GetAtt SQS.Arn
+      # Allow access to the RDS instances and master secrets
+      - PolicyName: "AllowRDS"
+        PolicyDocument:
+          Version: "2012-10-17"
+          Statement:
+            - Effect: Allow
+              Action:
+                - rds:DescribeDBInstances
+              Resource: "*"
+              # Condition:
+              #   StringEquals:
+              #     "aws:ResourceTag/datadoghq.com/install-dbm": "true"
+      # Allow access to the RDS instances and master secrets
+      - PolicyName: "AllowSecrets"
+        PolicyDocument:
+          Version: "2012-10-17"
+          Statement:
+            - Effect: Allow
+              Action:
+                - secretsmanager:GetSecretValue
+                - secretsmanager:DescribeSecret
+              Resource: "*"
+              Condition:
+                StringEquals:
+                  "aws:ResourceTag/aws:secretsmanager:owningService": "rds"
+
+  LambdaSecurityGroup:
+    Type: "AWS::EC2::SecurityGroup"
+    Properties:
+      GroupDescription: "Lambda traffic"
+      VpcId: !Ref VPC
+      SecurityGroupEgress:
+      - IpProtocol: "-1"
+        CidrIp: "0.0.0.0/0"
+
+  LambdaFunction:
+    Type: AWS::Lambda::Function
+    Properties:
+      Handler: index.handler
+      Role: !GetAtt LambdaExecutionRole.Arn
+      Code:
+        S3Bucket: !Ref LambdaBucket
+        S3Key: !Ref LambdaKey
+      # Environment:
+      #   Variables:
+      #     ADMIN_USER_SECRET_ARN: !Ref AdminUserSecretArn
+      #     DATADOG_USER_SECRET_ARN: !Ref DatadogUserSecretArn
+      Runtime: nodejs22.x
+      Timeout: 10
+      MemorySize: 128
+      VpcConfig:
+        SecurityGroupIds:
+          - !Ref LambdaSecurityGroup
+        SubnetIds:
+          - !Ref SubnetId
+
+  # The Lambda itself inherits the EC2 permissions so we need to deny them to the Lambda function
+  # This is a separate policy to avoid a circular dependency
+  LambdaFunctionDenyPolicy:
+    Type: AWS::IAM::Policy
+    Properties:
+      PolicyName: "LambdaFunctionDenyPolicy"
+      Roles:
+        - !Ref LambdaExecutionRole
+      PolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Effect: Deny
+            Action:
+              - ec2:CreateNetworkInterface
+              - ec2:DescribeNetworkInterfaces
+              - ec2:DescribeSubnets
+              - ec2:DeleteNetworkInterface
+              - ec2:AssignPrivateIpAddresses
+              - ec2:UnassignPrivateIpAddresses
+            Resource: "*"
+            Condition:
+              ArnEquals:
+                lambda:SourceFunctionArn: !GetAtt LambdaFunction.Arn

static/resources/yaml/dbm/rds-auto-install/deploy.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+aws cloudformation deploy \
+--template-file cloudformation.yaml \
+--stack-name rds-auto-install \
+--parameter-overrides \
+"VPC=$(aws cloudformation describe-stacks --stack-name "rds-auto-install-demo" --query "Stacks[0].Outputs[?OutputKey=='VPCId'].OutputValue" --output text)" \
+"SubnetId=$(aws cloudformation describe-stacks --stack-name "rds-auto-install-demo" --query "Stacks[0].Outputs[?OutputKey=='SubnetAId'].OutputValue" --output text)" \
+"LambdaBucket=rds-auto-install-demo" \
+"LambdaKey=$(tar c -C lambda index.js package.json package-lock.json | md5 -q).zip" \
+--capabilities CAPABILITY_IAM

static/resources/yaml/dbm/rds-auto-install/lambda/.gitignore

@@ -0,0 +1,2 @@
+node_modules
+zips

static/resources/yaml/dbm/rds-auto-install/lambda/deploy.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Prep and upload Lambda zip file
+# Avoid including the parent directories in the zip file
+echo "Installing npm dependencies"
+npm ci
+hashFile="$(tar c index.js package.json package-lock.json | md5 -q).zip"
+echo "Zipping lambda function"
+mkdir -p zips
+zip -r "zips/$hashFile" ./* >/dev/null
+aws s3 cp "zips/$hashFile" "s3://rds-auto-install-demo/$hashFile"