Calculate instances required for containers

app/jobs/deploy_runner_job.rb

@@ -9,7 +9,7 @@ def perform(heritage, without_before_deploy:, description: "")
     @heritage = heritage
     heritage.with_lock do
       if other_deploy_in_progress?(heritage)
-        notify(heritage, level: :error, message: "The other deployment is in progress. Stopped deploying.")
+        notify(level: :error, message: "The other deployment is in progress. Stopped deploying.")
         return
       end
 

app/models/backend/ecs/v1/service.rb

@@ -60,6 +60,10 @@ def applied?
       !(ecs_service.nil? || ecs_service.status != "ACTIVE")
     end
 
+    def task_definition
+      fetch_ecs_service.task_definition
+    end
+
     def register_task
       task_definition = HeritageTaskDefinition.service_definition(service).to_task_definition
       aws.ecs.register_task_definition(task_definition)

app/models/container_instance.rb

@@ -13,6 +13,9 @@ def user_data
       # Embed SHA2 hash dockercfg so that instance replacement happens when dockercfg is updated
       "# #{Digest::SHA256.hexdigest(district.dockercfg.to_s)}",
 
+      # Get IMDSv2 token that expires in 1 hour
+      'IMDSTOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 3600"`',
+
       # Setup swap
       "MEMSIZE=`cat /proc/meminfo | grep MemTotal | awk '{print $2}'`",
       "if [ $MEMSIZE -lt 2097152 ]; then",
@@ -26,6 +29,7 @@ def user_data
 
       "AWS_REGION=#{district.region}",
       "aws configure set s3.signature_version s3v4",
+
       "aws s3 cp s3://#{district.s3_bucket_name}/#{district.name}/ecs.config /etc/ecs/ecs.config",
       "chmod 600 /etc/ecs/ecs.config",
 
@@ -35,7 +39,7 @@ def user_data
       "service sshd restart",
 
       # Configure AWS CloudWatch Logs
-      "ec2_id=$(curl http://169.254.169.254/latest/meta-data/instance-id)",
+      'ec2_id=$(curl -H "X-aws-ec2-metadata-token: $IMDSTOKEN" -v http://169.254.169.254/latest/meta-data/instance-id)',
       'sed -i -e "s/{ec2_id}/$ec2_id/g" /etc/awslogs/awslogs.conf',
       'sed -i -e "s/us-east-1/'+district.region+'/g" /etc/awslogs/awscli.conf',
       "systemctl start awslogsd",

app/models/district.rb

@@ -109,14 +109,26 @@ def subnets(network = "Private")
     ).subnets
   end
 
+  def container_instance_arns
+    @container_instance_arns ||= aws.ecs.list_container_instances(
+                                   cluster: name
+                                 ).container_instance_arns
+  end
+
+  def cluster_container_instances
+    return [] if container_instance_arns.blank?
+
+    @cluster_container_instances ||= aws.ecs.describe_container_instances(
+                                       cluster: name,
+                                       container_instances: container_instance_arns
+                                     ).container_instances
+  end
+
   def container_instances
-    arns = aws.ecs.list_container_instances(cluster: name).container_instance_arns
-    return [] if arns.blank?
-    container_instances = aws.ecs.
-                          describe_container_instances(cluster: name, container_instances: arns).
-                          container_instances
+    return [] if cluster_container_instances.blank?
+
     instances = {}
-    container_instances.each do |ci|
+    cluster_container_instances.each do |ci|
       instance = {
         status: ci.status,
         container_instance_arn: ci.container_instance_arn,
@@ -129,7 +141,7 @@ def container_instances
     end
 
     ec2_instances = aws.ec2.describe_instances(
-      instance_ids: container_instances.map(&:ec2_instance_id)
+      instance_ids: cluster_container_instances.map(&:ec2_instance_id)
     ).reservations.map(&:instances).flatten
 
     ec2_instances.each do |ins|
@@ -210,6 +222,77 @@ def update_notification_stack
 
   private
 
+  def total_registered(resource)
+    container_instances.pluck(:registered_resources)
+                       .flatten
+                       .select {|x| x.name == resource.to_s.upcase}
+                       .sum {|x| x.integer_value}
+  end
+
+  def demand_structure(resource)
+    heritages.flat_map(&:services).flat_map do |service|
+      # map all the containers' memory or cpu
+      backend = service.send(:backend)
+
+      if backend.nil?
+        puts "service #{service.name} of H #{service.heritage.name} has no backend"
+
+        next {
+          count: 0,
+          amount: 0
+        }
+      end
+
+      ecs_service = backend.send(:ecs_service)
+
+      if ecs_service.nil?
+        puts "service #{service.name} of H #{service.heritage.name} has no ecs"
+
+        next {
+          count: 0,
+          amount: 0
+        }
+      end
+
+      definition = ecs_service.task_definition
+
+      # read the total amount requested by definition
+      total_resource = aws.ecs.describe_task_definition(task_definition: definition)
+                          .task_definition
+                          .container_definitions.sum { |condef| condef.send(resource.to_sym) }
+      {
+        count: service.desired_count,
+        amount: total_resource
+      }
+
+    end.inject({}) do |x, i|
+      # aggregate all particular counts into a map
+      x[i[:amount]] ||= 0
+      x[i[:amount]] += i[:count]
+      x
+    end
+  end
+
+  def total_demanded(resource)
+    demand_structure(resource).sum{|amount, count| count * amount}
+  end 
+
+  def instance_count_demanded(resource)
+    per_instance = total_registered(resource) / container_instances.count
+
+    # naively determine the number of instances needed for each service.
+    # this algo gives at worst n + 2 servers where n is the number of types
+    # of service memory requirements and at best the exact number of instances.
+    # please see tests for details.
+    demand_structure(resource).map do |k, v|
+      (k / per_instance.to_f * v).ceil + 1
+    end.sum
+  end
+
+  def instances_recommended
+    [instance_count_demanded(:cpu), instance_count_demanded(:memory)].max
+  end
+
   def validate_cidr_block
     if IPAddr.new(cidr_block).to_range.count < 65536
       errors.add(:cidr_block, "subnet mask bits must be smaller than or equal to 16")

lib/barcelona/network/autoscaling_builder.rb

@@ -4,21 +4,21 @@ class AutoscalingBuilder < CloudFormation::Builder
       # http://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-optimized_AMI.html
       # amzn2-ami-ecs-hvm-2.0
       ECS_OPTIMIZED_AMI_IDS = {
-        "us-east-1"      => "ami-0128839b21d19300e",
-        "us-east-2"      => "ami-0583ca2f3ce809fcb",
-        "us-west-1"      => "ami-0ac6a4a6e7e0949c4",
-        "us-west-2"      => "ami-030c9d6616d98227e",
-        "eu-west-1"      => "ami-0383e6ac19943cf6a",
-        "eu-west-2"      => "ami-0491c71e39d336e96",
-        "eu-west-3"      => "ami-06068eac7923b976b",
-        "eu-central-1"      => "ami-039bcbdcc961c4e81",
-        "ap-northeast-1"      => "ami-08c834e58473d808d",
-        "ap-northeast-2"      => "ami-0c0c0b030baf86093",
-        "ap-southeast-1"      => "ami-0791c84a135845cef",
-        "ap-southeast-2"      => "ami-0579b3efbc3a6c3e2",
-        "ca-central-1"      => "ami-0d0785328bd0eb34a",
-        "ap-south-1"      => "ami-01ab67467126a45fb",
-        "sa-east-1"      => "ami-0a339e14c13e704df",
+        "us-east-1"      => "ami-0ec7896dee795dfa9",
+        "us-east-2"      => "ami-02ef98ccecbf47e86",
+        "us-west-1"      => "ami-0c95b81c98a196de2",
+        "us-west-2"      => "ami-006d48b829793b507",
+        "eu-west-1"      => "ami-0cdce788baec293cb",
+        "eu-west-2"      => "ami-0a7f94d6f878fdd02",
+        "eu-west-3"      => "ami-08b42a2167e4c521f",
+        "eu-central-1"      => "ami-027d55743533d8658",
+        "ap-northeast-1"      => "ami-0ee0c841e0940c58f",
+        "ap-northeast-2"      => "ami-098340641fcc77afb",
+        "ap-southeast-1"      => "ami-002281dd675dedcbf",
+        "ap-southeast-2"      => "ami-016f6cf165ef55d02",
+        "ca-central-1"      => "ami-0cde1f5ee149df291",
+        "ap-south-1"      => "ami-018ae918c152249f0",
+        "sa-east-1"      => "ami-02b0be10b5499d608",
       }
 
       def ebs_optimized_by_default?
@@ -36,6 +36,9 @@ def build_resources
           j.SecurityGroups [ref("InstanceSecurityGroup")]
           j.UserData instance_user_data
           j.EbsOptimized ebs_optimized_by_default?
+          j.MetadataOptions do |m|
+            m.HttpTokens 'required'
+          end
           j.BlockDeviceMappings [
             # Root volume
             # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/al2ami-storage-config.html

lib/barcelona/network/bastion_builder.rb

@@ -6,21 +6,21 @@ class BastionBuilder < CloudFormation::Builder
       # You can see the latest version stored in public SSM parameter store
       # https://ap-northeast-1.console.aws.amazon.com/systems-manager/parameters/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2/description?region=ap-northeast-1
       AMI_IDS = {
-        "us-east-1"      => "ami-04d29b6f966df1537",
-        "us-east-2"      => "ami-09558250a3419e7d0",
-        "us-west-1"      => "ami-08d9a394ac1c2994c",
-        "us-west-2"      => "ami-0e472933a1395e172",
-        "eu-west-1"      => "ami-0ce1e3f77cd41957e",
-        "eu-west-2"      => "ami-08b993f76f42c3e2f",
-        "eu-west-3"      => "ami-0e9c91a3fc56a0376",
-        "eu-central-1"      => "ami-0bd39c806c2335b95",
-        "ap-northeast-1"      => "ami-00f045aed21a55240",
-        "ap-northeast-2"      => "ami-03461b78fdba0ff9d",
-        "ap-southeast-1"      => "ami-0d728fd4e52be968f",
-        "ap-southeast-2"      => "ami-09f765d333a8ebb4b",
-        "ca-central-1"      => "ami-0fca0f98dc87d39df",
-        "ap-south-1"      => "ami-08f63db601b82ff5f",
-        "sa-east-1"      => "ami-0096398577720a4a3",
+        "us-east-1"      => "ami-0915bcb5fa77e4892",
+        "us-east-2"      => "ami-09246ddb00c7c4fef",
+        "us-west-1"      => "ami-066c82dabe6dd7f73",
+        "us-west-2"      => "ami-09c5e030f74651050",
+        "eu-west-1"      => "ami-096f43ef67d75e998",
+        "eu-west-2"      => "ami-0ffd774e02309201f",
+        "eu-west-3"      => "ami-0ec28fc9814fce254",
+        "eu-central-1"      => "ami-02f9ea74050d6f812",
+        "ap-northeast-1"      => "ami-09d28faae2e9e7138",
+        "ap-northeast-2"      => "ami-006e2f9fa7597680a",
+        "ap-southeast-1"      => "ami-0d06583a13678c938",
+        "ap-southeast-2"      => "ami-075a72b1992cb0687",
+        "ca-central-1"      => "ami-0df612970f825f04c",
+        "ap-south-1"      => "ami-0eeb03e72075b9bcc",
+        "sa-east-1"      => "ami-0a0bc0fa94d632c94",
       }
 
       def build_resources
@@ -103,6 +103,9 @@ def build_resources
           j.SecurityGroups [ref("SecurityGroupBastion")]
           j.AssociatePublicIpAddress true
           j.UserData user_data
+          j.MetadataOptions do |m|
+            m.HttpTokens 'required'
+          end
         end
 
         add_resource(BastionAutoScaling, "BastionAutoScaling",
@@ -140,10 +143,13 @@ def user_data
         EOS
 
         ud.run_commands += [
+          # imdsv2
+          'IMDSTOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 3600"`',
+
           # awslogs
-          "ec2_id=$(curl http://169.254.169.254/latest/meta-data/instance-id)",
+          'ec2_id=$(curl -H "X-aws-ec2-metadata-token: $IMDSTOKEN" -v http://169.254.169.254/latest/meta-data/instance-id)',
           # There are cases when we must wait for meta-data
-          'while [ "$ec2_id" = "" ]; do sleep 1 ; ec2_id=$(curl http://169.254.169.254/latest/meta-data/instance-id) ; done',
+          'while [ "$ec2_id" = "" ]; do sleep 1 ; ec2_id=$(curl -H "X-aws-ec2-metadata-token: $IMDSTOKEN" -v http://169.254.169.254/latest/meta-data/instance-id) ; done',
           'sed -i -e "s/{ec2_id}/$ec2_id/g" /etc/awslogs/awslogs.conf',
           'sed -i -e "s/us-east-1/'+district.region+'/g" /etc/awslogs/awscli.conf',
           "systemctl start awslogsd",

lib/barcelona/network/nat_builder.rb

@@ -40,6 +40,15 @@ def build_resources
             ]
           end
 
+          add_resource("AWS::EC2::LaunchTemplate", nat_launch_template_name) do |j|
+            j.LaunchTemplateName nat_launch_template_name
+            j.LaunchTemplateData do |d|
+              d.MetadataOptions do |m|
+                m.HttpTokens 'required'
+              end
+            end
+          end
+
           add_resource("AWS::EC2::Instance", nat_name,
                        depends_on: ["VPCGatewayAttachment"]) do |j|
             j.InstanceType options[:instance_type] || 't3.nano'
@@ -53,6 +62,10 @@ def build_resources
                 "GroupSet" => [ref("SecurityGroupNAT")]
               }
             ]
+            j.LaunchTemplate do |t|
+              t.LaunchTemplateName nat_launch_template_name
+              t.Version get_attr(nat_launch_template_name, "LatestVersionNumber")
+            end
             j.Tags [
               tag("barcelona", stack.district.name),
               tag("barcelona-role", "nat"),
@@ -95,6 +108,10 @@ def eip_name
       def nat_name
         "NAT#{options[:type].to_s.classify}#{options[:nat_id]}"
       end
+
+      def nat_launch_template_name
+        "NAT#{options[:type].to_s.classify}#{options[:nat_id]}LaunchTemplate"
+      end
     end
   end
 end

lib/barcelona/plugins/datadog_logs_plugin.rb

@@ -4,9 +4,12 @@ class DatadogLogsPlugin < Base
       LOCAL_LOGGER_PORT = 514
       SYSTEM_PACKAGES = %w[rsyslog-gnutls ca-certificates]
       RUN_COMMANDS = [
+        # imdsv2
+        'IMDSTOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 3600"`',
+
         # set up hostname properly on the host so we don't end up identifying the host differently
         # and pay an extra 23 dollars for each phantom host that posts logs on datadog
-        'sed "s/{{HOSTNAME}}/`curl http://169.254.169.254/latest/meta-data/instance-id`/g" /etc/rsyslog.d/datadog.conf > temp' ,
+        'sed "s/{{HOSTNAME}}/`curl -H "X-aws-ec2-metadata-token: $IMDSTOKEN" -v http://169.254.169.254/latest/meta-data/instance-id`/g" /etc/rsyslog.d/datadog.conf > temp' ,
         'cp temp /etc/rsyslog.d/datadog.conf',
         'rm temp',
         "service rsyslog restart"

lib/barcelona/plugins/pcidss_plugin.rb

@@ -47,8 +47,11 @@ def manager_user_data
           # Install AWS Inspector agent
           "curl https://d1wk0tztpsntt1.cloudfront.net/linux/latest/install | bash",
 
+          # imdsv2
+          'IMDSTOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 3600"`',
+
           # awslogs
-          "ec2_id=$(curl http://169.254.169.254/latest/meta-data/instance-id)",
+          'ec2_id=$(curl -H "X-aws-ec2-metadata-token: $IMDSTOKEN" -v http://169.254.169.254/latest/meta-data/instance-id)',
           'sed -i -e "s/{ec2_id}/$ec2_id/g" /etc/awslogs/awslogs.conf',
           'sed -i -e "s/us-east-1/'+district.region+'/g" /etc/awslogs/awscli.conf',
           "systemctl start awslogsd",
@@ -65,11 +68,11 @@ def manager_user_data
 
           # Attach OSSEC volume
           "volume_id=$(aws ec2 describe-volumes --region ap-northeast-1 --filters Name=tag-key,Values=ossec-manager-volume Name=tag:barcelona,Values=#{district.name} | jq -r '.Volumes[0].VolumeId')",
-          "instance_id=$(curl http://169.254.169.254/latest/meta-data/instance-id)",
+          'instance_id=$(curl -H "X-aws-ec2-metadata-token: $IMDSTOKEN" -v http://169.254.169.254/latest/meta-data/instance-id)',
           "aws ec2 attach-volume --region ap-northeast-1 --volume-id $volume_id --instance-id $instance_id --device /dev/xvdh",
 
           # Register its private IP to Route53
-          "private_ip=$(curl http://169.254.169.254/latest/meta-data/local-ipv4)",
+          'private_ip=$(curl -H "X-aws-ec2-metadata-token: $IMDSTOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)',
           "change_batch=$(echo '#{change_batch}' | sed -e \"s/{private_ip}/$private_ip/\")",
           "aws route53 change-resource-record-sets --hosted-zone-id #{district.private_hosted_zone_id} --change-batch $change_batch",
 
@@ -214,6 +217,9 @@ def build_resources
               }
             },
           ]
+          j.MetadataOptions do |m|
+            m.HttpTokens 'required'
+          end
         end
 
         add_resource("AWS::IAM::Role", "OSSECManagerRole") do |j|