From c05ae6944b862874facad6e8ac54b94e30537325 Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Tue, 24 Nov 2020 21:48:01 +0900
Subject: [PATCH 01/24] routines to calculate instances required

---
 app/models/district.rb | 72 ++++++++++++++++++++++++++++++++++++++----
 1 file changed, 65 insertions(+), 7 deletions(-)

diff --git a/app/models/district.rb b/app/models/district.rb
index 480f17e7..963de8d0 100644
--- a/app/models/district.rb
+++ b/app/models/district.rb
@@ -109,14 +109,26 @@ def subnets(network = "Private")
     ).subnets
   end
 
+  def container_instance_arns
+    @container_instance_arns ||= aws.ecs.list_container_instances(
+                                   cluster: name
+                                 ).container_instance_arns
+  end
+
+  def cluster_container_instances
+    return [] if container_instance_arns.blank?
+
+    @cluster_container_instances ||= aws.ecs.describe_container_instances(
+                                       cluster: name,
+                                       container_instances: container_instance_arns
+                                     ).container_instances
+  end
+
   def container_instances
-    arns = aws.ecs.list_container_instances(cluster: name).container_instance_arns
-    return [] if arns.blank?
-    container_instances = aws.ecs.
-                          describe_container_instances(cluster: name, container_instances: arns).
-                          container_instances
+    return [] if cluster_container_instances.blank?
+
     instances = {}
-    container_instances.each do |ci|
+    cluster_container_instances.each do |ci|
       instance = {
         status: ci.status,
         container_instance_arn: ci.container_instance_arn,
@@ -129,7 +141,7 @@ def container_instances
     end
 
     ec2_instances = aws.ec2.describe_instances(
-      instance_ids: container_instances.map(&:ec2_instance_id)
+      instance_ids: cluster_container_instances.map(&:ec2_instance_id)
     ).reservations.map(&:instances).flatten
 
     ec2_instances.each do |ins|
@@ -210,6 +222,52 @@ def update_notification_stack
 
   private
 
+  def total_registered(resource)
+    container_instances.pluck(:registered_resources)
+                       .flatten
+                       .select {|x| x.name == resource.to_s.upcase}
+                       .sum {|x| x.integer_value}
+  end
+
+  def demand_structure(resource)
+    heritages.flat_map(&:services).flat_map do |service|
+      # map all the containers' memory or cpu
+      definition = service.send(:backend).send(:ecs_service).task_definition
+
+      # read the total amount requested by definition
+      total_resource = aws.ecs.describe_task_definition(task_definition: definition)
+                          .task_definition
+                          .container_definitions.sum { |condef| condef.send(resource.to_sym) }
+      {
+        count: service.desired_count,
+        amount: total_resource
+      }
+
+    end.inject({}) do |x, i|
+      # aggregate all particular counts into a map
+      x[i[:amount]] ||= 0
+      x[i[:amount]] += i[:count]
+      x
+    end
+  end
+
+  def total_demanded(resource)
+    demand_structure(resource).sum{|amount, count| count * amount}
+  end 
+
+  def instance_count_demanded(resource)
+    per_instance = total_registered(resource) / container_instances.count
+
+    # naively determine the number of instances needed for each service
+    demand_structure(resource).map do |k, v|
+      (k / 1699.to_f * v).ceil
+    end.sum
+  end
+
+  def instances_recommended
+    [instance_count_demanded(:cpu), instance_count_demanded(:memory)].max
+  end
+
   def validate_cidr_block
     if IPAddr.new(cidr_block).to_range.count < 65536
       errors.add(:cidr_block, "subnet mask bits must be smaller than or equal to 16")

From 31baba0886e137ee29af16f1b9ebe0754b71d9b5 Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Wed, 25 Nov 2020 23:11:28 +0900
Subject: [PATCH 02/24] algorithm tests

---
 app/models/district.rb       |  7 ++++--
 spec/models/district_spec.rb | 48 ++++++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/app/models/district.rb b/app/models/district.rb
index 963de8d0..aea8a609 100644
--- a/app/models/district.rb
+++ b/app/models/district.rb
@@ -258,9 +258,12 @@ def total_demanded(resource)
   def instance_count_demanded(resource)
     per_instance = total_registered(resource) / container_instances.count
 
-    # naively determine the number of instances needed for each service
+    # naively determine the number of instances needed for each service.
+    # this algo gives at worst n + 2 servers where n is the number of types
+    # of service memory requirements and at best the exact number of instances.
+    # please see tests for details.
     demand_structure(resource).map do |k, v|
-      (k / 1699.to_f * v).ceil
+      (k / per_instance.to_f * v).ceil + 1
     end.sum
   end
 
diff --git a/spec/models/district_spec.rb b/spec/models/district_spec.rb
index 2575e3d2..eec11269 100644
--- a/spec/models/district_spec.rb
+++ b/spec/models/district_spec.rb
@@ -147,4 +147,52 @@
       district.publish_sns("message")
     end
   end
+
+  describe '#instances_recommended' do
+    it 'gives the maximum from cpu and memory requirements' do
+      allow(district).to receive(:instance_count_demanded).with(:cpu) { 100 }
+      allow(district).to receive(:instance_count_demanded).with(:memory) { 10 }
+
+      expect(district.send(:instances_recommended)).to eq 100
+    end
+  end
+
+  describe '#instance_count_demanded' do
+    before do
+      # set some constants
+      allow(district).to receive(:container_instances) { [1] }
+      allow(district).to receive(:total_registered) { 1000 }
+    end
+
+    it 'gives 1 more server than required if we have only 1 service type with exact occupancy' do
+      allow(district).to receive(:demand_structure) { { 1000 => 3 } }
+
+      expect(district.send(:instance_count_demanded, :something)).to eq 4
+    end
+
+    it 'gives one more server required if we have 1 service type with less than half occupancy' do
+      allow(district).to receive(:demand_structure) { { 400 => 3 } }
+
+      expect(district.send(:instance_count_demanded, :something)).to eq 3
+    end
+
+    it 'gives exactly the number of servers required if we have 1 service type with more than half occupancy' do
+      allow(district).to receive(:demand_structure) { { 600 => 3 } }
+
+      expect(district.send(:instance_count_demanded, :something)).to eq 3
+    end
+
+    it 'gives two more servers than required if we have 1 service type with less than half occupancy and a minor type' do
+      allow(district).to receive(:demand_structure) { { 400 => 3, 100 => 3 } }
+
+      expect(district.send(:instance_count_demanded, :something)).to eq 5
+    end
+
+    it 'gives two more servers than required if we have 1 service type with more than half occupancy and a minor type' do
+      allow(district).to receive(:demand_structure) { { 600 => 3, 100 => 3 } }
+
+      expect(district.send(:instance_count_demanded, :something)).to eq 5
+    end
+
+  end
 end

From f4a1cee4844cd2ffef4e409a000fddb94bc48b64 Mon Sep 17 00:00:00 2001
From: David Siaw <davidsiaw@gmail.com>
Date: Tue, 27 Oct 2020 21:34:28 +0900
Subject: [PATCH 03/24] transition to using imdsv2 for querying instance
 metadata

---
 app/models/container_instance.rb             | 6 +++++-
 lib/barcelona/network/bastion_builder.rb     | 7 +++++--
 lib/barcelona/plugins/datadog_logs_plugin.rb | 5 ++++-
 lib/barcelona/plugins/pcidss_plugin.rb       | 9 ++++++---
 4 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/app/models/container_instance.rb b/app/models/container_instance.rb
index 1adbd9fc..d47806ab 100644
--- a/app/models/container_instance.rb
+++ b/app/models/container_instance.rb
@@ -13,6 +13,9 @@ def user_data
       # Embed SHA2 hash dockercfg so that instance replacement happens when dockercfg is updated
       "# #{Digest::SHA256.hexdigest(district.dockercfg.to_s)}",
 
+      # Get IMDSv2 token that expires in 1 hour
+      'IMDSTOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 3600"`',
+
       # Setup swap
       "MEMSIZE=`cat /proc/meminfo | grep MemTotal | awk '{print $2}'`",
       "if [ $MEMSIZE -lt 2097152 ]; then",
@@ -26,6 +29,7 @@ def user_data
 
       "AWS_REGION=#{district.region}",
       "aws configure set s3.signature_version s3v4",
+
       "aws s3 cp s3://#{district.s3_bucket_name}/#{district.name}/ecs.config /etc/ecs/ecs.config",
       "chmod 600 /etc/ecs/ecs.config",
 
@@ -35,7 +39,7 @@ def user_data
       "service sshd restart",
 
       # Configure AWS CloudWatch Logs
-      "ec2_id=$(curl http://169.254.169.254/latest/meta-data/instance-id)",
+      'ec2_id=$(curl -H "X-aws-ec2-metadata-token: $IMDSTOKEN" -v http://169.254.169.254/latest/meta-data/instance-id)',
       'sed -i -e "s/{ec2_id}/$ec2_id/g" /etc/awslogs/awslogs.conf',
       'sed -i -e "s/us-east-1/'+district.region+'/g" /etc/awslogs/awscli.conf',
       "systemctl start awslogsd",
diff --git a/lib/barcelona/network/bastion_builder.rb b/lib/barcelona/network/bastion_builder.rb
index ea68952f..fb579973 100644
--- a/lib/barcelona/network/bastion_builder.rb
+++ b/lib/barcelona/network/bastion_builder.rb
@@ -140,10 +140,13 @@ def user_data
         EOS
 
         ud.run_commands += [
+          # imdsv2
+          'IMDSTOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 3600"`',
+
           # awslogs
-          "ec2_id=$(curl http://169.254.169.254/latest/meta-data/instance-id)",
+          'ec2_id=$(curl -H "X-aws-ec2-metadata-token: $IMDSTOKEN" -v http://169.254.169.254/latest/meta-data/instance-id)',
           # There are cases when we must wait for meta-data
-          'while [ "$ec2_id" = "" ]; do sleep 1 ; ec2_id=$(curl http://169.254.169.254/latest/meta-data/instance-id) ; done',
+          'while [ "$ec2_id" = "" ]; do sleep 1 ; ec2_id=$(curl -H "X-aws-ec2-metadata-token: $IMDSTOKEN" -v http://169.254.169.254/latest/meta-data/instance-id) ; done',
           'sed -i -e "s/{ec2_id}/$ec2_id/g" /etc/awslogs/awslogs.conf',
           'sed -i -e "s/us-east-1/'+district.region+'/g" /etc/awslogs/awscli.conf',
           "systemctl start awslogsd",
diff --git a/lib/barcelona/plugins/datadog_logs_plugin.rb b/lib/barcelona/plugins/datadog_logs_plugin.rb
index 4b60556c..d281133d 100644
--- a/lib/barcelona/plugins/datadog_logs_plugin.rb
+++ b/lib/barcelona/plugins/datadog_logs_plugin.rb
@@ -4,9 +4,12 @@ class DatadogLogsPlugin < Base
       LOCAL_LOGGER_PORT = 514
       SYSTEM_PACKAGES = %w[rsyslog-gnutls ca-certificates]
       RUN_COMMANDS = [
+        # imdsv2
+        'IMDSTOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 3600"`',
+
         # set up hostname properly on the host so we don't end up identifying the host differently
         # and pay an extra 23 dollars for each phantom host that posts logs on datadog
-        'sed "s/{{HOSTNAME}}/`curl http://169.254.169.254/latest/meta-data/instance-id`/g" /etc/rsyslog.d/datadog.conf > temp' ,
+        'sed "s/{{HOSTNAME}}/`curl -H "X-aws-ec2-metadata-token: $IMDSTOKEN" -v http://169.254.169.254/latest/meta-data/instance-id`/g" /etc/rsyslog.d/datadog.conf > temp' ,
         'cp temp /etc/rsyslog.d/datadog.conf',
         'rm temp',
         "service rsyslog restart"
diff --git a/lib/barcelona/plugins/pcidss_plugin.rb b/lib/barcelona/plugins/pcidss_plugin.rb
index bf0a69cf..58ca76c9 100644
--- a/lib/barcelona/plugins/pcidss_plugin.rb
+++ b/lib/barcelona/plugins/pcidss_plugin.rb
@@ -47,8 +47,11 @@ def manager_user_data
           # Install AWS Inspector agent
           "curl https://d1wk0tztpsntt1.cloudfront.net/linux/latest/install | bash",
 
+          # imdsv2
+          'IMDSTOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 3600"`',
+
           # awslogs
-          "ec2_id=$(curl http://169.254.169.254/latest/meta-data/instance-id)",
+          'ec2_id=$(curl -H "X-aws-ec2-metadata-token: $IMDSTOKEN" -v http://169.254.169.254/latest/meta-data/instance-id)',
           'sed -i -e "s/{ec2_id}/$ec2_id/g" /etc/awslogs/awslogs.conf',
           'sed -i -e "s/us-east-1/'+district.region+'/g" /etc/awslogs/awscli.conf',
           "systemctl start awslogsd",
@@ -65,11 +68,11 @@ def manager_user_data
 
           # Attach OSSEC volume
           "volume_id=$(aws ec2 describe-volumes --region ap-northeast-1 --filters Name=tag-key,Values=ossec-manager-volume Name=tag:barcelona,Values=#{district.name} | jq -r '.Volumes[0].VolumeId')",
-          "instance_id=$(curl http://169.254.169.254/latest/meta-data/instance-id)",
+          'instance_id=$(curl -H "X-aws-ec2-metadata-token: $IMDSTOKEN" -v http://169.254.169.254/latest/meta-data/instance-id)',
           "aws ec2 attach-volume --region ap-northeast-1 --volume-id $volume_id --instance-id $instance_id --device /dev/xvdh",
 
           # Register its private IP to Route53
-          "private_ip=$(curl http://169.254.169.254/latest/meta-data/local-ipv4)",
+          'private_ip=$(curl -H "X-aws-ec2-metadata-token: $IMDSTOKEN" -v http://169.254.169.254/latest/meta-data/local-ipv4)',
           "change_batch=$(echo '#{change_batch}' | sed -e \"s/{private_ip}/$private_ip/\")",
           "aws route53 change-resource-record-sets --hosted-zone-id #{district.private_hosted_zone_id} --change-batch $change_batch",
 

From 4921473934edbeaffe2e7a4781fead07042419a2 Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Mon, 9 Nov 2020 14:34:03 +0900
Subject: [PATCH 04/24] disable imdsv1 for new districts

---
 lib/barcelona/network/auto_scaling_group.rb  | 1 +
 lib/barcelona/network/autoscaling_builder.rb | 1 +
 2 files changed, 2 insertions(+)

diff --git a/lib/barcelona/network/auto_scaling_group.rb b/lib/barcelona/network/auto_scaling_group.rb
index 2e696a7b..2ee1cd0f 100644
--- a/lib/barcelona/network/auto_scaling_group.rb
+++ b/lib/barcelona/network/auto_scaling_group.rb
@@ -20,6 +20,7 @@ def define_resource(json)
           j.HealthCheckType "EC2"
           j.LaunchConfigurationName ref("ContainerInstanceLaunchConfiguration")
           j.VPCZoneIdentifier [ref("SubnetTrusted1"), ref("SubnetTrusted2")]
+          j.DisableIMDSv1 true
           j.Tags [
             {
               "Key" => "Name",
diff --git a/lib/barcelona/network/autoscaling_builder.rb b/lib/barcelona/network/autoscaling_builder.rb
index 9641cbc0..84daa9ef 100644
--- a/lib/barcelona/network/autoscaling_builder.rb
+++ b/lib/barcelona/network/autoscaling_builder.rb
@@ -37,6 +37,7 @@ def build_resources
           j.UserData instance_user_data
           j.EbsOptimized ebs_optimized_by_default?
           j.BlockDeviceMappings [
+          j.DisableIMDSv1 true
             # Root volume
             # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/al2ami-storage-config.html
             {

From 354480e0a3eeb61dd70913113eab037569abf8ff Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Tue, 10 Nov 2020 15:13:22 +0900
Subject: [PATCH 05/24] fix

---
 lib/barcelona/network/autoscaling_builder.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/barcelona/network/autoscaling_builder.rb b/lib/barcelona/network/autoscaling_builder.rb
index 84daa9ef..4fa6e162 100644
--- a/lib/barcelona/network/autoscaling_builder.rb
+++ b/lib/barcelona/network/autoscaling_builder.rb
@@ -36,8 +36,8 @@ def build_resources
           j.SecurityGroups [ref("InstanceSecurityGroup")]
           j.UserData instance_user_data
           j.EbsOptimized ebs_optimized_by_default?
-          j.BlockDeviceMappings [
           j.DisableIMDSv1 true
+          j.BlockDeviceMappings [
             # Root volume
             # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/al2ami-storage-config.html
             {

From c763d01240c4bd81e3b1cdb6afa1442ada80738b Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Tue, 10 Nov 2020 16:01:01 +0900
Subject: [PATCH 06/24] use something cloudformation understands

---
 lib/barcelona/network/auto_scaling_group.rb  | 4 +++-
 lib/barcelona/network/autoscaling_builder.rb | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/lib/barcelona/network/auto_scaling_group.rb b/lib/barcelona/network/auto_scaling_group.rb
index 2ee1cd0f..a8f2825a 100644
--- a/lib/barcelona/network/auto_scaling_group.rb
+++ b/lib/barcelona/network/auto_scaling_group.rb
@@ -20,7 +20,9 @@ def define_resource(json)
           j.HealthCheckType "EC2"
           j.LaunchConfigurationName ref("ContainerInstanceLaunchConfiguration")
           j.VPCZoneIdentifier [ref("SubnetTrusted1"), ref("SubnetTrusted2")]
-          j.DisableIMDSv1 true
+          j.MetadataOption do |m|
+            m.HttpTokens 'required'
+          end
           j.Tags [
             {
               "Key" => "Name",
diff --git a/lib/barcelona/network/autoscaling_builder.rb b/lib/barcelona/network/autoscaling_builder.rb
index 4fa6e162..a6d405da 100644
--- a/lib/barcelona/network/autoscaling_builder.rb
+++ b/lib/barcelona/network/autoscaling_builder.rb
@@ -36,7 +36,9 @@ def build_resources
           j.SecurityGroups [ref("InstanceSecurityGroup")]
           j.UserData instance_user_data
           j.EbsOptimized ebs_optimized_by_default?
-          j.DisableIMDSv1 true
+          j.MetadataOption do |m|
+            m.HttpTokens 'required'
+          end
           j.BlockDeviceMappings [
             # Root volume
             # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/al2ami-storage-config.html

From dc46c6d33424dcde2254562f3caa99832bbdad93 Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Tue, 10 Nov 2020 16:17:37 +0900
Subject: [PATCH 07/24] plural

---
 lib/barcelona/network/auto_scaling_group.rb  | 2 +-
 lib/barcelona/network/autoscaling_builder.rb | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/barcelona/network/auto_scaling_group.rb b/lib/barcelona/network/auto_scaling_group.rb
index a8f2825a..9d5d7deb 100644
--- a/lib/barcelona/network/auto_scaling_group.rb
+++ b/lib/barcelona/network/auto_scaling_group.rb
@@ -20,7 +20,7 @@ def define_resource(json)
           j.HealthCheckType "EC2"
           j.LaunchConfigurationName ref("ContainerInstanceLaunchConfiguration")
           j.VPCZoneIdentifier [ref("SubnetTrusted1"), ref("SubnetTrusted2")]
-          j.MetadataOption do |m|
+          j.MetadataOptions do |m|
             m.HttpTokens 'required'
           end
           j.Tags [
diff --git a/lib/barcelona/network/autoscaling_builder.rb b/lib/barcelona/network/autoscaling_builder.rb
index a6d405da..89d0652b 100644
--- a/lib/barcelona/network/autoscaling_builder.rb
+++ b/lib/barcelona/network/autoscaling_builder.rb
@@ -36,7 +36,7 @@ def build_resources
           j.SecurityGroups [ref("InstanceSecurityGroup")]
           j.UserData instance_user_data
           j.EbsOptimized ebs_optimized_by_default?
-          j.MetadataOption do |m|
+          j.MetadataOptions do |m|
             m.HttpTokens 'required'
           end
           j.BlockDeviceMappings [

From d2d8a799e59ed16f42ff3d6877036a8fa04bdbad Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Wed, 11 Nov 2020 19:04:41 +0900
Subject: [PATCH 08/24] remove options from unknown are

---
 lib/barcelona/network/auto_scaling_group.rb | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/lib/barcelona/network/auto_scaling_group.rb b/lib/barcelona/network/auto_scaling_group.rb
index 9d5d7deb..2e696a7b 100644
--- a/lib/barcelona/network/auto_scaling_group.rb
+++ b/lib/barcelona/network/auto_scaling_group.rb
@@ -20,9 +20,6 @@ def define_resource(json)
           j.HealthCheckType "EC2"
           j.LaunchConfigurationName ref("ContainerInstanceLaunchConfiguration")
           j.VPCZoneIdentifier [ref("SubnetTrusted1"), ref("SubnetTrusted2")]
-          j.MetadataOptions do |m|
-            m.HttpTokens 'required'
-          end
           j.Tags [
             {
               "Key" => "Name",

From 48f53d5f494f21ed1b896cc2bd7110f70701ff80 Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Thu, 26 Nov 2020 17:15:51 +0900
Subject: [PATCH 09/24] mandate imdsv2 for bastion servers as well

---
 lib/barcelona/network/bastion_builder.rb | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/barcelona/network/bastion_builder.rb b/lib/barcelona/network/bastion_builder.rb
index fb579973..6c5e6e71 100644
--- a/lib/barcelona/network/bastion_builder.rb
+++ b/lib/barcelona/network/bastion_builder.rb
@@ -103,6 +103,9 @@ def build_resources
           j.SecurityGroups [ref("SecurityGroupBastion")]
           j.AssociatePublicIpAddress true
           j.UserData user_data
+          j.MetadataOptions do |m|
+            m.HttpTokens 'required'
+          end
         end
 
         add_resource(BastionAutoScaling, "BastionAutoScaling",

From 07058edacf1089b4bd4a19024e5840b8c4d53198 Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Thu, 28 Jan 2021 17:35:59 +0900
Subject: [PATCH 10/24] fix tests

---
 lib/barcelona/network/nat_builder.rb             | 3 +++
 lib/barcelona/plugins/pcidss_plugin.rb           | 3 +++
 spec/lib/barcelona/network/network_stack_spec.rb | 2 ++
 3 files changed, 8 insertions(+)

diff --git a/lib/barcelona/network/nat_builder.rb b/lib/barcelona/network/nat_builder.rb
index 920c3c28..48ebb289 100644
--- a/lib/barcelona/network/nat_builder.rb
+++ b/lib/barcelona/network/nat_builder.rb
@@ -53,6 +53,9 @@ def build_resources
                 "GroupSet" => [ref("SecurityGroupNAT")]
               }
             ]
+            j.MetadataOptions do |m|
+              m.HttpTokens 'required'
+            end
             j.Tags [
               tag("barcelona", stack.district.name),
               tag("barcelona-role", "nat"),
diff --git a/lib/barcelona/plugins/pcidss_plugin.rb b/lib/barcelona/plugins/pcidss_plugin.rb
index 58ca76c9..c03d8303 100644
--- a/lib/barcelona/plugins/pcidss_plugin.rb
+++ b/lib/barcelona/plugins/pcidss_plugin.rb
@@ -217,6 +217,9 @@ def build_resources
               }
             },
           ]
+          j.MetadataOptions do |m|
+            m.HttpTokens 'required'
+          end
         end
 
         add_resource("AWS::IAM::Role", "OSSECManagerRole") do |j|
diff --git a/spec/lib/barcelona/network/network_stack_spec.rb b/spec/lib/barcelona/network/network_stack_spec.rb
index b15b8598..ce58da70 100644
--- a/spec/lib/barcelona/network/network_stack_spec.rb
+++ b/spec/lib/barcelona/network/network_stack_spec.rb
@@ -142,6 +142,7 @@
           "IamInstanceProfile" => {"Ref"=>"ECSInstanceProfile"},
           "ImageId" => kind_of(String),
           "InstanceType" => "t3.small",
+          "MetadataOptions"=>{"HttpTokens"=>"required"},
           "SecurityGroups" => [{"Ref"=>"InstanceSecurityGroup"}],
           "UserData" => instance_of(String),
           "EbsOptimized" => true,
@@ -366,6 +367,7 @@
         "Type" => "AWS::AutoScaling::LaunchConfiguration",
         "Properties" => {
           "InstanceType" => "t3.micro",
+          "MetadataOptions"=>{"HttpTokens"=>"required"},
           "IamInstanceProfile" => {"Ref" => "BastionProfile"},
           "ImageId" => kind_of(String),
           "UserData" => anything,

From 9c0d2df85bc9d044f0bfc193c7e222614babc80d Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Wed, 3 Feb 2021 16:38:57 +0900
Subject: [PATCH 11/24] fix deploy runner error and test

---
 app/jobs/deploy_runner_job.rb       |  2 +-
 spec/jobs/deploy_runner_job_spec.rb | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)
 create mode 100644 spec/jobs/deploy_runner_job_spec.rb

diff --git a/app/jobs/deploy_runner_job.rb b/app/jobs/deploy_runner_job.rb
index fabe7e13..8a7aebc5 100644
--- a/app/jobs/deploy_runner_job.rb
+++ b/app/jobs/deploy_runner_job.rb
@@ -9,7 +9,7 @@ def perform(heritage, without_before_deploy:, description: "")
     @heritage = heritage
     heritage.with_lock do
       if other_deploy_in_progress?(heritage)
-        notify(heritage, level: :error, message: "The other deployment is in progress. Stopped deploying.")
+        notify(level: :error, message: "The other deployment is in progress. Stopped deploying.")
         return
       end
 
diff --git a/spec/jobs/deploy_runner_job_spec.rb b/spec/jobs/deploy_runner_job_spec.rb
new file mode 100644
index 00000000..0bc54d99
--- /dev/null
+++ b/spec/jobs/deploy_runner_job_spec.rb
@@ -0,0 +1,18 @@
+require 'rails_helper'
+
+describe DeployRunnerJob, type: :job do
+  it 'creates an event if another deploy is in progress' do
+    job = DeployRunnerJob.new
+
+    district_object = create :district
+    heritage = create :heritage, district: district_object
+    allow(job).to receive(:other_deploy_in_progress?) { true }
+
+    event_object = double("Event")
+    expect(event_object).to receive(:notify)
+
+    expect(Event).to receive(:new).with(district_object) { event_object }
+
+    job.perform(heritage, without_before_deploy: true, description: "meow")
+  end
+end

From 115794648879dc766abe82955f740131f2d36441 Mon Sep 17 00:00:00 2001
From: Taku Nakajima <tnakajima@degica.com>
Date: Thu, 4 Feb 2021 10:29:14 +0900
Subject: [PATCH 12/24] Update AMI to 2.0.20210202

---
 lib/barcelona/network/autoscaling_builder.rb | 30 ++++++++++----------
 lib/barcelona/network/bastion_builder.rb     | 30 ++++++++++----------
 2 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/lib/barcelona/network/autoscaling_builder.rb b/lib/barcelona/network/autoscaling_builder.rb
index 89d0652b..df542428 100644
--- a/lib/barcelona/network/autoscaling_builder.rb
+++ b/lib/barcelona/network/autoscaling_builder.rb
@@ -4,21 +4,21 @@ class AutoscalingBuilder < CloudFormation::Builder
       # http://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-optimized_AMI.html
       # amzn2-ami-ecs-hvm-2.0
       ECS_OPTIMIZED_AMI_IDS = {
-        "us-east-1"      => "ami-0128839b21d19300e",
-        "us-east-2"      => "ami-0583ca2f3ce809fcb",
-        "us-west-1"      => "ami-0ac6a4a6e7e0949c4",
-        "us-west-2"      => "ami-030c9d6616d98227e",
-        "eu-west-1"      => "ami-0383e6ac19943cf6a",
-        "eu-west-2"      => "ami-0491c71e39d336e96",
-        "eu-west-3"      => "ami-06068eac7923b976b",
-        "eu-central-1"      => "ami-039bcbdcc961c4e81",
-        "ap-northeast-1"      => "ami-08c834e58473d808d",
-        "ap-northeast-2"      => "ami-0c0c0b030baf86093",
-        "ap-southeast-1"      => "ami-0791c84a135845cef",
-        "ap-southeast-2"      => "ami-0579b3efbc3a6c3e2",
-        "ca-central-1"      => "ami-0d0785328bd0eb34a",
-        "ap-south-1"      => "ami-01ab67467126a45fb",
-        "sa-east-1"      => "ami-0a339e14c13e704df",
+        "us-east-1"      => "ami-0e5b37ba2c8e7cc82",
+        "us-east-2"      => "ami-09c93f5e8e4b50e05",
+        "us-west-1"      => "ami-0306f5737181bb754",
+        "us-west-2"      => "ami-0927d80c641f8d8bb",
+        "eu-west-1"      => "ami-0c15700b4e6bf474e",
+        "eu-west-2"      => "ami-0f82969826859fb14",
+        "eu-west-3"      => "ami-0b2b26f34eb9f6482",
+        "eu-central-1"      => "ami-0f8ee411ba3a66276",
+        "ap-northeast-1"      => "ami-08aba6714243b1bf9",
+        "ap-northeast-2"      => "ami-0d9ea717f56829882",
+        "ap-southeast-1"      => "ami-093df4839e8df694e",
+        "ap-southeast-2"      => "ami-0122a3618e52b6418",
+        "ca-central-1"      => "ami-0c9bfd3e97bd089e9",
+        "ap-south-1"      => "ami-08fcbb6bcfbc5ced8",
+        "sa-east-1"      => "ami-0e32d15591985c079",
       }
 
       def ebs_optimized_by_default?
diff --git a/lib/barcelona/network/bastion_builder.rb b/lib/barcelona/network/bastion_builder.rb
index 6c5e6e71..df16bd53 100644
--- a/lib/barcelona/network/bastion_builder.rb
+++ b/lib/barcelona/network/bastion_builder.rb
@@ -6,21 +6,21 @@ class BastionBuilder < CloudFormation::Builder
       # You can see the latest version stored in public SSM parameter store
       # https://ap-northeast-1.console.aws.amazon.com/systems-manager/parameters/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2/description?region=ap-northeast-1
       AMI_IDS = {
-        "us-east-1"      => "ami-04d29b6f966df1537",
-        "us-east-2"      => "ami-09558250a3419e7d0",
-        "us-west-1"      => "ami-08d9a394ac1c2994c",
-        "us-west-2"      => "ami-0e472933a1395e172",
-        "eu-west-1"      => "ami-0ce1e3f77cd41957e",
-        "eu-west-2"      => "ami-08b993f76f42c3e2f",
-        "eu-west-3"      => "ami-0e9c91a3fc56a0376",
-        "eu-central-1"      => "ami-0bd39c806c2335b95",
-        "ap-northeast-1"      => "ami-00f045aed21a55240",
-        "ap-northeast-2"      => "ami-03461b78fdba0ff9d",
-        "ap-southeast-1"      => "ami-0d728fd4e52be968f",
-        "ap-southeast-2"      => "ami-09f765d333a8ebb4b",
-        "ca-central-1"      => "ami-0fca0f98dc87d39df",
-        "ap-south-1"      => "ami-08f63db601b82ff5f",
-        "sa-east-1"      => "ami-0096398577720a4a3",
+        "us-east-1"      => "ami-047a51fa27710816e",
+        "us-east-2"      => "ami-01aab85a5e4a5a0fe",
+        "us-west-1"      => "ami-005c06c6de69aee84",
+        "us-west-2"      => "ami-0e999cbd62129e3b1",
+        "eu-west-1"      => "ami-0fc970315c2d38f01",
+        "eu-west-2"      => "ami-098828924dc89ea4a",
+        "eu-west-3"      => "ami-0ea4a063871686f37",
+        "eu-central-1"      => "ami-0a6dc7529cd559185",
+        "ap-northeast-1"      => "ami-0992fc94ca0f1415a",
+        "ap-northeast-2"      => "ami-09282971cf2faa4c9",
+        "ap-southeast-1"      => "ami-0e2e44c03b85f58b3",
+        "ap-southeast-2"      => "ami-04f77aa5970939148",
+        "ca-central-1"      => "ami-075cfad2d9805c5f2",
+        "ap-south-1"      => "ami-08e0ca9924195beba",
+        "sa-east-1"      => "ami-089aac6323aa08aee",
       }
 
       def build_resources

From 4649651af90ad39914edb2532bb128aed9c6f2dc Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Tue, 2 Feb 2021 20:01:07 +0900
Subject: [PATCH 13/24] wait for object to be available

---
 lib/cloud_formation/executor.rb           | 22 +++++++++++++++++++---
 spec/lib/cloud_formation/executor_spec.rb | 13 +++++++++++++
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/lib/cloud_formation/executor.rb b/lib/cloud_formation/executor.rb
index ccd4b27b..ec54a088 100644
--- a/lib/cloud_formation/executor.rb
+++ b/lib/cloud_formation/executor.rb
@@ -56,13 +56,29 @@ def template_name
     end
 
     def upload_to_s3!
-      resp = @s3_client.put_object({
-        body: stack.target!,
+      params = {
         bucket: @bucket,
         key: template_name,
+      }
+
+      resp = @s3_client.put_object({
+        body: stack.target!,
+        **params
       })
-      Rails.logger.info "Uploaded stack template to bucket"
       Rails.logger.info resp
+
+      begin
+        @s3_client.wait_until(:object_exists, params, 
+          before_wait: -> (attempts, response) do
+            Rails.logger.info "Waiting for stack template to be uploaded"
+          end
+        )
+      rescue Aws::Waiters::Errors::WaiterFailed => e
+        Rails.logger.warn "Upload failed: #{e.message}"
+        raise e
+      end
+
+      Rails.logger.info "Uploaded stack template to bucket"
     end
 
     def stack_options
diff --git a/spec/lib/cloud_formation/executor_spec.rb b/spec/lib/cloud_formation/executor_spec.rb
index b284cfac..99beec6c 100644
--- a/spec/lib/cloud_formation/executor_spec.rb
+++ b/spec/lib/cloud_formation/executor_spec.rb
@@ -11,6 +11,7 @@
   describe "#update" do
     it "creates a change set if true" do
       expect(s3).to receive(:put_object)
+      expect(s3).to receive(:wait_until).with(:object_exists, anything, anything)
       expect(client).to receive(:create_change_set)
       expect(client).to_not receive(:update_stack)
       executor.update(change_set: true)
@@ -18,9 +19,21 @@
 
     it "updates the stack directly if false" do
       expect(s3).to receive(:put_object)
+      expect(s3).to receive(:wait_until).with(:object_exists, anything, anything)
       expect(client).to_not receive(:create_change_set)
       expect(client).to receive(:update_stack)
       executor.update(change_set: false)
     end
+
+    it "creates a change set if true" do
+      expect(s3).to receive(:put_object)
+      expect(s3).to receive(:wait_until).with(:object_exists, anything, anything) do
+        raise Aws::Waiters::Errors::WaiterFailed
+      end
+      expect(client).to_not receive(:create_change_set)
+      expect(client).to_not receive(:update_stack)
+      expect(Rails.logger).to receive(:warn).with("Upload failed: Aws::Waiters::Errors::WaiterFailed")
+      expect { executor.update(change_set: true) }.to raise_error Aws::Waiters::Errors::WaiterFailed
+    end
   end
 end

From 2968d456072f3e3c452c96bd404a01d96d765843 Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Thu, 4 Feb 2021 12:19:59 +0900
Subject: [PATCH 14/24] review fix

---
 spec/lib/cloud_formation/executor_spec.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spec/lib/cloud_formation/executor_spec.rb b/spec/lib/cloud_formation/executor_spec.rb
index 99beec6c..856b1a59 100644
--- a/spec/lib/cloud_formation/executor_spec.rb
+++ b/spec/lib/cloud_formation/executor_spec.rb
@@ -25,7 +25,7 @@
       executor.update(change_set: false)
     end
 
-    it "creates a change set if true" do
+    it "passes on any failures from s3 wait" do
       expect(s3).to receive(:put_object)
       expect(s3).to receive(:wait_until).with(:object_exists, anything, anything) do
         raise Aws::Waiters::Errors::WaiterFailed

From 5935046e01ea79a3bede9eb48c4b0737e7511161 Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Thu, 4 Feb 2021 13:01:28 +0900
Subject: [PATCH 15/24] add a sleep

---
 lib/cloud_formation/executor.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/cloud_formation/executor.rb b/lib/cloud_formation/executor.rb
index ec54a088..f476b1a7 100644
--- a/lib/cloud_formation/executor.rb
+++ b/lib/cloud_formation/executor.rb
@@ -67,6 +67,7 @@ def upload_to_s3!
       })
       Rails.logger.info resp
 
+      sleep(5)
       begin
         @s3_client.wait_until(:object_exists, params, 
           before_wait: -> (attempts, response) do

From 64e53cf4813dfde78df3bacde2e1f457c43fea41 Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Thu, 4 Feb 2021 19:40:23 +0900
Subject: [PATCH 16/24] log note

---
 lib/cloud_formation/executor.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/cloud_formation/executor.rb b/lib/cloud_formation/executor.rb
index f476b1a7..15df251a 100644
--- a/lib/cloud_formation/executor.rb
+++ b/lib/cloud_formation/executor.rb
@@ -67,6 +67,7 @@ def upload_to_s3!
       })
       Rails.logger.info resp
 
+      Rails.logger.info "Waiting for stack template to be uploaded"
       sleep(5)
       begin
         @s3_client.wait_until(:object_exists, params, 

From 8bd7d9207a6e2b98dd0c82e6300ab0ff7de39ed4 Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Thu, 4 Feb 2021 20:34:38 +0900
Subject: [PATCH 17/24] wait longer?

---
 lib/cloud_formation/executor.rb | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/cloud_formation/executor.rb b/lib/cloud_formation/executor.rb
index 15df251a..dd00eb5e 100644
--- a/lib/cloud_formation/executor.rb
+++ b/lib/cloud_formation/executor.rb
@@ -8,6 +8,8 @@ def initialize(stack, district)
       @client = district.aws.cloudformation
       @s3_client = district.aws.s3
       @bucket = district.s3_bucket_name
+
+      upload_to_s3!
     end
 
     def describe
@@ -68,7 +70,7 @@ def upload_to_s3!
       Rails.logger.info resp
 
       Rails.logger.info "Waiting for stack template to be uploaded"
-      sleep(5)
+      sleep(30)
       begin
         @s3_client.wait_until(:object_exists, params, 
           before_wait: -> (attempts, response) do
@@ -84,7 +86,6 @@ def upload_to_s3!
     end
 
     def stack_options
-      upload_to_s3!
       {
         stack_name: stack.name,
         capabilities: ["CAPABILITY_IAM"],

From 496c6cf61b9289686c89e42dcfb2638a6756c862 Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Fri, 5 Feb 2021 14:17:46 +0900
Subject: [PATCH 18/24] fix template name changing every second

---
 lib/cloud_formation/executor.rb           |  8 +++-----
 lib/cloud_formation/stack.rb              |  4 ++++
 spec/lib/cloud_formation/executor_spec.rb | 10 ++++++++++
 spec/rails_helper.rb                      |  1 +
 4 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/lib/cloud_formation/executor.rb b/lib/cloud_formation/executor.rb
index dd00eb5e..61ee4643 100644
--- a/lib/cloud_formation/executor.rb
+++ b/lib/cloud_formation/executor.rb
@@ -8,8 +8,6 @@ def initialize(stack, district)
       @client = district.aws.cloudformation
       @s3_client = district.aws.s3
       @bucket = district.s3_bucket_name
-
-      upload_to_s3!
     end
 
     def describe
@@ -54,7 +52,7 @@ def create_change_set(type: "UPDATE")
     end
 
     def template_name
-      "stack_templates/#{stack.name}/#{Time.current.strftime("%Y-%m-%d-%H%M%S")}.template"
+      @template_name ||= "stack_templates/#{stack.name}/#{Time.current.strftime("%Y-%m-%d-%H%M%S")}.template"
     end
 
     def upload_to_s3!
@@ -70,7 +68,6 @@ def upload_to_s3!
       Rails.logger.info resp
 
       Rails.logger.info "Waiting for stack template to be uploaded"
-      sleep(30)
       begin
         @s3_client.wait_until(:object_exists, params, 
           before_wait: -> (attempts, response) do
@@ -86,10 +83,11 @@ def upload_to_s3!
     end
 
     def stack_options
+      upload_to_s3!
       {
         stack_name: stack.name,
         capabilities: ["CAPABILITY_IAM"],
-        template_url: "https://#{@bucket}.s3.amazonaws.com/#{template_name}"
+        template_url: "https://s3.#{stack.region}.amazonaws.com/#{@bucket}/#{template_name}"
       }
     end
 
diff --git a/lib/cloud_formation/stack.rb b/lib/cloud_formation/stack.rb
index 88b8a4b3..1a297399 100644
--- a/lib/cloud_formation/stack.rb
+++ b/lib/cloud_formation/stack.rb
@@ -29,6 +29,10 @@ def build
       end
     end
 
+    def region
+      @options[:region]
+    end
+
     def build_parameters(json)
     end
 
diff --git a/spec/lib/cloud_formation/executor_spec.rb b/spec/lib/cloud_formation/executor_spec.rb
index 856b1a59..96be3281 100644
--- a/spec/lib/cloud_formation/executor_spec.rb
+++ b/spec/lib/cloud_formation/executor_spec.rb
@@ -35,5 +35,15 @@
       expect(Rails.logger).to receive(:warn).with("Upload failed: Aws::Waiters::Errors::WaiterFailed")
       expect { executor.update(change_set: true) }.to raise_error Aws::Waiters::Errors::WaiterFailed
     end
+
+    it "has template name that is not time dependent (regression)" do
+      expect(executor).to receive(:stack) { double(name: 'foobar') }
+      travel_to DateTime.new(2017, 7, 7)
+      expect(executor.template_name).to eq "stack_templates/foobar/2017-07-07-000000.template"
+      travel_back
+      travel_to DateTime.new(2019, 9, 9)
+      expect(executor.template_name).to eq "stack_templates/foobar/2017-07-07-000000.template"
+      travel_back
+    end
   end
 end
diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb
index 4d271781..a26899d6 100644
--- a/spec/rails_helper.rb
+++ b/spec/rails_helper.rb
@@ -34,6 +34,7 @@
   config.use_transactional_fixtures = true
   config.include FactoryBot::Syntax::Methods
   config.include StubEnv::Helpers
+  config.include ActiveSupport::Testing::TimeHelpers
 
   # Shoulda matchers
   config.include(Shoulda::Matchers::ActiveModel, type: :model)

From b7d9bda725d6dfc75cc99c2fd6b043758e59635c Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Fri, 5 Feb 2021 15:33:50 +0900
Subject: [PATCH 19/24] review fix

---
 lib/cloud_formation/executor.rb           |  9 +++------
 spec/lib/cloud_formation/executor_spec.rb | 10 ----------
 2 files changed, 3 insertions(+), 16 deletions(-)

diff --git a/lib/cloud_formation/executor.rb b/lib/cloud_formation/executor.rb
index 61ee4643..66e8d48c 100644
--- a/lib/cloud_formation/executor.rb
+++ b/lib/cloud_formation/executor.rb
@@ -51,11 +51,7 @@ def create_change_set(type: "UPDATE")
       client.create_change_set(options)
     end
 
-    def template_name
-      @template_name ||= "stack_templates/#{stack.name}/#{Time.current.strftime("%Y-%m-%d-%H%M%S")}.template"
-    end
-
-    def upload_to_s3!
+    def upload_to_s3!(template_name)
       params = {
         bucket: @bucket,
         key: template_name,
@@ -83,7 +79,8 @@ def upload_to_s3!
     end
 
     def stack_options
-      upload_to_s3!
+      template_name = "stack_templates/#{stack.name}/#{Time.current.strftime("%Y-%m-%d-%H%M%S")}.template"
+      upload_to_s3!(template_name)
       {
         stack_name: stack.name,
         capabilities: ["CAPABILITY_IAM"],
diff --git a/spec/lib/cloud_formation/executor_spec.rb b/spec/lib/cloud_formation/executor_spec.rb
index 96be3281..856b1a59 100644
--- a/spec/lib/cloud_formation/executor_spec.rb
+++ b/spec/lib/cloud_formation/executor_spec.rb
@@ -35,15 +35,5 @@
       expect(Rails.logger).to receive(:warn).with("Upload failed: Aws::Waiters::Errors::WaiterFailed")
       expect { executor.update(change_set: true) }.to raise_error Aws::Waiters::Errors::WaiterFailed
     end
-
-    it "has template name that is not time dependent (regression)" do
-      expect(executor).to receive(:stack) { double(name: 'foobar') }
-      travel_to DateTime.new(2017, 7, 7)
-      expect(executor.template_name).to eq "stack_templates/foobar/2017-07-07-000000.template"
-      travel_back
-      travel_to DateTime.new(2019, 9, 9)
-      expect(executor.template_name).to eq "stack_templates/foobar/2017-07-07-000000.template"
-      travel_back
-    end
   end
 end

From 587a2d805fdea28774ad12a8e064577757ccf3f7 Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Mon, 8 Feb 2021 11:52:36 +0900
Subject: [PATCH 20/24] quick fix for executor

---
 lib/cloud_formation/executor.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/cloud_formation/executor.rb b/lib/cloud_formation/executor.rb
index 66e8d48c..c7d55d97 100644
--- a/lib/cloud_formation/executor.rb
+++ b/lib/cloud_formation/executor.rb
@@ -84,7 +84,7 @@ def stack_options
       {
         stack_name: stack.name,
         capabilities: ["CAPABILITY_IAM"],
-        template_url: "https://s3.#{stack.region}.amazonaws.com/#{@bucket}/#{template_name}"
+        template_url: "https://#{@bucket}.s3.amazonaws.com/#{template_name}"
       }
     end
 

From fc54c5aba5e6ecafe29aaee9aad91024908e2185 Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Mon, 8 Feb 2021 11:46:24 +0900
Subject: [PATCH 21/24] imdsv2 for nat

---
 lib/barcelona/network/nat_builder.rb | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/lib/barcelona/network/nat_builder.rb b/lib/barcelona/network/nat_builder.rb
index 48ebb289..1a2178c0 100644
--- a/lib/barcelona/network/nat_builder.rb
+++ b/lib/barcelona/network/nat_builder.rb
@@ -40,6 +40,15 @@ def build_resources
             ]
           end
 
+          add_resource("AWS::EC2::LaunchTemplate", nat_launch_template_name) do |j|
+            j.LaunchTemplateName nat_launch_template_name
+            j.LaunchTemplateData do |d|
+              d.MetadataOptions do |m|
+                m.HttpTokens 'required'
+              end
+            end
+          end
+
           add_resource("AWS::EC2::Instance", nat_name,
                        depends_on: ["VPCGatewayAttachment"]) do |j|
             j.InstanceType options[:instance_type] || 't3.nano'
@@ -53,8 +62,9 @@ def build_resources
                 "GroupSet" => [ref("SecurityGroupNAT")]
               }
             ]
-            j.MetadataOptions do |m|
-              m.HttpTokens 'required'
+            j.LaunchTemplate do |t|
+              t.LaunchTemplateName nat_launch_template_name
+              t.Version get_attr(nat_launch_template_name, "LatestVersionNumber")
             end
             j.Tags [
               tag("barcelona", stack.district.name),
@@ -98,6 +108,10 @@ def eip_name
       def nat_name
         "NAT#{options[:type].to_s.classify}#{options[:nat_id]}"
       end
+
+      def nat_launch_template_name
+        "NAT#{options[:type].to_s.classify}#{options[:nat_id]}LaunchTemplate"
+      end
     end
   end
 end

From 34b3185b3d303d19fd6646ae47d1d2f67b05d9aa Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Thu, 18 Feb 2021 15:36:09 +0900
Subject: [PATCH 22/24] fix bootstrap

---
 lib/tasks/bootstrap.rake | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/lib/tasks/bootstrap.rake b/lib/tasks/bootstrap.rake
index 7cc86427..61494fcb 100644
--- a/lib/tasks/bootstrap.rake
+++ b/lib/tasks/bootstrap.rake
@@ -14,6 +14,10 @@ namespace :bcn do
     end
   end
 
+  def secret_key_base
+    ENV["SECRET_KEY_BASE"] || SecureRandom.hex(64)
+  end
+
   desc "Deploy Barcelona to the specified ECS cluster(local)"
   task :bootstrap => ["db:setup", :environment] do
     access_key_id = ENV["AWS_ACCESS_KEY_ID"]
@@ -64,6 +68,7 @@ namespace :bcn do
       image_tag: "master"
     )
     heritage.env_vars.build(key: "DATABASE_URL", value: ENV["BOOTSTRAP_DATABASE_URL"], secret: true)
+    heritage.env_vars.build(key: "SECRET_KEY_BASE", value: secret_key_base, secret: true)
     heritage.env_vars.build(key: "DISABLE_DATABASE_ENVIRONMENT_CHECK", value: "1", secret: false)
     heritage.env_vars.build(key: "AWS_REGION", value: region, secret: false)
     heritage.env_vars.build(key: "AWS_ACCESS_KEY_ID", value: access_key_id, secret: false)
@@ -142,7 +147,7 @@ EOS
           {key: "RAILS_LOG_TO_STDOUT", value: "true", secret: false},
           {key: "GITHUB_ORGANIZATION", value: ENV['GITHUB_ORGANIZATION'], secret: false},
           {key: "DATABASE_URL",  value: ENV["DATABASE_URL"], secret: true},
-          {key: "SECRET_KEY_BASE", value: SecureRandom.hex(64), secret: true},
+          {key: "SECRET_KEY_BASE", value: secret_key_base, secret: true},
           {key: "ENCRYPTION_KEY", value: ENV["ENCRYPTION_KEY"], secret: true}
         ],
         services_attributes: [

From 6d71b96234fd16c1b75d230d172a699bc4eec4c0 Mon Sep 17 00:00:00 2001
From: Taku Nakajima <tnakajima@degica.com>
Date: Thu, 4 Mar 2021 16:41:20 +0900
Subject: [PATCH 23/24] Update AMI to -2.0.20210301

---
 lib/barcelona/network/autoscaling_builder.rb | 30 ++++++++++----------
 lib/barcelona/network/bastion_builder.rb     | 30 ++++++++++----------
 2 files changed, 30 insertions(+), 30 deletions(-)

diff --git a/lib/barcelona/network/autoscaling_builder.rb b/lib/barcelona/network/autoscaling_builder.rb
index df542428..d7844c5c 100644
--- a/lib/barcelona/network/autoscaling_builder.rb
+++ b/lib/barcelona/network/autoscaling_builder.rb
@@ -4,21 +4,21 @@ class AutoscalingBuilder < CloudFormation::Builder
       # http://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs-optimized_AMI.html
       # amzn2-ami-ecs-hvm-2.0
       ECS_OPTIMIZED_AMI_IDS = {
-        "us-east-1"      => "ami-0e5b37ba2c8e7cc82",
-        "us-east-2"      => "ami-09c93f5e8e4b50e05",
-        "us-west-1"      => "ami-0306f5737181bb754",
-        "us-west-2"      => "ami-0927d80c641f8d8bb",
-        "eu-west-1"      => "ami-0c15700b4e6bf474e",
-        "eu-west-2"      => "ami-0f82969826859fb14",
-        "eu-west-3"      => "ami-0b2b26f34eb9f6482",
-        "eu-central-1"      => "ami-0f8ee411ba3a66276",
-        "ap-northeast-1"      => "ami-08aba6714243b1bf9",
-        "ap-northeast-2"      => "ami-0d9ea717f56829882",
-        "ap-southeast-1"      => "ami-093df4839e8df694e",
-        "ap-southeast-2"      => "ami-0122a3618e52b6418",
-        "ca-central-1"      => "ami-0c9bfd3e97bd089e9",
-        "ap-south-1"      => "ami-08fcbb6bcfbc5ced8",
-        "sa-east-1"      => "ami-0e32d15591985c079",
+        "us-east-1"      => "ami-0ec7896dee795dfa9",
+        "us-east-2"      => "ami-02ef98ccecbf47e86",
+        "us-west-1"      => "ami-0c95b81c98a196de2",
+        "us-west-2"      => "ami-006d48b829793b507",
+        "eu-west-1"      => "ami-0cdce788baec293cb",
+        "eu-west-2"      => "ami-0a7f94d6f878fdd02",
+        "eu-west-3"      => "ami-08b42a2167e4c521f",
+        "eu-central-1"      => "ami-027d55743533d8658",
+        "ap-northeast-1"      => "ami-0ee0c841e0940c58f",
+        "ap-northeast-2"      => "ami-098340641fcc77afb",
+        "ap-southeast-1"      => "ami-002281dd675dedcbf",
+        "ap-southeast-2"      => "ami-016f6cf165ef55d02",
+        "ca-central-1"      => "ami-0cde1f5ee149df291",
+        "ap-south-1"      => "ami-018ae918c152249f0",
+        "sa-east-1"      => "ami-02b0be10b5499d608",
       }
 
       def ebs_optimized_by_default?
diff --git a/lib/barcelona/network/bastion_builder.rb b/lib/barcelona/network/bastion_builder.rb
index df16bd53..8951eb17 100644
--- a/lib/barcelona/network/bastion_builder.rb
+++ b/lib/barcelona/network/bastion_builder.rb
@@ -6,21 +6,21 @@ class BastionBuilder < CloudFormation::Builder
       # You can see the latest version stored in public SSM parameter store
       # https://ap-northeast-1.console.aws.amazon.com/systems-manager/parameters/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2/description?region=ap-northeast-1
       AMI_IDS = {
-        "us-east-1"      => "ami-047a51fa27710816e",
-        "us-east-2"      => "ami-01aab85a5e4a5a0fe",
-        "us-west-1"      => "ami-005c06c6de69aee84",
-        "us-west-2"      => "ami-0e999cbd62129e3b1",
-        "eu-west-1"      => "ami-0fc970315c2d38f01",
-        "eu-west-2"      => "ami-098828924dc89ea4a",
-        "eu-west-3"      => "ami-0ea4a063871686f37",
-        "eu-central-1"      => "ami-0a6dc7529cd559185",
-        "ap-northeast-1"      => "ami-0992fc94ca0f1415a",
-        "ap-northeast-2"      => "ami-09282971cf2faa4c9",
-        "ap-southeast-1"      => "ami-0e2e44c03b85f58b3",
-        "ap-southeast-2"      => "ami-04f77aa5970939148",
-        "ca-central-1"      => "ami-075cfad2d9805c5f2",
-        "ap-south-1"      => "ami-08e0ca9924195beba",
-        "sa-east-1"      => "ami-089aac6323aa08aee",
+        "us-east-1"      => "ami-0915bcb5fa77e4892",
+        "us-east-2"      => "ami-09246ddb00c7c4fef",
+        "us-west-1"      => "ami-066c82dabe6dd7f73",
+        "us-west-2"      => "ami-09c5e030f74651050",
+        "eu-west-1"      => "ami-096f43ef67d75e998",
+        "eu-west-2"      => "ami-0ffd774e02309201f",
+        "eu-west-3"      => "ami-0ec28fc9814fce254",
+        "eu-central-1"      => "ami-02f9ea74050d6f812",
+        "ap-northeast-1"      => "ami-09d28faae2e9e7138",
+        "ap-northeast-2"      => "ami-006e2f9fa7597680a",
+        "ap-southeast-1"      => "ami-0d06583a13678c938",
+        "ap-southeast-2"      => "ami-075a72b1992cb0687",
+        "ca-central-1"      => "ami-0df612970f825f04c",
+        "ap-south-1"      => "ami-0eeb03e72075b9bcc",
+        "sa-east-1"      => "ami-0a0bc0fa94d632c94",
       }
 
       def build_resources

From 222f209f8101bcf5dfc93066ef680dfbe1b4aa18 Mon Sep 17 00:00:00 2001
From: David Siaw <dsiaw@degica.com>
Date: Mon, 27 Nov 2023 14:34:18 +0900
Subject: [PATCH 24/24] instance calc

---
 app/models/backend/ecs/v1/service.rb |  4 ++++
 app/models/district.rb               | 24 +++++++++++++++++++++++-
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/app/models/backend/ecs/v1/service.rb b/app/models/backend/ecs/v1/service.rb
index 1ddf5ecc..37068d13 100644
--- a/app/models/backend/ecs/v1/service.rb
+++ b/app/models/backend/ecs/v1/service.rb
@@ -60,6 +60,10 @@ def applied?
       !(ecs_service.nil? || ecs_service.status != "ACTIVE")
     end
 
+    def task_definition
+      fetch_ecs_service.task_definition
+    end
+
     def register_task
       task_definition = HeritageTaskDefinition.service_definition(service).to_task_definition
       aws.ecs.register_task_definition(task_definition)
diff --git a/app/models/district.rb b/app/models/district.rb
index aea8a609..e6ae4a1d 100644
--- a/app/models/district.rb
+++ b/app/models/district.rb
@@ -232,7 +232,29 @@ def total_registered(resource)
   def demand_structure(resource)
     heritages.flat_map(&:services).flat_map do |service|
       # map all the containers' memory or cpu
-      definition = service.send(:backend).send(:ecs_service).task_definition
+      backend = service.send(:backend)
+
+      if backend.nil?
+        puts "service #{service.name} of H #{service.heritage.name} has no backend"
+
+        next {
+          count: 0,
+          amount: 0
+        }
+      end
+
+      ecs_service = backend.send(:ecs_service)
+
+      if ecs_service.nil?
+        puts "service #{service.name} of H #{service.heritage.name} has no ecs"
+
+        next {
+          count: 0,
+          amount: 0
+        }
+      end
+
+      definition = ecs_service.task_definition
 
       # read the total amount requested by definition
       total_resource = aws.ecs.describe_task_definition(task_definition: definition)