tf: add standalone apm server profiles with NVMe SSDs and io2 EBS (#15624)

Improve benchmarking tf setup to facilitate benchmarking on standalone apm server setup with faster disks (e.g. local NVMe SSD)
- Add NVMe c6id ec2 instance profiles
- Use NVMe disk for data directory for instance types with d., e.g. c6id.2xlarge
- Configurable disk IOPS which is a required field for io2 volume type
- Bigger worker instance for 32GB profile to avoid OOM

(cherry picked from commit 3afe1fd)

Author: carsonip

testing/benchmark/main.tf

@@ -157,6 +157,7 @@ module "standalone_apm_server" {
   apm_instance_type   = var.standalone_apm_server_instance_size
   apm_volume_type     = var.standalone_apm_server_volume_type
   apm_volume_size     = var.apm_server_tail_sampling ? coalesce(var.standalone_apm_server_volume_size, 60) : var.standalone_apm_server_volume_size
+  apm_iops            = var.standalone_apm_server_iops
   apm_server_bin_path = var.apm_server_bin_path
   ea_managed          = false
 

testing/benchmark/system-profiles/16GB_NVMe-x2zone.tfvars

@@ -0,0 +1,23 @@
+user_name = "USER"
+
+# APM bench
+
+worker_instance_type = "c6i.2xlarge"
+
+# Elastic Cloud
+
+# The number of AZs the APM Server should span.
+apm_server_zone_count = 1
+# The Elasticsearch cluster node size.
+elasticsearch_size = "128g"
+# The number of AZs the Elasticsearch cluster should have.
+elasticsearch_zone_count = 2
+# APM server instance size
+apm_server_size = "16g"
+# Number of shards for the ES indices
+apm_shards = 4
+
+# Standalone
+
+standalone_apm_server_instance_size = "c6id.2xlarge"
+standalone_moxy_instance_size       = "c6i.4xlarge"

testing/benchmark/system-profiles/1GB_NVMe-x1zone.tfvars

@@ -0,0 +1,21 @@
+user_name = "USER"
+
+# APM bench
+
+worker_instance_type = "c6i.large"
+
+# Elastic Cloud
+
+# The number of AZs the APM Server should span.
+apm_server_zone_count = 1
+# The Elasticsearch cluster node size.
+elasticsearch_size = "16g"
+# The number of AZs the Elasticsearch cluster should have.
+elasticsearch_zone_count = 2
+# APM server instance size
+apm_server_size = "1g"
+
+# Standalone
+
+standalone_apm_server_instance_size = "c6id.large"
+standalone_moxy_instance_size       = "c6i.xlarge"

testing/benchmark/system-profiles/2GB_NVMe-x1zone.tfvars

@@ -0,0 +1,21 @@
+user_name = "USER"
+
+# APM bench
+
+worker_instance_type = "c6i.large"
+
+# Elastic Cloud
+
+# The number of AZs the APM Server should span.
+apm_server_zone_count = 1
+# The Elasticsearch cluster node size.
+elasticsearch_size = "16g"
+# The number of AZs the Elasticsearch cluster should have.
+elasticsearch_zone_count = 2
+# APM server instance size
+apm_server_size = "2g"
+
+# Standalone
+
+standalone_apm_server_instance_size = "c6id.large"
+standalone_moxy_instance_size       = "c6i.xlarge"

testing/benchmark/system-profiles/32GB_NVMe-x2zone.tfvars

@@ -0,0 +1,25 @@
+user_name = "USER"
+
+# APM bench
+
+worker_instance_type = "c6i.4xlarge"
+
+# Elastic Cloud
+
+# The number of AZs the APM Server should span.
+apm_server_zone_count = 1
+# The Elasticsearch cluster node size.
+elasticsearch_size = "256g"
+# The number of AZs the Elasticsearch cluster should have.
+elasticsearch_zone_count = 2
+# Run the cluster with a dedicated master
+elasticsearch_dedicated_masters = true
+# APM server instance size
+apm_server_size = "32g"
+# Number of shards for the ES indices
+apm_shards = 4
+
+# Standalone
+
+standalone_apm_server_instance_size = "c6id.4xlarge"
+standalone_moxy_instance_size       = "c6i.8xlarge"

testing/benchmark/system-profiles/32GBx2zone.tfvars

@@ -2,7 +2,7 @@ user_name = "USER"
 
 # APM bench
 
-worker_instance_type = "c6i.2xlarge"
+worker_instance_type = "c6i.4xlarge"
 
 # Elastic Cloud
 

testing/benchmark/system-profiles/4GB_NVMe-x1zone.tfvars

@@ -0,0 +1,21 @@
+user_name = "USER"
+
+# APM bench
+
+worker_instance_type = "c6i.large"
+
+# Elastic Cloud
+
+# The number of AZs the APM Server should span.
+apm_server_zone_count = 1
+# The Elasticsearch cluster node size.
+elasticsearch_size = "32g"
+# The number of AZs the Elasticsearch cluster should have.
+elasticsearch_zone_count = 2
+# APM server instance size
+apm_server_size = "4g"
+
+# Standalone
+
+standalone_apm_server_instance_size = "c6id.large"
+standalone_moxy_instance_size       = "c6i.xlarge"

testing/benchmark/system-profiles/8GB_NVMe-x1zone.tfvars

@@ -0,0 +1,21 @@
+user_name = "USER"
+
+# APM bench
+
+worker_instance_type = "c6i.xlarge"
+
+# Elastic Cloud
+
+# The number of AZs the APM Server should span.
+apm_server_zone_count = 1
+# The Elasticsearch cluster node size.
+elasticsearch_size = "64g"
+# The number of AZs the Elasticsearch cluster should have.
+elasticsearch_zone_count = 2
+# APM server instance size
+apm_server_size = "8g"
+
+# Standalone
+
+standalone_apm_server_instance_size = "c6id.xlarge"
+standalone_moxy_instance_size       = "c6i.2xlarge"

testing/benchmark/system-profiles/8GB_io2_6000iops-x1zone.tfvars

@@ -0,0 +1,23 @@
+user_name = "USER"
+
+# APM bench
+
+worker_instance_type = "c6i.xlarge"
+
+# Elastic Cloud
+
+# The number of AZs the APM Server should span.
+apm_server_zone_count = 1
+# The Elasticsearch cluster node size.
+elasticsearch_size = "64g"
+# The number of AZs the Elasticsearch cluster should have.
+elasticsearch_zone_count = 2
+# APM server instance size
+apm_server_size = "8g"
+
+# Standalone
+
+standalone_apm_server_instance_size = "c6i.xlarge"
+standalone_apm_server_volume_type   = "io2"
+standalone_apm_server_iops          = 6000
+standalone_moxy_instance_size       = "c6i.2xlarge"

testing/benchmark/variables.tf

@@ -142,6 +142,12 @@ variable "standalone_apm_server_volume_size" {
   description = "Optional volume size in GB to use for APM Server VM"
 }
 
+variable "standalone_apm_server_iops" {
+  default     = null
+  type        = number
+  description = "Optional disk IOPS in GB to use for APM Server VM"
+}
+
 ## VPC Network settings
 
 variable "vpc_cidr" {

testing/infra/terraform/modules/standalone_apm_server/main.tf

@@ -172,6 +172,7 @@ resource "aws_instance" "apm" {
   root_block_device {
     volume_type = var.apm_volume_type
     volume_size = var.apm_volume_size
+    iops        = var.apm_iops
   }
 
   connection {
@@ -181,6 +182,18 @@ resource "aws_instance" "apm" {
     private_key = file("${var.aws_provisioner_key_name}")
   }
 
+  // For instance types with 'd.' e.g. c6id.2xlarge, use the NVMe ssd as data disk.
+  provisioner "remote-exec" {
+    inline = length(regexall("d[.]", self.instance_type)) > 0 ? [
+      "sudo mkfs -t xfs /dev/nvme1n1",
+      "mkdir ~/data",
+      "sudo mount /dev/nvme1n1 ~/data",
+      "sudo chown $USER:$USER ~/data",
+      ] : [
+      ":", // no-op
+    ]
+  }
+
   provisioner "file" {
     source      = "${var.apm_server_bin_path}/apm-server"
     destination = local.bin_path

testing/infra/terraform/modules/standalone_apm_server/variables.tf

@@ -22,6 +22,12 @@ variable "apm_volume_size" {
   description = "Optional apm server volume size in GB override"
 }
 
+variable "apm_iops" {
+  default     = null
+  type        = number
+  description = "Optional apm server disk IOPS override"
+}
+
 variable "vpc_id" {
   description = "VPC ID to provision the EC2 instance"
   type        = string