diff --git a/ansible/deploy-clickhouse-proxy.yml b/ansible/deploy-clickhouse-proxy.yml index 4d6309c3..2c00023d 100644 --- a/ansible/deploy-clickhouse-proxy.yml +++ b/ansible/deploy-clickhouse-proxy.yml @@ -5,12 +5,17 @@ become: true roles: - role: bootstrap + - role: dehydrated + vars: + ssl_domains: + - clickhouseproxy.dev.ooni.io - role: nginx tags: nginx - role: clickhouse_proxy vars: clickhouse_url: "clickhouse3.prod.ooni.io" clickhouse_port: 9000 + clickhouse_proxy_public_fqdn: "clickhouseproxy.dev.ooni.io" - role: dehydrated vars: ssl_domains: "clickhouseproxy.dev.ooni.io" diff --git a/ansible/roles/clickhouse_proxy/defaults/main.yml b/ansible/roles/clickhouse_proxy/defaults/main.yml new file mode 100644 index 00000000..14c3bf27 --- /dev/null +++ b/ansible/roles/clickhouse_proxy/defaults/main.yml @@ -0,0 +1 @@ +tls_cert_dir: /var/lib/dehydrated/certs \ No newline at end of file diff --git a/ansible/roles/clickhouse_proxy/tasks/main.yml b/ansible/roles/clickhouse_proxy/tasks/main.yml index 194c64e8..5b5fd094 100644 --- a/ansible/roles/clickhouse_proxy/tasks/main.yml +++ b/ansible/roles/clickhouse_proxy/tasks/main.yml @@ -9,6 +9,17 @@ notify: - reload nftables +# For prometheus scrape requests +- name: Allow traffic on port 9200 + tags: prometheus-proxy + blockinfile: + path: /etc/ooni/nftables/tcp/9200.nft + create: yes + block: | + add rule inet filter input tcp dport 9200 counter accept comment "prometheus" + notify: + - reload nftables + - name: Create the modules-enabled directory if not exists tags: webserv ansible.builtin.file: @@ -28,3 +39,14 @@ notify: - reload nginx - restart nginx + +- name: Add prometheus proxy nginx config + tags: webserv + template: + src: templates/prometheus-proxy.conf + dest: /etc/nginx/conf.d/prometheus-proxy.conf + mode: 0755 + owner: root + notify: + - reload nginx + - restart nginx diff --git a/ansible/roles/clickhouse_proxy/templates/prometheus-proxy.conf b/ansible/roles/clickhouse_proxy/templates/prometheus-proxy.conf new file mode 100644 index 00000000..2ea41815 --- /dev/null +++ b/ansible/roles/clickhouse_proxy/templates/prometheus-proxy.conf @@ -0,0 +1,16 @@ +server { + listen 9200 ssl; + + server_name {{ clickhouse_proxy_public_fqdn }}; + + include /etc/nginx/ssl_intermediate.conf; + + ssl_certificate {{tls_cert_dir}}/{{inventory_hostname}}/fullchain.pem; + ssl_certificate_key {{tls_cert_dir}}/{{inventory_hostname}}/privkey.pem; + ssl_trusted_certificate {{tls_cert_dir}}/{{inventory_hostname}}/chain.pem; + + proxy_ssl_server_name on; + location ~ /([a-zA-Z0-9_\.]+)/(.*) { + proxy_pass http://$1:9100/$2$is_args$args; + } +} diff --git a/ansible/roles/prometheus/templates/prometheus.yml b/ansible/roles/prometheus/templates/prometheus.yml index c1f53654..3e8fa796 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml +++ b/ansible/roles/prometheus/templates/prometheus.yml @@ -214,4 +214,38 @@ scrape_configs: static_configs: - targets: - backend-hel.ooni.org:444 + + # EC2 instances monitoring: + - job_name: 'ooni-aws-ec2' + scrape_interval: 5s + scheme: https + metrics_path: "/metrics" + + # Node level metrics for cluster nodes + ec2_sd_configs: + - access_key: "{{prometheus_aws_access_key_dev}}" + secret_key: "{{prometheus_aws_secret_key_dev}}" + region: "eu-central-1" + port: 9100 # should be the proxy + relabel_configs: # Change the host to the proxy host with relabeling + - source_labels: [__address__] + regex: "([0-9\\.]+):([0-9]+)" # :" + replacement: "$1" + target_label: "ec2_host" + action: "replace" + - source_labels: [__address__] + regex: "([0-9\\.]+):([0-9]+)" # : + replacement: "{{clickhouse_proxy_host_dev}}:9200/${1}/metrics" + target_label: "proxy_host" + action: "replace" + - source_labels: [proxy_host] + regex: "([^/]*)/(.*)" + replacement: "$1" + target_label: "__address__" + action: "replace" + - source_labels: [proxy_host] + regex: "([^/]*)/(.*)" + replacement: "/$2" + target_label: "__metrics_path__" + action: "replace" ... diff --git a/ansible/roles/prometheus/vars/main.yml b/ansible/roles/prometheus/vars/main.yml index d8774c47..3e3fd359 100644 --- a/ansible/roles/prometheus/vars/main.yml +++ b/ansible/roles/prometheus/vars/main.yml @@ -150,3 +150,12 @@ blackbox_jobs: - name: icmp module: icmp targets: "{{ dom0_hosts | list }}" + +prometheus_aws_access_key_dev: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/ooni_monitoring/access_key', profile='oonidevops_user_dev') }}" +prometheus_aws_secret_key_dev: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/ooni_monitoring/secret_key', profile='oonidevops_user_dev') }}" + +prometheus_aws_access_key_prod: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/ooni_monitoring/access_key', profile='oonidevops_user_prod') }}" +prometheus_aws_secret_key_prod: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/ooni_monitoring/secret_key', profile='oonidevops_user_prod') }}" + +clickhouse_proxy_host_dev: "clickhouseproxy.dev.ooni.io" +clickhouse_proxy_host_prod: "clickhouseproxy.dev.ooni.io" # TODO Change for prod diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 14e02f71..f9ed1c58 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -301,6 +301,13 @@ module "ooniapi_cluster" { instance_type = "t3a.micro" + monitoring_sg_ids = [ + # The clickhouse proxy has an nginx configuration + # to proxy requests from the monitoring server + # to the cluster instances + module.ooni_clickhouse_proxy.ec2_sg_id + ] + tags = merge( local.tags, { Name = "ooni-tier0-api-ecs-cluster" } @@ -411,6 +418,10 @@ module "ooniapi_reverseproxy" { ) } +data "dns_a_record_set" "monitoring_host" { + host = "monitoring.ooni.org" +} + module "ooni_clickhouse_proxy" { source = "../../modules/ec2" @@ -426,31 +437,37 @@ module "ooni_clickhouse_proxy" { name = "oonickprx" ingress_rules = [{ - from_port = 22, - to_port = 22, - protocol = "tcp", + from_port = 22, + to_port = 22, + protocol = "tcp", cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 80, - to_port = 80, - protocol = "tcp", + }, { + from_port = 80, + to_port = 80, + protocol = "tcp", cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 9000, - to_port = 9000, - protocol = "tcp", + }, { + from_port = 9000, + to_port = 9000, + protocol = "tcp", cidr_blocks = module.network.vpc_subnet_private[*].cidr_block, + }, { + // For the prometheus proxy: + from_port = 9200, + to_port = 9200, + protocol = "tcp" + cidr_blocks = [for ip in flatten(data.dns_a_record_set.monitoring_host.*.addrs) : "${tostring(ip)}/32"] }] egress_rules = [{ - from_port = 0, - to_port = 0, - protocol = "-1", + from_port = 0, + to_port = 0, + protocol = "-1", cidr_blocks = ["0.0.0.0/0"], - }, { - from_port = 0, - to_port = 0, - protocol = "-1", + }, { + from_port = 0, + to_port = 0, + protocol = "-1", ipv6_cidr_blocks = ["::/0"] }] @@ -792,6 +809,9 @@ resource "aws_acm_certificate_validation" "ooniapi_frontend" { ### Ooni monitoring module "ooni_monitoring" { - source = "../../modules/ooni_monitoring" - tags = local.tags -} \ No newline at end of file + source = "../../modules/ooni_monitoring" + environment = local.environment + aws_region = var.aws_region + + tags = local.tags +} diff --git a/tf/modules/ec2/outputs.tf b/tf/modules/ec2/outputs.tf index 4a99b3c8..55488632 100644 --- a/tf/modules/ec2/outputs.tf +++ b/tf/modules/ec2/outputs.tf @@ -5,3 +5,7 @@ output "aws_instance_id" { output "aws_instance_public_dns" { value = aws_instance.ooni_ec2.public_dns } + +output "ec2_sg_id" { + value = aws_security_group.ec2_sg.id +} \ No newline at end of file diff --git a/tf/modules/ecs_cluster/main.tf b/tf/modules/ecs_cluster/main.tf index 54ed820f..1de525b2 100644 --- a/tf/modules/ecs_cluster/main.tf +++ b/tf/modules/ecs_cluster/main.tf @@ -120,6 +120,14 @@ resource "aws_security_group" "container_host" { ] } + ingress { + protocol = "tcp" + from_port = 9100 + to_port = 9100 + + security_groups = var.monitoring_sg_ids + } + egress { from_port = 0 to_port = 0 diff --git a/tf/modules/ecs_cluster/variables.tf b/tf/modules/ecs_cluster/variables.tf index 1d8a5cd5..5604b418 100644 --- a/tf/modules/ecs_cluster/variables.tf +++ b/tf/modules/ecs_cluster/variables.tf @@ -62,6 +62,10 @@ variable "instance_volume_size" { default = "5" } +variable "monitoring_sg_ids" { + default = [] +} + variable "node_exporter_port" { default = "9100" } \ No newline at end of file diff --git a/tf/modules/ooni_monitoring/main.tf b/tf/modules/ooni_monitoring/main.tf index 22021f0e..a3615f73 100644 --- a/tf/modules/ooni_monitoring/main.tf +++ b/tf/modules/ooni_monitoring/main.tf @@ -1,3 +1,11 @@ +locals { + name = "ecs-service-discovery-${var.environment}" + + tags = { + Name = local.name + Environment = var.environment + } +} resource "aws_iam_user" "ooni_monitoring" { name = "oonidevops-monitoring" } @@ -34,4 +42,4 @@ resource "aws_ssm_parameter" "ooni_monitoring_secret_key" { name = "/oonidevops/secrets/ooni_monitoring/secret_key" type = "SecureString" value = aws_iam_access_key.ooni_monitoring.secret -} \ No newline at end of file +} diff --git a/tf/modules/ooni_monitoring/variables.tf b/tf/modules/ooni_monitoring/variables.tf index c9769271..f49676a0 100644 --- a/tf/modules/ooni_monitoring/variables.tf +++ b/tf/modules/ooni_monitoring/variables.tf @@ -2,4 +2,19 @@ variable "tags" { description = "tags to apply to the resources" default = {} type = map(string) -} \ No newline at end of file +} + +variable "environment" { + type = string +} + +variable "task_memory" { + description = "How much memory to allocate for this task" + type = number + default = 64 +} + +variable "aws_region" { + description = "AWS region" + type = string +}