refactor: infrastructure code from another folder

infrastructure-platform/src/app.tf

@@ -0,0 +1,64 @@
+locals {
+    app_ami             = var.ami
+    app_key_pair        = var.key_pair
+    app_server_name     = var.app_server_name
+    app_instance_type   = var.app_instance_type
+    app_ssh_port        = 22
+    app_protocol        = "tcp"
+    app_any_port        = 0
+    app_any_protocol    = "-1"
+    db_instance_dns     = module.influxdb_server.private_dns
+}
+
+module "app_server" {
+    source = "./modules/instances"
+
+    ami                     = local.app_ami
+    instance_type           = local.app_instance_type
+    key_pair                = local.app_key_pair
+    enable_public_facing    = false
+    subnet_id               = lookup(module.app_subnets.subnets, "_private_subnet_a").id
+    security_groups         = [aws_security_group.app_security_group.id]
+    server_name             = local.app_server_name
+    iam_instance_profile    = aws_iam_instance_profile.ec2_cloudwatch_instance_profile.name
+
+    user_data               = templatefile("${path.cwd}/../../bootstrap/app-bootstrap.tmpl", { db_instance_dns = local.db_instance_dns })
+
+    depends_on = [module.influxdb_server]
+}
+
+resource "aws_security_group" "app_security_group" {
+    name        = local.app_server_name
+    description = "The SG to assign to the app instance"
+    vpc_id      = module.app_vpc.vpc_id
+}
+
+resource "aws_security_group_rule" "app_allow_ssh" {
+    type                = "ingress"
+    description         = "allow ssh"
+    from_port           = local.app_ssh_port
+    to_port             = local.app_ssh_port
+    protocol            = local.app_protocol
+    cidr_blocks         = ["10.0.2.0/24"]
+    security_group_id   = aws_security_group.app_security_group.id
+}
+
+resource "aws_security_group_rule" "app_inbound_traffic" {
+    type                = "ingress"
+    description         = "allowing inbound access to the app"
+    from_port           = var.app_port
+    to_port             = var.app_port
+    protocol            = local.app_protocol
+    cidr_blocks         = ["10.0.2.0/24"]
+    security_group_id   = aws_security_group.app_security_group.id
+}
+
+resource "aws_security_group_rule" "app_default_outbound" {
+    type                = "egress"
+    description         = "default_outbound"
+    from_port           = local.app_any_port
+    to_port             = local.app_any_port
+    protocol            = local.app_any_protocol
+    cidr_blocks         = ["0.0.0.0/0"]
+    security_group_id   = aws_security_group.app_security_group.id
+}

infrastructure-platform/src/bootstrap/app-bootstrap.tmpl

@@ -0,0 +1,33 @@
+#!/bin/bash
+yum update -y
+
+# Cloudwatch Agent to gather metrics like Memory Usage
+wget https://s3.amazonaws.com/amazoncloudwatch-agent/amazon_linux/amd64/latest/amazon-cloudwatch-agent.rpm
+rpm -U ./amazon-cloudwatch-agent.rpm
+/opt/aws/amazon-cloudwatch-agent/bin/amazon-cloudwatch-agent-ctl -a start -m ec2 -c default
+
+# setup cron jobs
+touch cron
+cat > cron <<EOL
+@reboot sudo service docker start
+@reboot sudo docker run --rm --name github-cloud-proxy -d -p 8080:8080 -e KPI_DATABASE_URL=http://${db_instance_dns}:8086 -e KPI_DATABASE_USERNAME=root -e KPI_DATABASE_PASSWORD=root colinbut/githubcloudproxy
+EOL
+chown ec2-user:ec2-user cron
+su ec2-user -c 'crontab cron'
+rm -f cron
+
+# su ec2-user -c 'crontab -l > cron; \ 
+# 	echo "@reboot sudo service docker start" >> cron; \
+# 	echo "@reboot sudo docker run --rm --name github-cloud-proxy -d -p 8080:8080 -e KPI_DATABASE_URL=http://influxdb:8086 -e KPI_DATABASE_USERNAME=root -e KPI_DATABASE_PASSWORD=root colinbut/githubcloudproxy" >> cron; \
+# 	crontab cron; \
+# 	rm cron;'
+
+
+# install docker & start the github-cloud-proxy app
+amazon-linux-extras install docker -y
+service docker start
+sleep 30
+
+docker run --rm --name github-cloud-proxy -d -p 8080:8080 -e KPI_DATABASE_URL=http://${db_instance_dns}:8086 -e KPI_DATABASE_USERNAME=root -e KPI_DATABASE_PASSWORD=root colinbut/githubcloudproxy
+
+

infrastructure-platform/src/bootstrap/db-bootstrap.tmpl

@@ -0,0 +1,56 @@
+#!/bin/bash
+yum update -y 
+yum install xfsprogs -y 
+
+# mount the volumes on startup
+xvdf_result=$(file -s /dev/xvdf)
+if [ "$xvdf_result" == "/dev/xvdf: data" ]
+then
+    mkfs -t xfs /dev/xvdf
+fi
+
+xvds_result=$(file -s /dev/xvds)
+if [ "$xvds_result" == "/dev/xvds: data" ]
+then
+    mkfs -t xfs /dev/xvds
+fi
+
+mkdir /{data,wal}
+mount /dev/xvdf /data
+mount /dev/xvds /wal
+
+# setup auto mounting of volumes upon restart
+UUID1=$(blkid | grep "/dev/xvda1" | awk '{printf substr($3,7,length($3) - 7)}')
+UUID2=$(blkid | grep "/dev/xvdf" | awk '{printf substr($2,7,length($2) - 7)}')
+cp /etc/fstab /etc/fstab.orig
+echo "UUID=$UUID1 /data xfs defaults,nofail 0 2" >> /etc/fstab
+echo "UUID=$UUID2 /wal xfs defaults,nofail 0 2" >> /etc/fstab
+
+# setup daily backup of db
+cd /home/ec2-user/
+touch influxdb-backup.sh
+chown ec2-user:ec2-user influxdb-backup.sh
+chmod +x influxdb-backup.sh
+cat > influxdb-backup.sh <<'EOL'
+#!/bin/bash
+backup=$(date '+%Y%m%d')
+mkdir -p /data/$backup/
+docker exec influxdb influxd backup -database githubcloudproxy -portable /data/$backup/
+cd /data && zip $backup.zip $backup/* 
+aws s3 mv $backup.zip s3://${db_backup_bucket_name}/$backup.zip
+EOL
+
+# setup cron jobs
+su ec2-user -c 'crontab -l > cron; \ 
+	echo "@reboot sudo service docker start" >> cron; \
+	echo "@reboot sudo docker run --rm --name influxdb -v /data:/data -v /wal:/wal -d -p 8086:8086 influxdb" >> cron; \
+	echo "00 00 * * * ./home/ec2-user/influxdb-backup.sh" >> cron; \
+	crontab cron; \
+	rm cron;'
+
+amazon-linux-extras install docker -y 
+service docker start
+
+sleep 30
+
+docker run --rm --name influxdb -v /data:/data -v /wal:/wal -d -p 8086:8086 influxdb

infrastructure-platform/src/database.tf

@@ -0,0 +1,115 @@
+locals {
+    db_ami                  = var.ami
+    db_key_pair             = var.key_pair
+    db_server_name          = var.db_server_name
+    db_instance_type        = var.db_instance_type
+    db_ssh_port             = 22
+    db_protocol             = "tcp"
+    db_any_port             = 0
+    db_any_protocol         = "-1"
+    db_backup_bucket_name   = aws_s3_bucket.influxdb-backup-bucket.bucket
+}
+
+resource "aws_s3_bucket" "influxdb-backup-bucket" {
+    bucket = var.db_backup_bucket_name
+
+    versioning {
+        enabled = true
+    }
+
+    server_side_encryption_configuration {
+        rule {
+            apply_server_side_encryption_by_default {
+                sse_algorithm = "AES256"
+            }
+        }
+    }
+}
+
+module "influxdb_server" {
+    source = "./modules/instances"
+
+    ami                     = local.db_ami
+    instance_type           = local.db_instance_type
+    key_pair                = local.db_key_pair
+    enable_public_facing    = false
+    subnet_id               = lookup(module.app_subnets.subnets, "_private_subnet_a").id
+    security_groups         = [aws_security_group.db_security_group.id]
+    iam_instance_profile    = aws_iam_instance_profile.ec2_s3_instance_profile.name
+    server_name             = local.db_server_name
+
+    user_data               = templatefile("${path.cwd}/../../bootstrap/db-bootstrap.tmpl", { db_backup_bucket_name = local.db_backup_bucket_name })
+
+    depends_on = [aws_s3_bucket.influxdb-backup-bucket]
+}
+
+resource "aws_security_group" "db_security_group" {
+    name        = local.db_server_name
+    description = "The SG to assign to the database instance"
+    vpc_id      = module.app_vpc.vpc_id
+}
+
+resource "aws_security_group_rule" "db_allow_ssh" {
+    type                = "ingress"
+    description         = "allow ssh"
+    from_port           = local.db_ssh_port
+    to_port             = local.db_ssh_port
+    protocol            = local.db_protocol
+    cidr_blocks         = ["10.0.2.0/24"]
+    security_group_id   = aws_security_group.db_security_group.id
+}
+
+resource "aws_security_group_rule" "db_inbound_traffic" {
+    type                = "ingress"
+    description         = "allowing inbound access to the db"
+    from_port           = var.db_port
+    to_port             = var.db_port
+    protocol            = local.db_protocol
+    cidr_blocks         = ["10.0.1.0/24"]
+    security_group_id   = aws_security_group.db_security_group.id
+}
+
+resource "aws_security_group_rule" "db_default_outbound" {
+    type                = "egress"
+    description         = "default_outbound"
+    from_port           = local.db_any_port
+    to_port             = local.db_any_port
+    protocol            = local.db_any_protocol
+    cidr_blocks         = ["0.0.0.0/0"]
+    security_group_id   = aws_security_group.db_security_group.id
+}
+
+resource "aws_ebs_volume" "data_volume" {
+    availability_zone   = "eu-west-1a"
+    size                = 100
+    type                = "gp2"
+
+    tags = {
+        Name = "InfluxDB-Data-Volume"
+    }
+}
+
+resource "aws_ebs_volume" "wal_volume" {
+    availability_zone   = "eu-west-1a"
+    size                = 40
+    type                = "io1"
+    iops                = 100
+
+    tags = {
+        Name = "InfluxDB-Wal-Volume"
+    }
+}
+
+resource "aws_volume_attachment" "data_ebs_att" {
+    device_name     = "/dev/xvdf"
+    volume_id       = aws_ebs_volume.data_volume.id
+    instance_id     = module.influxdb_server.instance_id
+    force_detach    = true
+}
+
+resource "aws_volume_attachment" "wal_ebs_att" {
+    device_name     = "/dev/xvds"
+    volume_id       = aws_ebs_volume.wal_volume.id
+    instance_id     = module.influxdb_server.instance_id
+    force_detach    = true
+}

infrastructure-platform/src/lambda-ec2-reboot.tf

@@ -0,0 +1,15 @@
+resource "aws_lambda_function" "lambda_ec2_reboot" {
+    filename        = "lambda_function_payload.zip"
+    function_name   = "lambda_ec2_reboot_instances"
+    role            = aws_iam_role.lambda_ec2_role.arn
+    handler         = "lambda_function.lambda_handler"
+    runtime         = "python3.7"
+    timeout         = 10
+
+    environment {
+        variables = {
+            region = "eu-west-1",
+            instance_id = module.app_server.instance_id
+        }
+    }
+}

infrastructure-platform/src/lambda_function.py

@@ -0,0 +1,10 @@
+import boto3
+import os
+
+region = os.environ['region']
+instances = [os.environ['instance_id']]
+ec2 = boto3.client('ec2', region_name=region)
+
+def lambda_handler(event, context):
+    ec2.reboot_instances(InstanceIds=instances)
+    print('rebooted instances: ' + str(instances))

infrastructure-platform/src/main.tf

@@ -0,0 +1,12 @@
+terraform {
+    required_version = "~> 0.13"
+    backend "s3" {
+        bucket = "github-cloud-proxy-terraform-state"
+        key = "infrastructure-platform/terraform.tfstate"
+        region = "eu-west-2"
+    }
+}
+
+provider "aws" {
+    region = "eu-west-2"
+}

infrastructure-platform/src/modules/instances/main.tf

@@ -0,0 +1,15 @@
+resource "aws_instance" "server" {
+    ami                         = var.ami
+    instance_type               = var.instance_type
+    key_name                    = var.key_pair
+    associate_public_ip_address = var.enable_public_facing
+    security_groups             = var.security_groups
+    subnet_id                   = var.subnet_id
+    iam_instance_profile        = var.iam_instance_profile
+
+    user_data                   = var.user_data
+
+    tags = {
+        Name = var.server_name
+    }
+}

infrastructure-platform/src/modules/instances/outputs.tf

@@ -0,0 +1,19 @@
+output "instance_id" {
+    value = aws_instance.server.id
+}
+
+output "public_ip" {
+    value = aws_instance.server.public_ip
+}
+
+output "public_dns" {
+    value = aws_instance.server.public_dns
+}
+
+output "private_ip" {
+    value = aws_instance.server.private_ip
+}
+
+output "private_dns" {
+    value = aws_instance.server.private_dns
+}