Add AWS Deployment

Makefile

@@ -1,9 +1,27 @@
 #### Main
 
-provision: tf.apply twitter-forwarder.build streams.build connectors.add.both twitter-forwarder.start
+provision: provision.app
 
-tf.apply:
-	terraform apply --auto-approve terraform
+provision.app: tf.apply.app connectors.add.both twitter-forwarder.start
+
+provision.aws: tf.apply.eks configure.aws.kubeconfig configure.helm.svcaccount tf.apply.app.aws connectors.add.both twitter-forwarder.start.aws
+
+tf.apply: tf.apply.app
+
+tf.apply.app:
+	terraform apply --auto-approve -var deploy_aws=false -target="module.app" terraform
+
+tf.apply.app.aws:
+	terraform apply --auto-approve -var deploy_aws=true -target="module.app" terraform
+
+tf.apply.eks:
+	terraform apply --auto-approve -var deploy_aws=true -target="module.eks" terraform
+
+configure.aws.kubeconfig:
+	aws eks update-kubeconfig --name kafka-demo
+
+configure.helm.svcaccount:
+	kubectl apply -f terraform/eks/tiller-user.yml && helm init --service-account tiller --upgrade
 
 tf.destroy:
 	terraform destroy
@@ -36,6 +54,10 @@ jenkins.password:
 jenkins.open: jenkins.password
 	open http://localhost:8081
 
+# jenkins.open.aws: jenkins.open
+# 	kubectl port-forward svc/jenkins 8081:8081 -n kafka
+
+
 #### Confluent Kafka Connect
 
 connectors.wait.for.confluent:
@@ -67,12 +89,15 @@ connectors.add.both: connectors.wait.for.confluent connector.source.add connecto
 #### Twitter Forwarder
 
 twitter-forwarder.build:
-	docker build twitter-forwarder -t sfo/twitter-forwarder
+	docker build twitter-forwarder -t twitter-forwarder
 
 twitter-forwarder.update: twitter-forwarder.build
 	terraform taint kubernetes_pod.twitter-forwarder && \
 	terraform apply -auto-approve terraform
 
+twitter-forwarder.start.aws: 
+	kubectl exec -it twitter-forwarder -n kafka -- sh -c 'curl -s http://localhost/twitter/on'
+
 twitter-forwarder.start:
 	curl -s http://localhost:3000/twitter/on
 
@@ -86,7 +111,7 @@ twitter-forwarder.logs:
 #### Tweets Transformer
 
 tweets-transformation.build:
-	docker build tweets-transformation -t sfo/tweets-transformation
+	docker build tweets-transformation -t tweets-transformation
 
 tweets-transformation.update: tweets-transformation.build
 	terraform taint kubernetes_pod.tweets-transformation && \
@@ -99,7 +124,7 @@ tweets-transformation.logs:
 #### Streams App
 
 streams.build:
-	docker build kafka-streams -t sfo/kafka-streams
+	docker build kafka-streams -t kafka-streams
 
 streams.update: streams.build
 	terraform taint kubernetes_pod.kafka-streams && \
@@ -130,6 +155,9 @@ grafana.password:
 grafana.open: grafana.password
 	open http://localhost:8083
 
+grafana.open.aws: grafana.open
+	kubectl port-forward svc/grafana 8083:8083 -n kafka 
+
 #### Other
 
 kube.proxy:

README.md

@@ -59,4 +59,88 @@ The Twitter forwarder is started by default, but you can also stop it.
 
 ### Console consumer for Twitter feed
 
-1. `make consumer.twitter`
+1. `make consumer.twitter`  
+
+---
+
+### AWS Deployment
+
+AWS Deployment is split into two sections, AWS cluster deployment module (eks) and resource deployment module (app). Local deployment (above) assumes local cluster has been provisioned and is available and only deploys the app terraform module. AWS Deployment will deploy both cluster and app modules to AWS.
+
+1. Setup local AWS CLI credentials (`aws configure`) and set your `AWS_PROFILE` environment variable if you are not using the `default` profile. Set your region environment variable `AWS_DEFAULT_REGION`, then initialize terraform with `terraform init terraform`.
+
+2. Build the Docker images ([kafka-streams](kafka-streams/), [tweets-transformation](tweets-transformation/), and [twitter-forwarder](twitter-forwarder/)) then deploy to AWS ECR. [(see AWS ECR Documentation for reference)](https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-basics.html). When you run the next step terraform will point to your AWS ECR images under the `latest` tag.
+
+3. Run `make provision.aws` and start the proxy in a separate terminal (`make kube.proxy`)  
+    It will take 10-15 minutes for the EKS cluster to provision in AWS, and an additional 10-20 minutes for the services to deploy and come up.
+
+    The `make provision.aws` command will do the following:  
+      * `tf.apply.eks` (EKS TF module)
+      * `configure.aws.kubeconfig` (Configures kubectl on your system for EKS endpoint)
+      * `configure.helm.svcaccount` (Configures helm service account in EKS)
+      * `tf.apply.local` (Deploys kafka-demo to the EKS cluster)
+      * `connectors.add.both` (Configures connectors)
+      * `twitter-forwarder.start.aws` (Starts the twitter forwarder service in EKS)
+
+Run `make tf.destroy` to remove both the kafka-demo resources and EKS cluster.
+
+#### Revert to local deployment
+
+Update kubectl to point to your local Kubernetes cluster by running:  
+  `kubectl config use-context docker-desktop` (if using docker-desktop)
+
+Follow instructions at top of the Readme (`make provision` or `make provision.app`)
+
+`make provision` (or `make provision.app`) will deploy only the local application resources in the `kube` module, and skip the AWS resources `eks` module. The deployment will use your local Docker images for `kafka-streams`, `tweets-transformation`, and `twitter-forwarder`.
+
+#### Terraform v0.11 / v0.12
+
+For use with Terraform v0.11 use the source `github.com/terraform-aws-modules/terraform-aws-eks?ref=v4.0.2` in `terraform/eks/eks.tf` eks module. For use with Terraform v0.12 use `terraform-aws-modules/eks/aws`. 
+
+#### Services  
+
+* Control Center  
+  Control center comes up by running :  
+  `make control-center.open`  
+
+* Grafana  
+  Grafana comes up by running :  
+   `make grafana.open.aws`  User: admin, password is in your clipboard.
+
+* Dashboard  
+  Dashboard does not seem to start correctly. The container fails to start in the default namespace, with errors in the logs. Likely permissions
+    ```
+    panic: secrets is forbidden: User "system:serviceaccount:default:kube-dashboard-kubernetes-dashboard" cannot create resource "secrets" in API group "" in the namespace "kube-system"
+    ```
+
+    Full Logs:
+    ```
+    2019/09/03 20:45:04 Starting overwatch
+    2019/09/03 20:45:04 Using in-cluster config to connect to apiserver
+    2019/09/03 20:45:04 Using service account token for csrf signing
+    2019/09/03 20:45:04 Successful initial request to the apiserver, version: v1.13.10-eks-5ac0f1
+    2019/09/03 20:45:04 Generating JWE encryption key
+    2019/09/03 20:45:04 New synchronizer has been registered: kubernetes-dashboard-key-holder-kube-system. Starting
+    2019/09/03 20:45:04 Starting secret synchronizer for kubernetes-dashboard-key-holder in namespace kube-system
+    2019/09/03 20:45:04 Synchronizer kubernetes-dashboard-key-holder-kube-system exited with error: unexpected object: &Secret{ObjectMeta:k8s_io_apimachinery_pkg_apis_meta_v1.ObjectMeta{Name:,GenerateName:,Namespace:,SelfLink:,UID:,ResourceVersion:,Generation:0,CreationTimestamp:0001-01-01 00:00:00 +0000 UTC,DeletionTimestamp:<nil>,DeletionGracePeriodSeconds:nil,Labels:map[string]string{},Annotations:map[string]string{},OwnerReferences:[],Finalizers:[],ClusterName:,Initializers:nil,},Data:map[string][]byte{},Type:,StringData:map[string]string{},}
+    2019/09/03 20:45:06 Restarting synchronizer: kubernetes-dashboard-key-holder-kube-system.
+    2019/09/03 20:45:06 Starting secret synchronizer for kubernetes-dashboard-key-holder in namespace kube-system
+    2019/09/03 20:45:06 Synchronizer kubernetes-dashboard-key-holder-kube-system exited with error: kubernetes-dashboard-key-holder-kube-system watch ended with timeout
+    2019/09/03 20:45:08 Restarting synchronizer: kubernetes-dashboard-key-holder-kube-system.
+    2019/09/03 20:45:08 Starting secret synchronizer for kubernetes-dashboard-key-holder in namespace kube-system
+    2019/09/03 20:45:08 Synchronizer kubernetes-dashboard-key-holder-kube-system exited with error: kubernetes-dashboard-key-holder-kube-system watch ended with timeout
+    2019/09/03 20:45:10 Storing encryption key in a secret
+    panic: secrets is forbidden: User "system:serviceaccount:default:kube-dashboard-kubernetes-dashboard" cannot create resource "secrets" in API group "" in the namespace "kube-system"
+
+    goroutine 1 [running]:
+    github.com/kubernetes/dashboard/src/app/backend/auth/jwe.(*rsaKeyHolder).init(0xc4203b0a80)
+            /home/travis/build/kubernetes/dashboard/.tmp/backend/src/github.com/kubernetes/dashboard/src/app/backend/auth/jwe/keyholder.go:131 +0x35e
+    github.com/kubernetes/dashboard/src/app/backend/auth/jwe.NewRSAKeyHolder(0x1367500, 0xc420269da0, 0xc420269da0, 0x1213a6e)
+            /home/travis/build/kubernetes/dashboard/.tmp/backend/src/github.com/kubernetes/dashboard/src/app/backend/auth/jwe/keyholder.go:170 +0x64
+    main.initAuthManager(0x13663e0, 0xc420122240, 0xc4204cfcd8, 0x1)
+            /home/travis/build/kubernetes/dashboard/.tmp/backend/src/github.com/kubernetes/dashboard/src/app/backend/dashboard.go:185 +0x12c
+    main.main()
+            /home/travis/build/kubernetes/dashboard/.tmp/backend/src/github.com/kubernetes/dashboard/src/app/backend/dashboard.go:103 +0x26b
+    ```
+
+

terraform/confluent.tf → terraform/app/confluent.tf

@@ -3,6 +3,7 @@ resource "helm_release" "confluent" {
   repository = "./"
   chart      = "cp-helm-charts"
   namespace  = "kafka"
+  timeout    = 600
 
   set {
     name  = "cp-kafka.persistence.enabled"

terraform/kafka-streams.tf → terraform/app/kafka-streams.tf

@@ -6,9 +6,14 @@ resource "kubernetes_pod" "kafka-streams" {
 
   spec {
     container {
-      image = "sfo/kafka-streams"
+      image = "${var.kafka-streams-img}"
       image_pull_policy = "IfNotPresent"
       name  = "kafka-streams-1"
     }
   }
-}
+}
+
+variable "kafka-streams-img" {
+  type = "string"
+  default = "kafka-streams:latest"
+}

terraform/main.tf → terraform/app/main.tf

@@ -1,13 +1,8 @@
-provider "helm" {}
-
 data "helm_repository" "stable" {
   name = "stable"
   url  = "https://kubernetes-charts.storage.googleapis.com"
 }
 
-provider "kubernetes" {
-}
-
 resource "kubernetes_namespace" "kafka" {
   metadata {
     name = "kafka"

terraform/monitoring.tf → terraform/app/monitoring.tf

@@ -3,13 +3,15 @@ resource "helm_release" "prometheus" {
   repository = "${data.helm_repository.stable.metadata.0.name}"
   chart      = "prometheus"
   namespace  = "kafka"
+  timeout    = 600
 }
 
 resource "helm_release" "grafana" {
   name       = "grafana"
   repository = "${data.helm_repository.stable.metadata.0.name}"
   chart      = "grafana"
   namespace  = "kafka"
+  timeout = 600
 
   values = [
     "${file("grafana/values.yaml")}",

terraform/postgres.tf → terraform/app/postgres.tf

@@ -3,6 +3,7 @@ resource "helm_release" "pg" {
   repository = "${data.helm_repository.stable.metadata.0.name}"
   chart      = "postgresql"
   namespace  = "kafka"
+  timeout    = 600
 
   values = [
     "${file("pg/pg-values.yaml")}",

terraform/tweets-transformation.tf → terraform/app/tweets-transformation.tf

@@ -9,9 +9,14 @@ resource "kubernetes_pod" "tweets-transformation" {
 
   spec {
     container {
-      image = "sfo/tweets-transformation"
+      image = "${var.tweets-transformation-img}"
       image_pull_policy = "IfNotPresent"
       name  = "tweets-transformation-1"
     }
   }
+}
+
+variable "tweets-transformation-img" {
+  type = "string"
+  default = "tweets-transformation:latest"
 }

terraform/twitter-forwarder.tf → terraform/app/twitter-forwarder.tf

@@ -9,7 +9,7 @@ resource "kubernetes_pod" "twitter-forwarder" {
 
   spec {
     container {
-      image = "sfo/twitter-forwarder"
+      image = "${var.twitter-forwarder-img}"
       image_pull_policy = "IfNotPresent"
       name  = "twitter-forwarder-1"
     }
@@ -32,4 +32,9 @@ resource "kubernetes_service" "twitter-forwarder" {
 
     type = "LoadBalancer"
   }
+}
+
+variable "twitter-forwarder-img" {
+  type = "string"
+  default = "twitter-forwarder:latest"
 }

terraform/eks/eks.tf

@@ -0,0 +1,41 @@
+module "cluster" {
+  source       = "github.com/terraform-aws-modules/terraform-aws-eks?ref=v4.0.2"
+  cluster_name = "kafka-demo"
+  subnets      = ["${module.vpc.public_subnets}"]
+  vpc_id       = "${module.vpc.vpc_id}"
+  cluster_create_timeout = "30m"
+  cluster_delete_timeout = "30m"
+
+  worker_groups = [
+    {
+      instance_type = "m4.large"
+      asg_max_size  = 2
+      asg_desired_capacity = 2
+    }
+  ]
+}
+
+data "aws_availability_zones" "available" {}
+data "aws_caller_identity" "current" {}
+data "aws_region" "current" {}
+
+output "account_id" {
+  value = "${data.aws_caller_identity.current.account_id}"
+}
+
+output "region" {
+  value = "${data.aws_region.current.name}"
+}
+
+module "vpc" {
+  source         = "terraform-aws-modules/vpc/aws"
+  version        = "1.60.0"
+  name           = "kafka-demo-vpc"
+  cidr           = "10.0.0.0/16"
+  azs            = ["${data.aws_availability_zones.available.names}"]
+  public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"]
+
+  tags = {
+    "kubernetes.io/cluster/kafka-demo" = "shared"
+  }
+}

terraform/eks/tiller-user.yml

@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: tiller
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: tiller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: tiller
+    namespace: kube-system

terraform/modules.tf

@@ -0,0 +1,34 @@
+
+provider "helm" {
+}
+
+provider "kubernetes" {
+}
+
+provider "aws" {
+  version = ">= 2.11"
+  region = "us-west-2"
+}
+
+module "eks" {
+  source = "./eks"
+  providers = {
+    aws = "aws"
+  }
+}
+
+module "app" {
+  source = "./app"
+  providers = {
+    helm = "helm"
+    kubernetes = "kubernetes"
+  }
+  kafka-streams-img = "${var.deploy_aws == true ? join("." , list(module.eks.account_id,"dkr.ecr",module.eks.region,"amazonaws.com/kafka-streams:latest")) : "kafka-streams:latest"}"
+  twitter-forwarder-img = "${var.deploy_aws == true ? join("." , list(module.eks.account_id,"dkr.ecr",module.eks.region,"amazonaws.com/twitter-forwarder:latest")) : "twitter-forwarder:latest"}"
+  tweets-transformation-img  = "${var.deploy_aws == true ? join("." , list(module.eks.account_id,"dkr.ecr",module.eks.region,"amazonaws.com/tweets-transformation:latest")) : "tweets-transformation:latest"}"
+}
+
+variable "deploy_aws" {
+  type = "string"
+  default = "false"
+}