extract variables

Neha-Sinha2305 · Aug 28, 2020 · 16a4b13 · 16a4b13
1 parent d6062fe
commit 16a4b13
Show file tree

Hide file tree

Showing 8 changed files with 172 additions and 57 deletions.
diff --git a/Makefile b/Makefile
@@ -49,3 +49,15 @@ clean: destroy-box remove-tmp
 update-box:
 	@SSL_CERT_FILE=${SSL_CERT_FILE} CURL_CA_BUNDLE=${CURL_CA_BUNDLE} vagrant box update || (echo '\n\nIf you get an SSL error you might be behind a transparent proxy. \nMore info https://github.com/fredrikhgrelland/vagrant-hashistack/blob/master/README.md#if-you-are-behind-a-transparent-proxy\n\n' && exit 2)
 
+# to-hivemetastore
+proxy-h:
+	consul connect proxy -service hivemetastore-local -upstream hive-metastore:9083 -log-level debug
+# to-minio
+proxy-m:
+	consul connect proxy -service minio-local -upstream minio:9000 -log-level debug
+# to-postgres
+proxy-p:
+	consul connect proxy -service postgres-local -upstream postgres:5432 -log-level debug
+# to-presto
+proxy-p:
+	consul connect proxy -service postgres-local -upstream presto:8080 -log-level debug
diff --git a/conf/nomad/presto.hcl b/conf/nomad/presto.hcl
@@ -1,6 +1,7 @@
-job "presto" {
+job "${nomad_job_name}" {
   type = "service"
-  datacenters = ["dc1"]
+  datacenters   = "${datacenters}"
+  namespace     = "${namespace}"
 
   update {
     max_parallel      = 1
@@ -21,18 +22,18 @@ job "presto" {
     }
 
     service {
-      name = "presto"
-      port = 8080
+      name = "${service_name}"
+      port = "${port}"
       connect {
         sidecar_service {
           proxy {
             upstreams {
-              destination_name = "hive-metastore"
-              local_bind_port  = 9083
+              destination_name = "${hivemetastore_service_name}"
+              local_bind_port  = "${hivemetastore_port}"
             }
             upstreams {
-              destination_name = "minio"
-              local_bind_port  = 9000
+              destination_name = "${minio_service_name}"
+              local_bind_port  = "${minio_port}"
             }
           }
         }
@@ -85,7 +86,7 @@ job "presto" {
       template {
         destination = "tmp/service.json"
         data = <<EOH
-          {{- service "hive-metastore" | toJSON -}}
+          {{- service "${hivemetastore_service_name}" | toJSON -}}
         EOH
       }
     }
@@ -111,7 +112,7 @@ job "presto" {
       template {
         destination = "tmp/service.json"
         data = <<EOH
-          {{- service "minio" | toJSON -}}
+          {{- service "${minio_service_name}" | toJSON -}}
         EOH
       }
     }
@@ -120,16 +121,16 @@ job "presto" {
       driver = "docker"
 
       config {
-        image = "prestosql/presto:333"
+        image = "${image}"
         volumes = [
           "local/presto/config.properties:/lib/presto/default/etc/config.properties",
           "local/presto/catalog/hive.properties:/lib/presto/default/etc/catalog/hive.properties",
         ]
       }
       template {
         data = <<EOH
-          MINIO_ACCESS_KEY = "minio"
-          MINIO_SECRET_KEY = "minio123"
+          MINIO_ACCESS_KEY = "${minio_access_key}"
+          MINIO_SECRET_KEY = "${minio_secret_key}"
           EOH
         destination = "secrets/.env"
         env         = true
@@ -141,15 +142,11 @@ job "presto" {
         destination = "/local/presto/catalog/hive.properties"
         data = <<EOH
 connector.name=hive-hadoop2
-hive.metastore.uri=thrift://{{ env "NOMAD_UPSTREAM_ADDR_hive-metastore" }}
+hive.metastore.uri=thrift://{{ env "NOMAD_UPSTREAM_ADDR_${hivemetastore_service_name}" }}
 hive.metastore-timeout=1m
-hive.s3.aws-access-key=minio
-hive.s3.aws-secret-key=minio123
-#hive.s3.aws-access-key={{ env "MINIO_ACCESS_KEY" }}
-#hive.s3.aws-secret-key={{ env "MINIO_SECRET_KEY" }}
-#hive.s3.aws-access-key=$$MINIO_ACCESS_KEY
-#hive.s3.aws-secret-key=$$MINIO_SECRET_KEY
-hive.s3.endpoint=http://{{ env "NOMAD_UPSTREAM_ADDR_minio" }}
+hive.s3.aws-access-key=${minio_access_key}
+hive.s3.aws-secret-key=${minio_secret_key}
+hive.s3.endpoint=http://{{ env "NOMAD_UPSTREAM_ADDR_${minio_service_name}" }}
 hive.s3.path-style-access=true
 hive.s3.ssl.enabled=false
 hive.s3.socket-timeout=15m
@@ -159,10 +156,18 @@ EOH
         destination   = "local/presto/config.properties"
         data = <<EOH
 node-scheduler.include-coordinator=true
-http-server.http.port=8080
+http-server.http.port=${port}
 discovery-server.enabled=true
-discovery.uri=http://127.0.0.1:8080
+discovery.uri=http://127.0.0.1:${port}
 EOH
+      }
+      template {
+        destination = "local/data/.additional-envs"
+        change_mode = "noop"
+        env         = true
+        data        = <<EOF
+${envs}
+EOF
       }
       resources {
         memory = 2048

diff --git a/example/main.tf b/example/main.tf
@@ -1,5 +1,5 @@
 locals {
-  nomad_namespace = "default"
+  nomad_namespace   = "default"
   nomad_datacenters = ["dc1"]
 }
 
@@ -10,70 +10,92 @@ module "presto" {
   ]
 
   source = "../."
+
+  nomad_job_name    = "presto"
+  nomad_datacenters = local.nomad_datacenters
+  nomad_namespace   = local.nomad_namespace
+
+  service_name = "presto"
+  port         = 8080
+  docker_image = "prestosql/presto:333"
+
+  #hivemetastore
+  hivemetastore = {
+    service_name = module.hive.service_name
+    port         = 9083 # todo: add output var
+  }
+
+  # minio
+  minio = {
+    service_name = module.minio.minio_service_name
+    port         = 9000 # todo: add output var
+    access_key   = module.minio.minio_access_key
+    secret_key   = module.minio.minio_secret_key
+  }
 }
 
 module "minio" {
-  source        = "github.com/fredrikhgrelland/terraform-nomad-minio.git?ref=0.0.2"
+  source = "github.com/fredrikhgrelland/terraform-nomad-minio.git?ref=0.0.2"
 
   # nomad
-  nomad_datacenters   = local.nomad_datacenters
-  nomad_namespace     = local.nomad_namespace
+  nomad_datacenters = local.nomad_datacenters
+  nomad_namespace   = local.nomad_namespace
 
   # minio
-  service_name = "minio"
-  host = "127.0.0.1"
-  port = 9000
-  container_image = "minio/minio:latest" # todo: avoid using tag latest in future releases
-  access_key = "minio"
-  secret_key = "minio123"
-  buckets = ["default", "hive"]
+  service_name                    = "minio"
+  host                            = "127.0.0.1"
+  port                            = 9000
+  container_image                 = "minio/minio:latest" # todo: avoid using tag latest in future releases
+  access_key                      = "minio"
+  secret_key                      = "minio123"
+  buckets                         = ["default", "hive"]
   container_environment_variables = ["JUST_EXAMPLE_VAR1=some-value", "ANOTHER_EXAMPLE2=some-other-value"]
 
   # mc
-  mc_service_name = "mc"
-  mc_container_image = "minio/mc:latest" # todo: avoid using tag latest in future releases
+  mc_service_name                    = "mc"
+  mc_container_image                 = "minio/mc:latest" # todo: avoid using tag latest in future releases
   mc_container_environment_variables = ["JUST_EXAMPLE_VAR3=some-value", "ANOTHER_EXAMPLE4=some-other-value"]
 }
 
 module "postgres" {
-  source                  = "github.com/fredrikhgrelland/terraform-nomad-postgres.git?ref=0.0.1"
+  source = "github.com/fredrikhgrelland/terraform-nomad-postgres.git?ref=0.0.1"
 
   # nomad
-  nomad_datacenters   = local.nomad_datacenters
-  nomad_namespace     = local.nomad_namespace
+  nomad_datacenters = local.nomad_datacenters
+  nomad_namespace   = local.nomad_namespace
 
   # postgres
-  postgres_service_name = "postgres"
-  postgres_container_image = "postgres:12-alpine"
-  postgres_container_port = 5432
-  postgres_admin_user     = "hive"
-  postgres_admin_password = "hive"
-  postgres_database       = "metastore"
+  postgres_service_name                    = "postgres"
+  postgres_container_image                 = "postgres:12-alpine"
+  postgres_container_port                  = 5432
+  postgres_admin_user                      = "hive"
+  postgres_admin_password                  = "hive"
+  postgres_database                        = "metastore"
   postgres_container_environment_variables = ["PGDATA=/var/lib/postgresql/data"]
 }
 
 module "hive" {
   source = "github.com/fredrikhgrelland/terraform-nomad-hive.git?ref=0.0.1"
 
   # nomad
-  nomad_datacenters   = local.nomad_datacenters
-  nomad_namespace     = local.nomad_namespace
+  nomad_datacenters = local.nomad_datacenters
+  nomad_namespace   = local.nomad_namespace
 
   nomad_job_switch_local = false
 
   # hive
-  hive_service_name = "hive-metastore"
-  hive_container_port = 9083
+  hive_service_name                    = "hive-metastore"
+  hive_container_port                  = 9083
   hive_container_environment_variables = ["SOME_EXAMPLE=example-value"]
 
   # hive - minio
   hive_bucket = {
-    default     = "default",
-    hive        = "hive"
+    default = "default",
+    hive    = "hive"
   }
   minio_service = {
     service_name = module.minio.minio_service_name,
-    port         = 9000,  # todo: minio 0.0.1 does not have output variable port
+    port         = 9000, # todo: minio 0.0.1 does not have output variable port
     access_key   = module.minio.minio_access_key,
     secret_key   = module.minio.minio_secret_key,
   }

diff --git a/main.tf b/main.tf
@@ -1,12 +1,38 @@
 locals {
   datacenters = join(",", var.nomad_datacenters)
+  presto_env_vars = join("\n",
+    concat([
+      "JUST_EXAMPLE_ENV=some-value",
+    ], var.container_environment_variables)
+  )
 }
 
 data "template_file" "template-nomad-job-presto" {
   template = file("${path.module}/conf/nomad/presto.hcl")
+
+  vars = {
+    nomad_job_name = var.nomad_job_name
+    datacenters    = local.datacenters
+    namespace      = var.nomad_namespace
+
+    service_name = var.service_name
+    port         = var.port
+    image        = var.docker_image
+    envs         = local.presto_env_vars
+
+    #hivemetastore
+    hivemetastore_service_name = var.hivemetastore.service_name
+    hivemetastore_port         = var.hivemetastore.port
+
+    # minio
+    minio_service_name = var.minio.service_name
+    minio_port         = var.minio.port
+    minio_access_key   = var.minio.access_key
+    minio_secret_key   = var.minio.secret_key
+  }
 }
 
 resource "nomad_job" "nomad-job-presto" {
   jobspec = data.template_file.template-nomad-job-presto.rendered
-  detach = false
+  detach  = false
 }
diff --git a/outputs.tf b/outputs.tf
@@ -1,10 +1,10 @@
 output "presto_service_name" {
   description = "Presto service name"
-  #todo
+  value       = data.template_file.template-nomad-job-presto.vars.service_name
 }
 
 output "presto_port" {
   description = "Presto local bind port"
-  #todo
+  value       = data.template_file.template-nomad-job-presto.vars.port
 }
 
diff --git a/test_example/example/nomad_acl_test.tf b/test_example/example/nomad_acl_test.tf
@@ -1,7 +1,7 @@
 data "vault_generic_secret" "nomad_secret_id" {
   # Set count of this data source to 1 if ACLs are enabled in Nomad, and 0 if not
   count = var.nomad_acl ? 1 : 0
-  path = "nomad/creds/write"
+  path  = "nomad/creds/write"
 }
 
 provider "nomad" {

diff --git a/test_example/main.tf b/test_example/main.tf
@@ -1,4 +1,4 @@
 resource "nomad_job" "countdash" {
   jobspec = file("${path.module}/conf/nomad/countdash.hcl")
-  detach = false
+  detach  = false
 }
diff --git a/variables.tf b/variables.tf
@@ -14,4 +14,54 @@ variable "nomad_namespace" {
   description = "[Enterprise] Nomad namespace"
   default     = "default"
 }
+variable "nomad_job_name" {
+  type        = string
+  description = "Nomad job name"
+  default     = "presto"
+}
+
+# presto
+variable "service_name" {
+  type        = string
+  description = "Presto service name"
+  default     = "presto"
+}
+
+variable "port" {
+  type        = number
+  description = "Presto http port"
+  default     = 8080
+}
 
+variable "docker_image" {
+  type        = string
+  description = "Presto docker image"
+  default     = "prestosql/presto:333"
+}
+variable "container_environment_variables" {
+  type        = list(string)
+  description = "Presto environment variables"
+  default     = [""]
+}
+
+variable "hivemetastore" {
+  type = object({
+    service_name = string,
+    port         = number,
+  })
+  default = {
+    service_name = "hive-metastore"
+    port         = 9083
+  }
+  description = "Hivemetastore data-object contains service_name and port"
+}
+
+variable "minio" {
+  type = object({
+    service_name = string,
+    port         = number,
+    access_key   = string,
+    secret_key   = string,
+  })
+  description = "Minio data-object contains service_name, port, access_key and secret_key"
+}