diff --git a/.gitignore b/.gitignore index ad7b2bf..aed02ae 100644 --- a/.gitignore +++ b/.gitignore @@ -150,5 +150,7 @@ helm/**/charts/**/* .vscode/settings.json .scratch +# Terraform +.terraform diff --git a/helm/cas-airflow/.helmignore b/helm/cas-airflow/.helmignore new file mode 100644 index 0000000..7ee5f7e --- /dev/null +++ b/helm/cas-airflow/.helmignore @@ -0,0 +1,31 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ + +# Terraform files from local and migration runs +.terraform/ +*.tfstate +*.tfvars +.terraform.lock.hcl +credentials.json +terraform/*.sh diff --git a/helm/cas-airflow/templates/jobs/terraform-apply.yaml b/helm/cas-airflow/templates/jobs/terraform-apply.yaml new file mode 100644 index 0000000..81ea7a4 --- /dev/null +++ b/helm/cas-airflow/templates/jobs/terraform-apply.yaml @@ -0,0 +1,77 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: terraform-apply + labels: + component: backend + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install, pre-upgrade +spec: + backoffLimit: 0 + activeDeadlineSeconds: 900 + template: + spec: + serviceAccountName: "terraform-kubernetes-service-account" + containers: + - name: terraform-apply + resources: {{ toYaml .Values.devops.resources | nindent 12 }} + image: "{{ .Values.devops.image.repository }}:{{ .Values.devops.sourceRepoImageTag | default .Values.devops.image.tag }}" + imagePullPolicy: {{ .Values.devops.image.pullPolicy }} + volumeMounts: + - mountPath: /etc/gcp + name: service-account-credentials-volume + readOnly: True + - mountPath: /etc/tf + name: terraform-backend-config-volume + readOnly: True + - name: tf-cache + mountPath: /working + readOnly: False + - name: terraform-modules + mountPath: /terraform + readOnly: False + env: + - name: TF_VAR_project_id + valueFrom: + secretKeyRef: + name: gcp-credentials-secret + key: gcp_project_id + - name: TF_VAR_openshift_namespace + value: {{ .Release.Namespace | quote }} + - name: TF_VAR_apps + value: '["airflow-backups", "airflow-logs"]' + - name: kubernetes_host + value: "https://api.silver.devops.gov.bc.ca:6443" + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/etc/gcp/credentials.json" + # Terraform was having an issue pulling kubernetes_host in as a TF_VAR, so we add it as a attribute to the command + command: + - /bin/sh + - -c + - | + set -euo pipefail; + cp -r /terraform/. /working; + cd working; + export TF_VAR_kubernetes_token=$( cat /var/run/secrets/kubernetes.io/serviceaccount/token ); + terraform init -backend-config=/etc/tf/gcs.tfbackend; + terraform apply -var="kubernetes_host=$kubernetes_host" -auto-approve; + restartPolicy: Never + volumes: + - name: service-account-credentials-volume + secret: + secretName: gcp-credentials-secret + items: + - key: sa_json + path: credentials.json + - name: terraform-backend-config-volume + secret: + secretName: gcp-credentials-secret + items: + - key: tf_backend + path: gcs.tfbackend + - name: tf-cache + emptyDir: {} + - name: terraform-modules + configMap: + name: terraform-modules diff --git a/helm/cas-airflow/templates/jobs/terraform-modules.yaml b/helm/cas-airflow/templates/jobs/terraform-modules.yaml new file mode 100644 index 0000000..90346a7 --- /dev/null +++ b/helm/cas-airflow/templates/jobs/terraform-modules.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: terraform-modules + namespace: {{ .Release.Namespace }} + # Because terraform-apply.yaml is pre-install, pre-upgrade, this configmap needs to be in place before it + annotations: + "helm.sh/hook": pre-install, pre-upgrade + "helm.sh/hook-weight": "-10" +binaryData: +{{- range $path, $data := .Files.Glob "terraform/**.tf" }} +{{ $path | base | indent 2 }}: >- +{{- $data | toString | b64enc | nindent 4 }} +{{ end }} diff --git a/helm/cas-airflow/templates/jobs/terraform-service-account.yaml b/helm/cas-airflow/templates/jobs/terraform-service-account.yaml new file mode 100644 index 0000000..663ab91 --- /dev/null +++ b/helm/cas-airflow/templates/jobs/terraform-service-account.yaml @@ -0,0 +1,35 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: "terraform-secret-admin" + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install, pre-upgrade +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "delete", "deletecollection", "get", "list", "patch", "update", "watch"] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "terraform-kubernetes-service-account" + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install, pre-upgrade +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: "terraform-kubernetes-service-account-secret-admin-binding" + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install, pre-upgrade +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: "terraform-secret-admin" +subjects: +- kind: ServiceAccount + name: "terraform-kubernetes-service-account" + namespace: {{ .Release.Namespace }} diff --git a/helm/cas-airflow/terraform/main.tf b/helm/cas-airflow/terraform/main.tf new file mode 100644 index 0000000..aa5fa3d --- /dev/null +++ b/helm/cas-airflow/terraform/main.tf @@ -0,0 +1,100 @@ +# Terraform workspace configuration. To apply changes to this file, use `make create_workspace` + +terraform { + required_version = ">=1.4.6" + + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.23" + } + google = { + source = "hashicorp/google" + version = "~> 5.2.0" + } + } + + backend "gcs" {} +} + +# Configure OCP infrastructure to setup the host and authentication token +provider "kubernetes" { + host = var.kubernetes_host + token = var.kubernetes_token +} + +# Configure GCP infrastructure to setup the credentials, default project and location (zone and/or region) for your resources +provider "google" { + project = var.project_id + region = local.region +} + +# Create GCS buckets +resource "google_storage_bucket" "bucket" { + for_each = { for v in var.apps : v => v } + name = "${var.openshift_namespace}-${each.value}" + location = local.region +} + +# Create GCP service accounts for each GCS bucket +resource "google_service_account" "account" { + for_each = { for v in var.apps : v => v } + account_id = "sa-${var.openshift_namespace}-${each.value}" + display_name = "${var.openshift_namespace}-${each.value} Service Account" + depends_on = [google_storage_bucket.bucket] +} + +# Assign Storage Admin role for the corresponding service accounts +resource "google_storage_bucket_iam_member" "admin" { + for_each = { for v in var.apps : v => v } + bucket = "${var.openshift_namespace}-${each.value}" + role = "roles/storage.admin" + member = "serviceAccount:${google_service_account.account[each.key].email}" + depends_on = [google_service_account.account] +} + +# Create viewer GCP service accounts for each GCS bucket +resource "google_service_account" "viewer_account" { + for_each = { for v in var.apps : v => v } + account_id = "ro-${var.openshift_namespace}-${each.value}" + display_name = "${var.openshift_namespace}-${each.value} Viewer Service Account" + depends_on = [google_storage_bucket.bucket] +} + +# Assign (manually created) Storage Viewer role for the corresponding service accounts +resource "google_storage_bucket_iam_member" "viewer" { + for_each = { for v in var.apps : v => v } + bucket = "${var.openshift_namespace}-${each.value}" + role = "projects/${var.project_id}/roles/${var.iam_storage_role_template_id}" + member = "serviceAccount:${google_service_account.viewer_account[each.key].email}" + depends_on = [google_service_account.viewer_account] +} + +# Create keys for the service accounts +resource "google_service_account_key" "key" { + for_each = { for v in var.apps : v => v } + service_account_id = google_service_account.account[each.key].name +} + +# Create keys for the viewer service accounts +resource "google_service_account_key" "viewer_key" { + for_each = { for v in var.apps : v => v } + service_account_id = google_service_account.viewer_account[each.key].name +} + +resource "kubernetes_secret" "secret_sa" { + for_each = { for v in var.apps : v => v } + metadata { + name = "gcp-${var.openshift_namespace}-${each.value}-service-account-key" + namespace = "${var.openshift_namespace}" + labels = { + created-by = "Terraform" + } + } + + data = { + "bucket_name" = "${var.openshift_namespace}-${each.value}" + "credentials.json" = base64decode(google_service_account_key.key[each.key].private_key) + "viewer_credentials.json" = base64decode(google_service_account_key.viewer_key[each.key].private_key) + } +} diff --git a/helm/cas-airflow/terraform/variables.tf b/helm/cas-airflow/terraform/variables.tf new file mode 100644 index 0000000..475e7d4 --- /dev/null +++ b/helm/cas-airflow/terraform/variables.tf @@ -0,0 +1,33 @@ +# Since variables could be overridden via environment variables, use local values to define immutable values +locals { + # The GCP region to create things in. https://cloud.google.com/compute/docs/regions-zones" + region = "northamerica-northeast1" # Montreal +} + +variable "project_id" { + description = "The ID of the GCP project" +} + +variable "kubernetes_host" { + description = "The hostname of the OCP cluster" +} + +variable "kubernetes_token" { + description = "The authentication token of the OCP cluster" +} + +variable "apps" { + type = list(string) + description = "The list of app names for the OCP project in a namespace" +} + +variable "openshift_namespace" { + type = string + description = "The OCP project namespace" +} + +variable "iam_storage_role_template_id" { + type = string + description = "ID for a custom IAM role template we manually created in GCP for Storage Viewers" + default = "casStorageViewer" +} diff --git a/helm/cas-airflow/values.yaml b/helm/cas-airflow/values.yaml index f609fc0..5e23741 100644 --- a/helm/cas-airflow/values.yaml +++ b/helm/cas-airflow/values.yaml @@ -212,3 +212,17 @@ cas-postgres: namespace: ~ gcs: bucketSuffix: ~ + +devops: + image: + repository: hashicorp/terraform + pullPolicy: Always + tag: "1.4.6" + + resources: + limits: + cpu: 1000m + memory: 512Mi + requests: + cpu: 100m + memory: 64Mi