diff --git a/README.md b/README.md index fad05b4..8a77f8c 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,6 @@ This repository contains a Terraform module for creating an EKS cluster and all - [ExternalDNS](#externaldns) - [cert-manager](#cert-manager) - [Customer's CA certificates](#customers-ca-certificates) - - [Velero Backups](#velero-backups) - - [Enabling backups on pre-existing clusters](#enabling-backups-on-pre-existing-clusters) - [Production cluster considerations](#production-cluster-considerations) - [Configuring a Terraform backend](#configuring-a-terraform-backend) - [Examples](#examples) @@ -333,25 +331,6 @@ tls_cert = "/opt/CA/cert.crt" tls_key = "LS0tLS1C....BLRVktLS0tLQo=" ``` -### Velero Backups - -This module can set up the resources required for running backups with Velero on your cluster by setting the flag `enable_backup` to `true`. - -#### Enabling backups on pre-existing clusters - -If your cluster is pre-existing and already contains a namespace named `velero`, then enabling backups will initially fail with an error that you are trying to create a namespace which already exists. - -``` -Error: namespaces "velero" already exists -``` - -If you get this error, consider it a warning - you may then adjust accordingly by importing that namespace to be managed by Terraform, deleting the previously existing ns if it wasn't actually in use, or setting `enable_backup` back to `false` to continue managing Velero in the previous manner. - -The recommended way is to import the namespace and then run another Terraform plan and apply: - -``` -terraform import module.eks-jx.module.backup.kubernetes_namespace.velero_namespace velero -``` ### Production cluster considerations The configuration, as seen in [Cluster provisioning](#cluster-provisioning), is not suited for creating and maintaining a production Jenkins X cluster. @@ -411,7 +390,6 @@ Each example generates a valid _jx-requirements.yml_ file that can be used to bo | Name | Source | Version | |------|--------|---------| -| [backup](#module\_backup) | ./modules/backup | n/a | | [cluster](#module\_cluster) | ./modules/cluster | n/a | | [dns](#module\_dns) | ./modules/dns | n/a | | [health](#module\_health) | ./modules/health | n/a | @@ -434,7 +412,7 @@ Each example generates a valid _jx-requirements.yml_ file that can be used to bo | [apex\_domain](#input\_apex\_domain) | The main domain to either use directly or to configure a subdomain from | `string` | `""` | no | | [asm\_role](#input\_asm\_role) | DEPRECATED: Use the new bot\_iam\_role input with he same semantics instead. | `string` | `""` | no | | [boot\_iam\_role](#input\_boot\_iam\_role) | Specify arn of the role to apply to the boot job service account | `string` | `""` | no | -| [boot\_secrets](#input\_boot\_secrets) | n/a |
list(object({| `[]` | no | +| [boot\_secrets](#input\_boot\_secrets) | n/a |
name = string
value = string
type = string
}))
list(object({| `[]` | no | | [cluster\_name](#input\_cluster\_name) | Variable to provide your desired name for the cluster | `string` | n/a | yes | | [cluster\_oidc\_issuer\_url](#input\_cluster\_oidc\_issuer\_url) | The oidc provider url for the clustrer | `string` | n/a | yes | | [create\_and\_configure\_subdomain](#input\_create\_and\_configure\_subdomain) | Flag to create an NS record set for the subdomain in the apex domain's Hosted Zone | `bool` | `false` | no | @@ -450,12 +428,8 @@ Each example generates a valid _jx-requirements.yml_ file that can be used to bo | [create\_pipeline\_vis\_role](#input\_create\_pipeline\_vis\_role) | Flag to control pipeline visualizer role | `bool` | `true` | no | | [create\_ssm\_role](#input\_create\_ssm\_role) | Flag to control AWS Parameter Store iam roles creation | `bool` | `false` | no | | [create\_tekton\_role](#input\_create\_tekton\_role) | Flag to control tekton iam role creation | `bool` | `true` | no | -| [create\_velero\_role](#input\_create\_velero\_role) | Flag to control velero iam role creation | `bool` | `true` | no | -| [eks\_cluster\_tags](#input\_eks\_cluster\_tags) | Add tags for the EKS Cluster | `map(any)` | `{}` | no | | [enable\_acl](#input\_enable\_acl) | Flag to enable ACL instead of bucket ownership for S3 storage | `bool` | `false` | no | -| [enable\_backup](#input\_enable\_backup) | Whether or not Velero backups should be enabled | `bool` | `false` | no | | [enable\_external\_dns](#input\_enable\_external\_dns) | Flag to enable or disable External DNS in the final `jx-requirements.yml` file | `bool` | `false` | no | -| [enable\_key\_rotation](#input\_enable\_key\_rotation) | Flag to enable kms key rotation | `bool` | `true` | no | | [enable\_logs\_storage](#input\_enable\_logs\_storage) | Flag to enable or disable long term storage for logs | `bool` | `true` | no | | [enable\_reports\_storage](#input\_enable\_reports\_storage) | Flag to enable or disable long term storage for reports | `bool` | `true` | no | | [enable\_repository\_storage](#input\_enable\_repository\_storage) | Flag to enable or disable the repository bucket storage | `bool` | `true` | no | @@ -470,7 +444,6 @@ Each example generates a valid _jx-requirements.yml_ file that can be used to bo | [jx\_bot\_username](#input\_jx\_bot\_username) | Bot username used to interact with the Jenkins X cluster git repository | `string` | `""` | no | | [jx\_git\_operator\_values](#input\_jx\_git\_operator\_values) | Extra values for jx-git-operator chart as a list of yaml formated strings | `list(string)` | `[]` | no | | [jx\_git\_url](#input\_jx\_git\_url) | URL for the Jenkins X cluster git repository | `string` | `""` | no | -| [local-exec-interpreter](#input\_local-exec-interpreter) | If provided, this is a list of interpreter arguments used to execute the command | `list(string)` |
name = string
value = string
type = string
}))
[| no | | [manage\_apex\_domain](#input\_manage\_apex\_domain) | Flag to control if apex domain should be managed/updated by this module. Set this to false,if your apex domain is managed in a different AWS account or different provider | `bool` | `true` | no | | [manage\_subdomain](#input\_manage\_subdomain) | Flag to control subdomain creation/management | `bool` | `true` | no | | [nginx\_chart\_version](#input\_nginx\_chart\_version) | nginx chart version | `string` | n/a | yes | @@ -483,7 +456,6 @@ Each example generates a valid _jx-requirements.yml_ file that can be used to bo | [s3\_extra\_tags](#input\_s3\_extra\_tags) | Add new tags for s3 buckets | `map(any)` | `{}` | no | | [s3\_kms\_arn](#input\_s3\_kms\_arn) | ARN of the kms key used for encrypting s3 buckets | `string` | `""` | no | | [subdomain](#input\_subdomain) | The subdomain to be added to the apex domain. If subdomain is set, it will be appended to the apex domain in `jx-requirements-eks.yml` file | `string` | `""` | no | -| [subnets](#input\_subnets) | The subnet ids to create EKS cluster in if create\_vpc is false | `list(string)` | `[]` | no | | [tls\_cert](#input\_tls\_cert) | TLS certificate encrypted with Base64 | `string` | `""` | no | | [tls\_email](#input\_tls\_email) | The email to register the LetsEncrypt certificate with. Added to the `jx-requirements.yml` file | `string` | `""` | no | | [tls\_key](#input\_tls\_key) | TLS key encrypted with Base64 | `string` | `""` | no | @@ -493,23 +465,17 @@ Each example generates a valid _jx-requirements.yml_ file that can be used to bo | [vault\_instance\_values](#input\_vault\_instance\_values) | Extra values for vault-instance chart as a list of yaml formated strings | `list(string)` | `[]` | no | | [vault\_operator\_values](#input\_vault\_operator\_values) | Extra values for vault-operator chart as a list of yaml formated strings | `list(string)` | `[]` | no | | [vault\_url](#input\_vault\_url) | URL to an external Vault instance in case Jenkins X does not create its own system Vault | `string` | `""` | no | -| [velero\_namespace](#input\_velero\_namespace) | Kubernetes namespace for Velero | `string` | `"velero"` | no | -| [velero\_schedule](#input\_velero\_schedule) | The Velero backup schedule in cron notation to be set in the Velero Schedule CRD (see [default-backup.yaml](https://github.com/jenkins-x/jenkins-x-boot-config/blob/master/systems/velero-backups/templates/default-backup.yaml)) | `string` | `"0 * * * *"` | no | -| [velero\_ttl](#input\_velero\_ttl) | The the lifetime of a velero backup to be set in the Velero Schedule CRD (see [default-backup.yaml](https://github.com/jenkins-x/jenkins-x-boot-config/blob/master/systems/velero-backups/templates/default-backup)) | `string` | `"720h0m0s"` | no | -| [velero\_username](#input\_velero\_username) | The username to be assigned to the Velero IAM user | `string` | `"velero"` | no | -| [vpc\_id](#input\_vpc\_id) | The VPC to create EKS cluster in if create\_vpc is false | `string` | `""` | no | #### Outputs | Name | Description | |------|-------------| -| [backup\_bucket\_url](#output\_backup\_bucket\_url) | The bucket where backups from velero will be stored | | [cert\_manager\_iam\_role](#output\_cert\_manager\_iam\_role) | The IAM Role that the Cert Manager pod will assume to authenticate | | [cluster\_asm\_iam\_role](#output\_cluster\_asm\_iam\_role) | The IAM Role that the External Secrets pod will assume to authenticate (Secrets Manager) | | [cluster\_autoscaler\_iam\_role](#output\_cluster\_autoscaler\_iam\_role) | The IAM Role that the Jenkins X UI pod will assume to authenticate | | [cluster\_name](#output\_cluster\_name) | The name of the created cluster | | [cluster\_ssm\_iam\_role](#output\_cluster\_ssm\_iam\_role) | The IAM Role that the External Secrets pod will assume to authenticate (Parameter Store) | | [cm\_cainjector\_iam\_role](#output\_cm\_cainjector\_iam\_role) | The IAM Role that the CM CA Injector pod will assume to authenticate | -| [connect](#output\_connect) | "The cluster connection string to use once Terraform apply finishes,
"/bin/bash",
"-c"
]
{| no | -| [s3\_extra\_tags](#input\_s3\_extra\_tags) | Add new tags for s3 buckets | `map(any)` | `{}` | no | -| [s3\_kms\_arn](#input\_s3\_kms\_arn) | ARN of the kms key used for encrypting s3 buckets | `string` | `""` | no | -| [use\_kms\_s3](#input\_use\_kms\_s3) | Flag to determine whether kms should be used for encrypting s3 buckets | `bool` | `false` | no | -| [velero\_namespace](#input\_velero\_namespace) | Kubernetes namespace for Velero | `string` | `"velero"` | no | -| [velero\_username](#input\_velero\_username) | The username to be assigned to the Velero IAM user | `string` | `"velero"` | no | -#### Outputs - -| Name | Description | -|------|-------------| -| [backup\_bucket\_url](#output\_backup\_bucket\_url) | n/a | - \ No newline at end of file diff --git a/modules/backup/main.tf b/modules/backup/main.tf deleted file mode 100644 index 85471f3..0000000 --- a/modules/backup/main.tf +++ /dev/null @@ -1,148 +0,0 @@ -// ---------------------------------------------------------------------------- -// Create bucket for storing Velero backups -// -// https://github.com/vmware-tanzu/velero -// https://www.terraform.io/docs/providers/aws/r/s3_bucket.html -// https://github.com/vmware-tanzu/velero-plugin-for-aws#create-s3-bucket -// ---------------------------------------------------------------------------- -locals { - encryption_algo = var.use_kms_s3 ? "aws:kms" : "AES256" -} - -resource "aws_s3_bucket" "backup_bucket" { - count = var.enable_backup ? 1 : 0 - bucket_prefix = "backup-${lower(var.cluster_name)}-" - tags = merge(var.s3_default_tags, var.s3_extra_tags) - force_destroy = var.force_destroy -} - -resource "aws_s3_bucket_acl" "backup_bucket" { - count = var.enable_backup && var.enable_acl ? 1 : 0 - bucket = aws_s3_bucket.backup_bucket[0].bucket - acl = "private" -} - -resource "aws_s3_bucket_ownership_controls" "backup_bucket" { - count = var.enable_backup && var.enable_acl ? 1 : 0 - bucket = aws_s3_bucket.backup_bucket[0].bucket - - rule { - object_ownership = "BucketOwnerEnforced" - } -} - -resource "aws_s3_bucket_server_side_encryption_configuration" "backup_bucket" { - count = var.enable_backup ? 1 : 0 - bucket = aws_s3_bucket.backup_bucket[0].bucket - - rule { - apply_server_side_encryption_by_default { - sse_algorithm = local.encryption_algo - kms_master_key_id = var.s3_kms_arn - } - } -} - -resource "aws_s3_bucket_lifecycle_configuration" "backup_bucket" { - count = var.enable_backup ? 1 : 0 - bucket = aws_s3_bucket.backup_bucket[0].id - rule { - status = "Enabled" - id = "abort_incomplete_uploads" - abort_incomplete_multipart_upload { - days_after_initiation = 7 - } - } -} - -// ---------------------------------------------------------------------------- -// Setup IAM User and Policies for Velero -// -// https://github.com/vmware-tanzu/velero-plugin-for-aws#set-permissions-for-velero -// https://github.com/vmware-tanzu/velero/issues/3143 -// ---------------------------------------------------------------------------- -resource "aws_iam_user" "velero" { - count = var.enable_backup ? 1 : 0 - name = var.velero_username -} - -resource "aws_iam_access_key" "velero" { - count = var.enable_backup ? 1 : 0 - user = aws_iam_user.velero[0].name - depends_on = [ - aws_iam_user.velero - ] -} - -data "aws_iam_policy_document" "velero" { - count = var.enable_backup && var.create_velero_role ? 1 : 0 - statement { - sid = "veleroPolicyEC2" - effect = "Allow" - - actions = [ - "ec2:DescribeVolumes", - "ec2:DescribeSnapshots", - "ec2:CreateTags", - "ec2:CreateVolume", - "ec2:CreateSnapshot", - "ec2:DeleteSnapshot" - ] - - resources = ["*"] - } - statement { - sid = "veleroPolicyS3Objects" - effect = "Allow" - - actions = [ - "s3:GetObject", - "s3:DeleteObject", - "s3:PutObject", - "s3:AbortMultipartUpload", - "s3:ListMultipartUploadParts" - ] - - resources = ["${aws_s3_bucket.backup_bucket[0].arn}/*"] - } - statement { - sid = "veleroPolicyS3Bucket" - effect = "Allow" - - actions = [ - "s3:ListBucket" - ] - - resources = [aws_s3_bucket.backup_bucket[0].arn] - } -} - -resource "aws_iam_user_policy" "velero" { - count = var.enable_backup && var.create_velero_role ? 1 : 0 - name = "velero" - user = aws_iam_user.velero[0].name - policy = data.aws_iam_policy_document.velero[0].json - depends_on = [ - aws_iam_user.velero - ] -} - -// ---------------------------------------------------------------------------- -// Setup Kubernetes Velero namespace and service account -// ---------------------------------------------------------------------------- - -resource "kubernetes_secret" "credentials-velero" { - count = var.enable_backup ? 1 : 0 - metadata { - name = "velero-secret" - namespace = var.velero_namespace - } - - data = { - "cloud" = <
"Owner": "Jenkins-x"
}
list(object({| `[]` | no | +| [boot\_secrets](#input\_boot\_secrets) | n/a |
name = string
value = string
type = string
}))
list(object({| `[]` | no | | [cluster\_name](#input\_cluster\_name) | n/a | `string` | n/a | yes | | [cluster\_oidc\_issuer\_url](#input\_cluster\_oidc\_issuer\_url) | The oidc provider url for the clustrer | `string` | n/a | yes | | [content](#input\_content) | Interpolated jx-requirements.yml | `string` | `""` | no | @@ -48,24 +47,19 @@ No requirements. | [enable\_logs\_storage](#input\_enable\_logs\_storage) | ---------------------------------------------------------------------------- Flag Variables ---------------------------------------------------------------------------- | `bool` | `true` | no | | [enable\_reports\_storage](#input\_enable\_reports\_storage) | n/a | `bool` | `true` | no | | [enable\_repository\_storage](#input\_enable\_repository\_storage) | n/a | `bool` | `true` | no | -| [enable\_worker\_group](#input\_enable\_worker\_group) | Flag to enable worker group. Setting this to false will provision a node group instead | `bool` | `true` | no | | [expire\_logs\_after\_days](#input\_expire\_logs\_after\_days) | Number of days objects in the logs bucket are stored | `number` | `90` | no | | [force\_destroy](#input\_force\_destroy) | Flag to determine whether storage buckets get forcefully destroyed. If set to false, empty the bucket first in the aws s3 console, else terraform destroy will fail with BucketNotEmpty error | `bool` | `false` | no | | [jx\_bot\_token](#input\_jx\_bot\_token) | Bot token used to interact with the Jenkins X cluster git repository | `string` | `""` | no | | [jx\_bot\_username](#input\_jx\_bot\_username) | Bot username used to interact with the Jenkins X cluster git repository | `string` | `""` | no | | [jx\_git\_operator\_values](#input\_jx\_git\_operator\_values) | Extra values for jx-git-operator chart as a list of yaml formated strings | `list(string)` | `[]` | no | | [jx\_git\_url](#input\_jx\_git\_url) | URL for the Jenins X cluster git repository | `string` | `""` | no | -| [local-exec-interpreter](#input\_local-exec-interpreter) | If provided, this is a list of interpreter arguments used to execute the command | `list(string)` |
name = string
value = string
type = string
}))
[| no | | [region](#input\_region) | The region to create the resources into | `string` | `"us-east-1"` | no | -| [s3\_default\_tags](#input\_s3\_default\_tags) | Default tags for s3 buckets | `map(any)` |
"/bin/bash",
"-c"
]
{| no | +| [s3\_default\_tags](#input\_s3\_default\_tags) | Default tags for s3 buckets | `map(any)` |
"Owner": "Jenkins-x"
}
{| no | | [s3\_extra\_tags](#input\_s3\_extra\_tags) | Add new tags for s3 buckets | `map(any)` | `{}` | no | | [s3\_kms\_arn](#input\_s3\_kms\_arn) | ARN of the kms key used for encrypting s3 buckets | `string` | `""` | no | -| [subnets](#input\_subnets) | The subnet ids to create EKS cluster in if create\_vpc is false | `list(string)` | `[]` | no | | [tls\_cert](#input\_tls\_cert) | Path to TLS certificate or base64-encrypted content | `string` | `""` | no | | [tls\_key](#input\_tls\_key) | Path to TLS key or base64-encrypted content | `string` | `""` | no | -| [use\_asm](#input\_use\_asm) | Flag to specify if AWS Secrets manager is being used | `bool` | `false` | no | | [use\_kms\_s3](#input\_use\_kms\_s3) | Flag to determine whether kms should be used for encrypting s3 buckets | `bool` | `false` | no | -| [vpc\_id](#input\_vpc\_id) | The VPC to create EKS cluster in if create\_vpc is false | `string` | `""` | no | #### Outputs | Name | Description | diff --git a/modules/cluster/charts.tf b/modules/cluster/charts.tf index 9fe399d..19a29d2 100644 --- a/modules/cluster/charts.tf +++ b/modules/cluster/charts.tf @@ -43,8 +43,4 @@ resource "helm_release" "jx-git-operator" { type = set.value["type"] } } - - depends_on = [ - null_resource.kubeconfig - ] } diff --git a/modules/cluster/main.tf b/modules/cluster/main.tf index 5d81a60..8e24303 100644 --- a/modules/cluster/main.tf +++ b/modules/cluster/main.tf @@ -1,23 +1,12 @@ data "aws_caller_identity" "current" {} -// ---------------------------------------------------------------------------- -// Update the kube configuration after the cluster has been created so we can -// connect to it and create the K8s resources -// ---------------------------------------------------------------------------- -resource "null_resource" "kubeconfig" { - provisioner "local-exec" { - command = "aws eks update-kubeconfig --name ${var.cluster_name} --region=${var.region}" - interpreter = var.local-exec-interpreter - } -} - // ---------------------------------------------------------------------------- // Add the Terraform generated jx-requirements.yml to a configmap so it can be // sync'd with the Git repository // // https://www.terraform.io/docs/providers/kubernetes/r/namespace.html // ---------------------------------------------------------------------------- -resource "kubernetes_config_map" "jenkins_x_requirements" { +resource "kubernetes_config_map_v1" "jenkins_x_requirements" { metadata { name = "terraform-jx-requirements" namespace = "default" diff --git a/modules/cluster/variables.tf b/modules/cluster/variables.tf index be85f68..545c10c 100644 --- a/modules/cluster/variables.tf +++ b/modules/cluster/variables.tf @@ -27,12 +27,6 @@ variable "expire_logs_after_days" { default = 90 } -variable "enable_worker_group" { - description = "Flag to enable worker group. Setting this to false will provision a node group instead" - type = bool - default = true -} - variable "enable_reports_storage" { type = bool default = true @@ -103,18 +97,6 @@ variable "jx_bot_token" { default = "" } -variable "vpc_id" { - description = "The VPC to create EKS cluster in if create_vpc is false" - type = string - default = "" -} - -variable "subnets" { - description = "The subnet ids to create EKS cluster in if create_vpc is false" - type = list(string) - default = [] -} - variable "create_tekton_role" { description = "Flag to control tekton iam role creation" type = bool @@ -181,12 +163,6 @@ variable "additional_tekton_role_policy_arns" { default = [] } -variable "local-exec-interpreter" { - description = "If provided, this is a list of interpreter arguments used to execute the command" - type = list(string) - default = ["/bin/bash", "-c"] -} - // ---------------------------------------------------------------------------- // Customer's Certificates // ---------------------------------------------------------------------------- @@ -213,12 +189,6 @@ variable "boot_secrets" { default = [] } -variable "use_asm" { - description = "Flag to specify if AWS Secrets manager is being used" - type = bool - default = false -} - variable "boot_iam_role" { description = "Specify arn of the role to apply to the boot job service account" type = string diff --git a/outputs.tf b/outputs.tf index 895abf5..1c37a4b 100644 --- a/outputs.tf +++ b/outputs.tf @@ -7,13 +7,8 @@ output "jx_requirements" { } // ---------------------------------------------------------------------------- -// Storage (backup, logs, reports, repo) +// Storage (logs, reports, repo) // ---------------------------------------------------------------------------- -output "backup_bucket_url" { - value = module.backup.backup_bucket_url - description = "The bucket where backups from velero will be stored" -} - output "lts_logs_bucket" { value = length(module.cluster.logs_jenkins_x) > 0 ? module.cluster.logs_jenkins_x[0] : "" description = "The bucket where logs from builds will be stored" @@ -99,9 +94,8 @@ output "subdomain_nameservers" { // ---------------------------------------------------------------------------- output "connect" { description = <
"Owner": "Jenkins-x"
}