diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..f50695d --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,8 @@ +# Changelog + +## [1.0.0] - 2024-11-14 +### Added +- Initial release of the Terraform AWS Glue Job module. +- Create a Glue Job of type Glue ETL / Python Shell. +- The Glue script is pulled from a S3 bucket. +- Tag the Glue ETL job. \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e69de29 diff --git a/README.md b/README.md index d7b56e4..950df2b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,83 @@ -# terraform-aws-glue-job -Private Terraform Registry Module - Glue Job +![](https://img.shields.io/github/commit-activity/t/subhamay-bhattacharyya/terraform-aws-glue-job) ![](https://img.shields.io/github/last-commit/subhamay-bhattacharyya/terraform-aws-glue-job) ![](https://img.shields.io/github/release-date/subhamay-bhattacharyya/terraform-aws-glue-job) ![](https://img.shields.io/github/repo-size/subhamay-bhattacharyya/terraform-aws-glue-job) ![](https://img.shields.io/github/directory-file-count/subhamay-bhattacharyya/terraform-aws-glue-job) ![](https://img.shields.io/github/issues/subhamay-bhattacharyya/terraform-aws-glue-job) ![](https://img.shields.io/github/languages/top/subhamay-bhattacharyya/terraform-aws-glue-job) ![](https://img.shields.io/github/commit-activity/m/subhamay-bhattacharyya/terraform-aws-glue-job) ![](https://img.shields.io/endpoint?url=https://gist.githubusercontent.com/bsubhamay/bd4e53c3a8f5097b10611aef73f6568c/raw/terraform-aws-glue-job.json?) + +# Terraform AWS Glue Job + +This Terraform module creates an AWS Glue Job with various configurations. + +## Usage + +```hcl +module "glue_etl_job" { + source = "app.terraform.io/subhamay-bhattacharyya/glue-job/aws" + version = "1.0.0" + + aws-region = "us-east-1" + project-name = "your-project" + environment-name = "your-environment" + glue-job-base-name = "example-glue-job" + glue-job-description = "An example Glue ETL Job" + glue-script-bucket-base-name = "example-script-bucket" + script-name = "example-script.py" + glue-job-role-name = "example-glue-role" + execution-class = "STANDARD" + glue-version = "2.0" + python-version = "3.0" + glue-job-command-name = "glueetl" + glue-job-timeout = 2880 + max-capacity = 0.0625 + worker-type = "Standard" + number-of-workers = 2 + max-concurrent-runs = 1 + default-arguments = { + "--job-bookmark-option" = "job-bookmark-enable" + "--enable-metrics" = "true" + } + glue-security-configuration-name = "security-configuration-name" + glue-job-tags = { + Environment = "dev" + ProjectName = "example-project" + GitHubRepository = "example-repo" + GitHubRef = "refs/heads/main" + GitHubURL = "https://github.com/example/example-repo" + GitHubSHA = "example-sha" + } + ci-build = "" +} +``` + +## Inputs + +| Name | Description | Type | Default | Required | +|---------------------------|-----------------------------------------------------------------|---------------|-------------|----------| +| aws-region | The AWS region to deploy resources | string | "us-east-1" | no | +| project-name | The name of the project | string | "gitops" | no | +| environment-name | The environment in which to deploy resources | string | "devl" | no | +| glue-job-base-name | The base name of the Glue ETL Job | string | n/a | yes | +| glue-job-description | The description of the Glue ETL Job | string | n/a | yes | +| glue-script-bucket-base-name | The base name of the Glue script S3 bucket | string | n/a | yes | +| script-name | The name of the Glue ETL Job script | string | n/a | yes | +| glue-job-role-name | The name of the Glue ETL Job role | string | n/a | yes | +| execution-class | The execution class of the Glue ETL Job | string | "STANDARD" | no | +| glue-version | The version of the Glue ETL Job | string | "2.0" | no | +| python-version | The Python version of the Glue ETL Job | string | "3.0" | no | +| glue-job-command-name | The command name of the Glue ETL job | string | "glueetl" | no | +| glue-job-timeout | The timeout period for the Glue job | number | 2880 | no | +| max-capacity | Glue Job max capacity | number | 0.0625 | no | +| worker-type | The worker type of the Glue ETL Job | string | null | no | +| number-of-workers | The number of workers of the Glue ETL Job | number | 2 | no | +| max-concurrent-runs | Maximum allowed concurrent runs | number | 1 | no | +| default-arguments | A map of default arguments used by Glue ETL Job | map(string) | {} | no | +| glue-security-configuration-name | Security configuration for the Glue ETL Job | string | "" | no | +| glue-job-tags | A map of tags to assign to the Glue ETL Job | map(string) | {} | no | +| ci-build | CI build identifier | string | "" | no | + +## Outputs + +| Name | Description | +|---------------------------|-----------------------------------------------------------------| +| glue-job-id | The ID of the created Glue ETL Job | +| glue-job-name | The name of the created Glue ETL Job | +| glue-job-arn | The ARN of the created Glue ETL Job | +| glue-job-role-arn | The ARN of the IAM role associated with the Glue ETL Job | +| glue-job-script-location | The S3 location of the Glue ETL Job script | +| glue-job-security-config | The security configuration of the Glue ETL Job | \ No newline at end of file diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..52f18d7 --- /dev/null +++ b/VERSION @@ -0,0 +1,15 @@ +# Version 1.0.0 + +Initial release of the Terraform AWS Glue Job module. +Version: 1.0.0 +Author: Subhamay Bhattacharyya +Created: 12-Nov-2024 +Updated: 13-Nov-2024 14:45 +Description: This module creates This module creates a Glue ETL Job using Terraform. + + +## Features +- Creates a Glue Job (Glue ETL / Python Shell). +- Create a Glue Job of type Glue ETL / Python Shell. +- The Glue script is pulled from a S3 bucket. +- Tag the Glue ETL job. \ No newline at end of file diff --git a/data.tf b/data.tf new file mode 100644 index 0000000..ea0cf3d --- /dev/null +++ b/data.tf @@ -0,0 +1,21 @@ +/* +#################################################################################################### +# Terraform Data Blocks Configuration +# +# Project: Nemesia +# Description: Working with Glue Workflow using Terraform, +# +# +# Author: Subhamay Bhattacharyya +# Created: 12-Nov-2024 Updated: 12-Nov-2024 14:27 +# Version: 1.0 +# +#################################################################################################### +*/ + +# --- root/data.tf --- + +# AWS Region and Caller Identity +data "aws_region" "current" {} + +data "aws_caller_identity" "current" {} \ No newline at end of file diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..b32e86f --- /dev/null +++ b/main.tf @@ -0,0 +1,40 @@ +/* +#################################################################################################### +# Terraform Glue Job Configuration +# +# Description: This module creates a Glue ETL Job using Terraform. +# +# Author: Subhamay Bhattacharyya +# Created: 11-Nov-2024 Updated: 13-Nov-2024 14:45 +# Version: 1.0 +# +#################################################################################################### +*/ + +# --- Glue Job +resource "aws_glue_job" "glue_job" { + + name = local.glue-job-name + description = var.glue-job-description + role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${var.glue-job-role-name}" + execution_class = var.glue-job-command-name == "glueetl" ? var.execution-class : null + glue_version = var.glue-job-command-name == "glueetl" ? var.glue-version : null + worker_type = var.glue-job-command-name == "glueetl" ? var.worker-type : null + number_of_workers = var.glue-job-command-name == "glueetl" ? var.number-of-workers : null + max_capacity = var.max-capacity + timeout = var.glue-job-timeout + security_configuration = var.glue-security-configuration-name + execution_property { + max_concurrent_runs = var.max-concurrent-runs + } + + command { + name = var.glue-job-command-name # "glueetl" + script_location = join("/", ["s3:/", local.glue-script-bucket-name, "glue-scripts", var.script-name]) + python_version = var.python-version + } + + default_arguments = var.default-arguments == null ? {} : var.default-arguments + + tags = var.glue-job-tags == null ? {} : var.glue-job-tags +} \ No newline at end of file diff --git a/outputs.tf b/outputs.tf new file mode 100644 index 0000000..1b5f4b6 --- /dev/null +++ b/outputs.tf @@ -0,0 +1,29 @@ +/* +/* +#################################################################################################### +# Terraform Glue Job Outputs Configuration +# +# Description: This module creates a Glue ETL Job using Terraform. +# +# Author: Subhamay Bhattacharyya +# Created: 11-Nov-2024 Updated: 12-Nov-2024 9:44 +# Version: 1.0 +# +#################################################################################################### +*/ + + +output "glue_job_arn" { + description = "The ARN of the Glue ETL Job" + value = aws_glue_job.glue_job.arn +} + +output "glue_job_id" { + description = "The name of the Glue ETL Job" + value = aws_glue_job.glue_job.id +} + +output "glue_tags_all" { + description = "The tags of the Glue ETL Job" + value = aws_glue_job.glue_job.tags_all +} diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..8b5e777 --- /dev/null +++ b/variables.tf @@ -0,0 +1,199 @@ +/* +################################################################################################### +# Terraform Variables Configuration +# +# Description: This module creates a Glue ETL Job using Terraform. +# +# Author: Subhamay Bhattacharyya +# Created: 11-Nov-2024 Updated: 12-Nov-2024 9:44 +# Version: 1.0 +# +#################################################################################################### +*/ + +######################################## AWS Configuration ######################################### +variable "aws-region" { + type = string + default = "us-east-1" +} + +######################################## Project Name ############################################## +variable "project-name" { + description = "The name of the project" + type = string + default = "gitops" +} + +######################################## Environment Name ########################################## +variable "environment-name" { + type = string + description = <