From eab331137ce4b7ef5248ad9eecb97a9b0eebd198 Mon Sep 17 00:00:00 2001 From: Teemu Matilainen Date: Tue, 29 Aug 2017 17:51:04 +0300 Subject: [PATCH] Initial public release --- .gitignore | 9 + .rspec | 1 + .rubocop.yml | 26 ++ .travis.yml | 7 + CHANGELOG.md | 3 + CODE_OF_CONDUCT.md | 74 +++ Gemfile | 4 + LICENSE.txt | 21 + README.md | 442 ++++++++++++++++++ Rakefile | 6 + bin/tf | 7 + examples/envs/prod.tfvars | 5 + examples/envs/test.tfvars | 2 + examples/main.tf | 10 + examples/tf.yaml | 4 + examples/tf_hooks/pre/get_current_hash.sh | 26 ++ examples/variables.tf | 22 + lib/yle_tf.rb | 70 +++ lib/yle_tf/action.rb | 29 ++ lib/yle_tf/action/builder.rb | 9 + lib/yle_tf/action/command.rb | 25 + lib/yle_tf/action/copy_root_module.rb | 22 + lib/yle_tf/action/generate_vars_file.rb | 27 ++ lib/yle_tf/action/load_config.rb | 17 + lib/yle_tf/action/terraform_init.rb | 63 +++ lib/yle_tf/action/tf_hooks.rb | 48 ++ lib/yle_tf/action/tmpdir.rb | 35 ++ lib/yle_tf/action/verify_terraform_version.rb | 41 ++ lib/yle_tf/action/verify_tf_env.rb | 24 + lib/yle_tf/backend_config.rb | 41 ++ lib/yle_tf/cli.rb | 88 ++++ lib/yle_tf/config.rb | 45 ++ lib/yle_tf/config/defaults.rb | 35 ++ lib/yle_tf/config/erb.rb | 22 + lib/yle_tf/config/file.rb | 26 ++ lib/yle_tf/config/loader.rb | 108 +++++ lib/yle_tf/error.rb | 4 + lib/yle_tf/logger.rb | 48 ++ lib/yle_tf/plugin.rb | 55 +++ lib/yle_tf/plugin/action_hook.rb | 23 + lib/yle_tf/plugin/loader.rb | 59 +++ lib/yle_tf/plugin/manager.rb | 49 ++ lib/yle_tf/system.rb | 22 + lib/yle_tf/tf_hook.rb | 90 ++++ lib/yle_tf/tf_hook/runner.rb | 48 ++ lib/yle_tf/vars_file.rb | 42 ++ lib/yle_tf/version.rb | 3 + lib/yle_tf/version_requirement.rb | 25 + lib/yle_tf_plugins/backends/file/command.rb | 31 ++ lib/yle_tf_plugins/backends/file/config.rb | 17 + lib/yle_tf_plugins/backends/file/plugin.rb | 16 + lib/yle_tf_plugins/backends/s3/command.rb | 19 + lib/yle_tf_plugins/backends/s3/plugin.rb | 16 + .../commands/__default/command.rb | 14 + .../commands/__default/plugin.rb | 14 + .../commands/_config/command.rb | 11 + lib/yle_tf_plugins/commands/_config/plugin.rb | 19 + lib/yle_tf_plugins/commands/_shell/command.rb | 15 + lib/yle_tf_plugins/commands/_shell/plugin.rb | 14 + lib/yle_tf_plugins/commands/help/command.rb | 55 +++ lib/yle_tf_plugins/commands/help/plugin.rb | 18 + .../commands/version/command.rb | 20 + lib/yle_tf_plugins/commands/version/plugin.rb | 18 + spec/fixtures/vars_files/envs/dau.tfvars | 0 spec/fixtures/vars_files/envs/diu.tfvars | 0 spec/spec_helper.rb | 2 + spec/yle_tf/tf_hook_spec.rb | 162 +++++++ spec/yle_tf/vars_file_spec.rb | 39 ++ spec/yle_tf_spec.rb | 7 + vendor/hash_deep_merge.rb | 59 +++ vendor/logger_level_patch.rb | 29 ++ vendor/middleware/LICENSE | 23 + vendor/middleware/builder.rb | 149 ++++++ vendor/middleware/runner.rb | 69 +++ yle_tf.gemspec | 37 ++ 75 files changed, 2785 insertions(+) create mode 100644 .gitignore create mode 100644 .rspec create mode 100644 .rubocop.yml create mode 100644 .travis.yml create mode 100644 CHANGELOG.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 Gemfile create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100644 Rakefile create mode 100755 bin/tf create mode 100644 examples/envs/prod.tfvars create mode 100644 examples/envs/test.tfvars create mode 100644 examples/main.tf create mode 100644 examples/tf.yaml create mode 100644 examples/tf_hooks/pre/get_current_hash.sh create mode 100644 examples/variables.tf create mode 100644 lib/yle_tf.rb create mode 100644 lib/yle_tf/action.rb create mode 100644 lib/yle_tf/action/builder.rb create mode 100644 lib/yle_tf/action/command.rb create mode 100644 lib/yle_tf/action/copy_root_module.rb create mode 100644 lib/yle_tf/action/generate_vars_file.rb create mode 100644 lib/yle_tf/action/load_config.rb create mode 100644 lib/yle_tf/action/terraform_init.rb create mode 100644 lib/yle_tf/action/tf_hooks.rb create mode 100644 lib/yle_tf/action/tmpdir.rb create mode 100644 lib/yle_tf/action/verify_terraform_version.rb create mode 100644 lib/yle_tf/action/verify_tf_env.rb create mode 100644 lib/yle_tf/backend_config.rb create mode 100644 lib/yle_tf/cli.rb create mode 100644 lib/yle_tf/config.rb create mode 100644 lib/yle_tf/config/defaults.rb create mode 100644 lib/yle_tf/config/erb.rb create mode 100644 lib/yle_tf/config/file.rb create mode 100644 lib/yle_tf/config/loader.rb create mode 100644 lib/yle_tf/error.rb create mode 100644 lib/yle_tf/logger.rb create mode 100644 lib/yle_tf/plugin.rb create mode 100644 lib/yle_tf/plugin/action_hook.rb create mode 100644 lib/yle_tf/plugin/loader.rb create mode 100644 lib/yle_tf/plugin/manager.rb create mode 100644 lib/yle_tf/system.rb create mode 100644 lib/yle_tf/tf_hook.rb create mode 100644 lib/yle_tf/tf_hook/runner.rb create mode 100644 lib/yle_tf/vars_file.rb create mode 100644 lib/yle_tf/version.rb create mode 100644 lib/yle_tf/version_requirement.rb create mode 100644 lib/yle_tf_plugins/backends/file/command.rb create mode 100644 lib/yle_tf_plugins/backends/file/config.rb create mode 100644 lib/yle_tf_plugins/backends/file/plugin.rb create mode 100644 lib/yle_tf_plugins/backends/s3/command.rb create mode 100644 lib/yle_tf_plugins/backends/s3/plugin.rb create mode 100644 lib/yle_tf_plugins/commands/__default/command.rb create mode 100644 lib/yle_tf_plugins/commands/__default/plugin.rb create mode 100644 lib/yle_tf_plugins/commands/_config/command.rb create mode 100644 lib/yle_tf_plugins/commands/_config/plugin.rb create mode 100644 lib/yle_tf_plugins/commands/_shell/command.rb create mode 100644 lib/yle_tf_plugins/commands/_shell/plugin.rb create mode 100644 lib/yle_tf_plugins/commands/help/command.rb create mode 100644 lib/yle_tf_plugins/commands/help/plugin.rb create mode 100644 lib/yle_tf_plugins/commands/version/command.rb create mode 100644 lib/yle_tf_plugins/commands/version/plugin.rb create mode 100644 spec/fixtures/vars_files/envs/dau.tfvars create mode 100644 spec/fixtures/vars_files/envs/diu.tfvars create mode 100644 spec/spec_helper.rb create mode 100644 spec/yle_tf/tf_hook_spec.rb create mode 100644 spec/yle_tf/vars_file_spec.rb create mode 100644 spec/yle_tf_spec.rb create mode 100644 vendor/hash_deep_merge.rb create mode 100644 vendor/logger_level_patch.rb create mode 100644 vendor/middleware/LICENSE create mode 100644 vendor/middleware/builder.rb create mode 100644 vendor/middleware/runner.rb create mode 100644 yle_tf.gemspec diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0cb6eeb --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +/.bundle/ +/.yardoc +/Gemfile.lock +/_yardoc/ +/coverage/ +/doc/ +/pkg/ +/spec/reports/ +/tmp/ diff --git a/.rspec b/.rspec new file mode 100644 index 0000000..4e1e0d2 --- /dev/null +++ b/.rspec @@ -0,0 +1 @@ +--color diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 0000000..a1d794f --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,26 @@ +Metrics/BlockLength: + Exclude: + - 'spec/**/*' + +Metrics/ClassLength: + Severity: warning + +Metrics/LineLength: + Max: 100 + Severity: warning + +Metrics/MethodLength: + Max: 12 + Severity: warning + +Style/Documentation: + Enabled: false + +Style/GuardClause: + MinBodyLength: 3 + +Style/NegatedIf: + Enabled: false + +Style/TrailingCommaInLiteral: + Enabled: false diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..d8a559b --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ +language: ruby +cache: bundler + +rvm: + - 2.4.1 + - 2.3.4 + - 2.2.7 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b7bc0ad --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,3 @@ +## 0.1.0 / 2017-08-29 + +- Initial public release diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..5172342 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,74 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or +advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at teemu.matilainen@reaktor.fi. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..e036d3c --- /dev/null +++ b/Gemfile @@ -0,0 +1,4 @@ +source 'https://rubygems.org' + +# Specify your gem's dependencies in yle_tf.gemspec +gemspec diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..3ff3ead --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2017 Yleisradio Oy + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..5c618ae --- /dev/null +++ b/README.md @@ -0,0 +1,442 @@ +# YleTf + +A wrapper for [Terraform](https://www.terraform.io/) with support for hooks and environments. + +Offers a command-line helper tool, `tf` + +## Getting started + +YleTf requires Terraform which can be installed according to their [instructions](https://www.terraform.io/intro/getting-started/install.html). On MacOS (and OSX) you can use [Homebrew](https://brew.sh/). You can also easily install and manage multiple versions of Terraform with [homebrew-terraforms](https://github.com/Yleisradio/homebrew-terraforms). + +## Installation + +YleTf can be installed as a gem: + +```sh +$ gem install yle_tf +``` + +## Usage + +The syntax to run YleTf is much like vanilla [Terraform](https://terraform.io). The main difference is that you must include desired environment as the first argument: + +``` +tf [] +``` + +For example: + +``` +$ tf test plan + +$ tf prod apply + +$ tf stage destroy +``` + +For a full list of available options, run `tf --help`. + +When `tf` is executed, it creates a temporary directory where your project is copied and initialized. + +### Example project + +The following is a really basic example on what you need in order to use YleTf. The project is pretty much the same as the [example project](https://www.terraform.io/intro/getting-started/build.html) in Terraform documentation but with the added support for environments. Introduction to [hooks](#hooks) is in it's own section. + +#### Project Config + +The root of your project directory will look like this: + +``` +. +├── envs +│   ├── prod.tfvars +│   └── test.tfvars +├── main.tf +├── tf.yaml +└── variables.tf +``` + +And here are the contents of the files: + +##### main.tf + +```hcl +provider "aws" { + access_key = "${var.aws_access_key}" + secret_key = "${var.aws_secret_key}" + region = "${var.region}" +} + +resource "aws_instance" "example" { + ami = "ami-2757f631" + instance_type = "t2.micro" +} +``` + +_Please note that the AMI identifier changes based on your desired OS and region. For more information see [AWS documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html)._ + +##### variables.tf + +```hcl +variable "region" { + description = "The AWS region for the resources" + default = "eu-west-1" +} + +variable "env" { + # passed by `tf` + description = "The environment" +} + +variable "aws_access_key" { + description = "Your AWS access key id" +} + +variable "aws_secret_key" { + description = "Your AWS secret key" +} + +variable "instance_type" { + description = "Instance type to use" + default = "t2.micro" +} +``` + +##### envs/test.tfvars + +```ini +aws_access_key = "TEST_ACCOUNT_ACCESS_KEY_HERE" +aws_secret_key = "TEST_ACCOUNT_SECRET_KEY_HERE" +``` + +##### tf.yaml + +```yaml +backend: + type: file +terraform: + version_requirement: "~> 0.9.11" +``` + +#### Execution + +With all the above in order, there's nothing more to do than to try and run the commands: + +First plan: + +``` +$ tf test plan +INFO: Symlinking state to '/usr/local/src/yle_tf/examples/examples_test.tfstate' + +Terraform has been successfully initialized! + +You may now begin working with Terraform. Try running "terraform plan" to see +any changes that are required for your infrastructure. All Terraform commands +should now work. + +If you ever set or change modules or backend configuration for Terraform, +rerun this command to reinitialize your environment. If you forget, other +commands will detect it and remind you to do so if necessary. +Refreshing Terraform state in-memory prior to plan... +The refreshed state will be used to calculate this plan, but will not be +persisted to local or remote state storage. + +The Terraform execution plan has been generated and is shown below. +Resources are shown in alphabetical order for quick scanning. Green resources +will be created (or destroyed and then created if an existing resource +exists), yellow resources are being changed in-place, and red resources +will be destroyed. Cyan entries are data sources to be read. + +Note: You didn't specify an "-out" parameter to save this plan, so when +"apply" is called, Terraform can't guarantee this is what will execute. + ++ aws_instance.example + ami: "ami-d7b9a2b1" + associate_public_ip_address: "" + availability_zone: "" + ebs_block_device.#: "" + ephemeral_block_device.#: "" + instance_state: "" + instance_type: "t2.micro" + ipv6_address_count: "" + ipv6_addresses.#: "" + key_name: "" + network_interface.#: "" + network_interface_id: "" + placement_group: "" + primary_network_interface_id: "" + private_dns: "" + private_ip: "" + public_dns: "" + public_ip: "" + root_block_device.#: "" + security_groups.#: "" + source_dest_check: "true" + subnet_id: "" + tenancy: "" + volume_tags.%: "" + vpc_security_group_ids.#: "" + + +Plan: 1 to add, 0 to change, 0 to destroy. +``` + +Then apply: + +``` +$ tf test apply + +< SNIP > + +aws_instance.example: Creating... + ami: "" => "ami-d7b9a2b1" + associate_public_ip_address: "" => "" + availability_zone: "" => "" + ebs_block_device.#: "" => "" + ephemeral_block_device.#: "" => "" + instance_state: "" => "" + instance_type: "" => "t2.micro" + ipv6_address_count: "" => "" + ipv6_addresses.#: "" => "" + key_name: "" => "" + network_interface.#: "" => "" + network_interface_id: "" => "" + placement_group: "" => "" + primary_network_interface_id: "" => "" + private_dns: "" => "" + private_ip: "" => "" + public_dns: "" => "" + public_ip: "" => "" + root_block_device.#: "" => "" + security_groups.#: "" => "" + source_dest_check: "" => "true" + subnet_id: "" => "" + tenancy: "" => "" + volume_tags.%: "" => "" + vpc_security_group_ids.#: "" => "" +aws_instance.example: Still creating... (10s elapsed) +aws_instance.example: Still creating... (20s elapsed) +aws_instance.example: Creation complete (ID: i-0f1327bbc53fd6bab) + +Apply complete! Resources: 1 added, 0 changed, 0 destroyed. + +< SNIP > +``` + +And when you're done using the infrastructure, destroy: + +``` +$ tf test destroy + +< SNIP > + +Do you really want to destroy? + Terraform will delete all your managed infrastructure. + There is no undo. Only 'yes' will be accepted to confirm. + + Enter a value: yes + +aws_instance.example: Refreshing state... (ID: i-0f1327bbc53fd6bab) +aws_instance.example: Destroying... (ID: i-0f1327bbc53fd6bab) +aws_instance.example: Still destroying... (ID: i-0f1327bbc53fd6bab, 10s elapsed) +aws_instance.example: Still destroying... (ID: i-0f1327bbc53fd6bab, 20s elapsed) +aws_instance.example: Still destroying... (ID: i-0f1327bbc53fd6bab, 30s elapsed) +aws_instance.example: Still destroying... (ID: i-0f1327bbc53fd6bab, 40s elapsed) +aws_instance.example: Still destroying... (ID: i-0f1327bbc53fd6bab, 50s elapsed) +aws_instance.example: Still destroying... (ID: i-0f1327bbc53fd6bab, 1m0s elapsed) +aws_instance.example: Still destroying... (ID: i-0f1327bbc53fd6bab, 1m10s elapsed) +aws_instance.example: Destruction complete + +Destroy complete! Resources: 1 destroyed. +``` + +#### Production environment + +Now that we've tried that everything works in the test environment, let's do the same in production. Note that you can override variables if needed: + +##### envs/prod.tfvars + +```ini +aws_access_key = "PRODUCTION_ACCOUNT_ACCESS_KEY_HERE" +aws_secret_key = "PRODUCTION_ACCOUNT_SECRET_KEY_HERE" + +# let's use a bigger instance type in prod +instance_type = "t2.medium" +``` + +And run the commands just like with _test_: + +``` +$ tf prod plan + +< SNIP > + +$ tf prod apply + +< SNIP > + +$ tf prod destroy + +< SNIP > +``` + +_Note that the instance type is now t2.medium_ + +## Default components + +### tf.yaml + +Here be dragons that configure your project. The tools serches for `tf.yaml`'s up your path all the way to root `/`. Configs made in subdirectories override those made upper in the path. This makes it easy to define common settings without having to edit `tf.yaml` in every project. + +By defalt the following configuration options are supported: +* [`hooks`](#hooks) - Pre and Post hooks. +* [`backend`](#backend) - Configuration of remote state location. +* [`terraform`](#terraform) - Terraform specific configuration. + +#### Hooks + +There are cases when it would be beneficial to run a task at the same time as terrafrom, but building native support for that would be quite cumbersome. The support for hooks was build into YleTf having those cases in mind. + +Essentially hooks are just scripts and small applications to extend the functionality of Terraform. Hooks can be either +* [local](#local-hooks) or +* [remote](#remote-hooks) i.e. stored into git. + +Real world usecases for hooks include at least the following: +* Automatically register ACM certificates and link them to desired resources +* Automatically generate dns record resources that are managed by separate configuration +* Package lambda applications for deployment via Terraform +* Manage SES authorisations and rules +* Modify parameters not yet supported by Terraform + +Currently two kinds of hooks are supported: +* `pre` - Hooks that run before terraform execution. +* `post` - Hooks that run after terraform execution. + +#### Local hooks + +For local hooks, add a directory called `tf_hooks` to your tf project root. You also need a folder to determine wether the hook is run before or after the execution of terraform. The folders are `pre` and/or `post`: + +``` +. +├── envs +│   ├── prod.tfvars +│   └── test.tfvars +├── main.tf +├── tf.yaml +├── tf_hooks +│   ├── pre +│   │ └── pre-hook.sh +│   └── post +│   │ └── post-hook.rb +└── variables.tf +``` + +For example, let's say you wish to have a variable `current_git_hash` and you want to populate it with a value of the latest git commit hash. The hook could be something like this: + +##### tf_hooks/pre/get_current_hash.sh + +```bash +#!/bin/bash + +set -eu + +current_hash() { + CURRENT_GIT_HASH="$(git rev-parse HEAD)" + + cat < current_git_hash.tf.json +{ + "variable": { + "current_git_hash": { + "value": "$CURRENT_GIT_HASH" + } + } +} +EOF +} + +# Execute only for commands `plan` and `apply` +case "$TF_COMMAND" in + plan|apply) + current_hash + ;; + *) + ;; +esac +``` + +_Please note that the script in the [examples](examples/) directory is intentionnally without the execute bit._ + +Once you set the hook script executable (`chmod +x tf_hooks/pre/get_current_hash.sh`) and run `tf` with either `plan` or `apply`, you'll have the following file during runtime: + +##### current_git_hash.tf.json + +```json +{ + "variable": { + "current_git_hash": { + "value": "c6a02baf0597e55d7698f78d70299ca4a65776cd" + } + } +} +``` + +_Naturraly the actual value varies according to your repository._ + +_**Please note that files generated into the temporary working directory while running hooks are removed once the execution has ended.**_ + +#### Remote hooks + +Hooks can also be stored in common git repositories. This is a handy way to avoid duplication in codebase. +To use a hook from git just add your script to a suitable repository (making sure it's still executable) and add following config to your `tf.yaml`: + +##### tf.yaml + +```yaml +hooks: + pre: + - description: "Get your current git commit hash" + source: "git@github.com:your-org/tf-hooks.git//git-hash/get_current_hash.sh" + vars: + defaults: + MESSAGE: "You can also define environment variables to your hooks" + test: + MESSAGE: "And the variables can also be environment spesific" +``` + +#### Backend + +Configure where Terraform [remote state](https://www.terraform.io/docs/state/remote.html) is stored + +* `type` - Backend type where the Terraform state is stored. +* `bucket` - The name of the S3 bucket. +* `file` - The name of the state file. +* `region` - The region of the S3 bucket. +* `encrypt` - Whether to enable server side encryption of the state file. + +```yaml +backend: + type: s3 +``` + +#### Terraform + +* `version_requirement` - The version requirement of Terraform in ruby gem syntax. + +```yaml +terraform: + version_requirement: "~> 0.9.11" +``` + +## Development + +After checking out the repo, run `bundle update` to install and update the dependencies. Then, run `bundle exec rake spec` to run the tests. + +To install this gem onto your local machine, run `bundle exec rake install`. + +## Contributing + +Bug reports and pull requests are welcome on GitHub at https://github.com/Yleisradio/yle_tf. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct. + +## License + +The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT). diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..4c774a2 --- /dev/null +++ b/Rakefile @@ -0,0 +1,6 @@ +require 'bundler/gem_tasks' +require 'rspec/core/rake_task' + +RSpec::Core::RakeTask.new(:spec) + +task default: :spec diff --git a/bin/tf b/bin/tf new file mode 100755 index 0000000..b82873a --- /dev/null +++ b/bin/tf @@ -0,0 +1,7 @@ +#!/usr/bin/env ruby + +require 'bundler/setup' if File.exist? File.expand_path('../../Gemfile', __FILE__) +require 'yle_tf/cli' + +cli = YleTf::CLI.new(ARGV) +cli.execute diff --git a/examples/envs/prod.tfvars b/examples/envs/prod.tfvars new file mode 100644 index 0000000..a7a0165 --- /dev/null +++ b/examples/envs/prod.tfvars @@ -0,0 +1,5 @@ +aws_access_key = "PRODUCTION_ACCOUNT_ACCESS_KEY_HERE" +aws_secret_key = "PRODUCTION_ACCOUNT_SECRET_KEY_HERE" + +# let's use a bigger instance type in prod +instance_type = "t2.medium" diff --git a/examples/envs/test.tfvars b/examples/envs/test.tfvars new file mode 100644 index 0000000..9346164 --- /dev/null +++ b/examples/envs/test.tfvars @@ -0,0 +1,2 @@ +aws_access_key = "TEST_ACCOUNT_ACCESS_KEY_HERE" +aws_secret_key = "TEST_ACCOUNT_SECRET_KEY_HERE" diff --git a/examples/main.tf b/examples/main.tf new file mode 100644 index 0000000..b06b619 --- /dev/null +++ b/examples/main.tf @@ -0,0 +1,10 @@ +provider "aws" { + access_key = "${var.aws_access_key}" + secret_key = "${var.aws_secret_key}" + region = "${var.region}" +} + +resource "aws_instance" "example" { + ami = "ami-d7b9a2b1" + instance_type = "${var.instance_type}" +} diff --git a/examples/tf.yaml b/examples/tf.yaml new file mode 100644 index 0000000..fc816e2 --- /dev/null +++ b/examples/tf.yaml @@ -0,0 +1,4 @@ +backend: + type: file +terraform: + version_requirement: "~> 0.9.11" diff --git a/examples/tf_hooks/pre/get_current_hash.sh b/examples/tf_hooks/pre/get_current_hash.sh new file mode 100644 index 0000000..2f737c9 --- /dev/null +++ b/examples/tf_hooks/pre/get_current_hash.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -eu + +current_hash() { + CURRENT_GIT_HASH="$(git rev-parse HEAD)" + + cat < current_git_hash.tf.json +{ + "variable": { + "current_git_hash": { + "value": "$CURRENT_GIT_HASH" + } + } +} +EOF +} + +# Execute only for commands `plan` and `apply` +case "$TF_COMMAND" in + plan|apply) + current_hash + ;; + *) + ;; +esac diff --git a/examples/variables.tf b/examples/variables.tf new file mode 100644 index 0000000..3d6a43a --- /dev/null +++ b/examples/variables.tf @@ -0,0 +1,22 @@ +variable "region" { + description = "The AWS region for the resources" + default = "eu-west-1" +} + +variable "env" { + # passed by `tf` + description = "The environment" +} + +variable "aws_access_key" { + description = "Your AWS access key id" +} + +variable "aws_secret_key" { + description = "Your AWS secret key" +} + +variable "instance_type" { + description = "Instance type to use" + default = "t2.micro" +} diff --git a/lib/yle_tf.rb b/lib/yle_tf.rb new file mode 100644 index 0000000..9b8f2cf --- /dev/null +++ b/lib/yle_tf.rb @@ -0,0 +1,70 @@ +require 'yle_tf/logger' +require 'yle_tf/version' + +class YleTf + autoload :Action, 'yle_tf/action' + autoload :Error, 'yle_tf/error' + autoload :Plugin, 'yle_tf/plugin' + + attr_reader :tf_env, :tf_command, :tf_command_args, :tf_options + attr_writer :actions + + def initialize(tf_options, tf_env, tf_command, tf_command_args = []) + Logger.debug("YleTf version: #{VERSION}") + Logger.debug("Ruby version: #{RUBY_VERSION}") + Logger.debug("tf_options: #{tf_options.inspect}") + Logger.debug("tf_env: #{tf_env.inspect}") + Logger.debug("tf_command: #{tf_command.inspect}") + Logger.debug("tf_command_args: #{tf_command_args.inspect}") + + @tf_options = tf_options + @tf_env = tf_env + @tf_command = tf_command + @tf_command_args = tf_command_args + + Plugin::Loader.load_plugins + end + + def run(env = {}) + Logger.debug('Building and running the stack') + apply_action_hooks + Logger.debug("actions: #{actions.inspect}") + env.merge!(action_env) + Logger.debug("env: #{env.inspect}") + actions.call(env) + end + + def actions + @actions ||= build_action_stack + end + + def build_action_stack + command_data = Plugin.manager.commands[tf_command] + command_proc = command_data[:proc] + command_proc.call + end + + def apply_action_hooks + hooks = Plugin.manager.action_hooks + Logger.debug("Applying #{hooks.length} action hooks") + Plugin::ActionHook.new(actions).tap do |h| + hooks.each { |hook_proc| hook_proc.call(h) } + end + end + + def action_env + { + tf_options: tf_options, + tf_env: tf_env, + tf_command: tf_command, + tf_command_args: tf_command_args, + tfvars: default_tfvars, + } + end + + def default_tfvars + { + 'env' => tf_env, + } + end +end diff --git a/lib/yle_tf/action.rb b/lib/yle_tf/action.rb new file mode 100644 index 0000000..f311683 --- /dev/null +++ b/lib/yle_tf/action.rb @@ -0,0 +1,29 @@ +class YleTf + module Action + autoload :Builder, 'yle_tf/action/builder' + autoload :Command, 'yle_tf/action/command' + autoload :CopyRootModule, 'yle_tf/action/copy_root_module' + autoload :GenerateVarsFile, 'yle_tf/action/generate_vars_file' + autoload :LoadConfig, 'yle_tf/action/load_config' + autoload :TerraformInit, 'yle_tf/action/terraform_init' + autoload :TfHooks, 'yle_tf/action/tf_hooks' + autoload :TmpDir, 'yle_tf/action/tmpdir' + autoload :VerifyTfEnv, 'yle_tf/action/verify_tf_env' + autoload :VerifyTerraformVersion, 'yle_tf/action/verify_terraform_version' + + def self.default_action_stack(command_class = nil) + Builder.new do + use LoadConfig + use VerifyTfEnv + use TmpDir + use VerifyTerraformVersion + use CopyRootModule + use GenerateVarsFile + use TfHooks + use TerraformInit + + use(Command, command_class) if command_class + end + end + end +end diff --git a/lib/yle_tf/action/builder.rb b/lib/yle_tf/action/builder.rb new file mode 100644 index 0000000..86d89ed --- /dev/null +++ b/lib/yle_tf/action/builder.rb @@ -0,0 +1,9 @@ +require_relative '../../../vendor/middleware/builder' + +class YleTf + module Action + # Middleware builder for the actions + class Builder < Middleware::Builder + end + end +end diff --git a/lib/yle_tf/action/command.rb b/lib/yle_tf/action/command.rb new file mode 100644 index 0000000..3b7a63f --- /dev/null +++ b/lib/yle_tf/action/command.rb @@ -0,0 +1,25 @@ +require 'yle_tf/logger' + +class YleTf + module Action + class Command + attr_reader :command + + def initialize(app, command) + @app = app + @command = command + end + + def call(env) + if env[:tf_options][:only_hooks] + Logger.debug "Skipping command #{command.class} due to `--only-hooks`" + else + Logger.debug "Executing command #{command.class} with env: #{env.inspect}" + command.new.execute(env) + end + + @app.call(env) + end + end + end +end diff --git a/lib/yle_tf/action/copy_root_module.rb b/lib/yle_tf/action/copy_root_module.rb new file mode 100644 index 0000000..1b021e9 --- /dev/null +++ b/lib/yle_tf/action/copy_root_module.rb @@ -0,0 +1,22 @@ +require 'fileutils' + +require 'yle_tf/logger' + +class YleTf + module Action + class CopyRootModule + def initialize(app) + @app = app + end + + def call(env) + config = env[:config] + + Logger.debug("Copying the Terraform module from '#{config.module_dir}' to '#{Dir.pwd}'") + FileUtils.cp_r("#{config.module_dir}/.", '.') + + @app.call(env) + end + end + end +end diff --git a/lib/yle_tf/action/generate_vars_file.rb b/lib/yle_tf/action/generate_vars_file.rb new file mode 100644 index 0000000..13a61a9 --- /dev/null +++ b/lib/yle_tf/action/generate_vars_file.rb @@ -0,0 +1,27 @@ +require 'yle_tf/logger' +require 'yle_tf/vars_file' + +class YleTf + module Action + class GenerateVarsFile + def initialize(app) + @app = app + end + + def call(env) + config = env[:config] + + Logger.debug("Generating 'terraform.tfvars'") + vars_file = VarsFile.new('terraform.tfvars') + vars_file.append_vars(tfvars(env)) + vars_file.append_file(VarsFile.find_env_vars_file(config)) + + @app.call(env) + end + + def tfvars(env) + env[:tfvars].merge(env[:config].fetch('tfvars')) + end + end + end +end diff --git a/lib/yle_tf/action/load_config.rb b/lib/yle_tf/action/load_config.rb new file mode 100644 index 0000000..aa3abd7 --- /dev/null +++ b/lib/yle_tf/action/load_config.rb @@ -0,0 +1,17 @@ +require 'yle_tf/config' + +class YleTf + module Action + class LoadConfig + def initialize(app) + @app = app + end + + def call(env) + env[:config] ||= Config.new(env[:tf_env]) + + @app.call(env) + end + end + end +end diff --git a/lib/yle_tf/action/terraform_init.rb b/lib/yle_tf/action/terraform_init.rb new file mode 100644 index 0000000..07a098b --- /dev/null +++ b/lib/yle_tf/action/terraform_init.rb @@ -0,0 +1,63 @@ +require 'yle_tf/error' +require 'yle_tf/logger' +require 'yle_tf/plugin' +require 'yle_tf/system' +require 'yle_tf/version_requirement' + +class YleTf + module Action + class TerraformInit + def initialize(app) + @app = app + end + + def call(env) + config = env[:config] + backend = backend_config(config) + + Logger.debug('Initializing Terraform with backend configuration:') + Logger.debug(backend.to_s) + + if VersionRequirement.pre_0_9?(env[:terraform_version]) + init_pre_0_9(backend) + else + init(backend) + end + + @app.call(env) + end + + def init_pre_0_9(backend) + cli_args = backend.cli_args + YleTf::System.cmd('terraform', 'remote', 'config', *cli_args) if cli_args + + Logger.debug('Fetching Terraform modules') + YleTf::System.cmd('terraform', 'get') + end + + def init(backend) + Logger.debug('Generating the backend configuration') + backend.generate_config do + Logger.debug('Initializing Terraform') + YleTf::System.cmd('terraform', 'init', '-no-color') + end + end + + def backend_config(config) + backend_type = config.fetch('backend', 'type').downcase + backend_proc = backend_proc(backend_type) + + klass = backend_proc.call + klass.new.backend_config(config) + end + + def backend_proc(backend_type) + backends = Plugin.manager.backends + backends.fetch(backend_type.to_sym) do + raise Error, "Unknown backend type '#{backend_type}'. " \ + "Supported backends: #{backends.keys.join(', ')}" + end + end + end + end +end diff --git a/lib/yle_tf/action/tf_hooks.rb b/lib/yle_tf/action/tf_hooks.rb new file mode 100644 index 0000000..751fcd9 --- /dev/null +++ b/lib/yle_tf/action/tf_hooks.rb @@ -0,0 +1,48 @@ +require 'yle_tf/logger' +require 'yle_tf/tf_hook/runner' + +require 'yle_tf/logger' +require 'yle_tf/tf_hook/runner' + +class YleTf + module Action + class TfHooks + def initialize(app) + @app = app + end + + def call(env) + @env = env + + hook_runner.run('pre') + @app.call(env) + hook_runner.run('post') + end + + def hook_runner + if run_hooks? + TfHook::Runner.new(@env[:config], hook_env) + else + NoRunner + end + end + + def hook_env + { + 'TF_COMMAND' => @env[:tf_command], + 'TF_ENV' => @env[:tf_env], + } + end + + def run_hooks? + !@env[:tf_options][:no_hooks] + end + + class NoRunner + def self.run(hook_type) + Logger.debug("Skipping #{hook_type} hooks due to `--no-hooks`") + end + end + end + end +end diff --git a/lib/yle_tf/action/tmpdir.rb b/lib/yle_tf/action/tmpdir.rb new file mode 100644 index 0000000..bdf283c --- /dev/null +++ b/lib/yle_tf/action/tmpdir.rb @@ -0,0 +1,35 @@ +require 'fileutils' +require 'tmpdir' + +require 'yle_tf/logger' + +class YleTf + module Action + class TmpDir + def initialize(app) + @app = app + end + + def call(env) + config = env[:config] + + tmpdir = Dir.mktmpdir(tmpdir_prefix(config)) + Logger.debug("Temporary Terraform directory: #{tmpdir}") + + Dir.chdir(tmpdir) do + @app.call(env) + end + ensure + FileUtils.rm_r(tmpdir) if tmpdir + end + + def tmpdir_prefix(config) + if config + "tf_#{config.module_dir.basename}_#{config.tf_env}_" + else + 'tf_' + end + end + end + end +end diff --git a/lib/yle_tf/action/verify_terraform_version.rb b/lib/yle_tf/action/verify_terraform_version.rb new file mode 100644 index 0000000..dfeeb6e --- /dev/null +++ b/lib/yle_tf/action/verify_terraform_version.rb @@ -0,0 +1,41 @@ +require 'yle_tf/error' +require 'yle_tf/logger' +require 'yle_tf/version_requirement' + +class YleTf + module Action + class VerifyTerraformVersion + def initialize(app) + @app = app + end + + def call(env) + Logger.debug('Verifying Terraform version') + + version = env[:terraform_version] = terraform_version + raise(Error, 'Terraform not found') if !version + + Logger.debug("Terraform version: #{version}") + verify_version(env) + + @app.call(env) + end + + def terraform_version + # TODO: move `command` to YleTf::System + Regexp.last_match(1) if `terraform version` =~ /^Terraform v([^\s]+)/ + rescue Errno::ENOENT + nil + end + + def verify_version(env) + version = env[:terraform_version] + requirement = env[:config].fetch('terraform', 'version_requirement') { nil } + + if !VersionRequirement.new(requirement).satisfied_by?(version) + raise Error, "Terraform version '#{requirement}' required, '#{version}' found" + end + end + end + end +end diff --git a/lib/yle_tf/action/verify_tf_env.rb b/lib/yle_tf/action/verify_tf_env.rb new file mode 100644 index 0000000..535f579 --- /dev/null +++ b/lib/yle_tf/action/verify_tf_env.rb @@ -0,0 +1,24 @@ +require 'yle_tf/error' +require 'yle_tf/vars_file' + +class YleTf + module Action + class VerifyTfEnv + def initialize(app) + @app = app + end + + def call(env) + config = env[:config] + all_envs = VarsFile.list_all_envs(config) + + if !all_envs.include?(config.tf_env) + raise Error, "Terraform vars file not found for the '#{config.tf_env}' " \ + " environment. Existing envs: #{all_envs.join(', ')}" + end + + @app.call(env) + end + end + end +end diff --git a/lib/yle_tf/backend_config.rb b/lib/yle_tf/backend_config.rb new file mode 100644 index 0000000..54d2605 --- /dev/null +++ b/lib/yle_tf/backend_config.rb @@ -0,0 +1,41 @@ +require 'json' + +class YleTf + class BackendConfig + BACKEND_CONFIG_FILE = '_backend.tf.json'.freeze + + attr_reader :type, :config + + def initialize(type, config) + @type = type + @config = config + end + + # Returns an `Array` of CLI args for Terraform pre 0.9 `init` command + def cli_args + args = ["-backend=#{type}"] + config.each do |key, value| + args << "-backend-config=#{key}=#{value}" + end + args + end + + # Generate backend configuration file for Terraform v0.9+ + def generate_config + data = { + terraform: [{ + backend: [to_h] + }] + } + File.write(BACKEND_CONFIG_FILE, JSON.pretty_generate(data)) + yield if block_given? + end + + # Returns the backend configuration as a `Hash` for Terraform v0.9+ + def to_h + { type => config } + end + + alias to_s to_h + end +end diff --git a/lib/yle_tf/cli.rb b/lib/yle_tf/cli.rb new file mode 100644 index 0000000..f5114ab --- /dev/null +++ b/lib/yle_tf/cli.rb @@ -0,0 +1,88 @@ +require 'yle_tf' + +class YleTf + class CLI + attr_reader :tf_options, :tf_command, :tf_command_args, :tf_env + + # YleTf option arguments + TF_OPTIONS = %w[--debug --no-hooks --only-hooks].freeze + + HELP_ARGS = %w[-h --help help].freeze + VERSION_ARGS = %w[-v --version version].freeze + + def initialize(argv) + @tf_options = {} + @tf_command_args = [] + split_args(argv) + end + + def execute + tf = YleTf.new(tf_options, tf_env, tf_command, tf_command_args) + tf.run + rescue YleTf::Error => e + raise e if debug? + + Logger.fatal e + exit 1 + end + + # rubocop:disable Metrics/AbcSize, Metrics/BlockLength, Metrics/MethodLength + # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity + def split_args(argv) + argv.each do |arg| + if @tf_env && @tf_command + if TF_OPTIONS.include?(arg) + @tf_options[key(arg)] = true + else + @tf_command_args << arg + end + elsif HELP_ARGS.include?(arg) + @tf_command = 'help' + @tf_env = '_' + break + elsif VERSION_ARGS.include?(arg) + @tf_command = 'version' + @tf_env = '_' + break + elsif arg.start_with?('-') + if TF_OPTIONS.include?(arg) + @tf_options[key(arg)] = true + else + STDERR.puts "Unknown option '#{arg}'" + @tf_command = 'help' + @tf_env = 'error' + break + end + elsif !@tf_env + @tf_env = arg + else + @tf_command = arg + end + end + + if !@tf_command || !@tf_env + @tf_command = 'help' + @tf_env = 'error' + end + + self.debug = true if @tf_options.include?(:debug) + end + + # Returns `Symbol` for the arg, e.g. `"--foo-bar"` -> `:foo_bar` + def key(arg) + arg.sub(/\A--?/, '').tr('-', '_').to_sym + end + + def debug=(value) + if value + ENV['TF_DEBUG'] = '1' + else + ENV.delete('TF_DEBUG') + end + end + + def debug? + ENV.key?('TF_DEBUG') + end + end +end diff --git a/lib/yle_tf/config.rb b/lib/yle_tf/config.rb new file mode 100644 index 0000000..4578376 --- /dev/null +++ b/lib/yle_tf/config.rb @@ -0,0 +1,45 @@ +require 'pathname' +require 'yaml' + +require 'yle_tf/config/loader' +require 'yle_tf/error' +require 'yle_tf/logger' + +class YleTf + # Configuration object to be used especially by the middleware stack + class Config + NotFoundError = Class.new(Error) + + attr_reader :config, :tf_env, :module_dir + + def initialize(tf_env) + Logger.debug("Initializing configuration for the #{tf_env.inspect} environment") + + @tf_env = tf_env + @module_dir = Pathname.pwd + @config = Loader.new(tf_env: tf_env, module_dir: module_dir).load + + Logger.debug(inspect) + end + + def to_s + YAML.dump(config) + end + + # Returns a value from the configuration hierarchy specified by a list of + # keys. If the key is not specified, return result of a specied block, or + # raise `NotFoundError` if none specified. + def fetch(*keys, &block) + block ||= DEFAULT_NOT_FOUND_BLOCK + + keys.inject(config) do |conf, key| + break block.call(keys) if !conf || !conf.key?(key) + conf[key] + end + end + + DEFAULT_NOT_FOUND_BLOCK = lambda do |keys| + raise NotFoundError, "Configuration key not found: #{keys.join(' > ')}" + end.freeze + end +end diff --git a/lib/yle_tf/config/defaults.rb b/lib/yle_tf/config/defaults.rb new file mode 100644 index 0000000..2d27d22 --- /dev/null +++ b/lib/yle_tf/config/defaults.rb @@ -0,0 +1,35 @@ +class YleTf + class Config + module Defaults + DEFAULT_CONFIG = { + 'hooks' => { + 'pre' => [], + 'post' => [] + }, + 'backend' => { + 'type' => 'file', + 'bucket' => nil, + 'file' => '<%= @module %>_<%= @env %>.tfstate', + 'region' => nil, + 'encrypt' => false, + }, + 'tfvars' => { + }, + 'terraform' => { + 'version_requirement' => nil + } + }.freeze + + def default_config + DEFAULT_CONFIG.dup + end + + def default_config_context + { + env: tf_env, + module: module_dir.basename.to_s, + } + end + end + end +end diff --git a/lib/yle_tf/config/erb.rb b/lib/yle_tf/config/erb.rb new file mode 100644 index 0000000..339b964 --- /dev/null +++ b/lib/yle_tf/config/erb.rb @@ -0,0 +1,22 @@ +require 'erb' + +class YleTf + class Config + module ERB + class Context + def initialize(vars) + vars.each { |key, value| instance_variable_set(:"@#{key}", value) } + end + + def binding + super + end + end + + def self.evaluate(string, vars = {}) + b = Context.new(vars).binding + ::ERB.new(string).result(b) + end + end + end +end diff --git a/lib/yle_tf/config/file.rb b/lib/yle_tf/config/file.rb new file mode 100644 index 0000000..c5b2a09 --- /dev/null +++ b/lib/yle_tf/config/file.rb @@ -0,0 +1,26 @@ +require 'yaml' + +require 'yle_tf/logger' + +class YleTf + class Config + class File + attr_reader :name + + def initialize(name) + @name = name.to_s + end + + def read + YAML.load_file(name) || {} + rescue StandardError => e + Logger.fatal("Failed to load or parse configuration from '#{name}'") + raise e + end + + def to_s + name + end + end + end +end diff --git a/lib/yle_tf/config/loader.rb b/lib/yle_tf/config/loader.rb new file mode 100644 index 0000000..626102f --- /dev/null +++ b/lib/yle_tf/config/loader.rb @@ -0,0 +1,108 @@ +require 'yle_tf/config/defaults' +require 'yle_tf/config/erb' +require 'yle_tf/config/file' +require 'yle_tf/logger' +require 'yle_tf/plugin' + +require_relative '../../../vendor/hash_deep_merge' + +class YleTf + class Config + class Loader + include Config::Defaults + + attr_reader :tf_env, :module_dir + + def initialize(opts) + @tf_env = opts.fetch(:tf_env) + @module_dir = opts.fetch(:module_dir) + end + + def load + Logger.debug('Loading default config') + config = default_config + Logger.debug(config.inspect) + + Logger.debug('Merging default configurations from plugins') + merge_plugin_configurations(config) + Logger.debug(config.inspect) + + Logger.debug('Merging configurations from files') + merge_config_files(config) + Logger.debug(config.inspect) + + Logger.debug('Evaluating the configuration strings') + eval_config(config) + end + + def config_context + @config_context ||= load_config_context + end + + def load_config_context + Logger.debug('Loading config context') + default_config_context.tap do |context| + Logger.debug('Merging configuration contexts from plugins') + merge_plugin_config_contexts(context) + Logger.debug("config_context: #{context.inspect}") + end + end + + def merge_plugin_config_contexts(context) + Plugin.manager.config_contexts.each do |plugin_context| + context.merge!(plugin_context) + end + end + + def merge_plugin_configurations(config) + Plugin.manager.default_configs.each do |plugin_config| + deep_merge( + config, plugin_config, + error_msg: + "Failed to merge a plugin's default configuration:\n" \ + "#{plugin_config.inspect}\ninto:\n#{config.inspect}" + ) + end + end + + def merge_config_files(config) + config_files do |file| + Logger.debug(" - #{file}") + deep_merge( + config, file.read, + error_msg: + "Failed to merge configuration from '#{file}' into:\n" \ + "#{config.inspect}" + ) + end + end + + def deep_merge(config, new_config, opts = {}) + config.deep_merge!(new_config) + rescue StandardError => e + Logger.fatal(opts[:error_msg]) if opts[:error_msg] + raise e + end + + def config_files + module_dir.descend do |dir| + file = dir.join('tf.yaml') + yield(Config::File.new(file)) if file.exist? + end + end + + def eval_config(config) + case config + when Hash + config.each_with_object({}) { |(key, value), h| h[key] = eval_config(value) } + when Array + config.map { |item| eval_config(item) } + when String + Config::ERB.evaluate(config, config_context) + else + config + end + end + end + end +end diff --git a/lib/yle_tf/error.rb b/lib/yle_tf/error.rb new file mode 100644 index 0000000..6a54380 --- /dev/null +++ b/lib/yle_tf/error.rb @@ -0,0 +1,4 @@ +class YleTf + # Base class for yle_tf errors + Error = Class.new(StandardError) +end diff --git a/lib/yle_tf/logger.rb b/lib/yle_tf/logger.rb new file mode 100644 index 0000000..e3a25ee --- /dev/null +++ b/lib/yle_tf/logger.rb @@ -0,0 +1,48 @@ +require 'forwardable' +require 'logger' +require 'rubygems' + +class YleTf + # Logger for debug, error, etc. outputs. + # Prints to STDERR, so it does not mess with e.g. `terraform output`. + module Logger + class << self + extend Forwardable + def_delegators :logger, :debug, :info, :warn, :error, :fatal + def_delegators :logger, :debug? + end + + def self.logger + @logger ||= ::Logger.new(STDERR).tap do |logger| + patch_for_old_ruby(logger) + logger.level = log_level + logger.formatter = log_formatter + end + end + + def self.log_level + (ENV['TF_DEBUG'] && 'DEBUG') || \ + ENV['TF_LOG'] || \ + 'INFO' + end + + def self.log_formatter + proc do |severity, _datetime, progname, msg| + if progname + "[#{progname}] #{severity}: #{msg}\n" + else + "#{severity}: #{msg}\n" + end + end + end + + # Patches the `::Logger` in older Ruby versions to + # accept log level as a `String` + def self.patch_for_old_ruby(logger) + if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3') + require_relative '../../vendor/logger_level_patch' + logger.extend(LoggerLevelPatch) + end + end + end +end diff --git a/lib/yle_tf/plugin.rb b/lib/yle_tf/plugin.rb new file mode 100644 index 0000000..75122ee --- /dev/null +++ b/lib/yle_tf/plugin.rb @@ -0,0 +1,55 @@ +class YleTf + class Plugin + autoload :ActionHook, 'yle_tf/plugin/action_hook' + autoload :Loader, 'yle_tf/plugin/loader' + autoload :Manager, 'yle_tf/plugin/manager' + + DEFAULT_COMMAND = Object.new.freeze + + def self.manager + @manager ||= Manager.new + end + + def self.register + Plugin.manager.register(self) + end + + def self.action_hooks + @action_hooks ||= [] + end + + def self.action_hook(&block) + action_hooks << block + end + + def self.commands + @commands ||= {} + end + + def self.command(name, synopsis, &block) + name = name.to_s if name.is_a?(Symbol) + commands[name] = { + synopsis: synopsis, + proc: block + } + end + + def self.default_config(config = nil) + @default_config = config if config + @default_config || {} + end + + def self.config_context(context = nil) + @config_context = context if context + @config_context || {} + end + + def self.backends + @backends ||= {} + end + + def self.backend(type, &block) + backends[type.to_sym] = block + end + end +end diff --git a/lib/yle_tf/plugin/action_hook.rb b/lib/yle_tf/plugin/action_hook.rb new file mode 100644 index 0000000..b423249 --- /dev/null +++ b/lib/yle_tf/plugin/action_hook.rb @@ -0,0 +1,23 @@ +class YleTf + class Plugin + class ActionHook + attr_reader :actions + + def initialize(actions) + @actions = actions + end + + def before(existing, new, *args, &block) + if actions.include?(existing) + actions.insert_before(existing, new, *args, &block) + end + end + + def after(existing, new, *args, &block) + if actions.include?(existing) + actions.insert_after(existing, new, *args, &block) + end + end + end + end +end diff --git a/lib/yle_tf/plugin/loader.rb b/lib/yle_tf/plugin/loader.rb new file mode 100644 index 0000000..c7823aa --- /dev/null +++ b/lib/yle_tf/plugin/loader.rb @@ -0,0 +1,59 @@ +require 'yle_tf/logger' + +class YleTf + class Plugin + module Loader + BUNDLER_PLUGIN_GROUP = :tf_plugins + + def self.load_plugins + load_core_plugins + load_bundler_plugins + load_user_plugins + end + + def self.load_core_plugins + core_plugins.each do |plugin_file| + Logger.debug("Loading core plugin: #{File.basename(plugin_file, '.rb')}") + load(plugin_file) + end + end + + def self.load_bundler_plugins + if defined?(Bundler) + print_bundler_plugin_list if Logger.debug? + Bundler.require(BUNDLER_PLUGIN_GROUP) + end + end + + def self.load_user_plugins + user_plugins.each do |plugin| + Logger.debug("Loading user plugin: #{plugin}") + require(plugin) + end + end + + def self.core_plugins + Dir.glob(File.expand_path('../../../yle_tf_plugins/**/plugin.rb', __FILE__)) + end + + def self.bundler_plugins + plugins = Bundler.definition.current_dependencies.select do |dep| + dep.groups.include?(BUNDLER_PLUGIN_GROUP) + end + plugins.map { |dep| Bundler.definition.specs[dep].first } + end + + def self.user_plugins + ENV.fetch('TF_PLUGINS', '').split(/[ ,]+/) + end + + def self.print_bundler_plugin_list + plugins = bundler_plugins + if !plugins.empty? + Logger.debug('Loading plugins via Bundler:') + plugins.each { |spec| Logger.debug(" - #{spec.name} = #{spec.version}") } + end + end + end + end +end diff --git a/lib/yle_tf/plugin/manager.rb b/lib/yle_tf/plugin/manager.rb new file mode 100644 index 0000000..342ab33 --- /dev/null +++ b/lib/yle_tf/plugin/manager.rb @@ -0,0 +1,49 @@ +require 'yle_tf/logger' + +class YleTf + class Plugin + class Manager + attr_reader :registered + + def initialize + @registered = [] + end + + def register(plugin) + if !registered.include?(plugin) + Logger.debug("Registered plugin: #{plugin}") + @registered << plugin + end + end + + def action_hooks + registered.map(&:action_hooks).flatten + end + + def commands + {}.tap do |commands| + registered.each do |plugin| + commands.merge!(plugin.commands) + end + commands.default = commands.delete(DEFAULT_COMMAND) + end + end + + def config_contexts + registered.map(&:config_context) + end + + def default_configs + registered.map(&:default_config) + end + + def backends + {}.tap do |backends| + registered.each do |plugin| + backends.merge!(plugin.backends) + end + end + end + end + end +end diff --git a/lib/yle_tf/system.rb b/lib/yle_tf/system.rb new file mode 100644 index 0000000..876a59a --- /dev/null +++ b/lib/yle_tf/system.rb @@ -0,0 +1,22 @@ +require 'shellwords' + +require 'yle_tf/error' +require 'yle_tf/logger' + +class YleTf + # Helpers to execute system commands with error handling + # + # TODO: Add way to wrap stdout of the commands and direct it to `Logger` + class System + ExecuteError = Class.new(YleTf::Error) + + def self.cmd(*args, **opts) + env = opts[:env] + YleTf::Logger.debug { "Calling `#{args.shelljoin}`#{" with env '#{env}'" if env}" } + + system(env || {}, *args) || + raise(ExecuteError, + "Failed to execute `#{args.shelljoin}`#{" with env '#{env}'" if env}") + end + end +end diff --git a/lib/yle_tf/tf_hook.rb b/lib/yle_tf/tf_hook.rb new file mode 100644 index 0000000..37f10b1 --- /dev/null +++ b/lib/yle_tf/tf_hook.rb @@ -0,0 +1,90 @@ +require 'fileutils' +require 'tmpdir' + +require 'yle_tf/error' +require 'yle_tf/logger' +require 'yle_tf/system' + +class YleTf + class TfHook + autoload :Runner, 'yle_tf/tf_hook/runner' + + # Returns a `TfHook` instance from configuration hash + def self.from_config(config, tf_env) + TfHook.new( + description: config['description'], + source: config['source'], + vars: merge_vars(config['vars'], tf_env) + ) + end + + # Returns a `Hook` instance from a local file path + def self.from_file(path) + TfHook.new( + description: File.basename(path), + path: path + ) + end + + attr_reader :description, :source, :path, :vars + + def initialize(opts = {}) + @description = opts[:description] + @source = opts[:source] + @path = opts[:path] + @vars = opts[:vars] || {} + @tmpdir = nil + end + + def run(tf_vars) + fetch if !path + + Logger.info("Running hook '#{description}'...") + YleTf::System.cmd(path, env: vars.merge(tf_vars)) + ensure + delete_tmpdir + end + + def parse_source_config + m = %r{^(?.+)//(?[^?]+)(\?ref=(?.*))?$}.match(source) + raise Error, "Invalid or missing `source` for hook '#{description}'" if !m + + { + uri: m[:uri], + path: m[:path], + ref: m[:ref] || 'master' + } + end + + def fetch + source_config = parse_source_config + source_config[:dir] = create_tmpdir + clone_git_repo(source_config) + @path = File.join(source_config[:dir], source_config[:path]) + end + + def clone_git_repo(config) + Logger.info("Cloning hook '#{description}' from #{config[:uri]} (#{config[:ref]})") + YleTf::System.cmd( + 'git', 'clone', '--no-progress', '--depth=1', '--branch', config[:ref], + '--', config[:uri], config[:dir] + ) + end + + def create_tmpdir + @tmpdir = Dir.mktmpdir('tf_hook_') + end + + def delete_tmpdir + FileUtils.rm_r(@tmpdir) if @tmpdir + @tmpdir = nil + end + + # Returns a hash with env specific vars merged into the default ones + def self.merge_vars(vars, tf_env) + vars ||= {} + defaults = vars['defaults'] || {} + defaults.merge(vars[tf_env] || {}) + end + end +end diff --git a/lib/yle_tf/tf_hook/runner.rb b/lib/yle_tf/tf_hook/runner.rb new file mode 100644 index 0000000..6a3a319 --- /dev/null +++ b/lib/yle_tf/tf_hook/runner.rb @@ -0,0 +1,48 @@ +require 'yle_tf/logger' +require 'yle_tf/tf_hook' + +class YleTf + class TfHook + class Runner + attr_reader :config, :hook_env + + def initialize(config, hook_env) + @config = config + @hook_env = hook_env + end + + def tf_env + @tf_env ||= config.tf_env + end + + def run(hook_type) + Logger.debug("Running #{hook_type} hooks") + hooks(hook_type).each do |hook| + hook.run(hook_env) + end + end + + def hooks(hook_type) + hook_confs(hook_type).map { |conf| TfHook.from_config(conf, tf_env) } + + hook_files(hook_type).map { |file| TfHook.from_file(file) } + end + + def hook_confs(hook_type) + config.fetch('hooks', hook_type).select do |hook| + if hook['envs'] && !hook['envs'].include?(tf_env) + Logger.debug("Skipping hook '#{hook['description']}' in env '#{tf_env}'") + false + else + true + end + end + end + + def hook_files(hook_type) + Dir.glob("tf_hooks/#{hook_type}/*").select do |file| + File.executable?(file) && !File.directory?(file) + end + end + end + end +end diff --git a/lib/yle_tf/vars_file.rb b/lib/yle_tf/vars_file.rb new file mode 100644 index 0000000..824a6ab --- /dev/null +++ b/lib/yle_tf/vars_file.rb @@ -0,0 +1,42 @@ +class YleTf + class VarsFile + # Returns the env specific tfvars file path if it exists + def self.find_env_vars_file(config) + path = "#{config.module_dir}/envs/#{config.tf_env}.tfvars" + VarsFile.new(path) if File.exist?(path) + end + + # Returns all envs that have tfvars files + def self.list_all_envs(config) + Dir.glob("#{config.module_dir}/envs/*.tfvars").map do |path| + File.basename(path, '.tfvars') + end + end + + attr_reader :path + + def initialize(path) + @path = path + end + + def read + IO.read(path) + end + + def append_file(vars_file) + File.open(path, 'a') do |file| + file.puts # ensure we don't append to an existing line + file.puts(vars_file.read) + end + end + + def append_vars(vars) + File.open(path, 'a') do |file| + file.puts # ensure we don't append to an existing line + vars.each do |key, value| + file.puts %(#{key} = "#{value}") + end + end + end + end +end diff --git a/lib/yle_tf/version.rb b/lib/yle_tf/version.rb new file mode 100644 index 0000000..a1059bf --- /dev/null +++ b/lib/yle_tf/version.rb @@ -0,0 +1,3 @@ +class YleTf + VERSION = '0.1.0'.freeze +end diff --git a/lib/yle_tf/version_requirement.rb b/lib/yle_tf/version_requirement.rb new file mode 100644 index 0000000..7e14a02 --- /dev/null +++ b/lib/yle_tf/version_requirement.rb @@ -0,0 +1,25 @@ +require 'rubygems' + +class YleTf + # Helper class for comparing versions + class VersionRequirement + # Checks if the specified Terrform version is older than 0.9 + def self.pre_0_9?(terraform_version) + new('< 0.9.0-beta').satisfied_by?(terraform_version) + end + + attr_reader :requirement + + def initialize(requirement) + @requirement = requirement && Gem::Requirement.new(*requirement) + end + + def satisfied_by?(version) + !requirement || requirement.satisfied_by?(Gem::Version.new(version)) + end + + def to_s + requirement.to_s + end + end +end diff --git a/lib/yle_tf_plugins/backends/file/command.rb b/lib/yle_tf_plugins/backends/file/command.rb new file mode 100644 index 0000000..9fc85fa --- /dev/null +++ b/lib/yle_tf_plugins/backends/file/command.rb @@ -0,0 +1,31 @@ +require_relative 'config' + +module YleTfPlugins + module Backends + module File + class Command + def backend_config(config) + local_path = Pathname.pwd.join('terraform.tfstate') + tfstate_path = tfstate_path(config) + + YleTf::Logger.info("Symlinking state to '#{tfstate_path}'") + local_path.make_symlink(tfstate_path) + tfstate_path.write(tfstate_template, perm: 0o644) if !tfstate_path.exist? + + Config.new( + 'file', + 'file' => tfstate_path.to_s + ) + end + + def tfstate_path(config) + config.module_dir.join(config.fetch('backend', 'file')) + end + + def tfstate_template + '{"version": 1}' + end + end + end + end +end diff --git a/lib/yle_tf_plugins/backends/file/config.rb b/lib/yle_tf_plugins/backends/file/config.rb new file mode 100644 index 0000000..0784fc0 --- /dev/null +++ b/lib/yle_tf_plugins/backends/file/config.rb @@ -0,0 +1,17 @@ +require 'yle_tf/backend_config' + +module YleTfPlugins + module Backends + module File + class Config < YleTf::BackendConfig + def generate_config + yield if block_given? + end + + def cli_args + nil + end + end + end + end +end diff --git a/lib/yle_tf_plugins/backends/file/plugin.rb b/lib/yle_tf_plugins/backends/file/plugin.rb new file mode 100644 index 0000000..ad6817f --- /dev/null +++ b/lib/yle_tf_plugins/backends/file/plugin.rb @@ -0,0 +1,16 @@ +require 'yle_tf' + +module YleTfPlugins + module Backends + module File + class Plugin < YleTf::Plugin + register + + backend('file') do + require_relative 'command' + Command + end + end + end + end +end diff --git a/lib/yle_tf_plugins/backends/s3/command.rb b/lib/yle_tf_plugins/backends/s3/command.rb new file mode 100644 index 0000000..7c66bea --- /dev/null +++ b/lib/yle_tf_plugins/backends/s3/command.rb @@ -0,0 +1,19 @@ +require 'yle_tf/backend_config' + +module YleTfPlugins + module Backends + module S3 + class Command + def backend_config(config) + YleTf::BackendConfig.new( + 's3', + 'region' => config.fetch('backend', 'region'), + 'bucket' => config.fetch('backend', 'bucket'), + 'key' => config.fetch('backend', 'file'), + 'encrypt' => config.fetch('backend', 'encrypt') + ) + end + end + end + end +end diff --git a/lib/yle_tf_plugins/backends/s3/plugin.rb b/lib/yle_tf_plugins/backends/s3/plugin.rb new file mode 100644 index 0000000..af8a985 --- /dev/null +++ b/lib/yle_tf_plugins/backends/s3/plugin.rb @@ -0,0 +1,16 @@ +require 'yle_tf' + +module YleTfPlugins + module Backends + module S3 + class Plugin < YleTf::Plugin + register + + backend('s3') do + require_relative 'command' + Command + end + end + end + end +end diff --git a/lib/yle_tf_plugins/commands/__default/command.rb b/lib/yle_tf_plugins/commands/__default/command.rb new file mode 100644 index 0000000..bdf6a58 --- /dev/null +++ b/lib/yle_tf_plugins/commands/__default/command.rb @@ -0,0 +1,14 @@ +require 'yle_tf/system' + +module YleTfPlugins + module CommandDefault + class Command + def execute(env) + command = env[:tf_command] + args = env[:tf_command_args] + + YleTf::System.cmd('terraform', command, *args) + end + end + end +end diff --git a/lib/yle_tf_plugins/commands/__default/plugin.rb b/lib/yle_tf_plugins/commands/__default/plugin.rb new file mode 100644 index 0000000..e2b7ed1 --- /dev/null +++ b/lib/yle_tf_plugins/commands/__default/plugin.rb @@ -0,0 +1,14 @@ +require 'yle_tf' + +module YleTfPlugins + module CommandDefault + class Plugin < YleTf::Plugin + register + + command(DEFAULT_COMMAND, 'Calls Terraform with the given subcommand') do + require_relative 'command' + YleTf::Action.default_action_stack(Command) + end + end + end +end diff --git a/lib/yle_tf_plugins/commands/_config/command.rb b/lib/yle_tf_plugins/commands/_config/command.rb new file mode 100644 index 0000000..4cde546 --- /dev/null +++ b/lib/yle_tf_plugins/commands/_config/command.rb @@ -0,0 +1,11 @@ +require 'yle_tf/logger' + +module YleTfPlugins + module CommandConfig + class Command + def execute(env) + puts env[:config] + end + end + end +end diff --git a/lib/yle_tf_plugins/commands/_config/plugin.rb b/lib/yle_tf_plugins/commands/_config/plugin.rb new file mode 100644 index 0000000..ef2bf6b --- /dev/null +++ b/lib/yle_tf_plugins/commands/_config/plugin.rb @@ -0,0 +1,19 @@ +require 'yle_tf' + +module YleTfPlugins + module CommandConfig + class Plugin < YleTf::Plugin + register + + command('_config', 'Prints the evaluated configuration') do + require_relative 'command' + + YleTf::Action::Builder.new do + use YleTf::Action::LoadConfig + use YleTf::Action::VerifyTfEnv + use YleTf::Action::Command, Command + end + end + end + end +end diff --git a/lib/yle_tf_plugins/commands/_shell/command.rb b/lib/yle_tf_plugins/commands/_shell/command.rb new file mode 100644 index 0000000..7e41e45 --- /dev/null +++ b/lib/yle_tf_plugins/commands/_shell/command.rb @@ -0,0 +1,15 @@ +module YleTfPlugins + module CommandShell + class Command + def execute(_env) + shell = ENV.fetch('SHELL', 'bash') + + puts "Executing shell '#{shell}' in the Terraform directory" + puts 'Use `exit` to quit' + puts + + system(shell) + end + end + end +end diff --git a/lib/yle_tf_plugins/commands/_shell/plugin.rb b/lib/yle_tf_plugins/commands/_shell/plugin.rb new file mode 100644 index 0000000..b5a059d --- /dev/null +++ b/lib/yle_tf_plugins/commands/_shell/plugin.rb @@ -0,0 +1,14 @@ +require 'yle_tf' + +module YleTfPlugins + module CommandShell + class Plugin < YleTf::Plugin + register + + command('_shell', 'Executes shell in the prepared environment') do + require_relative 'command' + YleTf::Action.default_action_stack(Command) + end + end + end +end diff --git a/lib/yle_tf_plugins/commands/help/command.rb b/lib/yle_tf_plugins/commands/help/command.rb new file mode 100644 index 0000000..f34a321 --- /dev/null +++ b/lib/yle_tf_plugins/commands/help/command.rb @@ -0,0 +1,55 @@ +require 'optparse' + +require 'yle_tf/plugin' + +module YleTfPlugins + module CommandHelp + class Command + def execute(env) + device(env).puts(opts.help) + exit 1 if error?(env) + end + + # rubocop:disable Metrics/AbcSize, Metrics/MethodLength + def opts + OptionParser.new do |o| + o.summary_width = 18 + o.banner = 'Usage: tf []' + o.separator '' + o.separator 'YleTf options:' + o.on('-h', '--help', 'Prints this help') + o.on('-v', '--version', 'Prints the version information') + o.on('--debug', 'Print debug information') + o.on('--no-hooks', 'Do not run any hooks') + o.on('--only-hooks', 'Only run the hooks') + o.separator '' + o.separator 'Special tf commands:' + o.separator tf_command_help + o.separator '' + o.separator 'Terraform commands:' + o.separator terraform_help + end + end + + def error?(env) + env[:tf_env] == 'error' + end + + def device(env) + error?(env) ? STDERR : STDOUT + end + + def tf_command_help + YleTf::Plugin.manager.commands.sort.map do |command, data| + " #{command.ljust(18)} #{data[:synopsis]}" + end + end + + def terraform_help + `terraform help`.lines.grep(/^ /) + rescue Errno::ENOENT + ' [Terraform not found]' + end + end + end +end diff --git a/lib/yle_tf_plugins/commands/help/plugin.rb b/lib/yle_tf_plugins/commands/help/plugin.rb new file mode 100644 index 0000000..12d5d0d --- /dev/null +++ b/lib/yle_tf_plugins/commands/help/plugin.rb @@ -0,0 +1,18 @@ +require 'yle_tf' + +module YleTfPlugins + module CommandHelp + class Plugin < YleTf::Plugin + register + + command('help', 'Prints this help') do + require_relative 'command' + + YleTf::Action::Builder.new do + use YleTf::Action::TmpDir + use YleTf::Action::Command, Command + end + end + end + end +end diff --git a/lib/yle_tf_plugins/commands/version/command.rb b/lib/yle_tf_plugins/commands/version/command.rb new file mode 100644 index 0000000..d1ac463 --- /dev/null +++ b/lib/yle_tf_plugins/commands/version/command.rb @@ -0,0 +1,20 @@ +require 'optparse' + +require 'yle_tf/version' + +module YleTfPlugins + module CommandVersion + class Command + def execute(_env) + puts "YleTf #{YleTf::VERSION}" + puts terraform_version + end + + def terraform_version + `terraform version`.lines.first + rescue Errno::ENOENT + '[Terraform not found]' + end + end + end +end diff --git a/lib/yle_tf_plugins/commands/version/plugin.rb b/lib/yle_tf_plugins/commands/version/plugin.rb new file mode 100644 index 0000000..7bc23d4 --- /dev/null +++ b/lib/yle_tf_plugins/commands/version/plugin.rb @@ -0,0 +1,18 @@ +require 'yle_tf' + +module YleTfPlugins + module CommandVersion + class Plugin < YleTf::Plugin + register + + command('version', 'Prints the version information') do + require_relative 'command' + + YleTf::Action::Builder.new do + use YleTf::Action::TmpDir + use YleTf::Action::Command, Command + end + end + end + end +end diff --git a/spec/fixtures/vars_files/envs/dau.tfvars b/spec/fixtures/vars_files/envs/dau.tfvars new file mode 100644 index 0000000..e69de29 diff --git a/spec/fixtures/vars_files/envs/diu.tfvars b/spec/fixtures/vars_files/envs/diu.tfvars new file mode 100644 index 0000000..e69de29 diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000..3ef6d3f --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,2 @@ +$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) +require 'yle_tf' diff --git a/spec/yle_tf/tf_hook_spec.rb b/spec/yle_tf/tf_hook_spec.rb new file mode 100644 index 0000000..d19ae2d --- /dev/null +++ b/spec/yle_tf/tf_hook_spec.rb @@ -0,0 +1,162 @@ +require 'spec_helper' + +require 'yle_tf/error' +require 'yle_tf/tf_hook' + +describe YleTf::TfHook do + describe '.from_config' do + subject(:hook) { described_class.from_config(conf, env) } + let(:conf) do + { + 'description' => description, + 'source' => source, + 'vars' => vars + } + end + let(:description) { 'Hook description' } + let(:source) { 'git@git.example.com:organization/repo.git//hook/path' } + let(:vars) {} + let(:env) { 'spec' } + + describe '#description' do + subject { hook.description } + it { is_expected.to eq(description) } + end + + describe '#source' do + subject { hook.source } + it { is_expected.to eq(source) } + end + + describe '#vars' do + subject { hook.vars } + + context 'without vars' do + let(:vars) { nil } + it { is_expected.to eq({}) } + end + + context 'with only default vars' do + let(:vars) { { 'defaults' => { 'FOO' => 'bar', 'MEMORY' => '5TB' } } } + it { is_expected.to eq(vars['defaults']) } + end + + context 'with only env specific vars' do + let(:vars) { { env => { 'DIU' => 'dau', 'MEMORY' => '2TB' } } } + it { is_expected.to eq(vars[env]) } + end + + context 'with default and env specific vars' do + let(:vars) do + { + 'defaults' => { 'FOO' => 'bar', 'MEMORY' => '5TB' }, + env => { 'DIU' => 'dau', 'MEMORY' => '2TB' }, + 'dummy' => { 'FOO' => 'baz' } + } + end + it 'returns the vars merged' do + expect(subject).to eq('FOO' => 'bar', 'DIU' => 'dau', 'MEMORY' => '2TB') + end + end + end + + describe '#run' do + before do + expect(YleTf::System).to receive(:system) + .with(expected_vars, expected_path) { return_value } + allow(hook).to receive(:create_tmpdir) { tmpdir } + expect(hook).to receive(:clone_git_repo) + + # silence the output + allow($stdout).to receive(:write) + allow($stderr).to receive(:write) + end + + let(:tf_vars) { { 'FOO' => 'xxx', 'BAR' => 'yyy' } } + let(:expected_vars) { tf_vars } + let(:expected_path) { "#{tmpdir}/hook/path" } + let(:tmpdir) { '/tmp/dir' } + + context 'when the hook suceeds' do + let(:return_value) { true } + + it { expect { hook.run(tf_vars) }.not_to raise_error } + + context 'with configured vars' do + let(:vars) do + { + 'defaults' => { 'FOO' => 'bar', 'MEMORY' => '5TB' }, + env => { 'DIU' => 'dau', 'MEMORY' => '2TB' } + } + end + let(:expected_vars) do + { + 'FOO' => 'xxx', 'BAR' => 'yyy', # from tf_vars + 'DIU' => 'dau', 'MEMORY' => '2TB' # from merged config vars + } + end + it 'merges all the vars' do + hook.run(tf_vars) + end + end + end + + context 'when the hook returns non-zero exit status' do + let(:return_value) { false } + it { expect { hook.run(tf_vars) }.to raise_error(YleTf::System::ExecuteError) } + end + + context 'when the hook execution fails' do + let(:return_value) { nil } + it { expect { hook.run(tf_vars) }.to raise_error(YleTf::System::ExecuteError) } + end + end + end + + describe '.from_file' do + subject(:hook) { described_class.from_file(path) } + let(:path) { 'some/local/hook_file_name' } + + describe '#description' do + subject { hook.description } + it { is_expected.to eq('hook_file_name') } + end + + describe '#path' do + subject { hook.path } + it { is_expected.to eq(path) } + end + + describe '#vars' do + subject { hook.vars } + it { is_expected.to eq({}) } + end + + describe '#run' do + let(:tf_vars) { { 'FOO' => 'xxx', 'BAR' => 'yyy' } } + before do + expect(YleTf::System).to receive(:system).with(tf_vars, path) { return_value } + expect(hook).not_to receive(:clone_git_repo) + + # silence the output + allow($stdout).to receive(:write) + allow($stderr).to receive(:write) + end + + context 'when the hook suceeds' do + let(:return_value) { true } + it { expect { hook.run(tf_vars) }.not_to raise_error } + end + + context 'when the hook returns non-zero exit status' do + let(:return_value) { false } + it { expect { hook.run(tf_vars) }.to raise_error(YleTf::Error) } + end + + context 'when the hook execution fails' do + let(:return_value) { nil } + it { expect { hook.run(tf_vars) }.to raise_error(YleTf::Error) } + end + end + end +end diff --git a/spec/yle_tf/vars_file_spec.rb b/spec/yle_tf/vars_file_spec.rb new file mode 100644 index 0000000..ca98856 --- /dev/null +++ b/spec/yle_tf/vars_file_spec.rb @@ -0,0 +1,39 @@ +require 'spec_helper' + +require 'ostruct' +require 'yle_tf/vars_file' + +describe YleTf::VarsFile do + describe '.find_env_vars_file' do + subject { described_class.find_env_vars_file(config) } + let(:config) { OpenStruct.new(module_dir: module_dir, tf_env: tf_env) } + let(:module_dir) { 'spec/fixtures/vars_files' } + + context 'with existing var file' do + let(:tf_env) { 'diu' } + it 'returns the corresponding tfvars file' do + expect(subject.path).to eq("#{module_dir}/envs/#{tf_env}.tfvars") + end + end + + context 'with non-existing var file' do + let(:tf_env) { 'foo' } + it { is_expected.to be_nil } + end + end + + describe '.list_all_envs' do + subject { described_class.list_all_envs(config) } + let(:config) { OpenStruct.new(module_dir: module_dir) } + + context 'with existing var files' do + let(:module_dir) { 'spec/fixtures/vars_files' } + it { is_expected.to eq(%w[dau diu]) } + end + + context 'with non-existing var files' do + let(:module_dir) { 'non_existing_dir' } + it { is_expected.to eq([]) } + end + end +end diff --git a/spec/yle_tf_spec.rb b/spec/yle_tf_spec.rb new file mode 100644 index 0000000..8ad5acd --- /dev/null +++ b/spec/yle_tf_spec.rb @@ -0,0 +1,7 @@ +require 'spec_helper' + +describe YleTf do + it 'has a version number' do + expect(YleTf::VERSION).not_to be nil + end +end diff --git a/vendor/hash_deep_merge.rb b/vendor/hash_deep_merge.rb new file mode 100644 index 0000000..8a15da2 --- /dev/null +++ b/vendor/hash_deep_merge.rb @@ -0,0 +1,59 @@ +# Copyright (c) 2005-2017 David Heinemeier Hansson +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +class Hash + # Returns a new hash with +self+ and +other_hash+ merged recursively. + # + # h1 = { a: true, b: { c: [1, 2, 3] } } + # h2 = { a: false, b: { x: [3, 4, 5] } } + # + # h1.deep_merge(h2) # => { a: false, b: { c: [1, 2, 3], x: [3, 4, 5] } } + # + # Like with Hash#merge in the standard library, a block can be provided + # to merge values: + # + # h1 = { a: 100, b: 200, c: { c1: 100 } } + # h2 = { b: 250, c: { c1: 200 } } + # h1.deep_merge(h2) { |key, this_val, other_val| this_val + other_val } + # # => { a: 100, b: 450, c: { c1: 300 } } + def deep_merge(other_hash, &block) + dup.deep_merge!(other_hash, &block) + end + + # Same as +deep_merge+, but modifies +self+. + def deep_merge!(other_hash, &block) + other_hash.each_pair do |current_key, other_value| + this_value = self[current_key] + + self[current_key] = if this_value.is_a?(Hash) && other_value.is_a?(Hash) + this_value.deep_merge(other_value, &block) + else + if block_given? && key?(current_key) + block.call(current_key, this_value, other_value) + else + other_value + end + end + end + + self + end +end diff --git a/vendor/logger_level_patch.rb b/vendor/logger_level_patch.rb new file mode 100644 index 0000000..a8d82fb --- /dev/null +++ b/vendor/logger_level_patch.rb @@ -0,0 +1,29 @@ +require 'logger' + +class YleTf + module LoggerLevelPatch + # Taken from Ruby 2.4.1 + def level=(severity) + if severity.is_a?(Integer) + @level = severity + else + case severity.to_s.downcase + when 'debug'.freeze + @level = ::Logger::DEBUG + when 'info'.freeze + @level = ::Logger::INFO + when 'warn'.freeze + @level = ::Logger::WARN + when 'error'.freeze + @level = ::Logger::ERROR + when 'fatal'.freeze + @level = ::Logger::FATAL + when 'unknown'.freeze + @level = ::Logger::UNKNOWN + else + raise ArgumentError, "invalid log level: #{severity}" + end + end + end + end +end diff --git a/vendor/middleware/LICENSE b/vendor/middleware/LICENSE new file mode 100644 index 0000000..05bd31f --- /dev/null +++ b/vendor/middleware/LICENSE @@ -0,0 +1,23 @@ +Copyright (c) 2012 Mitchell Hashimoto +Copyright (c) 2017 Teemu Matilainen + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/middleware/builder.rb b/vendor/middleware/builder.rb new file mode 100644 index 0000000..1aac9a9 --- /dev/null +++ b/vendor/middleware/builder.rb @@ -0,0 +1,149 @@ +require_relative 'runner' + +module Middleware + # This provides a DSL for building up a stack of middlewares. + # + # This code is based heavily off of `Rack::Builder` and + # `ActionDispatch::MiddlewareStack` in Rack and Rails, respectively. + # + # # Usage + # + # Building a middleware stack is very easy: + # + # app = Middleware::Builder.new do + # use A + # use B + # end + # + # # Call the middleware + # app.call(7) + # + class Builder + # Initializes the builder. An optional block can be passed which + # will be evaluated in the context of the instance. + # + # Example: + # + # Builder.new do + # use A + # use B + # end + # + # @param [Hash] opts Options hash + # @option opts [Class] :runner_class The class to wrap the middleware stack + # in which knows how to run them. + # @yield [] Evaluated in this instance which allows you to use methods + # like {#use} and such. + def initialize(opts=nil, &block) + opts ||= {} + @runner_class = opts[:runner_class] || Runner + instance_eval(&block) if block_given? + end + + # Returns a mergeable version of the builder. If `use` is called with + # the return value of this method, then the stack will merge, instead + # of being treated as a separate single middleware. + def flatten + lambda do |env| + self.call(env) + end + end + + # Adds a middleware class to the middleware stack. Any additional + # args and a block, if given, are saved and passed to the initializer + # of the middleware. + # + # @param [Class] middleware The middleware class + def use(middleware, *args, &block) + if middleware.kind_of?(Builder) + # Merge in the other builder's stack into our own + self.stack.concat(middleware.stack) + else + self.stack << [middleware, args, block] + end + + self + end + + # Inserts a middleware at the given index or directly before the + # given middleware object. + def insert(index, middleware, *args, &block) + index = self.index(index) unless index.is_a?(Integer) + raise "no such middleware to insert before: #{index.inspect}" unless index + + if middleware.kind_of?(Builder) + middleware.stack.reverse.each do |stack_item| + stack.insert(index, stack_item) + end + else + stack.insert(index, [middleware, args, block]) + end + end + + alias_method :insert_before, :insert + + # Inserts a middleware after the given index or middleware object. + def insert_after(index, middleware, *args, &block) + index = self.index(index) unless index.is_a?(Integer) + raise "no such middleware to insert after: #{index.inspect}" unless index + insert(index + 1, middleware, *args, &block) + end + + # Replaces the given middlware object or index with the new + # middleware. + def replace(index, middleware, *args, &block) + if index.is_a?(Integer) + delete(index) + insert(index, middleware, *args, &block) + else + insert_before(index, middleware, *args, &block) + delete(index) + end + end + + # Deletes the given middleware object or index + def delete(index) + index = self.index(index) unless index.is_a?(Integer) + stack.delete_at(index) + end + + # Runs the builder stack with the given environment. + def call(env=nil) + to_app.call(env) + end + + # Returns truish if the given middleware object exists in the stack. + def include?(object) + index(object) + end + + protected + + # Returns the numeric index for the given middleware object. + # + # @param [Object] object The item to find the index for + # @return [Integer] + def index(object) + stack.each_with_index do |item, i| + return i if item[0] == object + end + + nil + end + + # Returns the current stack of middlewares. You probably won't + # need to use this directly, and it's recommended that you don't. + # + # @return [Array] + def stack + @stack ||= [] + end + + # Converts the builder stack to a runnable action sequence. + # + # @return [Object] A callable object + def to_app + @runner_class.new(stack.dup) + end + end +end diff --git a/vendor/middleware/runner.rb b/vendor/middleware/runner.rb new file mode 100644 index 0000000..802b76e --- /dev/null +++ b/vendor/middleware/runner.rb @@ -0,0 +1,69 @@ +module Middleware + # This is a basic runner for middleware stacks. This runner does + # the default expected behavior of running the middleware stacks + # in order, then reversing the order. + class Runner + # A middleware which does nothing + EMPTY_MIDDLEWARE = lambda { |env| } + + # Build a new middleware runner with the given middleware + # stack. + # + # Note: This class usually doesn't need to be used directly. + # Instead, take a look at using the {Builder} class, which is + # a much friendlier way to build up a middleware stack. + # + # @param [Array] stack An array of the middleware to run. + def initialize(stack) + # We need to take the stack of middleware and initialize them + # all so they call the proper next middleware. + @kickoff = build_call_chain(stack) + end + + # Run the middleware stack with the given state bag. + # + # @param [Object] env The state to pass into as the initial + # environment data. This is usual a hash of some sort. + def call(env) + # We just call the kickoff middleware, which is responsible + # for properly calling the next middleware, and so on and so + # forth. + @kickoff.call(env) + end + + protected + + # This takes a stack of middlewares and initializes them in a way + # that each middleware properly calls the next middleware. + def build_call_chain(stack) + # We need to instantiate the middleware stack in reverse + # order so that each middleware can have a reference to + # the next middleware it has to call. The final middleware + # is always the empty middleware, which does nothing but return. + stack.reverse.inject(EMPTY_MIDDLEWARE) do |next_middleware, current_middleware| + # Unpack the actual item + klass, args, block = current_middleware + + # Default the arguments to an empty array. Otherwise in Ruby 1.8 + # a `nil` args will actually pass `nil` into the class. Not what + # we want! + args ||= [] + + if klass.is_a?(Class) + # If the klass actually is a class, then instantiate it with + # the app and any other arguments given. + klass.new(next_middleware, *args, &block) + elsif klass.respond_to?(:call) + # Make it a lambda which calls the item then forwards up + # the chain. + lambda do |env| + klass.call(env) + next_middleware.call(env) + end + else + raise "Invalid middleware, doesn't respond to `call`: #{action.inspect}" + end + end + end + end +end diff --git a/yle_tf.gemspec b/yle_tf.gemspec new file mode 100644 index 0000000..9101e0f --- /dev/null +++ b/yle_tf.gemspec @@ -0,0 +1,37 @@ +# coding: utf-8 + +lib = File.expand_path('../lib', __FILE__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'yle_tf/version' + +Gem::Specification.new do |spec| + spec.name = 'yle_tf' + spec.version = YleTf::VERSION + spec.summary = 'Tooling for Terraform' + spec.description = 'Tooling for Terraform to support environments, hooks, etc.' + spec.homepage = 'https://github.com/Yleisradio/yle_tf' + spec.license = 'MIT' + + spec.authors = [ + 'Yleisradio', + 'Teemu Matilainen', + 'Antti Forsell', + ] + spec.email = [ + 'devops@yle.fi', + 'teemu.matilainen@iki.fi', + 'antti.forsell@iki.fi', + ] + + spec.files = `git ls-files -z`.split("\x0").reject do |f| + f.match(%r{^(test|spec|features)/}) + end + + spec.bindir = 'bin' + spec.executables = ['tf'] + spec.require_paths = ['lib'] + + spec.add_development_dependency 'bundler', '~> 1.13' + spec.add_development_dependency 'rake', '~> 12.0' + spec.add_development_dependency 'rspec', '~> 3.5' +end