diff --git a/.gitignore b/.gitignore index 3be1e9f..ef43978 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,7 @@ terraform.tfstate* .terraform .minio.sys tmp -docker/conf/certificates/*.crt \ No newline at end of file +docker/conf/certificates/*.crt +*.log +# any presto binary +presto* diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f9aae5..952faf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [0.0.2 UNRELEASED] + +### Added + +- Data examples & create tables +- Documentation + ## [0.0.1] ### Added diff --git a/Makefile b/Makefile index 92b235e..8b617e7 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,10 @@ include dev/.env export export PATH := $(shell pwd)/tmp:$(PATH) +# Presto cli version +PRESTO_VERSION = 340 +presto := ./presto_${PRESTO_VERSION} + .ONESHELL .PHONY: up update-box destroy-box remove-tmp clean example .DEFAULT_GOAL := up @@ -9,6 +13,9 @@ export PATH := $(shell pwd)/tmp:$(PATH) install: mkdir -p tmp;(cd tmp; git clone --depth=1 https://github.com/fredrikhgrelland/vagrant-hashistack.git; cd vagrant-hashistack; make install); rm -rf tmp/vagrant-hashistack +linter: + docker run -e RUN_LOCAL=true -v "${PWD}:/tmp/lint/" github/super-linter + #### Development #### # start commands dev: update-box @@ -36,14 +43,6 @@ else cd template_example; SSL_CERT_FILE=${SSL_CERT_FILE} CURL_CA_BUNDLE=${CURL_CA_BUNDLE} CUSTOM_CA=${CUSTOM_CA} ANSIBLE_ARGS='--extra-vars "local_test=true"' vagrant up --provision endif -template-example: custom_ca -ifdef CI # CI is set in Github Actions - cd template_example; SSL_CERT_FILE=${SSL_CERT_FILE} CURL_CA_BUNDLE=${CURL_CA_BUNDLE} vagrant up --provision -else - if [ -f "docker/conf/certificates/*.crt" ]; then cp -f docker/conf/certificates/*.crt template_example/docker/conf/certificates; fi - cd template_example; SSL_CERT_FILE=${SSL_CERT_FILE} CURL_CA_BUNDLE=${CURL_CA_BUNDLE} CUSTOM_CA=${CUSTOM_CA} ANSIBLE_ARGS='--extra-vars "local_test=true"' vagrant up --provision -endif - # clean commands destroy-box: vagrant destroy -f @@ -53,9 +52,11 @@ remove-tmp: clean: destroy-box remove-tmp +status: + vagrant global-status # helper commands update-box: - @SSL_CERT_FILE=${SSL_CERT_FILE} CURL_CA_BUNDLE=${CURL_CA_BUNDLE} vagrant box update || (echo '\n\nIf you get an SSL error you might be behind a transparent proxy. \nMore info https://github.com/fredrikhgrelland/vagrant-hashistack/blob/master/README.md#if-you-are-behind-a-transparent-proxy\n\n' && exit 2) + @SSL_CERT_FILE=${SSL_CERT_FILE} CURL_CA_BUNDLE=${CURL_CA_BUNDLE} vagrant box update || (echo '\n\nIf you get an SSL error you might be behind a transparent proxy. \nMore info https://github.com/fredrikhgrelland/vagrant-hashistack/blob/master/README.md#proxy\n\n' && exit 2) # to-hivemetastore proxy-h: @@ -64,8 +65,19 @@ proxy-h: proxy-m: consul connect proxy -service minio-local -upstream minio:9000 -log-level debug # to-postgres -proxy-p: +proxy-pg: consul connect proxy -service postgres-local -upstream postgres:5432 -log-level debug + # to-presto -proxy-p: - consul connect proxy -service postgres-local -upstream presto:8080 -log-level debug +proxy-presto: + consul connect proxy -service presto-local -upstream presto:8080 -log-level debug + +download-presto-cli: + curl https://repo1.maven.org/maven2/io/prestosql/presto-cli/${PRESTO_VERSION}/presto-cli-${PRESTO_VERSION}-executable.jar -o ./${presto} + chmod +x ./${presto} + +presto-cli: +ifeq (,$(wildcard ./${presto})) + $(MAKE) download-presto-cli +endif + ${presto} --server localhost:8080 --catalog hive --schema default --user presto diff --git a/README.md b/README.md index 9d9c12b..cdf58d1 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,285 @@ -# Terraform-nomad-presto +

+

Terraform-nomad-presto

+

Terraform module with example

+ + Releases + +

+# + +## Contents +0. [Prerequisites](#prerequisites) +1. [Compatibility](#compatibility) +2. [Usage](#usage) + 1. [Requirements](#requirements) + 1. [Required software](#required-software) + 2. [Providers](#providers) +3. [Inputs](#inputs) +4. [Outputs](#outputs) +5. [Examples](#examples) +6. [Authors](#authors) +7. [License](#license) + +## Prerequisites +Please follow [this section in original template](https://github.com/fredrikhgrelland/vagrant-hashistack-template#install-prerequisites) + +## Compatibility +|Software|OSS Version|Enterprise Version| +|:---|:---|:---| +|Terraform|0.13.1 or newer|| +|Consul|1.8.3 or newer|1.8.3 or newer| +|Vault|1.5.2.1 or newer|1.5.2.1 or newer| +|Nomad|0.12.3 or newer|0.12.3 or newer| + +## Usage + +```text +make up +``` + +Check the example of terraform-nomad-presto documentation [here](./example) + +### Requirements + +#### Required software +See [template README's prerequisites](template_README.md#install-prerequisites). + +Local only. +For verification and debugging: +- [consul](https://releases.hashicorp.com/consul/) binary available on `PATH` on the local machine +- java11 +- [presto.jar](https://prestosql.io/docs/current/installation/cli.html) in root is version 340 + +### Providers +This module uses the [Nomad](https://registry.terraform.io/providers/hashicorp/nomad/latest/docs) provider. + +## Inputs +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| nomad\_provider\_address | Nomad provider address | string | "http://127.0.0.1:4646" | yes | +| nomad\_data\_center | Nomad data centers | list(string) | ["dc1"] | yes | +| nomad\_namespace | [Enterprise] Nomad namespace | string | "default" | yes | +| nomad\_job\_name | Nomad job name | string | "presto" | yes | +| service\_name | Presto service name | string | "presto" | yes | +| port | Presto http port | number | 8080 | yes | +| docker\_image | Presto docker image | string | "prestosql/presto:333" | yes | +| container\_environment\_variables | Presto environment variables | list(string) | [""] | no | +| hivemetastore.service_name | Hive metastore service name | string | "hive-metastore" | yes | +| hivemetastore.port | Hive metastore port | number | 9083 | yes | +| minio.service_name | minio service name | string | | yes | +| minio.port | minio port | number | | yes | +| minio.access_key | minio access key | string | | yes | +| minio.secret_key | minio secret key | string | | yes | + + +## Outputs +| Name | Description | Type | +|------|-------------|------| +| presto\_service\_name | Presto service name | string | +| presto\_port | Presto port | number | + +## Examples +```hcl +module "presto" { + depends_on = [ + module.minio, + module.hive + ] + + source = "github.com/fredrikhgrelland/terraform-nomad-presto.git?ref=0.0.1" + + nomad_job_name = "presto" + nomad_datacenters = ["dc1"] + nomad_namespace = "default" + + service_name = "presto" + port = 8080 + docker_image = "prestosql/presto:333" + + #hivemetastore + hivemetastore = { + service_name = module.hive.service_name + port = 9083 + } + + # minio + minio = { + service_name = module.minio.minio_service_name + port = 9000 + access_key = module.minio.minio_access_key + secret_key = module.minio.minio_secret_key + } +} +``` + +For detailed information check [example/](./example) directory. + +### Verifying setup + +You can verify successful run with next steps: + +#### Option 1 [hive-metastore and nomad] + +* Go to [http://localhost:4646/ui/exec/hive-metastore](http://localhost:4646/ui/exec/hive-metastore) +* Chose metastoreserver -> metastoreserver and click enter. +* Connect using beeline cli +```text +# from metastore (loopback) +beeline -u jdbc:hive2:// +``` +* Query existing tables (beeline-cli) + +```text +SHOW DATABASES; +SHOW TABLES IN ; +DROP DATABASE ; +SELECT * FROM ; + +# examples +SHOW TABLES; +SELECT * FROM iris; +SELECT * FROM tweets; +``` + +#### Option 2 [presto and nomad] +* Go to [http://localhost:4646/ui/exec/presto](http://localhost:4646/ui/exec/presto) +* Chose standalone -> server and click enter. +* Connect using presto-cli +```text +presto +``` +* Query existing tables (presto-cli) +```text +SHOW CATALOGS [ LIKE pattern ] +SHOW SCHEMAS [ FROM catalog ] [ LIKE pattern ] +SHOW TABLES [ FROM schema ] [ LIKE pattern ] + +# examples +SHOW CATALOGS; +SHOW SCHEMAS IN hive; +SHOW TABLES IN hive.default; +SELECT * FROM hive.default.iris; +``` + +#### Option 3 [local presto-cli] +`NB!` Check [required software section](#required-software) first. + +* create local proxy to presto instance with `consul` binary. +```text +make proxy-presto +``` + +* in another terminal run `presto-cli` session +```text +presto --server localhost:8080 --catalog hive --schema default --user presto +``` + +* Query tables (3 tables should be available) +```text +show tables; +select * from ; +``` + +To debug or continue developing you can use [presto cli](https://prestosql.io/docs/current/installation/cli.html) locally. +Some useful commands. +```text +# manual table creation for different file types +presto --server localhost:8080 --catalog hive --schema default --user presto --file ./example/resources/query/csv_create_table.sql +presto --server localhost:8080 --catalog hive --schema default --user presto --file ./example/resources/query/json_create_table.sql +presto --server localhost:8080 --catalog hive --schema default --user presto --file ./example/resources/query/avro_tweets_create_table.sql +``` + +## Authors + +## License + +________ + +## References +- [Blog post](https://towardsdatascience.com/load-and-query-csv-file-in-s3-with-presto-b0d50bc773c9) +- Presto, so far (release 340), [supports only varchar columns](https://github.com/prestosql/presto/pull/920#issuecomment-517593414). + +## File types + +### CSV +```sql +CREATE TABLE iris ( + sepal_length varchar, + sepal_width varchar, + petal_length varchar, + petal_width varchar, + species varchar +) +WITH ( + format = 'CSV', + external_location='s3a://hive/data/csv/', + skip_header_line_count=1 +); +``` + +`NB!` Hive supports csv int types for columns. +You can create a table for `csv` file format using `hive-metastore`. +```sql +CREATE EXTERNAL TABLE iris (sepal_length DECIMAL, sepal_width DECIMAL, +petal_length DECIMAL, petal_width DECIMAL, species STRING) +ROW FORMAT DELIMITED +FIELDS TERMINATED BY ',' +LINES TERMINATED BY '\n' +LOCATION 's3a://hive/data/csv/' +TBLPROPERTIES ("skip.header.line.count"="1"); +``` + +### JSON + +```sql +CREATE TABLE somejson ( + description varchar, + foo ROW ( + bar varchar, + quux varchar, + level1 ROW ( + l2string varchar, + l2struct ROW ( + level3 varchar + ) + ) + ), + wibble varchar, + wobble ARRAY ( + ROW ( + entry int, + EntryDetails ROW ( + details varchar, + details2 int + ) + ) + ) +) +WITH ( + format = 'JSON', + external_location = 's3a://hive/data/json/' +); +``` + +### AVRO + +```sql +CREATE TABLE tweets ( + username varchar, + tweet varchar, + timestamp bigint +) +WITH ( + format = 'AVRO', + external_location='s3a://hive/data/avro-tweet/' +); +``` + +### PROTOBUF +Reference to [using-protobuf-parquet](https://costimuraru.wordpress.com/2018/04/26/using-protobuf-parquet-with-aws-athena-presto-or-hive/) + +todo +```sql + +``` diff --git a/README_template.md b/README_template.md deleted file mode 100644 index fb0e273..0000000 --- a/README_template.md +++ /dev/null @@ -1,51 +0,0 @@ - -

- -> :warning: The text under headlines are examples, and should be removed. -> -> :warning: Remove this line and the one above! - -# Terraform-module name - -## Compatibility -List of compatible versions. E.g. -1. Terraform 0.12.2 or newer -2. Nomad 0.12 or newer - -## Usage - -### Requirements -#### Required software -1. Software 1 -2. Software 2 - -#### Other -Any other requirements. E.g. "This needs to be run on a Debian system" - -### Providers -A description of the providers that the module uses. E.g. "This module uses the [Nomad](https://registry.terraform.io/providers/hashicorp/nomad/latest/docs) and [Vault](https://registry.terraform.io/providers/hashicorp/vault/latest/docs) providers" - -## Inputs -|Name |Description |Type |Default |Required | -|:--|:--|:--|:-:|:-:| -| | |bool |true |yes | - -## Outputs -|Name |Description |Type |Default |Required | -|:--|:--|:--|:-:|:-:| -| | |bool |true |yes | - -### Example -Example-code that shows how to use the module, and, if applicable, its different use cases. -```hcl-terraform -module "example"{ - source = "./" -} -``` - -### Verifying setup -Description of expected end result and how to check it. E.g. "After a successful run Presto should be available at localhost:8080". - -## Authors - -## License \ No newline at end of file diff --git a/conf/nomad/presto.hcl b/conf/nomad/presto.hcl index 732cb79..6f35e87 100644 --- a/conf/nomad/presto.hcl +++ b/conf/nomad/presto.hcl @@ -38,15 +38,15 @@ job "${nomad_job_name}" { } } } -// check { -// task = "server" -// name = "presto-hive-availability" -// type = "script" -// command = "presto" -// args = ["--execute", "SHOW TABLES IN hive.default"] -// interval = "30s" -// timeout = "15s" -// } + check { + task = "server" + name = "presto-hive-availability" + type = "script" + command = "presto" + args = ["--execute", "SHOW TABLES IN hive.default"] + interval = "30s" + timeout = "15s" + } check { expose = true name = "presto-info" diff --git a/dev/ansible/01_build_presto_cli_image.yml b/dev/ansible/01_build_presto_cli_image.yml new file mode 100644 index 0000000..8dcb136 --- /dev/null +++ b/dev/ansible/01_build_presto_cli_image.yml @@ -0,0 +1,8 @@ +- name: Build docker image + docker_image: + name: fredrikhgrelland/presto-cli + build: + dockerfile: Dockerfile.presto-cli + path: /vagrant/docker + pull: false + source: build diff --git a/dev/ansible/03_upload_files.yml b/dev/ansible/03_upload_files.yml new file mode 100644 index 0000000..cbc90a1 --- /dev/null +++ b/dev/ansible/03_upload_files.yml @@ -0,0 +1,55 @@ +- name: Start local proxy on port 9999 to minio service + shell: nohup consul connect proxy -service minio-local -upstream minio:9999 -log-level debug /dev/null 2>&1 & + register: test_output + tags: example-upload + +# To quickly verify that proxy available (without any credentials) +# https://github.com/minio/minio/issues/4851#issuecomment-324543717 +- name: Minio curl example. Verify that proxy to minio available + uri: + url: http://127.0.0.1:9999 + status_code: 403 + register: curl_output + until: curl_output.status == 403 + retries: 10 + delay: 5 + tags: example-upload + +- name: Debug curl_output + debug: + msg: "{{curl_output}}" + tags: example-upload + +- name: Add minio config + shell: mc config host add minio-container http://127.0.0.1:9999 minio minio123 + tags: example-upload + +- name: Upload csv file to `/hive/some/prefix/iris-file.csv` + shell: mc cp ../../example/resources/data/csv/iris.csv minio-container/hive/some/prefix/iris-file.csv + register: csv_copy + retries: 10 + delay: 5 + until: csv_copy.rc == 0 + tags: example-upload + +- name: Verify that file exist on target `/hive/some/prefix/iris-file.csv` + shell: mc cat minio-container/hive/some/prefix/iris-file.csv + failed_when: csv_copy.rc == 1 + tags: example-upload + +- name: Upload `json` examples to target `/hive/prefix/json/` + shell: mc cp --recursive ../../example/resources/data/ minio-container/hive/data/ + register: batch_upload + tags: example-upload + +- name: Verify folder `proto-message/` is uploaded + shell: mc ls minio-container/hive/data --json | jq -s . | jq ".[].key" | grep "proto-message/" + register: uploaded + retries: 10 + delay: 5 + until: uploaded.rc == 0 + tags: example-upload + + + + diff --git a/dev/ansible/05_presto_create_tables.yml b/dev/ansible/05_presto_create_tables.yml new file mode 100644 index 0000000..4a0281d --- /dev/null +++ b/dev/ansible/05_presto_create_tables.yml @@ -0,0 +1,65 @@ +- name: Presto healthchecks pass + uri: + url: http://localhost:8500/v1/health/checks/presto?filter={{ '(Status=="passing")' | urlencode }} + method: GET + return_content: yes + status_code: 200 + body_format: json + register: result_presto + retries: 15 + delay: 15 + until: result_presto.json | length == 3 + tags: example-upload + +- name: Start local proxy on port 8888 to presto service + shell: nohup consul connect proxy -service presto-local -upstream presto:8888 -log-level debug /dev/null 2>&1 & + register: test_output + tags: example-upload + +# To quickly verify that proxy available (without any credentials) +- name: Presto curl example. Verify that proxy to presto available + uri: + url: http://localhost:8888/ui/login.html + status_code: 200 + register: curl_output + until: curl_output.status == 200 + retries: 10 + delay: 5 + tags: example-upload + +- name: Debug curl_output + debug: + msg: "{{curl_output}}" + tags: example-upload + +### Create tables using docker image with presto-cli + +- name: Create CSV table + shell: docker run --network host -v "/vagrant/example/resources/query/csv_create_table.sql:/csv_create_table.sql" fredrikhgrelland/presto-cli ./presto --server localhost:8888 --catalog hive --schema default --user presto --file /csv_create_table.sql + register: docker_output_cmd_csv + tags: example-upload + +- name: Create JSON table + shell: docker run --network host -v "/vagrant/example/resources/query/json_create_table.sql:/json_create_table.sql" fredrikhgrelland/presto-cli ./presto --server localhost:8888 --catalog hive --schema default --user presto --file /json_create_table.sql + register: docker_output_cmd_json + tags: example-upload + +- name: Create AVRO table + shell: docker run --network host -v "/vagrant/example/resources/query/avro_tweets_create_table.sql:/avro_tweets_create_table.sql" fredrikhgrelland/presto-cli ./presto --server localhost:8888 --catalog hive --schema default --user presto --file /avro_tweets_create_table.sql + register: docker_output_cmd_avro + tags: example-upload + +- name: Verify CSV table - available 150 records + shell: docker run --network host fredrikhgrelland/presto-cli ./presto --server localhost:8888 --catalog hive --schema default --user presto --execute "SELECT COUNT(*) FROM iris;" + register: query_csv_result + until: query_csv_result.stdout == "\"150\"" + +- name: Verify JSON table - available 2 records + shell: docker run --network host fredrikhgrelland/presto-cli ./presto --server localhost:8888 --catalog hive --schema default --user presto --execute "SELECT COUNT(*) FROM somejson;" + register: query_json_result + until: query_json_result.stdout == "\"2\"" + +- name: Verify AVRO table - available 2 records + shell: docker run --network host fredrikhgrelland/presto-cli ./presto --server localhost:8888 --catalog hive --schema default --user presto --execute "SELECT COUNT(*) FROM tweets;" + register: query_avro_result + until: query_avro_result.stdout == "\"2\"" diff --git a/docker/Dockerfile.presto-cli b/docker/Dockerfile.presto-cli new file mode 100644 index 0000000..afd9cfb --- /dev/null +++ b/docker/Dockerfile.presto-cli @@ -0,0 +1,10 @@ +FROM openjdk:11-slim + +ENV PRESTO_VERSION=${PRESTO_VERSION:-340} + +# copy certs if has any +COPY conf/certificates/* /usr/local/share/ca-certificates/ +RUN apt-get update && apt-get install -y ca-certificates curl bash && rm -rf /var/lib/apt/lists/* +RUN update-ca-certificates 2>/dev/null || true && echo "NOTE: CA warnings suppressed." +RUN curl https://repo1.maven.org/maven2/io/prestosql/presto-cli/$PRESTO_VERSION/presto-cli-$PRESTO_VERSION-executable.jar -o ./presto +RUN chmod +x ./presto diff --git a/docker/README.md b/docker/README.md index d1ca9db..c9004ca 100644 --- a/docker/README.md +++ b/docker/README.md @@ -2,6 +2,11 @@ Put your Dockerfile and other files relating to a docker-build here. +## [Presto-cli image](./Dockerfile.presto-cli) +Presto-cli image is required for run the example of terraform-nomad-presto module. Main usage is to create tables for uploaded data to Minio. +- Ansible builds local image in [dev/ansible/01_build_presto_cli_image.yml](../dev/ansible/01_build_presto_cli_image.yml) +- Ansible create tables in [dev/ansible/05_presto_create_tables.yml](../dev/ansible/05_presto_create_tables.yml) + ## Building docker image locally If you have docker installed on your machine, you may `cd docker; docker build -t my_image:local .` and build the image. diff --git a/example/README.md b/example/README.md index 77d4817..82cf0f7 100644 --- a/example/README.md +++ b/example/README.md @@ -1,6 +1,20 @@ -# This is a terraform module example +# This is a Presto terraform module example The current directory contains terraform related files that use the module in `../`. See [template_example](../template_example/example/). +## Services +![img](./terraform-nomad-presto.png) + +## Example of uploaded files +Directory `/resources` contains data example with will be loaded to technology stack in the current example. + +```text +├── resources +│ ├── data/ # files that are uploaded to minio +│ ├── query/ # presto query example for uploaded data +│ └── schema/ # schema(s) for data serializers/deserializers +├── ... +``` + ## References - [Creating Modules - official terraform documentation](https://www.terraform.io/docs/modules/index.html) diff --git a/example/main.tf b/example/main.tf index bea8f15..619c9b5 100644 --- a/example/main.tf +++ b/example/main.tf @@ -35,7 +35,7 @@ module "presto" { } module "minio" { - source = "github.com/fredrikhgrelland/terraform-nomad-minio.git?ref=0.0.2" + source = "github.com/fredrikhgrelland/terraform-nomad-minio.git?ref=0.0.3" # nomad nomad_datacenters = local.nomad_datacenters @@ -75,7 +75,7 @@ module "postgres" { } module "hive" { - source = "github.com/fredrikhgrelland/terraform-nomad-hive.git?ref=0.0.1" + source = "github.com/fredrikhgrelland/terraform-nomad-hive.git?ref=0.0.2" # nomad nomad_datacenters = local.nomad_datacenters @@ -86,7 +86,12 @@ module "hive" { # hive hive_service_name = "hive-metastore" hive_container_port = 9083 - hive_container_environment_variables = ["SOME_EXAMPLE=example-value"] + + // support CSV -> https://towardsdatascience.com/load-and-query-csv-file-in-s3-with-presto-b0d50bc773c9 + // metastore.storage.schema.reader.impl=org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader + hive_container_environment_variables = [ + "HIVE_SITE_CONF_metastore_storage_schema_reader_impl=org.apache.hadoop.hive.metastore.SerDeStorageSchemaReader" + ] # hive - minio hive_bucket = { diff --git a/example/resources/data/avro-tweet/tweet1.snappy.avro b/example/resources/data/avro-tweet/tweet1.snappy.avro new file mode 100644 index 0000000..a159efd Binary files /dev/null and b/example/resources/data/avro-tweet/tweet1.snappy.avro differ diff --git a/example/resources/data/avro-tweet/tweet2.snappy.avro b/example/resources/data/avro-tweet/tweet2.snappy.avro new file mode 100644 index 0000000..54d7b7d Binary files /dev/null and b/example/resources/data/avro-tweet/tweet2.snappy.avro differ diff --git a/example/resources/data/csv/iris.csv b/example/resources/data/csv/iris.csv new file mode 100644 index 0000000..6464198 --- /dev/null +++ b/example/resources/data/csv/iris.csv @@ -0,0 +1,151 @@ +sepal_length,sepal_width,petal_length,petal_width,species +5.1,3.5,1.4,0.2,setosa +4.9,3,1.4,0.2,setosa +4.7,3.2,1.3,0.2,setosa +4.6,3.1,1.5,0.2,setosa +5,3.6,1.4,0.2,setosa +5.4,3.9,1.7,0.4,setosa +4.6,3.4,1.4,0.3,setosa +5,3.4,1.5,0.2,setosa +4.4,2.9,1.4,0.2,setosa +4.9,3.1,1.5,0.1,setosa +5.4,3.7,1.5,0.2,setosa +4.8,3.4,1.6,0.2,setosa +4.8,3,1.4,0.1,setosa +4.3,3,1.1,0.1,setosa +5.8,4,1.2,0.2,setosa +5.7,4.4,1.5,0.4,setosa +5.4,3.9,1.3,0.4,setosa +5.1,3.5,1.4,0.3,setosa +5.7,3.8,1.7,0.3,setosa +5.1,3.8,1.5,0.3,setosa +5.4,3.4,1.7,0.2,setosa +5.1,3.7,1.5,0.4,setosa +4.6,3.6,1,0.2,setosa +5.1,3.3,1.7,0.5,setosa +4.8,3.4,1.9,0.2,setosa +5,3,1.6,0.2,setosa +5,3.4,1.6,0.4,setosa +5.2,3.5,1.5,0.2,setosa +5.2,3.4,1.4,0.2,setosa +4.7,3.2,1.6,0.2,setosa +4.8,3.1,1.6,0.2,setosa +5.4,3.4,1.5,0.4,setosa +5.2,4.1,1.5,0.1,setosa +5.5,4.2,1.4,0.2,setosa +4.9,3.1,1.5,0.1,setosa +5,3.2,1.2,0.2,setosa +5.5,3.5,1.3,0.2,setosa +4.9,3.1,1.5,0.1,setosa +4.4,3,1.3,0.2,setosa +5.1,3.4,1.5,0.2,setosa +5,3.5,1.3,0.3,setosa +4.5,2.3,1.3,0.3,setosa +4.4,3.2,1.3,0.2,setosa +5,3.5,1.6,0.6,setosa +5.1,3.8,1.9,0.4,setosa +4.8,3,1.4,0.3,setosa +5.1,3.8,1.6,0.2,setosa +4.6,3.2,1.4,0.2,setosa +5.3,3.7,1.5,0.2,setosa +5,3.3,1.4,0.2,setosa +7,3.2,4.7,1.4,versicolor +6.4,3.2,4.5,1.5,versicolor +6.9,3.1,4.9,1.5,versicolor +5.5,2.3,4,1.3,versicolor +6.5,2.8,4.6,1.5,versicolor +5.7,2.8,4.5,1.3,versicolor +6.3,3.3,4.7,1.6,versicolor +4.9,2.4,3.3,1,versicolor +6.6,2.9,4.6,1.3,versicolor +5.2,2.7,3.9,1.4,versicolor +5,2,3.5,1,versicolor +5.9,3,4.2,1.5,versicolor +6,2.2,4,1,versicolor +6.1,2.9,4.7,1.4,versicolor +5.6,2.9,3.6,1.3,versicolor +6.7,3.1,4.4,1.4,versicolor +5.6,3,4.5,1.5,versicolor +5.8,2.7,4.1,1,versicolor +6.2,2.2,4.5,1.5,versicolor +5.6,2.5,3.9,1.1,versicolor +5.9,3.2,4.8,1.8,versicolor +6.1,2.8,4,1.3,versicolor +6.3,2.5,4.9,1.5,versicolor +6.1,2.8,4.7,1.2,versicolor +6.4,2.9,4.3,1.3,versicolor +6.6,3,4.4,1.4,versicolor +6.8,2.8,4.8,1.4,versicolor +6.7,3,5,1.7,versicolor +6,2.9,4.5,1.5,versicolor +5.7,2.6,3.5,1,versicolor +5.5,2.4,3.8,1.1,versicolor +5.5,2.4,3.7,1,versicolor +5.8,2.7,3.9,1.2,versicolor +6,2.7,5.1,1.6,versicolor +5.4,3,4.5,1.5,versicolor +6,3.4,4.5,1.6,versicolor +6.7,3.1,4.7,1.5,versicolor +6.3,2.3,4.4,1.3,versicolor +5.6,3,4.1,1.3,versicolor +5.5,2.5,4,1.3,versicolor +5.5,2.6,4.4,1.2,versicolor +6.1,3,4.6,1.4,versicolor +5.8,2.6,4,1.2,versicolor +5,2.3,3.3,1,versicolor +5.6,2.7,4.2,1.3,versicolor +5.7,3,4.2,1.2,versicolor +5.7,2.9,4.2,1.3,versicolor +6.2,2.9,4.3,1.3,versicolor +5.1,2.5,3,1.1,versicolor +5.7,2.8,4.1,1.3,versicolor +6.3,3.3,6,2.5,virginica +5.8,2.7,5.1,1.9,virginica +7.1,3,5.9,2.1,virginica +6.3,2.9,5.6,1.8,virginica +6.5,3,5.8,2.2,virginica +7.6,3,6.6,2.1,virginica +4.9,2.5,4.5,1.7,virginica +7.3,2.9,6.3,1.8,virginica +6.7,2.5,5.8,1.8,virginica +7.2,3.6,6.1,2.5,virginica +6.5,3.2,5.1,2,virginica +6.4,2.7,5.3,1.9,virginica +6.8,3,5.5,2.1,virginica +5.7,2.5,5,2,virginica +5.8,2.8,5.1,2.4,virginica +6.4,3.2,5.3,2.3,virginica +6.5,3,5.5,1.8,virginica +7.7,3.8,6.7,2.2,virginica +7.7,2.6,6.9,2.3,virginica +6,2.2,5,1.5,virginica +6.9,3.2,5.7,2.3,virginica +5.6,2.8,4.9,2,virginica +7.7,2.8,6.7,2,virginica +6.3,2.7,4.9,1.8,virginica +6.7,3.3,5.7,2.1,virginica +7.2,3.2,6,1.8,virginica +6.2,2.8,4.8,1.8,virginica +6.1,3,4.9,1.8,virginica +6.4,2.8,5.6,2.1,virginica +7.2,3,5.8,1.6,virginica +7.4,2.8,6.1,1.9,virginica +7.9,3.8,6.4,2,virginica +6.4,2.8,5.6,2.2,virginica +6.3,2.8,5.1,1.5,virginica +6.1,2.6,5.6,1.4,virginica +7.7,3,6.1,2.3,virginica +6.3,3.4,5.6,2.4,virginica +6.4,3.1,5.5,1.8,virginica +6,3,4.8,1.8,virginica +6.9,3.1,5.4,2.1,virginica +6.7,3.1,5.6,2.4,virginica +6.9,3.1,5.1,2.3,virginica +5.8,2.7,5.1,1.9,virginica +6.8,3.2,5.9,2.3,virginica +6.7,3.3,5.7,2.5,virginica +6.7,3,5.2,2.3,virginica +6.3,2.5,5,1.9,virginica +6.5,3,5.2,2,virginica +6.2,3.4,5.4,2.3,virginica +5.9,3,5.1,1.8,virginica diff --git a/example/resources/data/json/ex1.json b/example/resources/data/json/ex1.json new file mode 100644 index 0000000..1830106 --- /dev/null +++ b/example/resources/data/json/ex1.json @@ -0,0 +1 @@ +{"description": "my doc", "foo": {"bar": "baz", "quux": "revlos", "level1": {"l2string": "l2val", "l2struct": {"level3": "l3val"}}}, "wibble": "123", "wobble": [{"entry": 1, "EntryDetails": {"details1": "lazybones", "details2": 414}}]} diff --git a/example/resources/data/json/ex2.json b/example/resources/data/json/ex2.json new file mode 100644 index 0000000..cd273b2 --- /dev/null +++ b/example/resources/data/json/ex2.json @@ -0,0 +1 @@ +{"description": "Another document", "foo": {"bar": "cure", "quux": "this is quux", "level1": {"l2string": "some string", "l2struct": {"level3": "deep lvl3"}}}, "wibble": "123", "wobble": [{"entry": 7, "EntryDetails": {"details1": "some details at 1", "details2": 777}}]} diff --git a/example/resources/data/proto-message/info1.data b/example/resources/data/proto-message/info1.data new file mode 100644 index 0000000..78475d9 --- /dev/null +++ b/example/resources/data/proto-message/info1.data @@ -0,0 +1 @@ +*yohoho and a bottle of rum"*+ \ No newline at end of file diff --git a/example/resources/data/proto-message/info2.data b/example/resources/data/proto-message/info2.data new file mode 100644 index 0000000..95c59dd --- /dev/null +++ b/example/resources/data/proto-message/info2.data @@ -0,0 +1 @@ +’!parrot says my name"îÉ \ No newline at end of file diff --git a/example/resources/query/avro_tweets_create_table.sql b/example/resources/query/avro_tweets_create_table.sql new file mode 100644 index 0000000..eee0a24 --- /dev/null +++ b/example/resources/query/avro_tweets_create_table.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS tweets ( + username varchar, + tweet varchar, + timestamp bigint +) +WITH ( + format = 'AVRO', + external_location='s3a://hive/data/avro-tweet/' +); diff --git a/example/resources/query/csv_create_table.sql b/example/resources/query/csv_create_table.sql new file mode 100644 index 0000000..445d3ef --- /dev/null +++ b/example/resources/query/csv_create_table.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS iris ( + sepal_length varchar, + sepal_width varchar, + petal_length varchar, + petal_width varchar, + species varchar +) +WITH ( + format = 'CSV', + external_location='s3a://hive/data/csv/', + skip_header_line_count=1 +); diff --git a/example/resources/query/json_create_table.sql b/example/resources/query/json_create_table.sql new file mode 100644 index 0000000..a0dcaa5 --- /dev/null +++ b/example/resources/query/json_create_table.sql @@ -0,0 +1,27 @@ +CREATE TABLE IF NOT EXISTS somejson ( + description varchar, + foo ROW ( + bar varchar, + quux varchar, + level1 ROW ( + l2string varchar, + l2struct ROW ( + level3 varchar + ) + ) + ), + wibble varchar, + wobble ARRAY ( + ROW ( + entry int, + EntryDetails ROW ( + details varchar, + details2 int + ) + ) + ) +) +WITH ( + format = 'JSON', + external_location = 's3a://hive/data/json/' +); diff --git a/example/resources/schema/.gitignore b/example/resources/schema/.gitignore new file mode 100644 index 0000000..1f0c37e --- /dev/null +++ b/example/resources/schema/.gitignore @@ -0,0 +1 @@ +cpp/ diff --git a/example/resources/schema/Makefile b/example/resources/schema/Makefile new file mode 100644 index 0000000..ef5ed4d --- /dev/null +++ b/example/resources/schema/Makefile @@ -0,0 +1,26 @@ +# avro +prereq: +ifeq (, $(shell which avro-tools)) + $(error "No avro-tools in $(PATH), install http://avro.apache.org/releases.html#Download ") +endif +ifeq (, $(shell which protoc)) + $(error "No protoc in $(PATH), install https://github.com/protocolbuffers/protobuf/releases ") +endif + +encode-json-2-avro: prereq + avro-tools fromjson --codec snappy --schema-file ./avro/twitter_scheme.avsc ./avro/tweet1.json > ../data/avro-tweet/tweet1.snappy.avro + avro-tools fromjson --codec snappy --schema-file ./avro/twitter_scheme.avsc ./avro/tweet2.json > ../data/avro-tweet/tweet2.snappy.avro + +# protofub +encode-text-2-proto: prereq + mkdir cpp | true + protoc --cpp_out=cpp ./proto/info.proto | true + protoc --encode=Info ./proto/info.proto < ./proto/info1.text > ../data/proto-message/info1.data + protoc --encode=Info ./proto/info.proto < ./proto/info2.text > ../data/proto-message/info2.data + rm -rf cpp +hex: prereq + hexdump -C ../data/proto-message/info1.data + hexdump -C ../data/proto-message/info2.data +decode-bin-2-text: prereq + protoc --decode Info ./proto/info.proto < ../data/proto-message/info1.data + protoc --decode Info ./proto/info.proto < ../data/proto-message/info2.data diff --git a/example/resources/schema/README.md b/example/resources/schema/README.md new file mode 100644 index 0000000..35b07f3 --- /dev/null +++ b/example/resources/schema/README.md @@ -0,0 +1,22 @@ +# Schema +Schema directory contains examples of different types of serialization. +For local testing check [Makefile](./Makefile) + +## AVRO notes +- [Long type problem](https://stackoverflow.com/a/40861211) +- [Union specific](https://stackoverflow.com/a/27499930) +- [How to encode/decode data with avro-tools blogpost](https://www.michael-noll.com/blog/2013/03/17/reading-and-writing-avro-files-from-the-command-line/) + +If you want to play with serialization/deserialization of data using cli: +- [avro-tools](http://avro.apache.org/releases.html#Download) + +`NB` data in directory `data/` is encoded with schemas together. + +## Protobuf notes + +- [Official documentation protobuf v3](https://developers.google.com/protocol-buffers/docs/proto3) +- [How to encode/decode data with protoc blogpost](https://medium.com/@at_ishikawa/cli-to-generate-protocol-buffers-c2cfdf633dce) +- [Protoc installation](https://grpc.io/docs/protoc-installation/) + +If you want to play with serialization/deserialization of data using cli: +- [protoc](https://github.com/protocolbuffers/protobuf/releases) diff --git a/example/resources/schema/avro/tweet1.json b/example/resources/schema/avro/tweet1.json new file mode 100644 index 0000000..b479d3c --- /dev/null +++ b/example/resources/schema/avro/tweet1.json @@ -0,0 +1 @@ +{"username":"miguno","tweet":"Rock: Nerf paper, scissors is fine.","timestamp": 1366150681} diff --git a/example/resources/schema/avro/tweet2.json b/example/resources/schema/avro/tweet2.json new file mode 100644 index 0000000..cb23d00 --- /dev/null +++ b/example/resources/schema/avro/tweet2.json @@ -0,0 +1 @@ +{"username":"BlizzardCS","tweet":"Works as intended. Terran is IMBA.","timestamp": 1366154481} diff --git a/example/resources/schema/avro/twitter_scheme.avsc b/example/resources/schema/avro/twitter_scheme.avsc new file mode 100644 index 0000000..3926609 --- /dev/null +++ b/example/resources/schema/avro/twitter_scheme.avsc @@ -0,0 +1,23 @@ +{ + "type" : "record", + "name" : "twitter_schema", + "namespace" : "no.fredrikhgrelland.terraform.nomad.minio.avro.twitter", + "fields" : [ + { + "name" : "username", + "type" : "string", + "doc" : "Name of the user account on Twitter.com" + }, + { + "name" : "tweet", + "type" : "string", + "doc" : "The content of the user's Twitter message" + }, + { + "name" : "timestamp", + "type" : "long", + "doc" : "Unix epoch time in seconds" + } + ], + "doc:" : "A basic schema for storing Twitter messages" +} diff --git a/example/resources/schema/proto/info.proto b/example/resources/schema/proto/info.proto new file mode 100644 index 0000000..57b681f --- /dev/null +++ b/example/resources/schema/proto/info.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; + +message Info { + int32 id = 1; + bool is_simple = 2; + string name = 3; + repeated int32 sample_lists = 4; +} diff --git a/example/resources/schema/proto/info1.text b/example/resources/schema/proto/info1.text new file mode 100644 index 0000000..d12c94b --- /dev/null +++ b/example/resources/schema/proto/info1.text @@ -0,0 +1,4 @@ +id: 42 +is_simple: false +name: "yohoho and a bottle of rum" +sample_lists: [42, 24, 43] diff --git a/example/resources/schema/proto/info2.text b/example/resources/schema/proto/info2.text new file mode 100644 index 0000000..596304d --- /dev/null +++ b/example/resources/schema/proto/info2.text @@ -0,0 +1,4 @@ +id: 4242 +is_simple: true +name: "parrot says my name" +sample_lists: [42222] diff --git a/example/terraform-nomad-presto.png b/example/terraform-nomad-presto.png new file mode 100644 index 0000000..776cb9e Binary files /dev/null and b/example/terraform-nomad-presto.png differ diff --git a/template_README.md b/template_README.md new file mode 100644 index 0000000..87dfec6 --- /dev/null +++ b/template_README.md @@ -0,0 +1,366 @@ + + +

+ Vagrant-hashistack +

Vagrant-hashistack Template

+

Starter template for fredrikhgrelland/vagrant-hashistack

+

+ + Build + + + Releases + + + Updated + +
+
+

+ + + +

+

+ +## Content +1. [Description - What & Why](#description---what--why) + 1. [Why Does This Exist?](#why-does-this-exist) + 2. [Services](#services) +2. [Install Prerequisites](#install-prerequisites) + 1. [Packages that needs to be pre-installed](#packages-that-needs-to-be-pre-installed) + 1. [MacOS Specific](#macos-specific) + 2. [Ubuntu Specific](#ubuntu-specific) +3. [Configuration](#configuration) + 1. [Startup Scheme](#startup-scheme) + 1. [Detailed Startup Procedure](#detailed-startup-procedure) + 2. [Pre and Post Hashistack Startup Procedure](#pre-and-post-hashistack-startup-procedure) + 1. [Ansible Playbooks Pre and Post Hashistack Startup](#ansible-playbooks-pre-and-post-hashistack-startup) + 2. [Bash Scripts Pre and Post Ansible Playbook](#bash-scripts-pre-and-post-ansible-playbook) + 3. [Pre-packaged Configuration Switches](#pre-packaged-configuration-switches) + 1. [Enterprise vs Open Source Software (OSS)](#enterprise-vs-open-source-software-oss) + 2. [Nomad](#nomad) + 3. [Consul](#consul) + 4. [Vault](#vault) + 1. [Consul Secrets Engine](#consul-secrets-engine) + 2. [Vagrant Box Resources](#vagrant-box-resources) +4. [Usage](#usage) + 1. [Commands](#commands) + 2. [MinIO](#minio) + 1. [Pushing Resources To MinIO With Ansible (Docker image)](#pushing-resources-to-minio-with-ansible-docker-image) + 2. [Fetching Resources From MinIO With Nomad (Docker image)](#fetching-resources-from-minio-with-nomad-docker-image) + 3. [Iteration of the Development Process](#iteration-of-the-development-process) +5. [Test Configuration and Execution](#test-configuration-and-execution) + + +## Description - What & Why +This template is a starting point, and example, on how to take advantage of the [Hashistack vagrant-box](https://app.vagrantup.com/fredrikhgrelland/boxes/hashistack) to create, develop, and test Terraform-modules within the Hashistack ecosystem. + +**Hashistack**, in current repository context, is a set of software products by [HashiCorp](https://www.hashicorp.com/). + + +> :bulb: If you found this in `fredrikhgrelland/vagrant-hashistack`, you may be interested in the separate repository [vagrant-hashistack-template](https://github.com/fredrikhgrelland/vagrant-hashistack-template/). + +> :warning: If you are reading this in your own repository, go to [If This Is in Your Own Repository](#if-this-is-in-your-own-repository) + +### Why Does This Exist? + This template aims to standardize workflow for building and testing terraform-nomad-modules, using the [fredrikhgrelland/hashistack](https://github.com/fredrikhgrelland/vagrant-hashistack) vagrant-box. + + +### Services +The default box will start Nomad, Vault, Consul and MinIO bound to loopback and advertising on the IP `10.0.3.10`, which should be available on your local machine. +Port-forwarding for `nomad` on port `4646` should bind to `127.0.0.1` and should allow you to use the nomad binary to post jobs directly. +Consul and Vault have also been port-forwarded and are available on `127.0.0.1` on ports `8500` and `8200` respectively. +Minio is started on port `9000` and shares the `/vagrant` (your repo) from within the vagrant box. + +|Service|URL|Token(s)| +|:---|:---:|:---:| +|Nomad| [http://10.0.3.10:4646](http://10.0.3.10:4646)|| +|Consul| [http://10.0.3.10:8500](http://10.0.3.10:8500)|master| +|Vault| [http://10.0.3.10:8200](http://10.0.3.10:8200)|master| +|Minio| [http://10.0.3.10:9000](http://10.0.3.10:9000)|minioadmin : minioadmin| + + +## Install Prerequisites + +```text +make install +``` + +The command, will install: +- [VirtualBox](https://www.virtualbox.org/) +- [Packer](https://www.packer.io/) +- [Vagrant](https://www.vagrantup.com/) with additional plugins +- [Additional software dependent on the OS (Linux, MacOS)](../install/Makefile) + +### Packages that needs to be pre-installed + +- [Make](https://man7.org/linux/man-pages/man1/make.1.html) +- [Git CLI](https://git-scm.com/book/en/v2/Getting-Started-The-Command-Line) + +#### MacOS Specific +- Virtualization must be enabled. [This is enabled by default on MacOS.](https://support.apple.com/en-us/HT203296) +- [Homebrew](https://brew.sh/) must be installed. + +#### Ubuntu Specific +- Virtualization must be enabled. [Error if it is not.](https://github.com/fredrikhgrelland/vagrant-hashistack/issues/136) +- Packages [gpg](http://manpages.ubuntu.com/manpages/xenial/man1/gpg.1.html) and [apt](http://manpages.ubuntu.com/manpages/bionic/man8/apt.8.html) must be installed. + +--- + +`NB` _Post installation you might need to reboot your system in order to start the virtual-provider (VirtualBox)_ + +--- + + +## Configuration + +### Startup Scheme +From a thousand foot view the startup scheme will: +1. Start the hashistack and MinIO +2. Run [playbook.yml](dev/ansible/playbook.yml), which in turn runs all ansible-playbooks inside [dev/ansible/](dev/ansible). + +> :bulb: Vagrantfile lines 8-11 run the first playbook on startup, and can be changed. + +> :bulb: Below is a detailed description of the _whole_ startup procedure, both user changeable and not. + +--- + +#### Detailed Startup Procedure +_box_ - Comes bundled with the box, not possible to change + +_system_ - Provided by the system in automated processes, not possible to change + +_user_ - Provided by the user to alter the box or template in some way + +|Seq number| What | Provided by | Description | +|:--:|:------------|:------------:|:-----| +|1 |`/home/vagrant/.env_default`|[ _box_ ]| default variables | +|2 |`/vagrant/.env`|[ _user_ ]| variables override, see [Pre-packaged Configuration Switches](#pre-packaged-configuration-switches) for details | +|3 |`/vagrant/.env_override`|[ _system_ ]| variables are overridden for test purposes | +|4 |`/vagrant/dev/vagrant/conf/pre_ansible.sh`|[ _user_ ]| script running before ansible bootstrap procedure, [details](dev/vagrant/conf/pre_bootstrap/README.md) | +|5 |`/vagrant/dev/vagrant/conf/pre_bootstrap/*.yml`|[ _user_ ]| pre bootstrap tasks, running before hashistack software starts, [details](dev/vagrant/conf/README.md) | +|6 |`/etc/ansible/bootstrap.yml`|[ _box_ ]| verify ansible variables and software configuration, run hashistack software and MinIO, & verify that it started correctly, [link](../ansible/bootstrap.yml) | +|7 |`/vagrant/conf/post_bootstrap/*.yml`|[ _user_ ]| poststart scripts, running after hashistack software has started, [details](dev/vagrant/conf/pre_bootstrap/README.md) | +|8 |`/vagrant/dev/conf/post_ansible.sh`|[ _user_ ]| script running after ansible bootstrap procedure, [details](dev/vagrant/conf/README.md) | +|9 |`/vagrant/ansible/*.yml`|[ _user_ ]| ansible tasks included in playbook, see [Pre-packaged Configuration Switches](#pre-packaged-configuration-switches) for details | + +--- + +### Pre and Post Hashistack Startup Procedure +#### Ansible Playbooks Pre and Post Hashistack Startup +You may change the hashistack configuration or add additional pre and post steps to the ansible startup procedure to match your needs. +Detailed documentation in [dev/vagrant/conf/README.md](dev/vagrant/conf/README.md) + +#### Bash Scripts Pre and Post Ansible Playbook +In addition to ansible playbooks, you can also add bash-scripts that will be run before and/or after the ansible provisioning step. This is useful for doing deeper changes to the box pertaining to your needs. Detailed documentation in [dev/vagrant/conf/README.md](dev/vagrant/conf/README.md) + + +### Pre-packaged Configuration Switches + +The box comes [with a set of configuration switches controlled by env variables](https://github.com/fredrikhgrelland/vagrant-hashistack#configuration) to simplify testing of different scenarios and enable staged development efforts. +To change any of these values from their defaults, you may add the environment variable to [.env](dev/.env). + +NB: All lowercase variables will automatically get a corresponding `TF_VAR_` prepended variant for use directly in terraform. [Script](../.github/action/create-env.py) + +#### Enterprise vs Open Source Software (OSS) +To use enterprise versions of the hashistack components set the software's corresponding Enterprise-variable to `true` (see below). + +#### Nomad + +| default | environment variable | value | +|:---------:|:----------------------|:-------:| +| | nomad_enterprise | true | +| x | nomad_enterprise | false | +| | nomad_acl | true | +| x | nomad_acl | false | + +When ACLs are enabled in Nomad the bootstrap token will be available in vault under `secret/nomad/management-token` with the two key-value pairs `accessor-id` and `secret-id`. `secret-id` is the token itself. These can be accessed in several ways: +- From inside the vagrant box with `vault kv get secret/nomad-bootstrap-token` +- From local machine with `vagrant ssh -c vault kv get secret/nomad-bootstrap-token"` +- By going to vault's UI on `localhost:8200`, and signing in with the root token. + +#### Consul + +| default | environment variable | value | +|:---------:|:---------------------------------|:-------:| +| | consul_enterprise | true | +| x | consul_enterprise | false | +| x | consul_acl | true | +| | consul_acl | false | +| x | consul_acl_default_policy | allow | +| | consul_acl_default_policy | deny | + +#### Vault + +| default | environment variable | value | +|:---------:|:---------------------------------|:-------:| +| | vault_enterprise | true | +| x | vault_enterprise | false | + +##### Consul Secrets Engine + +If `consul_acl_default_policy` has value `deny`, it will also enable [consul secrets engine](https://www.vaultproject.io/docs/secrets/consul) in vault. +Ansible will provision additional custom roles (admin-team, dev-team), [policies](../ansible/templates/consul-policies) and tokens for test purpose with different access level. + +How to generate token: +```text +# generate token for dev team member +vagrant ssh -c 'vault read consul/creds/dev-team' + +# generate token for admin team member +vagrant ssh -c 'vault read consul/creds/admin-team' +``` + +> :bulb: Tokens can be used to access UI (different access level depends on policy attached to the token) + +### Vagrant Box Resources +If you get the error message `Dimension memory exhausted on 1 node` or `Dimension CPU exhausted on 1 node`, you might want to increase resources dedicated to your vagrant-box. +To overwrite the default resource-configuration you can add the lines +```hcl +Vagrant.configure("2") do |config| + config.vm.provider "virtualbox" do |vb| + vb.memory = 2048 + vb.cpu = 2 + end +end +``` +to the bottom of your [Vagrantfile](Vagrantfile), and change `vb.memory` and `vb.cpu` to suit your needs. Any configuration in [Vagrantfile](Vagrantfile) will overwrite the defaults if there is any. [More configuration options](https://www.vagrantup.com/docs/providers/virtualbox/configuration.html). + +> :bulb: The defaults can be found in [Vagrantfile.default](Vagrantfile.default). + + +## Usage +### Commands +There are several commands that help to run the vagrant-box: +- `make install` installs all prerequisites. Run once. + +- `make up` provisions a [vagrant-hashistack](https://github.com/fredrikhgrelland/vagrant-hashistack/) box on your machine. After the machine and hashistack are set up it will run the [Startup Scheme](#startup-scheme). + +- `make clean` takes down the provisioned box if there is any. + +- `make update` downloads the newest version of the [vagrant-hashistack box](https://github.com/fredrikhgrelland/vagrant-hashistack/) from [vagrantcloud](https://vagrantcloud.com/fredrikhgrelland/hashistack). + +- `make example` runs the example in [template_example/](template_example) + +> :bulb: For full info, check [`template/Makefile`](./Makefile). +> :warning: Makefile commands are not idempotent in the context of vagrant-box. You could face the error of port collisions. Most of the cases it could happen because of the vagrant box has already been running. Run `vagrant destroy -f` to destroy the box. + +Once vagrant-box is running, you can use other [options like the Nomad- and Terraform-CLIs to iterate over the deployment in the development stage](#iteration-of-the-development-process). + +### MinIO +Minio S3 can be used as a general artifact repository while building and testing within the scope of the vagrantbox to push, pull and store resources for further deployments. + +> :warning: Directory `/vagrant` is mounted to minio. Only first level of sub-directories become bucket names. + +Resource examples: +- docker images +- compiled binaries +- jar files +- etc... + +#### Pushing Resources To MinIO With Ansible (Docker image) +Push(archive) of docker image. +```yaml +# NB! Folder /vagrant is mounted to Minio +# Folder `dev` is going to be a bucket name +- name: Create tmp if it does not exist + file: + path: /vagrant/dev/tmp + state: directory + mode: '0755' + owner: vagrant + group: vagrant + +- name: Archive docker image + docker_image: + name: docker_image + tag: local + archive_path: /vagrant/dev/tmp/docker_image.tar + source: local +``` +[Full example](template_example/dev/ansible/01_build_docker_image.yml) + +#### Fetching Resources From MinIO With Nomad (Docker image) +> :bulb: [The artifact stanza](https://www.nomadproject.io/docs/job-specification/artifact) instructs Nomad to fetch and unpack a remote resource, such as a file, tarball, or binary. + +Example: +```hcl +task "web" { + driver = "docker" + artifact { + source = "s3::http://127.0.0.1:9000/dev/tmp/docker_image.tar" + options { + aws_access_key_id = "minioadmin" + aws_access_key_secret = "minioadmin" + } + } + config { + load = "docker_image.tar" + image = "docker_image:local" + } +} +``` +[Full example](./template_example/conf/nomad/countdash.hcl) + +### Iteration of the Development Process + +Once you start the box with one of the commands `make dev`, `make up` or `make example`, +you need a simple way how to continuously deploy development changes. + +There are several options: + +1. **From the local machine**. You can install Hashicorp binaries on the local machine, such as terraform and nomad. +Then you can deploy changes to the vagrant-box using these binaries. + +Example terraform: +```text +terraform init +terraform apply +``` + +Example nomad: +```text +nomad job run countdash.hcl +``` + +> :warning: _Your local binaries and the binaries in the box might not be the same versions, and may behave differently. [Box versions.](../ansible/group_vars/all/variables.yml) + +2. **Using vagrant**. Box instance has all binaries are installed and available in the PATH. +You can use `vagrant ssh` to place yourself inside of the vagrantbox and run commands. + +```text +# remote command execution +vagrant ssh default -c 'cd /vagrant; terraform init; terraform apply' + +# ssh inside the box, local command execution +vagrant ssh default +cd /vagrant +terraform init +terraform apply +``` + +> :bulb: `default` is the name of running VM. You could also use VM `id`. +To get vm `id` check `vagrant global-status`. + +## Test Configuration and Execution +The tests are run using [Github Actions](https://github.com/features/actions) feature which makes it possible to automate, customize, and execute the software development workflows right in the repository. We utilize the **matrix testing strategy** to cover all the possible and logical combinations of the different properties and values that the components support. The .env_override file is used by the tests to override the values that are available in the .env_default file, as well as the user configurable .env file. + + +As of today, the following tests are executed: + +| Test name | Consul Acl | Consul Acl Policy | Nomad Acl | Hashicorp binary +|:------------------------------------------------------------------------------------------:|:------------|:-------------------:|:-------------:|:---------------:| +| test (consul_acl_enabled, consul_acl_deny, nomad_acl_enabled, hashicorp_oss) | true | deny | true | Open source | +| test (consul_acl_enabled, consul_acl_deny, nomad_acl_enabled, hashicorp_enterprise) | true | deny | true | enterprise | +| test (consul_acl_enabled, consul_acl_deny, nomad_acl_disabled, hashicorp_oss) | true | deny | false | Open source | +| test (consul_acl_enabled, consul_acl_deny, nomad_acl_disabled, hashicorp_enterprise) | true | deny | false | enterprise | +| test (consul_acl_disabled, consul_acl_deny, nomad_acl_enabled, hashicorp_oss) | false | deny | true | Open source | +| test (consul_acl_disabled, consul_acl_deny, nomad_acl_enabled, hashicorp_enterprise) | false | deny | true | enterprise | +| test (consul_acl_disabled, consul_acl_deny, nomad_acl_disabled, hashicorp_oss) | false | deny | false | Open source | +| test (consul_acl_disabled, consul_acl_deny, nomad_acl_disabled, hashicorp_enterprise) | false | deny | false | enterprise | + +The latest test results can be looked up under the **Actions** tab. + +## If This Is in Your Own Repository +If you are reading this from your own repository you should _delete_ this `README.md`, fill out `README_template.md`, and rename `README_template.md` to `README.md`. diff --git a/test_example/.env b/test_example/.env deleted file mode 100644 index 16ca13c..0000000 --- a/test_example/.env +++ /dev/null @@ -1,4 +0,0 @@ -#Control box features -#consul_acl=true -#consul_acl_default_policy=allow -#nomad_acl=false \ No newline at end of file diff --git a/test_example/Vagrantfile b/test_example/Vagrantfile deleted file mode 100644 index e9d6a3d..0000000 --- a/test_example/Vagrantfile +++ /dev/null @@ -1,11 +0,0 @@ -Vagrant.configure("2") do |config| - config.vm.box = "fredrikhgrelland/hashistack" - config.vm.provider "virtualbox" do |vb| - vb.linked_clone = true - vb.memory = 2048 - end - config.vm.provision "ansible_local" do |ansible| - ansible.provisioning_path = "/vagrant/dev/ansible" - ansible.playbook = "playbook.yml" # Note this playbook is, in this context, /ansible/playbook.yml - end -end diff --git a/test_example/conf/nomad/countdash.hcl b/test_example/conf/nomad/countdash.hcl deleted file mode 100644 index d7a2890..0000000 --- a/test_example/conf/nomad/countdash.hcl +++ /dev/null @@ -1,84 +0,0 @@ -job "countdash" { - datacenters = ["dc1"] - group "api" { - network { - mode = "bridge" - } - - service { - name = "count-api" - port = "9001" - - connect { - sidecar_service {} - } - check { - expose = true - name = "api-alive" - type = "http" - path = "/health" - interval = "10s" - timeout = "2s" - } - } - - task "web" { - driver = "docker" - artifact { - source = "s3::http://127.0.0.1:9000/dev/tmp/docker_image.tar" - options { - aws_access_key_id = "minioadmin" - aws_access_key_secret = "minioadmin" - } - } - config { - load = "docker_image.tar" - image = "docker_image:local" - } - } - } - - group "dashboard" { - network { - mode ="bridge" - port "http" { - static = 9002 - to = 9002 - } - } - - service { - name = "count-dashboard" - port = "9002" - - connect { - sidecar_service { - proxy { - upstreams { - destination_name = "count-api" - local_bind_port = 8080 - } - } - } - } - check { - expose = true - name = "dashboard-alive" - type = "http" - path = "/health" - interval = "10s" - timeout = "2s" - } - } - - task "dashboard" { - driver = "docker" - env { - COUNTING_SERVICE_URL = "http://${NOMAD_UPSTREAM_ADDR_count_api}" - } - config { - image = "hashicorpnomad/counter-dashboard:v1" - } - } - } -} diff --git a/test_example/dev/ansible/01_build_docker_image.yml b/test_example/dev/ansible/01_build_docker_image.yml deleted file mode 100644 index db4e7cd..0000000 --- a/test_example/dev/ansible/01_build_docker_image.yml +++ /dev/null @@ -1,32 +0,0 @@ -- name: Remove docker image - docker_image: - name: docker_image - tag: local - force_absent: true - state: absent - -- name: Build docker image - docker_image: - name: docker_image - tag: local - build: - path: /vagrant/docker - pull: false - args: - TEST_DOWNLOAD_BUILD_ARGUMENT: https://nrk.no - source: build - -- name: Create tmp if it does not exist - file: - path: /vagrant/dev/tmp - state: directory - mode: '0755' - owner: vagrant - group: vagrant - -- name: Archive docker image - docker_image: - name: docker_image - tag: local - archive_path: /vagrant/dev/tmp/docker_image.tar - source: local \ No newline at end of file diff --git a/test_example/dev/ansible/02_run_terraform.yml b/test_example/dev/ansible/02_run_terraform.yml deleted file mode 100644 index cc381f2..0000000 --- a/test_example/dev/ansible/02_run_terraform.yml +++ /dev/null @@ -1,10 +0,0 @@ -- name: Terraform - terraform: - project_path: ../../example - force_init: true - state: present - register: terraform - -- name: Terraform stdout - debug: - msg: "{{terraform.stdout}}" \ No newline at end of file diff --git a/test_example/dev/ansible/playbook.yml b/test_example/dev/ansible/playbook.yml deleted file mode 100644 index 5ccb21b..0000000 --- a/test_example/dev/ansible/playbook.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -- hosts: all - become: yes - tasks: - - name: Include and run tasks - include_tasks: "{{ item }}" - loop: "{{ query('fileglob', \"/vagrant/dev/ansible/*.yml\", wantlist=true) | reject('search','playbook.yml') | list | sort }}" \ No newline at end of file diff --git a/test_example/dev/vagrant/conf/consul/99-override.hcl b/test_example/dev/vagrant/conf/consul/99-override.hcl deleted file mode 100644 index d2b8041..0000000 --- a/test_example/dev/vagrant/conf/consul/99-override.hcl +++ /dev/null @@ -1 +0,0 @@ -#Any vaild configuration from https://www.consul.io/docs/agent/options.html#configuration_files \ No newline at end of file diff --git a/test_example/dev/vagrant/conf/nomad/99-override.hcl b/test_example/dev/vagrant/conf/nomad/99-override.hcl deleted file mode 100644 index d817340..0000000 --- a/test_example/dev/vagrant/conf/nomad/99-override.hcl +++ /dev/null @@ -1 +0,0 @@ -#Any vaild configuration from https://www.nomadproject.io/docs/configuration#general-parameters \ No newline at end of file diff --git a/test_example/dev/vagrant/conf/post_ansible.sh b/test_example/dev/vagrant/conf/post_ansible.sh deleted file mode 100644 index d09910c..0000000 --- a/test_example/dev/vagrant/conf/post_ansible.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -echo "Running /vagrant/dev/vagrant/conf/post_ansible.sh" \ No newline at end of file diff --git a/test_example/dev/vagrant/conf/post_bootstrap/00-poststart-example.yml b/test_example/dev/vagrant/conf/post_bootstrap/00-poststart-example.yml deleted file mode 100644 index 7bf4f56..0000000 --- a/test_example/dev/vagrant/conf/post_bootstrap/00-poststart-example.yml +++ /dev/null @@ -1,3 +0,0 @@ -- name: Task that shows usage of poststart - debug: - msg: This is the first poststart task ( 00-poststart-example.yml ) \ No newline at end of file diff --git a/test_example/dev/vagrant/conf/post_bootstrap/01-poststart-example.yml b/test_example/dev/vagrant/conf/post_bootstrap/01-poststart-example.yml deleted file mode 100644 index 4fc2a1b..0000000 --- a/test_example/dev/vagrant/conf/post_bootstrap/01-poststart-example.yml +++ /dev/null @@ -1,3 +0,0 @@ -- name: Task that shows usage of poststart - debug: - msg: This is the second poststart task ( 01-poststart-example.yml ) \ No newline at end of file diff --git a/test_example/dev/vagrant/conf/post_bootstrap/02-poststart-example.yml b/test_example/dev/vagrant/conf/post_bootstrap/02-poststart-example.yml deleted file mode 100644 index fc045ea..0000000 --- a/test_example/dev/vagrant/conf/post_bootstrap/02-poststart-example.yml +++ /dev/null @@ -1,3 +0,0 @@ -- name: Task that shows usage of poststart - debug: - msg: This is the third poststart task ( 02-poststart-example.yml ) \ No newline at end of file diff --git a/test_example/dev/vagrant/conf/pre_ansible.sh b/test_example/dev/vagrant/conf/pre_ansible.sh deleted file mode 100644 index 4695070..0000000 --- a/test_example/dev/vagrant/conf/pre_ansible.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -echo "Running /vagrant/dev/vagrant/conf/pre_ansible.sh" \ No newline at end of file diff --git a/test_example/dev/vagrant/conf/pre_bootstrap/00-prestart-example.yml b/test_example/dev/vagrant/conf/pre_bootstrap/00-prestart-example.yml deleted file mode 100644 index 4bcb7a7..0000000 --- a/test_example/dev/vagrant/conf/pre_bootstrap/00-prestart-example.yml +++ /dev/null @@ -1,3 +0,0 @@ -- name: Task that shows usage of prestart - debug: - msg: This is the first prestart task ( 00-poststart-example.yml ) \ No newline at end of file diff --git a/test_example/dev/vagrant/conf/pre_bootstrap/01-prestart-example.yml b/test_example/dev/vagrant/conf/pre_bootstrap/01-prestart-example.yml deleted file mode 100644 index 07002cc..0000000 --- a/test_example/dev/vagrant/conf/pre_bootstrap/01-prestart-example.yml +++ /dev/null @@ -1,3 +0,0 @@ -- name: Task that shows usage of prestart - debug: - msg: This is the second prestart task ( 01-poststart-example.yml ) \ No newline at end of file diff --git a/test_example/dev/vagrant/conf/pre_bootstrap/02-prestart-example.yml b/test_example/dev/vagrant/conf/pre_bootstrap/02-prestart-example.yml deleted file mode 100644 index deeb4d3..0000000 --- a/test_example/dev/vagrant/conf/pre_bootstrap/02-prestart-example.yml +++ /dev/null @@ -1,3 +0,0 @@ -- name: Task that shows usage of prestart - debug: - msg: This is the third prestart task ( 02-poststart-example.yml ) \ No newline at end of file diff --git a/test_example/dev/vagrant/conf/vault/99-override.hcl b/test_example/dev/vagrant/conf/vault/99-override.hcl deleted file mode 100644 index 2a828c6..0000000 --- a/test_example/dev/vagrant/conf/vault/99-override.hcl +++ /dev/null @@ -1 +0,0 @@ -#Any vaild configuration from https://www.vaultproject.io/docs/configuration \ No newline at end of file diff --git a/test_example/docker/.dockerignore b/test_example/docker/.dockerignore deleted file mode 100644 index bdef5a7..0000000 --- a/test_example/docker/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -* -!bin -!lib -!conf \ No newline at end of file diff --git a/test_example/docker/Dockerfile b/test_example/docker/Dockerfile deleted file mode 100644 index b8ca278..0000000 --- a/test_example/docker/Dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -########################################## -########### READ THIS FIRST ############## -########################################## -## This docker build is special... ## -## It wil leverage multi stage builds ## -## to test centos/debian/alpine ## -## certificate trust before building ## -## the docker image used in the example ## -########################################## - -FROM centos:8 - -# Allow buildtime config -ARG TEST_DOWNLOAD_BUILD_ARGUMENT=https://nrk.no - -#Add ca_certificates to the image ( if trust is not already added through base image ) -COPY conf/certificates /usr/share/pki/ca-trust-source/anchors/ - -#Install certs -RUN \ - #Update CA_Certs - update-ca-trust 2>/dev/null || true && echo "NOTE: CA warnings suppressed." \ - #Test download ( does ssl trust work ) - && curl -s -L -o /dev/null ${TEST_DOWNLOAD_BUILD_ARGUMENT} || printf "\n###############\nERROR: You are probably behind a corporate proxy. Add your custom ca .crt in the conf/certificates docker build folder\n###############\n" - -FROM debian:stretch - -# Allow buildtime config -ARG TEST_DOWNLOAD_BUILD_ARGUMENT=https://nrk.no - -#Add ca_certificates to the image ( if trust is not already added through base image ) -COPY conf/certificates /usr/local/share/ca-certificates - -#Install certs -# hadolint ignore=DL3015 -RUN \ - #Update CA_Certs - apt-get update && apt-get install -y curl=7.52.1-5+deb9u11 && rm -rf /var/lib/apt/lists/* \ - && update-ca-certificates 2>/dev/null || true && echo "NOTE: CA warnings suppressed." \ - #Test download ( does ssl trust work ) - && curl -s -L -o /dev/null ${TEST_DOWNLOAD_BUILD_ARGUMENT} || printf "\n###############\nERROR: You are probably behind a corporate proxy. Add your custom ca .crt in the conf/certificates docker build folder\n###############\n" - - -FROM hashicorpnomad/counter-api:v1 - -# Allow buildtime config -ARG TEST_DOWNLOAD_BUILD_ARGUMENT=https://nrk.no - -#Add ca_certificates to the image ( if trust is not already added through base image ) -COPY conf/certificates /usr/local/share/ca-certificates - -RUN apk --no-cache add curl=~7 ca-certificates=~20190108 \ - && find /usr/local/share/ca-certificates -not -name "*.crt" -type f -delete \ - && update-ca-certificates 2>/dev/null || true && echo "NOTE: CA warnings suppressed." \ - # Test download - && curl -s -L -o /dev/null ${TEST_DOWNLOAD_BUILD_ARGUMENT} || printf "\n###############\nERROR: You are probably behind a corporate proxy. Add your custom ca .crt in the conf/certificates docker build folder\n###############\n" diff --git a/test_example/docker/conf/certificates/.gitignore b/test_example/docker/conf/certificates/.gitignore deleted file mode 100644 index ee0cb61..0000000 --- a/test_example/docker/conf/certificates/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.crt \ No newline at end of file diff --git a/test_example/docker/conf/certificates/README.md b/test_example/docker/conf/certificates/README.md deleted file mode 100644 index 5829d33..0000000 --- a/test_example/docker/conf/certificates/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# Custom CA certificates -If the env variable `CUSTOM_CA` is pointing to a certificate file, it will be copied into this directory by the Makefile. \ No newline at end of file diff --git a/test_example/example/main.tf b/test_example/example/main.tf deleted file mode 100644 index 6bb90a1..0000000 --- a/test_example/example/main.tf +++ /dev/null @@ -1,7 +0,0 @@ -module "countdash" { - source = "./.." -} - -provider "vault" { - address = "http://127.0.0.1:8200" -} \ No newline at end of file diff --git a/test_example/example/nomad_acl_test.tf b/test_example/example/nomad_acl_test.tf deleted file mode 100644 index bcf3edf..0000000 --- a/test_example/example/nomad_acl_test.tf +++ /dev/null @@ -1,12 +0,0 @@ -data "vault_generic_secret" "nomad_secret_id" { - # Set count of this data source to 1 if ACLs are enabled in Nomad, and 0 if not - count = var.nomad_acl ? 1 : 0 - path = "nomad/creds/write" -} - -provider "nomad" { - address = "http://127.0.0.1:4646" - # Add a secret_id if ACLs are enabled in nomad - secret_id = var.nomad_acl ? data.vault_generic_secret.nomad_secret_id[0].data.secret_id : null -} - diff --git a/test_example/example/variables.tf b/test_example/example/variables.tf deleted file mode 100644 index 55cbd1b..0000000 --- a/test_example/example/variables.tf +++ /dev/null @@ -1,3 +0,0 @@ -variable "nomad_acl" { - type = bool -} \ No newline at end of file diff --git a/test_example/main.tf b/test_example/main.tf deleted file mode 100644 index 3a0df53..0000000 --- a/test_example/main.tf +++ /dev/null @@ -1,4 +0,0 @@ -resource "nomad_job" "countdash" { - jobspec = file("${path.module}/conf/nomad/countdash.hcl") - detach = false -} \ No newline at end of file diff --git a/test_example/outputs.tf b/test_example/outputs.tf deleted file mode 100644 index a711302..0000000 --- a/test_example/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "nomad_job" { - value = nomad_job.countdash -} \ No newline at end of file diff --git a/test_example/variables.tf b/test_example/variables.tf deleted file mode 100644 index e69de29..0000000