Updates to make the sidecare do all the things (#5)

.envrc

@@ -0,0 +1,3 @@
+# shellcheck shell=bash
+
+dotenv

.gitignore

@@ -0,0 +1 @@
+.env

Dockerfile

@@ -1,11 +1,4 @@
-# Download and verify the integrity of the download first
-FROM sethvargo/hashicorp-installer:0.1.3 AS installer
-ARG CONSUL_VERSION='1.0.6'
-ARG VAULT_VERSION='0.10.4'
-#RUN /install-hashicorp-tool "vault" "$VAULT_VERSION"
-RUN /install-hashicorp-tool "consul" "$CONSUL_VERSION"
-
-FROM alpine:3.6
+FROM asicsdigital/hermes:stable
 RUN apk -v --update --no-cache add \
         bash \
         python \
@@ -22,11 +15,9 @@ RUN apk -v --update --no-cache add \
         dumb-init \
         && \
     pip install --upgrade awscli==1.14.5 s3cmd==2.0.1 python-magic && \
-    apk -v --purge del py-pip && \
-    rm /var/cache/apk/*
+    apk -v --purge del py-pip
+    #rm /var/cache/apk/*
 
-COPY --from=installer /software/consul /bin/consul
-#COPY --from=installer /software/vault /bin/vault
 COPY scripts/*.sh /usr/local/bin/
 COPY check_definitions/*.sh /usr/local/bin/check_definitions/
 COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh

README.md

@@ -18,6 +18,13 @@ This deploys the check definition into a local `/consul_check_definitions` direc
 
 - `CHECKS` - A JSON list of check names to activate. in the form of `CHECKS="foo bar"`
 
+#### Supported checks
+
+- `backup` - This check adds a backup job to the local container that backs up the consul node to an s3 bucket. This check requires that `S3_BUCKET` env var is passed and that the ECS Task has permissions to write to the bucket. It adds the check to the ECS Cluster. If the script can't find an ECS Cluster, it will create a service called default.
+
+- `ecs-cluster` - this check creates a service in Consul with the ECS Cluster name, and adds a number of checks, AMI Status, to validate that the AMI is the latest AMI. ECS CloudWatch, which is a cloudwatch metric used for tracking Cluster availibility, and Instance Status, a check used to terminate the consul process in the case of the underlying instance terminating.
+
+
 Usage
 -----
 

check_definitions/backup.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+ECS_CLUSTER=$(curl -s http://localhost:51678/v1/metadata | jq -r .Cluster)
+if  [ -z "$ECS_CLUSTER" ]; then
+   ECS_CLUSTER="default"
+fi
+DOCKER_ID=$(awk -F/ '{ print $NF }' /proc/1/cpuset)
+BACKUP_INTERVAL=${1:-3600}
+
+_SERVICE=$(cat <<EOT
+{
+  "service": {
+    "name": "${ECS_CLUSTER}",
+    "address": "",
+    "tags": [
+      "backup"
+    ],
+    "checks": [
+      {
+        "id": "consul-backup-job",
+        "name": "Consul Backups",
+        "notes": "Job that run to create a consul snapshot and backup to s3",
+        "docker_container_id": "${DOCKER_ID}",
+        "shell": "/bin/bash",
+        "args": ["/usr/local/bin/backup.sh"],
+        "interval": "${BACKUP_INTERVAL}s",
+        "status": "passing"
+      }
+    ]
+  }
+}
+EOT
+)
+
+echo ${_SERVICE}
+
+
+
+#{
+#  "service": {
+#    "name": "ConsulBackup",
+#    "checks": [
+#      {
+#        "script": "/usr/local/bin/consul-backup.sh",
+#        "status": "passing",
+#        "interval": "3600s"
+#      }
+#    ]
+#  }
+#}

check_definitions/docker-test-critical.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-DOCKER_ID=$(head -1 /proc/self/cgroup  | cut -d'/' -f4)
+DOCKER_ID=$(awk -F/ '{ print $NF }' /proc/1/cpuset)
 
 _CHECK=$(cat <<EOT
 {

check_definitions/docker-test-warning.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-DOCKER_ID=$(head -1 /proc/self/cgroup  | cut -d'/' -f4)
+DOCKER_ID=$(awk -F/ '{ print $NF }' /proc/1/cpuset)
 
 _CHECK=$(cat <<EOT
 {

check_definitions/docker-test.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-DOCKER_ID=$(head -1 /proc/self/cgroup  | cut -d'/' -f4)
+DOCKER_ID=$(awk -F/ '{ print $NF }' /proc/1/cpuset)
 
 _CHECK=$(cat <<EOT
 {

check_definitions/ecs-cluster.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 ECS_CLUSTER=$(curl -s http://localhost:51678/v1/metadata | jq -r .Cluster)
-DOCKER_ID=$(head -1 /proc/self/cgroup  | cut -d'/' -f4)
+DOCKER_ID=$(awk -F/ '{ print $NF }' /proc/1/cpuset)
 
 _SERVICE=$(cat <<EOT
 {

docker-compose.yml

@@ -7,7 +7,17 @@ services:
         source: dummy-check
         target: /consul_check_definitions
       - "/var/run/docker.sock:/var/run/docker.sock"
+      - "${HOME}/.aws/credentials:/root/.aws/credentials"
+    environment:
+      #CHECKS: docker-test-warning docker-test
+      CHECKS: docker-test backup
+      AWS_PROFILE: ${AWS_PROFILE}
+      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID}
+      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY}
+      AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION}
+      S3_BUCKET: somebucket-${USER}
   consul:
+    #image: "fitnesskeeper/consul"
     image: "consul:1.0.3"
     environment:
       CONSUL_BIND_INTERFACE: eth0

scripts/backup.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+: ${S3_BUCKET}
+CONSUL_HTTP_ADDR="http://$(dudewheresmy hostip):8500"
+export CONSUL_HTTP_ADDR
+DC=$(curl -s ${CONSUL_HTTP_ADDR}/v1/catalog/datacenters | jq -r .[])
+FILE="/tmp/$(hostname).snap"
+
+if [ ${S3_BUCKET} ]; then
+  consul snapshot save $FILE
+  /usr/bin/aws s3 mv ${FILE} s3://${S3_BUCKET}/${DC}/consul/
+else
+  echo "S3_BUCKET is not set, consul-backup.sh is disabled."
+  exit 0
+fi

scripts/instance-status-handler.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+# shellcheck shell=sh
+# We probably want to replace this with something that eats json and do this in a non-horrible way
+
+sleep 10
+/usr/local/bin/instance-status.sh
+
+RC=$?
+
+if [ $RC = 255 ]; then
+  sleep 90
+  /bin/consul maint -enable
+  /bin/consul leave
+else
+  echo InstanceStatus is ok
+  exit 0
+fi