Merge branch 'release/1.11.0'

README.rst

@@ -59,13 +59,13 @@ Instead of installing **ecs-deploy** locally, which requires a Python environmen
 
 Running **ecs-deploy** via Docker is easy as::
 
-    docker run fabfuel/ecs-deploy:1.7.1
+    docker run fabfuel/ecs-deploy:1.10.2
     
-In this example, the stable version 1.7.1 is executed. Alternatively you can use Docker tags ``master`` or ``latest`` for the latest stable version or Docker tag ``develop`` for the newest development version of **ecs-deploy**.
+In this example, the stable version 1.10.2 is executed. Alternatively you can use Docker tags ``master`` or ``latest`` for the latest stable version or Docker tag ``develop`` for the newest development version of **ecs-deploy**.
 
 Please be aware, that when running **ecs-deploy** via Docker, the configuration - as described below - does not apply. You have to provide credentials and the AWS region via the command as attributes or environment variables::
 
-    docker run fabfuel/ecs-deploy:1.7.1 ecs deploy my-cluster my-service --region eu-central-1 --access-key-id ABC --secret-access-key ABC
+    docker run fabfuel/ecs-deploy:1.10.2 ecs deploy my-cluster my-service --region eu-central-1 --access-key-id ABC --secret-access-key ABC
 
 
 Configuration
@@ -359,7 +359,13 @@ Or implicitly via environment variables ``NEW_RELIC_API_KEY`` and ``NEW_RELIC_AP
     $ export NEW_RELIC_APP_ID=1234567890
     $ ecs deploy my-cluster my-service
 
-Optionally you can provide an additional comment to the deployment via ``--comment "New feature X"`` and the name of the user who deployed with ``--user john.doe``
+Optionally you can provide additional information for the deployment:
+
+- ``--comment "New feature X"`` - comment to the deployment
+- ``--user john.doe`` - the name of the user who deployed with
+- ``--newrelic-revision 1.0.0`` - explicitly set the revison to use for the deployment
+
+Note: If neither ``--tag`` nor ``--newrelic-revision`` are provided, the deployment will not be recorded.
 
 
 Troubleshooting

ecs_deploy/__init__.py

@@ -1 +1 @@
-VERSION = '1.10.5'
+VERSION = '1.11.0'

ecs_deploy/cli.py

@@ -7,7 +7,6 @@
 import json
 import getpass
 from datetime import datetime, timedelta
-
 from ecs_deploy import VERSION
 from ecs_deploy.ecs import DeployAction, ScaleAction, RunAction, EcsClient, DiffAction, \
     TaskPlacementError, EcsError, UpdateAction, LAUNCH_TYPE_EC2, LAUNCH_TYPE_FARGATE
@@ -32,6 +31,7 @@ def get_client(access_key_id, secret_access_key, region, profile):
 @click.option('-i', '--image', type=(str, str), multiple=True, help='Overwrites the image for a container: <container> <image>')
 @click.option('-c', '--command', type=(str, str), multiple=True, help='Overwrites the command in a container: <container> <command>')
 @click.option('-e', '--env', type=(str, str, str), multiple=True, help='Adds or changes an environment variable: <container> <name> <value>')
+@click.option('--env-file', type=(str, str), default=((None, None),), multiple=True, required=False, help='Load environment variables from .env-file')
 @click.option('-s', '--secret', type=(str, str, str), multiple=True, help='Adds or changes a secret environment variable from the AWS Parameter Store (Not available for Fargate): <container> <name> <parameter name>')
 @click.option('-r', '--role', type=str, help='Sets the task\'s role ARN: <task role ARN>')
 @click.option('-x', '--execution-role', type=str, help='Sets the execution\'s role ARN: <execution role ARN>')
@@ -45,6 +45,7 @@ def get_client(access_key_id, secret_access_key, region, profile):
 @click.option('--newrelic-apikey', required=False, help='New Relic API Key for recording the deployment. Can also be defined via environment variable NEW_RELIC_API_KEY')
 @click.option('--newrelic-appid', required=False, help='New Relic App ID for recording the deployment. Can also be defined via environment variable NEW_RELIC_APP_ID')
 @click.option('--newrelic-region', required=False, help='New Relic region: US or EU (default: US). Can also be defined via environment variable NEW_RELIC_REGION')
+@click.option('--newrelic-revision', required=False, help='New Relic revision for recording the deployment (default: --tag value). Can also be defined via environment variable NEW_RELIC_REVISION')
 @click.option('--comment', required=False, help='Description/comment for recording the deployment')
 @click.option('--user', required=False, help='User who executes the deployment (used for recording)')
 @click.option('--diff/--no-diff', default=True, help='Print which values were changed in the task definition (default: --diff)')
@@ -55,7 +56,7 @@ def get_client(access_key_id, secret_access_key, region, profile):
 @click.option('--sleep-time', default=1, type=int, help='Amount of seconds to wait between each check of the service (default: 1)')
 @click.option('--slack-url', required=False, help='Webhook URL of the Slack integration. Can also be defined via environment variable SLACK_URL')
 @click.option('--slack-service-match', default=".*", required=False, help='A regular expression for defining, which services should be notified. (default: .* =all). Can also be defined via environment variable SLACK_SERVICE_MATCH')
-def deploy(cluster, service, tag, image, command, env, secret, role, execution_role, task, region, access_key_id, secret_access_key, profile, timeout, newrelic_apikey, newrelic_appid, newrelic_region, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, sleep_time, slack_url, slack_service_match='.*'):
+def deploy(cluster, service, tag, image, command, env, env_file, secret, role, execution_role, task, region, access_key_id, secret_access_key, profile, timeout, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, sleep_time, slack_url, slack_service_match='.*'):
     """
     Redeploy or modify a service.
 
@@ -75,7 +76,7 @@ def deploy(cluster, service, tag, image, command, env, secret, role, execution_r
         td = get_task_definition(deployment, task)
         td.set_images(tag, **{key: value for (key, value) in image})
         td.set_commands(**{key: value for (key, value) in command})
-        td.set_environment(env, exclusive_env)
+        td.set_environment(env, exclusive_env, env_file)
         td.set_secrets(secret, exclusive_secrets)
         td.set_role_arn(role)
         td.set_execution_role_arn(execution_role)
@@ -116,7 +117,7 @@ def deploy(cluster, service, tag, image, command, env, secret, role, execution_r
             else:
                 raise
 
-        record_deployment(tag, newrelic_apikey, newrelic_appid, newrelic_region, comment, user)
+        record_deployment(tag, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user)
 
         slack.notify_success(cluster, td.revision, service=service)
 
@@ -133,22 +134,25 @@ def deploy(cluster, service, tag, image, command, env, secret, role, execution_r
 @click.option('-t', '--tag', help='Changes the tag for ALL container images')
 @click.option('-c', '--command', type=(str, str), multiple=True, help='Overwrites the command in a container: <container> <command>')
 @click.option('-e', '--env', type=(str, str, str), multiple=True, help='Adds or changes an environment variable: <container> <name> <value>')
+@click.option('--env-file', type=(str, str), default=((None, None),), multiple=True, required=False, help='Load environment variables from .env-file')
 @click.option('-r', '--role', type=str, help='Sets the task\'s role ARN: <task role ARN>')
 @click.option('--region', help='AWS region (e.g. eu-central-1)')
 @click.option('--access-key-id', help='AWS access key id')
 @click.option('--secret-access-key', help='AWS secret access key')
 @click.option('--newrelic-apikey', required=False, help='New Relic API Key for recording the deployment. Can also be defined via environment variable NEW_RELIC_API_KEY')
 @click.option('--newrelic-appid', required=False, help='New Relic App ID for recording the deployment. Can also be defined via environment variable NEW_RELIC_APP_ID')
 @click.option('--newrelic-region', required=False, help='New Relic region: US or EU (default: US). Can also be defined via environment variable NEW_RELIC_REGION')
+@click.option('--newrelic-revision', required=False, help='New Relic revision for recording the deployment (default: --tag value). Can also be defined via environment variable NEW_RELIC_REVISION')
 @click.option('--comment', required=False, help='Description/comment for recording the deployment')
 @click.option('--user', required=False, help='User who executes the deployment (used for recording)')
 @click.option('--profile', help='AWS configuration profile name')
 @click.option('--diff/--no-diff', default=True, help='Print what values were changed in the task definition')
 @click.option('--deregister/--no-deregister', default=True, help='Deregister or keep the old task definition (default: --deregister)')
 @click.option('--rollback/--no-rollback', default=False, help='Rollback to previous revision, if deployment failed (default: --no-rollback)')
+@click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers')
 @click.option('--slack-url', required=False, help='Webhook URL of the Slack integration. Can also be defined via environment variable SLACK_URL')
 @click.option('--slack-service-match', default=".*", required=False, help='A regular expression for defining, deployments of which crons should be notified. (default: .* =all). Can also be defined via environment variable SLACK_SERVICE_MATCH')
-def cron(cluster, task, rule, image, tag, command, env, role, region, access_key_id, secret_access_key, newrelic_apikey, newrelic_appid, newrelic_region, comment, user, profile, diff, deregister, rollback, slack_url, slack_service_match):
+def cron(cluster, task, rule, image, tag, command, env, env_file, role, region, access_key_id, secret_access_key, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, profile, diff, deregister, rollback, exclusive_env, slack_url, slack_service_match):
     """
     Update a scheduled task.
 
@@ -166,7 +170,7 @@ def cron(cluster, task, rule, image, tag, command, env, role, region, access_key
 
         td.set_images(tag, **{key: value for (key, value) in image})
         td.set_commands(**{key: value for (key, value) in command})
-        td.set_environment(env)
+        td.set_environment(env, exclusive_env, env_file)
         td.set_role_arn(role)
 
         slack = SlackNotification(
@@ -190,7 +194,7 @@ def cron(cluster, task, rule, image, tag, command, env, role, region, access_key
 
         slack.notify_success(cluster, td.revision, rule=rule)
 
-        record_deployment(tag, newrelic_apikey, newrelic_appid, newrelic_region, comment, user)
+        record_deployment(tag, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user)
 
         if deregister:
             deregister_task_definition(action, td)
@@ -206,6 +210,7 @@ def cron(cluster, task, rule, image, tag, command, env, role, region, access_key
 @click.option('-t', '--tag', help='Changes the tag for ALL container images')
 @click.option('-c', '--command', type=(str, str), multiple=True, help='Overwrites the command in a container: <container> <command>')
 @click.option('-e', '--env', type=(str, str, str), multiple=True, help='Adds or changes an environment variable: <container> <name> <value>')
+@click.option('--env-file', type=(str, str), default=((None, None),), multiple=True, required=False, help='Load environment variables from .env-file')
 @click.option('-s', '--secret', type=(str, str, str), multiple=True, help='Adds or changes a secret environment variable from the AWS Parameter Store (Not available for Fargate): <container> <name> <parameter name>')
 @click.option('-r', '--role', type=str, help='Sets the task\'s role ARN: <task role ARN>')
 @click.option('--region', help='AWS region (e.g. eu-central-1)')
@@ -216,7 +221,7 @@ def cron(cluster, task, rule, image, tag, command, env, role, region, access_key
 @click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers')
 @click.option('--exclusive-secrets', is_flag=True, default=False, help='Set the given secrets exclusively and remove all other pre-existing secrets from all containers')
 @click.option('--deregister/--no-deregister', default=True, help='Deregister or keep the old task definition (default: --deregister)')
-def update(task, image, tag, command, env, secret, role, region, access_key_id, secret_access_key, profile, diff, exclusive_env, exclusive_secrets, deregister):
+def update(task, image, tag, command, env, env_file, secret, role, region, access_key_id, secret_access_key, profile, diff, exclusive_env, exclusive_secrets, deregister):
     """
     Update a task definition.
 
@@ -232,7 +237,7 @@ def update(task, image, tag, command, env, secret, role, region, access_key_id,
 
         td.set_images(tag, **{key: value for (key, value) in image})
         td.set_commands(**{key: value for (key, value) in command})
-        td.set_environment(env, exclusive_env)
+        td.set_environment(env, exclusive_env, env_file)
         td.set_secrets(secret, exclusive_secrets)
         td.set_role_arn(role)
 
@@ -299,6 +304,7 @@ def scale(cluster, service, desired_count, access_key_id, secret_access_key, reg
 @click.argument('count', required=False, default=1)
 @click.option('-c', '--command', type=(str, str), multiple=True, help='Overwrites the command in a container: <container> <command>')
 @click.option('-e', '--env', type=(str, str, str), multiple=True, help='Adds or changes an environment variable: <container> <name> <value>')
+@click.option('--env-file', type=(str, str), default=((None, None),), multiple=True, required=False, help='Load environment variables from .env-file')
 @click.option('-s', '--secret', type=(str, str, str), multiple=True, help='Adds or changes a secret environment variable from the AWS Parameter Store (Not available for Fargate): <container> <name> <parameter name>')
 @click.option('--launchtype', type=click.Choice([LAUNCH_TYPE_EC2, LAUNCH_TYPE_FARGATE]), default=LAUNCH_TYPE_EC2, help='ECS Launch type (default: EC2)')
 @click.option('--subnet', type=str, multiple=True, help='A subnet ID to launch the task within. Required for launch type FARGATE (multiple values possible)')
@@ -309,8 +315,9 @@ def scale(cluster, service, desired_count, access_key_id, secret_access_key, reg
 @click.option('--access-key-id', help='AWS access key id')
 @click.option('--secret-access-key', help='AWS secret access key')
 @click.option('--profile', help='AWS configuration profile name')
+@click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers')
 @click.option('--diff/--no-diff', default=True, help='Print what values were changed in the task definition')
-def run(cluster, task, count, command, env, secret, launchtype, subnet, securitygroup, public_ip, platform_version, region, access_key_id, secret_access_key, profile, diff):
+def run(cluster, task, count, command, env, env_file, secret, launchtype, subnet, securitygroup, public_ip, platform_version, region, access_key_id, secret_access_key, profile, exclusive_env, diff):
     """
     Run a one-off task.
 
@@ -325,7 +332,7 @@ def run(cluster, task, count, command, env, secret, launchtype, subnet, security
 
         td = action.get_task_definition(task)
         td.set_commands(**{key: value for (key, value) in command})
-        td.set_environment(env)
+        td.set_environment(env, exclusive_env, env_file)
         td.set_secrets(secret)
 
         if diff:
@@ -513,12 +520,16 @@ def rollback_task_definition(deployment, old, new, timeout=600, sleep_time=1):
     )
 
 
-def record_deployment(revision, api_key, app_id, region, comment, user):
+def record_deployment(tag, api_key, app_id, region, revision, comment, user):
     api_key = getenv('NEW_RELIC_API_KEY', api_key)
     app_id = getenv('NEW_RELIC_APP_ID', app_id)
     region = getenv('NEW_RELIC_REGION', region)
+    revision = getenv('NEW_RELIC_REVISION', revision) or tag
 
     if not revision or not api_key or not app_id:
+        if api_key:
+            click.secho('Missing required parameters for recording New Relic deployment.' \
+                        'Please see https://github.com/fabfuel/ecs-deploy#new-relic')
         return False
 
     user = user or getpass.getuser()

ecs_deploy/ecs.py

@@ -15,11 +15,24 @@
 except AttributeError:
     JSONDecodeError = ValueError
 
-
 LAUNCH_TYPE_EC2 = 'EC2'
 LAUNCH_TYPE_FARGATE = 'FARGATE'
 
 
+def read_env_file(container_name,file):
+    env_vars = []
+    try:
+        with open(file) as f:
+            for line in f:
+                if line.startswith('#') or not line.strip() or '=' not in line:
+                    continue
+                key, value = line.strip().split('=', 1)
+                env_vars.append((container_name,key,value))
+    except Exception as e:
+        raise EcsTaskDefinitionCommandError(str(e))
+    return tuple(env_vars)
+
+
 class EcsClient(object):
     def __init__(self, access_key_id=None, secret_access_key=None,
                  region=None, profile=None, session_token=None):
@@ -248,16 +261,20 @@ def diff_raw(self, task_b):
         requirements_b = sorted([r['name'] for r in task_b.requires_attributes])
 
         for container in containers_a:
-            containers_a[container]['environment'] = {e['name']: e['value'] for e in containers_a[container].get('environment', {})}
+            containers_a[container]['environment'] = {e['name']: e['value'] for e in
+                                                      containers_a[container].get('environment', {})}
 
         for container in containers_b:
-            containers_b[container]['environment'] = {e['name']: e['value'] for e in containers_b[container].get('environment', {})}
+            containers_b[container]['environment'] = {e['name']: e['value'] for e in
+                                                      containers_b[container].get('environment', {})}
 
         for container in containers_a:
-            containers_a[container]['secrets'] = {e['name']: e['valueFrom'] for e in containers_a[container].get('secrets', {})}
+            containers_a[container]['secrets'] = {e['name']: e['valueFrom'] for e in
+                                                  containers_a[container].get('secrets', {})}
 
         for container in containers_b:
-            containers_b[container]['secrets'] = {e['name']: e['valueFrom'] for e in containers_b[container].get('secrets', {})}
+            containers_b[container]['secrets'] = {e['name']: e['valueFrom'] for e in
+                                                  containers_b[container].get('secrets', {})}
 
         composite_a = {
             'containers': containers_a,
@@ -305,7 +322,7 @@ def parse_command(command):
                 raise EcsTaskDefinitionCommandError(
                     "command should be valid JSON list. Got following "
                     "command: {} resulting in error: {}"
-                    .format(command, str(e)))
+                        .format(command, str(e)))
 
         return command.split()
 
@@ -360,9 +377,12 @@ def set_commands(self, **commands):
                 self._diff.append(diff)
                 container[u'command'] = self.parse_command(new_command)
 
-    def set_environment(self, environment_list, exclusive=False):
+    def set_environment(self, environment_list, exclusive=False, env_file=((None, None),)):
         environment = {}
-
+        if None not in env_file[0]:
+            for env in env_file:
+                l = read_env_file(env[0], env[1])
+                environment_list = l + environment_list
         for env in environment_list:
             environment.setdefault(env[0], {})
             environment[env[0]][env[1]] = env[2]