Integrate migration alerts into deploy

src/commcare_cloud/fab/fabfile.py

@@ -30,40 +30,28 @@
 import datetime
 import functools
 import os
+import pipes
 import posixpath
+from distutils.util import strtobool
 from getpass import getpass
 
-import pipes
 import pytz
-from distutils.util import strtobool
 
+from commcare_cloud.environment.main import get_environment
+from commcare_cloud.environment.paths import get_available_envs
 from fabric import utils
-from fabric.api import roles, execute, task, sudo, env, parallel
-from fabric.colors import blue, red, magenta
+from fabric.api import env, execute, parallel, roles, sudo, task
+from fabric.colors import blue, magenta, red
 from fabric.context_managers import cd
-from fabric.contrib import files, console
+from fabric.contrib import console, files
 from fabric.operations import require
 
-from commcare_cloud.environment.main import get_environment
-from commcare_cloud.environment.paths import get_available_envs
-
-from .const import (
-    ROLES_ALL_SRC,
-    ROLES_ALL_SERVICES,
-    ROLES_PILLOWTOP,
-    ROLES_DJANGO,
-    ROLES_DEPLOY,
-)
+from .checks import check_servers
+from .const import ROLES_ALL_SERVICES, ROLES_ALL_SRC, ROLES_DEPLOY, ROLES_DJANGO, ROLES_PILLOWTOP
 from .exceptions import PreindexNotFinished
-from .operations import (
-    db,
-    staticfiles,
-    supervisor,
-    formplayer,
-    release,
-    offline as offline_ops,
-    airflow
-)
+from .operations import airflow, db, formplayer
+from .operations import offline as offline_ops
+from .operations import release, staticfiles, supervisor
 from .utils import (
     DeployMetadata,
     cache_deploy_state,
@@ -73,9 +61,6 @@
     retrieve_cached_deploy_env,
     traceback_string,
 )
-from .checks import (
-    check_servers,
-)
 
 if env.ssh_config_path and os.path.isfile(os.path.expanduser(env.ssh_config_path)):
     env.use_ssh_config = True
@@ -332,6 +317,13 @@ def _confirm_translated():
     )
 
 
+def _confirm_changes():
+    env.deploy_metadata.diff.warn_of_migrations()
+    return console.confirm(
+        'Are you sure you want to preindex and deploy to '
+        '{env.deploy_env}?'.format(env=env), default=False)
+
+
 @task
 def kill_stale_celery_workers():
     """
@@ -647,9 +639,7 @@ def deploy_commcare(confirm="yes", resume='no', offline='no', skip_record='no'):
     _require_target()
     if strtobool(confirm) and (
         not _confirm_translated() or
-        not console.confirm(
-            'Are you sure you want to preindex and deploy to '
-            '{env.deploy_env}?'.format(env=env), default=False)
+        not _confirm_changes()
     ):
         utils.abort('Deployment aborted.')
 

src/commcare_cloud/fab/operations/release.py

@@ -353,7 +353,7 @@ def record_successful_deploy():
             'virtualenv_current': env.py3_virtualenv_current,
             'user': env.user,
             'environment': env.deploy_env,
-            'url': env.deploy_metadata.diff_url,
+            'url': env.deploy_metadata.diff.url,
             'minutes': str(int(delta.total_seconds() // 60))
         })
 

src/commcare_cloud/fab/utils.py

@@ -60,7 +60,6 @@ def __init__(self, code_branch, environment):
         self.timestamp = datetime.datetime.utcnow().strftime(DATE_FMT)
         self._deploy_tag = None
         self._max_tags = 100
-        self._last_tag = None
         self._code_branch = code_branch
         self._environment = environment
 
@@ -70,25 +69,9 @@ def repo(self):
 
     @memoized_property
     def last_commit_sha(self):
-        if self.last_tag:
-            return self.last_tag.commit.sha
-
         with cd(env.code_current):
             return sudo('git rev-parse HEAD')
 
-    @memoized_property
-    def last_tag(self):
-        pattern = ".*-{}-deploy".format(re.escape(self._environment))
-        for tag in self.repo.get_tags()[:self._max_tags]:
-            if re.match(pattern, tag.name):
-                return tag
-
-        print(magenta('Warning: No previous tag found in last {} tags for {}'.format(
-            self._max_tags,
-            self._environment
-        )))
-        return None
-
     def tag_commit(self):
         if env.offline:
             self._offline_tag_commit()
@@ -124,25 +107,6 @@ def _offline_tag_commit(self):
                 tag=tag_name,
             ))
 
-    @property
-    def diff_url(self):
-        if env.offline:
-            return '"No diff url for offline deploy"'
-        if not env.tag_deploy_commits:
-            return '"Diff URLs are not enabled for this environment"'
-
-        if _github_auth_provided() and self._deploy_tag is None:
-            raise Exception("You haven't tagged anything yet.")
-
-        from_ = self.last_tag.name if self.last_tag and self.last_tag.name else self.last_commit_sha
-        if not from_:
-            return '"Previous deploy not found, cannot make comparison"'
-
-        return "https://github.com/dimagi/commcare-hq/compare/{}...{}".format(
-            from_,
-            self._deploy_tag or self.deploy_ref,
-        )
-
     @memoized_property
     def deploy_ref(self):
         if env.offline:
@@ -172,20 +136,27 @@ def tag_setup_release(self):
     def current_ref_is_different_than_last(self):
         return self.deploy_ref != self.last_commit_sha
 
+    @memoized_property
+    def diff(self):
+        return DeployDiff(self.repo, self.last_commit_sha, self.deploy_ref)
+
 
 @memoized
 def _get_github_credentials():
-    if not env.tag_deploy_commits:
-        return (None, None)
     try:
         from .config import GITHUB_APIKEY
     except ImportError:
+        print(red("Github credentials not found!"))
+        print("This deploy script uses the Github API to display a summary of changes to be deployed.")
+        if env.tag_deploy_commits:
+            print("You're deploying an environment which uses release tags. "
+                  "Provide Github auth details to enable release tags.")
         print((
             "You can add a config file to automate this step:\n"
             "    $ cp {project_root}/config.example.py {project_root}/config.py\n"
             "Then edit {project_root}/config.py"
         ).format(project_root=PROJECT_ROOT))
-        username = input('Github username (leave blank for no auth): ') or None
+        username = input('Github username (leave blank to skip): ') or None
         password = getpass('Github password: ') if username else None
         return (username, password)
     else:
@@ -195,11 +166,6 @@ def _get_github_credentials():
 @memoized
 def _get_github():
     login_or_token, password = _get_github_credentials()
-    if env.tag_deploy_commits and not login_or_token:
-        print(magenta(
-            "Warning: Creation of release tags is disabled. "
-            "Provide Github auth details to enable release tags."
-        ))
     return Github(login_or_token=login_or_token, password=password)
 
 
@@ -286,62 +252,70 @@ def bower_command(command, production=True, config=None):
     sudo(cmd)
 
 
-def warn_of_migrations(last_deploy_sha, current_deploy_sha):
-    pr_numbers = _get_pr_numbers(last_deploy_sha, current_deploy_sha)
-    pool = Pool(5)
-    pr_infos = [_f for _f in pool.map(_get_pr_info, pr_numbers) if _f]
-
-    print("List of PRs since last deploy:")
-    _print_prs_formatted(pr_infos)
-
-    prs_by_label = _get_prs_by_label(pr_infos)
-    if prs_by_label:
-        print(red('You are about to deploy the following PR(s), which will trigger a reindex or migration. Click the URL for additional context.'))
-        _print_prs_formatted(prs_by_label['reindex/migration'])
-
-
-def _get_pr_numbers(last_deploy_sha, current_deploy_sha):
-    repo = _get_github().get_organization('dimagi').get_repo('commcare-hq')
-    comparison = repo.compare(last_deploy_sha, current_deploy_sha)
-
-    return [
-        int(re.search(r'Merge pull request #(\d+)',
-                      repo_commit.commit.message).group(1))
-        for repo_commit in comparison.commits
-        if repo_commit.commit.message.startswith('Merge pull request')
-    ]
-
-
-def _get_pr_info(pr_number):
-    repo = _get_github().get_organization('dimagi').get_repo('commcare-hq')
-    pr_response = repo.get_pull(pr_number)
-    if not pr_response.number:
-        # Likely rate limited by Github API
-        return None
-    assert pr_number == pr_response.number, (pr_number, pr_response.number)
-
-    labels = [label.name for label in pr_response.labels]
-
-    return {
-        'title': pr_response.title,
-        'url': pr_response.html_url,
-        'labels': labels,
-    }
+class DeployDiff:
+    def __init__(self, repo, last_commit, deploy_commit):
+        self.repo = repo
+        self.last_commit = last_commit
+        self.deploy_commit = deploy_commit
 
+    @property
+    def url(self):
+        """Human-readable diff URL"""
+        return "{}/compare/{}...{}".format(self.repo.html_url, self.last_commit, self.deploy_commit)
 
-def _get_prs_by_label(pr_infos):
-    prs_by_label = defaultdict(list)
-    for pr in pr_infos:
-        for label in pr['labels']:
-            if label in LABELS_TO_EXPAND:
-                prs_by_label[label].append(pr)
-    return dict(prs_by_label)
+    def warn_of_migrations(self):
+        if not _github_auth_provided():
+            return
 
+        pr_numbers = self._get_pr_numbers()
+        pool = Pool(5)
+        pr_infos = [_f for _f in pool.map(self._get_pr_info, pr_numbers) if _f]
+
+        print("List of PRs since last deploy:")
+        self._print_prs_formatted(pr_infos)
+
+        prs_by_label = self._get_prs_by_label(pr_infos)
+        if prs_by_label:
+            print(red('You are about to deploy the following PR(s), which will trigger a reindex or migration. Click the URL for additional context.'))
+            self._print_prs_formatted(prs_by_label['reindex/migration'])
+
+    def _get_pr_numbers(self):
+        comparison = self.repo.compare(self.last_commit, self.deploy_commit)
+        return [
+            int(re.search(r'Merge pull request #(\d+)',
+                          repo_commit.commit.message).group(1))
+            for repo_commit in comparison.commits
+            if repo_commit.commit.message.startswith('Merge pull request')
+        ]
+
+    def _get_pr_info(self, pr_number):
+        pr_response = self.repo.get_pull(pr_number)
+        if not pr_response.number:
+            # Likely rate limited by Github API
+            return None
+        assert pr_number == pr_response.number, (pr_number, pr_response.number)
+
+        labels = [label.name for label in pr_response.labels]
+
+        return {
+            'title': pr_response.title,
+            'url': pr_response.html_url,
+            'labels': labels,
+        }
+
+    def _get_prs_by_label(self, pr_infos):
+        prs_by_label = defaultdict(list)
+        for pr in pr_infos:
+            for label in pr['labels']:
+                if label in LABELS_TO_EXPAND:
+                    prs_by_label[label].append(pr)
+        return dict(prs_by_label)
+
+    def _print_prs_formatted(self, pr_list):
+        i = 1
+        for pr in pr_list:
+            print("{0}. ".format(i), end="")
+            print("{title} {url} | ".format(**pr), end="")
+            print(" ".join(label for label in pr['labels']))
+            i += 1
 
-def _print_prs_formatted(pr_list):
-    i = 1
-    for pr in pr_list:
-        print("{0}. ".format(i), end="")
-        print("{title} {url} | ".format(**pr), end="")
-        print(" ".join(label for label in pr['labels']))
-        i += 1