chords_control

#!python3

"""
The CHORDS management script. Use to manage the CHORDS portal configuration,
running/stopping the portal, and updating the portal software.

The CHORDS configuration is kept in a YAML file (default chords.yml).
This file contains standard configuration items, followed by user
created non-standard options. The later are for developer use only.
There is a one-to-one corresponence between the CHORDS configuration file
and .env. The .env simply simply contains environment variables which
will be passed to docker-compose. In most cases, .env is created from
the configuration file. However, the "backwards" command allows the configuration
file to be created from .env.

In configuration mode (-c), the existing configuration will be
read from the configuration file, the user will be prompted for changes,
and the configuration file will be re-written. For each configuration item,
the user may select the current value (hit enter), select the default
value (enter a period), or change the value (enter a new value). The configuration
may be initialized to complete default vaues by using the -d switch in conjunction
with -c.

If the CHORDs configuration file does not exist, then it will be created. Thus,
to create an initial default configuration, use:
   ./chords_control --config --default

A .env file is also created by the configuration mode. It contains
 environment variable commands, with one for each configuration item.
This .env file is used by docker-compose. The non-standard options will
be included in .env, allowing developers to test additional environment variables,
without having to edit the standardoptions specified in this script.

The -r and -s switches are used to run/stop a portal.

The -u switch updates the portal software by pulling the docker images.

Use -t to see the current portal status.

The devmode (-m) enables development mode, where the current directory
is mounted as the CHORDS Rails source.

The the --renew option makes a backup copy of this script and the docker-compose files, and pulls down
a new version from github.
"""
from __future__ import print_function

from builtins import str
from builtins import object
import os
import stat
import sys
import re
import shutil
import datetime
import argparse
import subprocess
import json
import platform
import tempfile
import string
import glob
import getpass
import socket
from collections import OrderedDict
from collections import namedtuple
# sh is not available on Windows, and the Linux commands are not there either!
if platform.system() != "Windows":
    import sh

# The definitions of standard configuration items. These are 5-tuples:
# [0]: The configuration item description. This will be printed as a prompt
#      during configuration, and included in the configuration file as a description.
# [1]: The name of the environment variable to be set in the .env file.
# [2]: The default value.
# [3]: Verify:True or False. If True, the user response is compared to the choices available,
#      verifying that a valid response was entered.
# [4]: If [3] is true, this is a list of valid choices. It may be modified/populated
#      dynamically by the script
#
# When you need to add required keys to the configuration, just define them here.
STD_CONFIG_ITEMS = [
      [
        "The CHORDS Release or GIT Branch where the docker-compose recipe will be fetched.\n"
        "Release names are prefixed with \'Release-\'.' Branches are only used by developers. \n"
        "Use the most recent release unless you have a good reason not to.\n",
        "GIT_BRANCH",
        "master",
        True,
        []
    ],
    [
        "The Docker version of the desired CHORDS release.\n"
        "For normal usage, this should match the CHORDS release that you just selected.\n",
        "DOCKER_TAG",
        "latest",
        True,
        []
    ],
    [
        "SSL is only required for systems that need to provide https (a really good idea).\n"
        "You will need to obtain a DNS name which points at your CHORDS instance,\n"
        "and obtain an SSL certificate (which your CHORDS instance will facilitate).\n"
        "Should SSL be enabled?\n",
        "SSL_ENABLED",
        "false",
        True,
        [
            "true",
            "false"
        ]
    ],
    [
        "The email that will be registered (with Letsencrypt), for SSL certification.\n"
        "Leave blank if SSL is not enabled.",
        "SSL_EMAIL",
        "",
        False,
        []
    ],
    [
        "The DNS resolvable host name. Leave blank if SSL is not enabled.",
        "SSL_HOST",
        "",
        False,
        []
    ],
    [
        "The PASSWORD for sysadmin access to CHORDS, mysql and influxdb in docker.\n"
        "(NOTE: This not the CHORDS website admin login)\n"
        "Replace this with a secure password.",
        "CHORDS_ADMIN_PW",
        "chords_ec_demo",
        False,
        []
    ],
    [
        "The PASSWORD for read-only access to influxdb.",
        "CHORDS_GUEST_PW",
        "guest",
        False,
        []
    ],
    [
        "An EMAIL ACCOUNT that will send CHORDS password reset instructions, \n"
        "Grafana alerts, etc. DO NOT use a personal or business account for this; \n"
        "instead set up an account specifically for CHORDS (e.g. at gmail).",
        "CHORDS_EMAIL_ADDRESS",
        "unknown@gmail.com",
        False,
        []
    ],
    [
        "The PASSWORD for the email account that will send CHORDS password reset instructions, \n"
        "Grafana alerts, etc. DO NOT use a personal or business account for this; \n"
        "instead set up an account specifically for CHORDS (e.g. at gmail).",
        "CHORDS_EMAIL_PASSWORD",
        "unknown",
        False,
        []
    ],
    [
        "The EMAIL SERVER that can relay CHORDS password reset instructions, \n"
        "Grafana alerts, etc. You must have an account on this service.",
        "CHORDS_EMAIL_SERVER",
        "smtp.gmail.com",
        False,
        []
    ],
    [
        "The SERVER PORT for the email server that can relay CHORDS password reset instructions, \n"
        "Grafana alerts, etc. You must have an account on this service.",
        "CHORDS_EMAIL_PORT",
        "587",
        False,
        []
    ],
    [
        "The IP name for the Grafana (and CHORDS) server.\n"
        "When Grafana sends an alert, it will include a link\n"
        "back to this server, so that a user can navigate to\n"
        "the Grafana panel which created the alert.",
        "GRAFANA_SERVER_DOMAIN",
        "localhost",
        False,
        []
    ],
    [
        "The PASSWORD for admin access to Grafana.\n"
        "Once Grafana is initialized with this password,\n"
        "it can only be changed from the Grafana admin web page.\n"
        "Replace this with a secure password.",
        "GRAFANA_ADMIN_PW",
        "admin",
        False,
        []
    ],
    [
        "A SECRET KEY BASE for Rails. Generate a\n"
        "secure value (e.g. 'openssl rand -hex 32').",
        "SECRET_KEY_BASE",
        "aaaaaaaaaaa",
        False,
        []
    ],
    [
        "The time series DATABASE RETENTION DURATION, e.g. 168h or 52w.\n"
        "Use \"inf\" for permanent. This value can be changed on\n"
        "successive restarts of a portal. Note: making it shorter\n"
        "will trim the existing time series database.",
        "DB_RETENTION",
        "inf",
        False,
        []
    ],
    [
        "CHORDS HTTP port.\n"
        "(Typically only changed if there are port conflicts or firewall restrictions).",
        "CHORDS_HTTP_PORT",
        "80",
        False,
        []
    ],
    [
        "Grafana port.\n"
        "(Typically only changed if there are port conflicts or firewall restrictions).",
        "GRAFANA_HTTP_PORT",
        "3000",
        False,
        []
    ],
    [
        "PROXY URL (e.g. http://proxy.myorg.com:8080).\n"
        "Leave blank if not needed.",
        "PROXY",
        "",
        False,
        []
    ],
    [
        "Enable InfluxData kapacitor.\n"
        "This is experimental.In general, do NOT enable,\n"
        "as it opens a security hole.",
        "KAPACITOR_ENABLED",
        "false",
        True,
        [
            "true",
            "false"
        ]
    ],
    [
        "The RAILS ENVIRONMENT. Unlikely to to be anything other than \"production\".",
        "RAILS_ENV",
        "production",
        True,
        [
            "production",
            "development",
            "test"
        ]
    ],
    [
        "The number of NGINX WORKERS. 4 is a good value",
        "WORKERS",
        4,
        False,
        []
    ]
]

# If set True, add some diagnostic printing.
VERBOSE = False

# Terminal control characters
TERM_BOLD_ON = '\033[1m'
TERM_BOLD_OFF = '\033[0m'
if os.name == 'nt':
    TERM_BOLD_ON = ''
    TERM_BOLD_OFF = ''

# Platform characteristics
OS_NAME = os.name
OS_ARCH = platform.uname()[4]

#####################################################################
def create_backup_file(filename, print_note=True):
    """
    Create a backup copy of the file. A timestamp is included
    in the new file name. If print_note is True, print
    out a  friendly message indicating that a backup copy was made.
    """
    # No need to backup a file that doesn't already exist or is zero size
    if not os.path.isfile(filename):
        return

    if os.stat(filename).st_size == 0:
        return

    # Create the backup file name
    fsplit = os.path.splitext(filename)
    backupfile = fsplit[0] + "-" + datetime.datetime.now().strftime('%Y-%m-%d-%H%M%S')
    if fsplit[1]:
        backupfile = backupfile + fsplit[1]
    # Copy the existing foile to the backup
    shutil.copyfile(filename, backupfile)

    # Make it accesible only to the use
    os.chmod(backupfile, stat.S_IRUSR | stat.S_IWUSR)

    if print_note:
        print("*** " + filename + " has been backed up to " + backupfile + ".")

#####################################################################
class CommandArgs(object):
    """
    Manage the command line options.
    The options are collated in a dictionary keyed on the option long name.
    The option dictionary will have None for options that aren't present.
    """
    def __init__(self):

        description = """
CHORDS configuration and operation management.
        
In configuration mode, you are prompted for configuration options. These will be
saved in the configuration file (default chords.yml) and in a corresponding .env file. 
Backup copies will be made of the existing configuration files. Use the --default option
to set the configuration to default values.
"""

        epilog = """
At least one and only one of --config, --env, --backwards, --run, --stop, 
--restart, --update, --backup, --restore or --renew can be specified. 
--default must be accompanied by --config. 
To create an initial default configuration: 
   'python chords_control --config --default'

chords_control is not available on Windows.

""" + "This operating system is " + OS_NAME + ", the architecture is " + OS_ARCH + "."

        parser = argparse.ArgumentParser(description=description, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter)
        parser.add_argument("--run",     help="run CHORDS",                                           default=False,        action="store_true")
        parser.add_argument("--stop",    help="stop CHORDS",                                          default=False,        action="store_true")
        parser.add_argument("--restart", help="restart CHORDS",                                       default=False,        action="store_true")
        parser.add_argument("--update",  help="update",                                               default=False,        action="store_true")
        parser.add_argument("--file",    help="configuration file (default chords.yml)",              default="chords.yml", action="store")
        parser.add_argument("--config",  help="prompt for configuration and write config/.env files", default=False,        action="store_true")
        parser.add_argument("--default", help="set all configuration values to defaults",             default=False,        action="store_true")
        parser.add_argument("--status",  help="status",                                               default=False,        action="store_true")
        parser.add_argument("--backup",  help="Dump a CHORDS portal to a backup file",                default=False,        action="store_true")
        parser.add_argument("--restore", help="Restore a portal from a CHORDS backup file",           default=None,         action="store")
        parser.add_argument("--env",     help="read config file and write CHORDS .env file",          default=False,        action="store_true")
        parser.add_argument("--backwards", help="read CHORDS .env file and write config file (use with caution!)", default=False, action="store_true")
        parser.add_argument("--proxy",   help="proxy for curl (e.g. proxy.myorg.com:8080)",           default="",           action="store")
        parser.add_argument("--devmode", help="run containers in source code development mode",       default=False,        action="store_true")
        parser.add_argument("--renew",   help="replace chords_control and docker configs with a new version (use with caution!)",  default=False, action="store_true")
        parser.add_argument("--verbose", help="enable VERBOSE",                                       default=False,        action="store_true")

        # If no switches, print the help.
        if not sys.argv[1:]:
            parser.print_help()
            parser.exit()

        # Parse the command line.
        args = parser.parse_args()
        self.options = vars(args)

        # Make sure that at most only one of these args was specified
        opt = self.options
        if [opt['config'],
                opt['run'],
                opt['stop'],
                opt['restart'],
                opt['update'],
                opt['env'],
                opt['backwards'],
                opt['renew'],
                opt['backup'],
                opt['restore']
           ].count(True) > 1:
            print(epilog)
            exit(1)

        if opt['default'] and not opt['config']:
            print(epilog)
            exit(1)

    def get_options(self):
        """
        Return the dictionary of existing options.
        """

        return self.options

#####################################################################
class ChordsConfig(OrderedDict):
    """
    Manage a CHORDS configuration. It is associated with the CHORDS configuration
    file, which is a YAML document containing comments and key:value pairs.
    Note: more complex YAML structuring is not supported.
    """
    def __init__(self, configfile):
        OrderedDict.__init__(self)

        # Initialize the configuration item descriptions. These are
        # used as comments in the output configuration file.
        self.init_config_items()

        # Fetch the configuration key:value pairs from the configuration file.
        # Add them to self.
        self.get_pairs(configfile)

    def get_pairs(self, config_file):
        """
        Return the configuration key:value pairs from the configuration file.
        """

        # Get the current configuration file
        if not os.path.isfile(config_file):
            cfile = open(OPTIONS["file"], 'w')
            cfile.close()
            print(config_file, 'has been created')

        items = config_items(config_file)

        tmp_config = OrderedDict()
        # Collect all of the standard items. Add them if they
        # weren't in the file
        for key in list(self.config_items.keys()):
            if key in list(items.keys()):
                tmp_config[key] = items[key]
                del items[key]
            else:
                tmp_config[key] = self.config_items[key]['default']
        for key, value in items.items():
            # Append any remaining items that aren't in the standard list
            tmp_config[key] = value
        for key in list(tmp_config.keys()):
            self[key] = tmp_config[key]

    def init_config_items(self):
        """
        Create a collection of config items which must be in the final configuration.
        These items define the default values which will be used if they haven't already been
        set in a configuration.
        """
        self.config_items = OrderedDict()

        for i in STD_CONFIG_ITEMS:
            self.config_items[i[1]] = ConfigItem(description=i[0], default=i[2], verify=i[3], choices=i[4])

    def to_yml(self):
        """
        Create the YML version of the configuration. Line terminators will be included.
        The standard items are wrtten first, followed by the extras.
        """
        yml = ''
        for key in list(self.keys()):
            if key in list(self.config_items.keys()):
                descripts = self.config_items[key]['description'].split('\n')
                for descript in descripts:
                    yml =  yml + '# ' + descript + '\n'
                yml = yml + key + ': ' + str(self[key]) + '\n'
        yml = yml +  '#' + '\n' + '# Non-standard options.' + '\n'
        for key in list(self.keys()):
            if key not in list(self.config_items.keys()):
                yml = yml + key + ': ' + str(self[key]) + '\n'
        return yml

    def query_values(self, usedefault):
        """
        Go through the configuration, asking the user if they want to
        change the values. The response can be a return, to accept the
        value, a new value to replace the value, or a period to use the default value.
        The items found in config_items are done first, followed by all other
        items.
        """
        print("Enter:\n   Return to keep the current value, or\n   Period (.) to select to the default value, or a\n   New value.")

        # Process config items that are part of the standard configuration
        for key in list(self.keys()):
            print()
            if key in list(self.config_items.keys()):
                description = self.config_items[key]['description']
                print()
                # If verify is enabled for this item, then we need to populate the valid choices.
                if self.config_items[key]['verify']:
                    description = description + 'Valid choices are: '
                    for choice in self.config_items[key]['choices']:
                        description = description + '\'' + choice + '\' '
                while True:
                    self.query_value(key=key,
                                     usedefault = usedefault,
                                     description = description,
                                     defaultval = self.config_items[key]['default'])
                    if not self.config_items[key]['verify']:
                        break
                    else:
                        if self[key] in self.config_items[key]['choices']:
                            break
                        else:
                            print('>>>>' + TERM_BOLD_ON + ' You entered an invalid value.' + TERM_BOLD_OFF +' Please try again (or use ctrl-C to exit)')

        # Process items that extras which are not part of the standard configuration
        for key in list(self.keys()):
            if key not in list(self.config_items.keys()):
                self.query_value(key=key, usedefault=usedefault)

    def query_value(self, key, usedefault, description=None, defaultval=None):
        """
        Query the user for a replacement value.
        """
        if description:
            print(description)
        print(key, end=' ')
        if defaultval:
            print("(default: " + str(defaultval) + ")", end=' ')
        print(TERM_BOLD_ON + '[' + str(self[key]) + ']'+ TERM_BOLD_OFF + '? ', end=' ')
        sys.stdout.flush()
        if not usedefault:
            response = sys.stdin.readline().strip()
        else:
            response = "."
            print(".", end=' ')
            sys.stdout.flush()
        # An empty line means retain value
        if response != "":
            # A period means use the default, if it is avaiable
            if response == ".":
                self[key] = defaultval
            # Other replace with the user entered value
            else:
                self[key] = response
        if usedefault:
            print()

    def write_yml_file(self, configfile):
        """
        Write the configuration to the file. The whole configuration is written,
        starting with the elements listed in the config_items. Configuration items
        are prefixed with the comments provided in the config_items.

        If specified, a backup copy of the original file is created.
        """

        print()
        create_backup_file(configfile)

        config_file = open(configfile, "w")
        config_file.write(self.to_yml())
        config_file.close()
        os.chmod(configfile, stat.S_IRUSR | stat.S_IWUSR)
        print("*** " + OPTIONS["file"] + " has been written with the new configuration.")

    def write_env_file(self):
        """
        Write the configuration to the .env file, in environment
        notation.

        A backup copy of the original file is created.
        """
        create_backup_file(".env")

        env_file = open(".env", 'w')
        for key in list(self.keys()):
            env_file.write(key + "=" + str(self[key])+"\n")
        env_file.close()
        os.chmod(".env", stat.S_IRUSR | stat.S_IWUSR)
        print("*** .env has been written with the new configuration.")


#####################################################################
class ConfigItem(OrderedDict):
    """
    An expected configuration item. The default value and description
    are recorded.
    """
    def __init__(self, default, description, verify, choices):
        OrderedDict.__init__(self)
        self["default"] = default
        self["description"] = description
        self["verify"] = verify
        self["choices"] = choices

#####################################################################
class ChordsGit(object):
    """
    Manage CHORDS git activities.
    """
    def __init__(self, proxy=""):
        # The path for the docker-compose configuration.
        self.proxy = proxy

    def fetch(self, release_or_branch, files):
        """
        Fetch files from the CHORDS git repository.
        release_or_branch - the git source branch or Release tag sha.
        files - a single file or a list of files
        """

        # This will be either a branch name or a commit sha.
        url_specifier = release_or_branch

        # If a Release specification, get the release tag shas.
        if "Release-" in release_or_branch:
          url_specifier = url_specifier.replace('Release-','')

        if not isinstance(files, list):
            files = [files]

        for file_name in files:
            file_url = 'https://raw.githubusercontent.com/earthcubeprojects-chords/chords/'+ url_specifier + '/' + file_name
#            if OS_ARCH[:3] == 'arm':
#                docker_compose_yml = 'https://raw.githubusercontent.com/earthcubeprojects-chords/chords/' + release_or_branch + '/rpi-docker-compose.yml'

            proxy_switch = []
            if self.proxy != '':
                proxy_switch = ['-xxx', self.proxy]

            print('Downloading ' + file_url + '.', end=' ')
            sys.stdout.flush()
            curl_cmd = [
                'curl', '-O', '-s', file_url
                ]
            if OS_NAME == 'nt':
                curl_cmd[1:1] = ['-k']

            # Insert proxy switch
            curl_cmd[1:1] = proxy_switch

            os_cmd(cmd=curl_cmd)

            print()

    def branches(self):
        """
        Return the available branches for github.com/earthcubeprojects-chords/chords.
        """
        git_cmd = [
            'git', 'ls-remote', '--heads', 'https://github.com/earthcubeprojects-chords/chords'
            ]
        heads = [e.replace('refs/heads/','') for e in os_cmd(cmd=git_cmd, printlines=False)[0].split()[1::2]]
        if 'gh-pages' in heads:
            heads.remove('gh-pages')
        return heads

    def tags_and_shas(self):
        """
        Return a dictionary of tags and associated shas for github.com/earthcubeprojects-chords/chords.

        Tags containing 'archive/' are ignored.
        """
        git_cmd = [
            'git', 'ls-remote', '--tags', '--refs', 'https://github.com/earthcubeprojects-chords/chords'
        ]
        shas_and_tags = os_cmd(cmd=git_cmd, printlines=False)[0].split()
        shas = shas_and_tags[0::2]
        tags = shas_and_tags[1::2]
        tags = [e.replace('refs/tags/', '') for e in tags]
        # Create a list of tuples (tag, sha)
        tags_shas = [[tags[i], shas[i]] for i in range(len(shas)-1)]
        # Remove those containing 'archive/'
        tags_shas = [e for e in tags_shas if 'archive/' not in e[0]]
        # Convert to a dictionary
        retval = {e[0]:e[1] for e in tags_shas}

        return retval

    def branches_and_releases(self):
      """
      Return a list of branch names and releases.

      Release names are prefixed with 'Release-'
      """
      # Add the branches
      branches = self.branches()

      # Add the tags
      tags_and_shas = self.tags_and_shas()
      releases = ['Release-'+k for k in tags_and_shas.keys()]

      return branches + releases

#####################################################################
class Docker(object):
    """
    Manage docker activities. docker-compose.yml and docker-compose-dev.yml are
    expected in the working directory.
    """

    def __init__(self, proxy=""):

        self.docker_compose_yml = 'docker-compose.yml'
        self.dockerhub_tags_uri = "https://registry.hub.docker.com/v2/repositories/earthcubechords/chords/tags"
        if OS_ARCH[:3] == 'arm':
            self.docker_compose_yml = 'rpi-docker-compose.yml'
            self.dockerhub_tags_uri = "https://registry.hub.docker.com/v1/repositories/earthcubechords/rpi-chords/tags"

        # Determine the path for docker-compose
        if OS_NAME == 'nt':
            self.docker_compose_cmd = 'docker-compose'
        else:
            cmd_choices = ['/usr/local/bin/docker-compose', '/usr/bin/docker-compose', '/bin/docker-compose']
            found_cmd = False
            for choice in cmd_choices:
                self.docker_compose_cmd = choice
                if check_file_exists(self.docker_compose_cmd, print_warning=False, exit_on_failure=False):
                    found_cmd = True
                    break
            if not found_cmd:
                print('docker-compose was not found in', cmd_choices)
                print('You must install docker-compose before proceeding.')
                exit(1)

        if not check_file_exists(self.docker_compose_yml, exit_on_failure=False):
            release_or_branch = choose_branch(self.docker_compose_yml)
            print(release_or_branch)
            ChordsGit(proxy=proxy).fetch(release_or_branch=release_or_branch, files=self.docker_compose_yml)

    def running_containers(self):
        """
        Return an array containing a dictionary for each currently running container.
        The dictionaries keys are a subset of the output columns from docker ps. They are:
            name:
            runningfor
            status
            createdat
            image
        """

        ps_cmd = [
            'docker',
            'ps',
            '--format',
            '\"name\":\"{{.Names}}\", \"runningfor\":\"{{.RunningFor}}\", \"status\":\"{{.Status}}\", \"createdat\":\"{{.CreatedAt}}\", \"image\":\"{{.Image}}\"']

        ps_result, _ = os_cmd(cmd=ps_cmd, printlines=False)

        ps_result = ps_result.split('\n')
        containers = []
        for result in ps_result:
            result = result.strip()
            if result != '':
                if not re.search("WARNING", result):
                    result = '{' + result + '}'
                    containers.append(json.loads(result))

        # Convert the json unicode to bytes
        return containers

    def docker_compose_up(self, devmode=False):
        """
        Bring the containers up with docker-compose.
        """
        # Make sure that .env exists
        if not os.path.isfile('.env'):
            print('*** The environment file .env is missing. Use chords_control to create it.')
            exit (1)

        # Find out the release that we are running
        d_tag = grep('DOCKER_TAG', '.env')
        d_tag = d_tag[0].split('=')
        if len(d_tag) == 2:
            print('*** Running the \'' + d_tag[1] + '\' release of CHORDS', end=' ')
            if not devmode:
                print('.')
            else:
                print(', in development mode.')
        else:
            print('.env file is incorrectly formatted (DOCKER_TAG is missing)')
            exit (1)

        if not devmode:
            up_cmd = [
                self.docker_compose_cmd,
                '-f', self.docker_compose_yml, '-p', 'chords', 'up', '-d']
        else:
            up_cmd = [
                self.docker_compose_cmd,
                '-p', 'chords',
                '-f', 'docker-compose.yml', '-f', 'docker-compose-dev.yml',
                'up', '-d']

        os_cmd(cmd=up_cmd, err_msg='Unable to start containers')

    def down(self):
        """
        Take the containers down with docker-compose.
        """
        dn_cmd = [
            self.docker_compose_cmd,
            '-f', self.docker_compose_yml, '-p', 'chords', 'down']
        os_cmd(cmd=dn_cmd, err_msg='Unable to stop containers')

    def pull(self):
        """
        Pull docker images.
        """

        # We need docker-compose.yml
        check_file_exists(self.docker_compose_yml)

        print("*** Pulling Docker images. This may take awhile...")
        pull_cmd = [
            self.docker_compose_cmd,
            '-f', self.docker_compose_yml, 'pull']
        os_cmd(cmd=pull_cmd, err_msg='Unable to pull Docker images')
        print("*** ...Docker pull is finished.")

    def tags(self):
        """
        Return an array of tag names for dockerhub images.
        """

        curl_cmd = [
            'curl', '-f', '-s', self.dockerhub_tags_uri
            ]
        if OS_NAME == 'nt':
            curl_cmd[1:1] = ['-k']

        curl_result, _ = os_cmd(
            cmd=curl_cmd,
            err_msg='Unable to fetch the Docker tags',
            printlines=False)

        tags = []
        for tag in json.loads(curl_result)['results']:
            tags.append(tag["name"])

        return tags
#####################################################################
class ChordsSslError(Exception):
    """
    Usage: Raise ChordsSslError("error msg").
    """

#####################################################################
class ChordsSSL(object):
    """
    Manage SSL certificates.

    The complete chain of certificate management activities
    is trigger just by instantiating this class.
    """
    def __init__(self, ssl_host, ssl_email):

        self.ssl_host = ssl_host
        self.ssl_email = ssl_email
        self.cert_dir = "/etc/letsencrypt/live/"+self.ssl_host

        print("*** Beginning SSL management. This is a complicated activity, with many steps.")
        print()
        print ("""
CHORDS should not be automatically restarted by a Linux
service during the SSL process. You should disable or stop
any such service before proceeding.""")
        print()
        reply = prompt("Have you stopped any CHORDS auto-restart services?", ['y', 'Y', 'n', 'N'])
        if reply == 'n':
            # Don't create cert now
            print()
            print("You will have to continue the SSL configuration later.")
            print()
            return

        if ssl_host == "" or ssl_email == "":
            raise ChordsSslError("When SSL is enabled, you must provide SSL_HOST and SSL_EMAIL. Rerun with --config to specify these.")
        print ("*** Checking DNS for your hostname...")
        self.host_dns()
        print ("*** Checking to see if you already have a valid SSL certificate...")

        result = self.check_cert()
        if result["valid_cert"]:
            print ("You have a valid certificate!")
        else:
            if result["test_cert"]:
                print ("You appear to have a test certificate.")
                reply = prompt("Do you wish to replace it with a valid certificate?", ['y', 'Y', 'n', 'N'])
                if reply == 'n':
                    # Don't create cert now
                    print()
                    print("""
The test certificate will be retained. The CHORDS portal will be using an
inoperative certificate. We suggest that you rerun the configuration
and set SSL_ENABLED to false.""")
                    print()
                    return

            # Create a new certificate
            print ("We need to create a new certificate.")
            self.create_cert()
            print()

    def create_cert(self):
        """
        Create a new certificate.

        CHORDS must not be running.

        A staging certificate will be created first, just to insure
        that the process is working.
        """
        print ("*** Creating the SSL certificate")

        self.stop_nginx()

        # docker_check() returns an empty string if the named instance is running.
        instances = [docker_check([s])=='' for s in ["chords_app", "chords_nginx", "chords_certbot"]]
        if True in instances:
            raise ChordsSslError(
                "SSL certificate creation can't be done while one of the CHORDS \n"
                "services (chords_app, chords_nginx, or chords_certbot) is running. \n"
                "Use python chords_control --stop."
            )

        self.make_dummy_cert()
        self.make_dh_params()
        self.request_cert(create_test_cert = True)

        reply = prompt("""

If the above text contains the phrase 
"Successfully received certificate. Your certificate and chain have been saved at ..."
then the test certificate was successfully created.

Did the test cert get created properly?""", ['y', 'Y', 'n', 'N'])
        if reply == 'n':
            print ("Test certificate creation failed. Try again later.")
            return
        print ("""
Test certificates can be created as often as you like.
However, you can only create 5 real certificates each WEEK.
If the certificate creation process is failing, do not
use up your quota of requests!
        """)

        reply = prompt("Do you want to create a real certificate", ['y', 'Y', 'n', 'N'])
        if reply == 'n':
            print ("Test certificate creation failed. Try again later.")
            return

        # Create the new cert
        self.request_cert(create_test_cert = False)

    def host_dns(self):
        """
        Determine the IP address of self.ssl_host.

        Raise ChordsSslError if self.ssl_host cannot be located.
        """

        try:
            out = socket.gethostbyname(self.ssl_host)
            print("Good news! Your host name (" + self.ssl_host + ") is registered with DNS as " + out + ".")
            print()
        except:
            raise ChordsSslError("The SSL_HOST " + self.ssl_host + " could not be located. Ensure that you have a DNS name assigned and that it is pointing to this computer.")

    def check_cert(self):
        """
        Return a dictionary containing the certificate status.
        """

        retval = {"valid_cert": False, "test_cert": False, "expired": None, "notBefore": None, "notAfter": None}

        # Ask certbot if there are any certificates.
        result = self.cert_inquire()
        print(result, "\n")

        retval["valid_cert"] =  (not
                                 ("No certs found." in result or "invalid" in result or "error" in result or "No certificates" in result) or
                                 "(VALID:" in result)
        retval["test_cert"] = ("INVALID: TEST_CERT" in result)
        if retval["test_cert"]:
            retval["valid_cert"] = False

        if retval["valid_cert"]:
            # Use openssl to get the timestamps from the certificate.
            cert_path = self.cert_dir + "/fullchain.pem"
            cmd = ["run", "--no-deps", "--entrypoint", "openssl x509 -dates -noout -in %s" % cert_path, "certbot"]
            try:
                result = docker_compose(cmd).split('\n')
                not_before = [s.split("=")[1] for s in result if "notBefore" in s]
                not_after = [s.split("=")[1] for s in result if "notAfter" in s]
                if not (retval["notBefore"] and retval["notAfter"]):
                    retval["notBefore"] = datetime.datetime.strptime(not_before[0], '%b %d %H:%M:%S %Y %Z')
                    retval["notAfter"]  = datetime.datetime.strptime(not_after[0], '%b %d %H:%M:%S %Y %Z')
                    retval["expired"]   = datetime.datetime.now() > retval["notAfter"]
                else:
                    retval["valid_cert"] = False
            except sh.ErrorReturnCode_1:
                retval["valid_cert"] = False

        return retval

    def make_dummy_cert(self):
        """
        Create a dummy key and certificate.

        This allows nginx to run in SSL mode.

        The key will be written to self.cert_dir/privkey.pem.
        The certificate will be written to self.cert_dir/fullchain.pem.
        """

        print("*** Creating a dummy certificate.")

        cmd = [
            "run",
            "--no-deps",
            "--entrypoint",
            "mkdir -p %s" % self.cert_dir,
            "certbot"]
        result = docker_compose(cmd)
        print(result)

        cmd = [
            "run",
            "--no-deps",
            "--entrypoint",
            "openssl req -x509 -nodes -newkey rsa:2048 -days 1 -keyout '%s/privkey.pem' -out '%s/fullchain.pem' -subj '/CN=localhost'" % (self.cert_dir, self.cert_dir),
            "certbot"]
        result = docker_compose(cmd)
        print(result)
        print("*** Dummy certificate created.")

    def make_dh_params(self):
        """
        Create the Diffie-Hellman parameters.

        They will be written to /etc/letsencrypt/chords-dhparam/dhparam-2048.pem.
        """
        print ("*** Creating the Diffie-Hellman parameters. This can take several minutes.")

        cmd = [
            "run",
            "--no-deps",
            "--entrypoint",
            "mkdir -p /etc/letsencrypt/chords-dhparam",
            "certbot"]
        result = docker_compose(cmd)
        print(result)

        cmd = [
            "run",
            "--no-deps",
            "--entrypoint",
            "openssl dhparam -out /etc/letsencrypt/chords-dhparam/dhparam-2048.pem 2048",
            "certbot"]
        result = docker_compose(cmd)
        print(result)
        print ("*** Diffie-Hellman parameters created.")

    def remove_certs(self):
        """
        Remove existing certificates.

        Remove the following:
        /etc/letsencrypt/live/*
        /etc/letsencrypt/archive/*
        /etc/letsencrypt/renewal/*
        """
        print ("*** Removing the existing certificates.")

        cmd = [
            "run",
            "--no-deps",
            "--entrypoint",
            "/bin/bash -c 'rm -rf /etc/letsencrypt/live/* /etc/letsencrypt/archive/* /etc/letsencrypt/renewal/*'",
            "certbot"]
        result = docker_compose(cmd)
        print(result)

    def request_cert(self, create_test_cert=True):
        """
        Obtain a test or real certificate from letsencrypt.
        """

        print ("*** Obtaining a new %s certificate from letsencrypt." % ("test" if create_test_cert else "real"))

        # Start nginx for the ACME challenge.
        self.start_nginx()

        # Remove the existing cert
        self.remove_certs()

        print("*** Sending certificate request to letsencrypt.")
        cmd = [
            "run",
            "--no-deps",
            "--entrypoint",
            "certbot certonly --webroot -w=/acme-challenge %s --email %s --agree-tos --no-eff-email -d %s" %
            (("--staging" if create_test_cert else ""), self.ssl_email, self.ssl_host),
            "certbot"]
        result = docker_compose(cmd)
        print(result)

        # Shutdown nginx
        self.stop_nginx()

    def start_nginx(self):
        """
        Start an nginx webserver, to service the ACME challenge.
        """

        print ("*** Starting nginx, to service the ACME challenge.")
        # Run nginx in certificate creation mode.
        cmd = [
            "run",
            "-e",
            "CERT_CREATE=1",
            "-e",
            "SSL_HOST=%s" % self.ssl_host,
            "-e",
            "SSL_EMAIL=%s" % self.ssl_email,
            "-p",
            "80:80",
            "-p",
            "443:443",
            "--no-deps",
            "-d",
            "nginx"
        ]
        result = docker_compose(cmd)
        print(result)

    def stop_nginx(self):
        """
        Stop the nginx container.
        """

        print("*** Stopping nginx")
        cmd = ["down"]
        result = docker_compose(cmd)
        print(result)

    def cert_inquire(self):
      # Ask certbot if there are any certificates.
      cmd = ["run", "--no-deps", "--entrypoint", "certbot certificates", "certbot"]
      result = docker_compose(cmd)
      return result

#####################################################################
class ChordsBackupError(Exception):
    """ Usage: raise ChordsBackupError("error msg"). """

class ChordsBackup(object):
    """
    Perform a CHORDS backup.

    The backup is performed when the class is instantiated.
    """

    def __init__(self):
        """
        Perform the backup.
        """

        self.id_check()
        self.config_files_check(files=["./.env", "chords.yml", "docker-compose.yml"])
        self.docker_check()

        time_stamp = datetime.datetime.now().replace(microsecond=0).isoformat()
        time_stamp = time_stamp.replace(":", "-")

        self.mysql_sql_file = 'mysql-' + time_stamp + '.sql'
        influx_dump_dir = 'influxdb-' + time_stamp
        self.influx_tar_file = 'influxdb-' + time_stamp + '.tar'
        self.grafana_tar_file = 'grafana-' + time_stamp + '.tar'
        self.manifest_file = "manifest-" + time_stamp + ".txt"
        self.chords_tar_file = 'chords-backup-' + self.nice_project_name() + '-' + time_stamp + '.tar'

        print("*** Saving mysql ***")
        mysql_file = open(self.mysql_sql_file, 'w')
        print(sh.docker('exec -t chords_mysql /usr/bin/mysqldump chords_demo_production'.split(),
                        _out=mysql_file).stdout.decode('utf-8'))

        print("*** Saving influxdb ***")
        print(docker_bash(
            "chords_influxdb", "cd /tmp && influxd backup -portable %s" % (influx_dump_dir)))
        print(docker_bash(
            "chords_influxdb", "cd /tmp && tar -cvf %s %s" % (self.influx_tar_file, influx_dump_dir)))
        docker_cp('chords_influxdb:%s' % ("/tmp/"+self.influx_tar_file), '.')
        print(docker_bash(
            "chords_influxdb", "cd /tmp && rm -rf %s %s" % (self.influx_tar_file, influx_dump_dir)))

        print("** Saving grafana ***")
        print(docker_bash(
            "chords_grafana", "cd / && tar -cvf /tmp/%s etc/grafana/grafana.ini var/lib/grafana/" %
            (self.grafana_tar_file)))
        docker_cp('chords_grafana:%s' % ("/tmp/"+self.grafana_tar_file), '.')
        print(docker_bash(
            "chords_grafana", "cd /tmp && rm %s" % (self.grafana_tar_file)))

        print("*** Manifest ***:")
        self.manifest()
        print("%s has been created." % (self.manifest_file))
        print("")

        print("*** Packaging files ***")
        tar_params = [
            '-czvf', self.chords_tar_file, ".env", "chords.yml",
            self.manifest_file,
            self.mysql_sql_file,
            self.influx_tar_file,
            self.grafana_tar_file]
        print(sh.tar(tar_params, _err_to_out=True).stdout.decode('utf-8'))
        chmod_params = [
            'go-rw', self.chords_tar_file
        ]
        print(sh.chmod(chmod_params, _err_to_out=True).stdout.decode('utf-8'))
        print(sh.rm(self.manifest_file, self.mysql_sql_file, self.influx_tar_file, self.grafana_tar_file))

        print("*** Success ***")
        print("%s has been created." % (self.chords_tar_file))
        print("For details on the CHORDS backup, examine the file: %s (contained in the tar file)." % (self.manifest_file))

    def nice_project_name(self):
        """ Return a nicely behaved project name for use in file name construction."""

        project_name = self.mysql_value(table="profiles", column="project")

        project_name = ''.join(ch for ch in project_name if ch not in set(string.punctuation))
        project_name = project_name.strip()
        project_name = " ".join(project_name.split())
        project_name = project_name.replace(' ','-')
        project_name = project_name.lower()

        return project_name

    def id_check(self):
        """ Verify that the user is running with the correct permissions. """

        if platform.system() == "Linux":
            if getpass.getuser() != 'root':
                raise ChordsBackupError("CHORDS load must be run as root user on Linux systems.")

    def config_files_check(self, files):
        """
        Verify that required configuration files are present.

        If not, raise ChordsretoreError.
        """

        error = False
        for file in files:
            if not check_file_exists(the_file=file, exit_on_failure=False):
                error = True
        if error:
            raise ChordsBackupError("""
Required configuration files are missing.
Are you in a CHORDS home directory (typically /var/lib/chords/)?
""")

    def docker_check(self):
        """ Verify that a CHORDS instance is running correctly. """

        if docker_check(containers=
                        ["chords_app", "chords_influxdb", "chords_mysql", "chords_grafana"]):
            raise ChordsBackupError("""
It appears that CHORDS is not running correctly.
A CHORDS backup cannot be performed.""")

    def manifest(self):
        """
        Create a manifest file.
        """

        manifest_file = open(self.manifest_file, "w")

        manifest_file.write("# CHORDS backup\n")
        manifest_file.write(
            "# Project: " + self.mysql_value(table="profiles", column="project") + "\n")
        manifest_file.write(
            "# Affiliation: " + self.mysql_value(table="profiles", column="affiliation") + "\n")
        manifest_file.write(
            "# Domain name: " + self.mysql_value(table="profiles", column="domain_name") + "\n")
        manifest_file.write("# CHORDS configuration file: chords.yml\n")
        manifest_file.write("# Docker environment file: .env\n")
        manifest_file.write("# MYQSL database dump file: %s\n" % (self.mysql_sql_file))
        manifest_file.write("# InfluxDB database dump file: %s\n" % (self.influx_tar_file))
        manifest_file.write("# Grafana configuration file: %s\n" % (self.grafana_tar_file))
        manifest_file.write("# Environment:\n")
        env_vars = docker_bash("chords_app", "grep export chords_env.sh | sed -e 's/export //'")
        manifest_file.write(env_vars)
        docker_tag = "DOCKER_TAG=" + docker_bash("chords_app", "printenv DOCKER_TAG")
        manifest_file.write(docker_tag)
        manifest_file.close()
        return

    def mysql_value(self, table, column):
        """ Fetch a mysql field. """
        value = docker_bash(
            "chords_mysql",
            '/usr/bin/mysql -s -N -e "use chords_demo_production; select %s from %s;"'
            % (column, table))
        value = value.strip()
        return value

#####################################################################
class ChordsRestoreError(Exception):
    """ Raise ChordsRestoreError("error msg"). """

class ChordsRestore(object):
    """
    Load a backup file into a running CHORDS instance.

    The CHORDS backup file was created by the ChordsBackup class.

    Requirements:
     - On Linux systems, you must have root privileges in order to access docker.

    """

    def __init__(self, dump_file, tmp_dir="/tmp"):
        """
        The constructor will perform the complete load process.

        If there are errors, ChordsRestoreError wil be thrown.
        """
        self.dump_file = dump_file

        self.config_files_check(files=["./.env", "chords.yml", "docker-compose.yml"])
        self.id_check()
        self.are_you_sure()

        self.tmp_dir = tempfile.mkdtemp(prefix="chords_restore-", dir=os.path.normpath(tmp_dir))
        self.file_paths = {}
        self.docker_containers = {}

        print("*** Docker ***")
        self.docker_check()
        self.docker_container_status()

        print("*** Dump file unpacking ***")
        self.backup_unpack()

        print("*** Environment restore ***")
        self.env_restore()

        print("*** Loading mysql ***")
        self.load_mysql()

        print("*** Loading influxdb ***")
        self.load_influxdb()

        print("*** Loading grafana ***")
        self.load_grafana()

        print("*** Cleanup ***")
        self.cleanup()

    def are_you_sure(self):
        """
        Verify that the user really wants to do a restore.

        If not, raise ChordsRestoreError.
        """

        print("*** Warning: this action will overwrite ALL data in the running portal.")
        print('Are you sure that you want restore from %s [Y/y]? ' % (self.dump_file), end=' ')
        sys.stdout.flush()
        response = sys.stdin.readline().strip()
        if response not in ['Y', 'y']:
            raise ChordsRestoreError("Restore cancelled.")

    def id_check(self):
        """ Verify that the user is running with the correct permissions. """

        if platform.system() == "Linux":
            if getpass.getuser() != 'root':
                raise ChordsRestoreError("CHORDS restore must be run as root user on Linux systems.")

    def config_files_check(self, files):
        """
        Verify that required configuration files are present.

        If not, raise ChordsRestoreError.
        """

        error = False
        for file in files:
            if not check_file_exists(the_file=file, exit_on_failure=False):
                error = True
        if error:
            raise ChordsRestoreError("""
Required configuration files are missing.
Are you in a CHORDS home directory (typically /var/lib/chords/)?
""")

    def docker_check(self):
        """ Verify that a CHORDS instance is running correctly. """

        if docker_check(containers=
                        ["chords_app", "chords_influxdb", "chords_mysql", "chords_grafana"]):
            raise ChordsRestoreError("""
It appears that CHORDS is not running correctly.
A CHORDS restore cannot be performed.""")

    def docker_container_status(self):
        """ Print the status of all containers. """

        print("Docker containers:")
        for name, container in list(self.docker_containers.items()):
            print(name + ": " + container.status)
        print("")

    def env_restore(self):
        """
        Restore the local environment and configuration files, if desired.

        The user may, or may not, wish to restore .env and chords.yml.
        Prompt and restore if desired.
        """

        config_files = (".env", "chords.yml")
        print("Since we are in a CHORDS home directory, you have already configured")
        print("CHORDS with items such as system passwords, email access and software versions.")
        print("However, we can restore the CHORDS configuration files %s" % (str(config_files)))
        print("from %s." % (self.dump_file))
        print("Backup copies of these files will be made, in case you need to reverse this action.")
        print('Would you like to restore these files [Y/y]? ')
        response = sys.stdin.readline().strip()
        if response not in ['Y', 'y']:
            print("%s WERE NOT restored." % (str(config_files)))
        else:
            for c_file in config_files:
                create_backup_file(c_file)
                src_file = os.path.abspath(self.tmp_dir + "/" + c_file)
                dst_file = os.path.abspath("./" + c_file)
                print("Restoring %s to %s" % (src_file, dst_file))
                sh.cp(src_file, dst_file)
        print(" ")
        # get the admin password, which will be needed to access the databases,etc.
        self.env_admin_pw = config_items(".env", delimiter="=")["CHORDS_ADMIN_PW"]

    def load_mysql(self, database_name="chords_demo_production"):
        """ Load the mysql database into the database in the chords_ysql container. """

        container="chords_mysql"
        mysql_user = "chords_demo_user"
        print("Container:%s, database:%s" % (container, database_name))

        print("Setting the user password for %s." % (mysql_user))
        docker_args = ['exec', '-t', container, '/usr/bin/mysql', '-e',
                       "set password for '%s' = '%s';"%(mysql_user, self.env_admin_pw)]
        try:
            print(
                sh.docker(
                    sh.cat(self.file_paths["mysql"]), docker_args, _err_to_out=True
                ).stdout.decode('utf-8')
            )
        except sh.ErrorReturnCode_1 as error:
            raise ChordsRestoreError(error)

        # Load the mysql database simply by streaming the database spl file into
        # the mysql app on the container.
        print("Restoring the database.")
        docker_args = ['exec', '-i', container, '/usr/bin/mysql', database_name]
        try:
            print(
                sh.docker(
                    sh.cat(self.file_paths["mysql"]), docker_args, _err_to_out=True
                ).stdout.decode('utf-8')
            )
        except sh.ErrorReturnCode_1 as error:
            raise ChordsRestoreError(error)

    def load_influxdb(self):
        """ Load the influxdb database into the database in the chords_influxdb container. """

        try:
            print("Setting the database password.")
            print(docker_bash("chords_influxdb",
                              "influx -username admin -password chords_ec_demo -execute \"set password for admin = \'%s\'\""
                              % (self.env_admin_pw)))
        except ChordsDockerError as error:
            try:
                print(docker_bash("chords_influxdb",
                                  "influx -username admin -password %s -execute \"set password for admin = \'%s\'\""
                                  % (self.env_admin_pw, self.env_admin_pw)))
            except ChordsDockerError as error:
                raise ChordsRestoreError(error)

        try:
            print("Current influxdb databases:")
            print(docker_bash("chords_influxdb",
                              "influx -username admin -password %s -execute 'show databases'"
                              % (self.env_admin_pw)))
        except ChordsDockerError as error:
            raise ChordsRestoreError(error)

        influx_tar_file_base = os.path.basename(self.file_paths["influxdb"])

        print("Loading from " + self.file_paths["influxdb"])
        docker_cp(self.file_paths["influxdb"], "chords_influxdb:/tmp/")
        try:
            docker_bash("chords_influxdb",
                        "cd /tmp && tar -xvf %s" % (influx_tar_file_base))
        except ChordsDockerError as error:
            raise ChordsRestoreError(error)

        try:
            print("Dropping influxdb database chords_ts_production")
            docker_bash(
                "chords_influxdb",
                "influx -username admin -password %s -execute 'drop database chords_ts_production'"
                % (self.env_admin_pw)
            )
            print("Loading databases")
            docker_bash(
                "chords_influxdb",
                "influxd restore -portable /tmp/%s" % (influx_tar_file_base.replace(".tar", ""))
            )
        except ChordsDockerError as error:
            raise ChordsRestoreError(error)

    def load_grafana(self):
        """ Load the grafana files into the grafana container. """

        grafana_tar_file_base = os.path.basename(self.file_paths["grafana"])

        print("Loading from " + self.file_paths["grafana"])
        docker_cp(self.file_paths["grafana"], "chords_grafana:/tmp/")
        docker_bash("chords_grafana",
                    "cd / && tar -xvpf /tmp/%s" % (grafana_tar_file_base), user='0')
        docker_bash("chords_grafana",
                    "rm /tmp/%s" % (grafana_tar_file_base), user='0')
        print(" ")

    def backup_unpack(self):
        """ Unpack the ChordsBackup created backup file to a temporary directoy. """

        print("*** Unpacking chords files ***")
        print("Temporary directory: " + self.tmp_dir)
        try:
            print(
                sh.tar('-xzvf', self.dump_file, '-C', self.tmp_dir, _err_to_out=True).stdout.decode('utf-8')
            )
        except Exception as sh_err:
            raise ChordsRestoreError(sh_err)
        self.file_check()

    def file_check(self):
        """
        Determine the dump file names for each database.

        The dictionary self.file_paths is initialized with the file name
        for each dataset type.

        Return an empty string if valid, or an error message if not.
        """

        err_msg = ""
        FileSpec = namedtuple('FileSpec', ['prefix', 'ext'])
        file_types = [FileSpec('mysql', 'sql'),
                      FileSpec('influxdb', 'tar'),
                      FileSpec('grafana', 'tar')]
        for file_type in file_types:
            files = glob.glob(
                self.tmp_dir + "/" + file_type.prefix + "*." + file_type.ext
            )
            if len(files) == 1:
                self.file_paths[file_type.prefix] = files[0]
            else:
                if len(files) > 1:
                    if err_msg:
                        err_msg += "\n"
                    err_msg += "More than one %s dump file was found: " % (file_type.prefix) + \
                        " ".join([os.path.basename(f) for f in files]) + "."
                else:
                    if err_msg:
                        err_msg += "\n"
                    err_msg += "No %s dump file was found." % (file_type.prefix)
        if err_msg:
            raise ChordsRestoreError(err_msg)

    def cleanup(self):
        """ Cleanup temporary files. """

        print("Removing the temporary directory " + self.tmp_dir)
        print(sh.rm('-rf', self.tmp_dir, _err_to_out=True).stdout.decode('utf-8'))

#####################################################################
class ChordsDockerError(Exception):
    """ Raise ChordsDockerError("error msg"). """

def docker_bash(container, script, user=None):
    """
    Run shell comands in a docker bash instance.

    This allows multiple commands to be &&'ed together.

    The user parameter must be a string, but it can be a uid such as '0'
    """

    if not user:
        retval = docker_sh('exec', '-t', container, '/bin/bash', '-c', script)
    else:
        retval = docker_sh('exec', '-t',  '-u', user, container, '/bin/bash', '-c', script)

    return retval

def docker_sh(*args):
    """ Run a docker command with the args, printing stdout and stderr. """

    try:
        if VERBOSE:
            print("Executing: docker", " ".join(args))
        result = sh.docker(args, _err_to_out=True).stdout.decode('utf-8')
        if VERBOSE:
            print("Result:", str(result))
        return str(result)
    except Exception as sh_err:
        raise ChordsDockerError(sh_err)

def docker_cp(src, dest):
    """
    Copy a file to/from container.

    src and/or dest will specify a container prefix when appropriate, e.g.
        docker_cp(database.sql chords_mysql:/tmp)
    """

    docker_sh('cp', src, dest)

def docker_compose(cmd, err_to_out=True):
    """
    Run a docker-compose command.

    By default, stderr is combined into stdout.
    """

    if VERBOSE:
        print("docker_compose:", cmd)

    try:
        result = sh.docker_compose(cmd, _err_to_out=err_to_out, _internal_bufsize=100).stdout.decode('utf-8')
    except sh.ErrorReturnCode as e:
        print('docker_compose command failed:', cmd, '\n', e.stdout, '/n', e.stderr)
        exit(1)
    return result

#####################################################################
def run():
    """
    Run CHORDS.
    """
    if OPTIONS['devmode']:
        check_file_exists('docker-compose-dev.yml')

    Docker(proxy=OPTIONS['proxy']).docker_compose_up(OPTIONS['devmode'])

def stop():
    """
    Stop CHORDS.
    """
    Docker(proxy=OPTIONS['proxy']).down()

def restart():
    """
    Stop, then start CHORDS.
    """
    stop()
    run()

def status():
    """
    Show the docker status for the running containers.
    """
    for container in Docker(proxy=OPTIONS['proxy']).running_containers():
        print(container['name'] + ': ' + str(container))

#####################################################################
def ssl(ssl_config):
    """ Manage SSL """
    try:
        if ssl_config["SSL_ENABLED"] == "true":
            ChordsSSL(ssl_host=ssl_config["SSL_HOST"], ssl_email=ssl_config["SSL_EMAIL"])
    except ChordsSslError as error:
        print(error)
        exit(1)

#####################################################################
def backup():
    """ Backup a CHORDS portal """
    try:
        ChordsBackup()
    except ChordsBackupError as error:
        print(error)
        exit(1)

#####################################################################
def restore(backup_file):
    """ Load a portal from a backup file. """

    try:
        ChordsRestore(backup_file)
    except ChordsRestoreError as restore_exception:
        print("Error processing %s" % (backup_file))
        print(restore_exception)
        exit(1)

    print("""
CHORDS databases have been restored.

CHORDS must be restarted by running:
python chords_control --stop
python chords_control --run
""")

#####################################################################
def docker_check(containers):
    """
    Verify that a CHORDS instance is running correctly.

    Print out a list of containers that are not running.
    Return the error message.

    A return string containing text indicates an error,
    which can thus be tested as a boolean, where True will indicate
    an error.
    """

    names = [c['name'].split("/")[-1] for c in Docker(proxy=OPTIONS['proxy']).running_containers()]
    err_msg = ""
    for container_name in containers:
        if container_name not in names:
            err_msg += "The %s docker container is not present.\n" % (container_name)
    if err_msg != '':
        print(err_msg)
    return err_msg

#####################################################################
def create_config_file_from_env(options_in):
    """
    Read the .env file, and convert it into a configuration file.
    options["file"] is the name of the new configuration file.
    If the named configuration file already exists, a backup copy is
    made when the new configuration is saved.
    Since we are using the ChordsConfig class, any values
    not present in .env will be given default values.
    """

    # create a new temporary configuration file.
    tfile = tempfile.mkstemp(prefix="chords_control_", text=True)[1]
    cfile = open(tfile, "w")
    if os.path.isfile(".env"):
        efile = open(".env")
        lines = efile.readlines()
        for line in lines:
            line = line.replace("=", ":")
            cfile.write(line)
    cfile.close()

    # Process the temporary configuration file, which will
    # merge in default values for variables that weren't in .env.
    config_tmp = ChordsConfig(configfile=tfile)

    # Save the new configuration file and update the .env file.
    config_tmp.write_yml_file(options_in["file"])
    config_tmp.write_env_file()

#####################################################################
def os_cmd(cmd, err_msg=None, printlines=True):
    """
    Run a shell command. The return value is a tuple.
    [0] is a single string, containing the command output,
    with each line delimited by a newline.
    [1] is the exit status of the command.
    If err is specified, it should contain an error message,
    and chk_cmd_status() will be called.
    """
    if VERBOSE:
        print(' '.join(str(x) for x in cmd))

    try:
        proc = subprocess.Popen(cmd,stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
        lines = ''
        while True:
            output = proc.stdout.readline().decode('utf-8')
            lines = lines + output
            if output == '' and proc.poll() is not None:
                break
            if output != '' and printlines:
                print(output.rstrip())
        exit_status = proc.poll()
    except (OSError, ValueError) as ex:
        print(str(ex) + ': ' + ' '.join(str(x) for x in cmd))

    if err_msg:
        check_cmd_status(exit_status=exit_status, err_msg=err_msg, cmd=cmd)
    return (lines, exit_status)

#####################################################################
def prompt(message, choices):
    '''
    Keep prompting until the user enters a valid choice.

    The lowercase of the accepted response will be returned.
    '''

    if not isinstance(choices, list):
        choices = [choices]

    choice_string = ""
    for i in range(len(choices)):
        choice_string += choices[i]
        if i != len(choices) - 1:
            choice_string += '/'
    print(message + " [" + choice_string + "]?", end=' ')
    sys.stdout.flush()

    response = ''
    while response not in choices:
        response = sys.stdin.readline().strip()
        if response not in choices:
            print ('Enter ' + choice_string, end=' ')
            sys.stdout.flush()
    response = response.lower()
    return response

#####################################################################
#def byteify(inputStr):
#    """
#    Convert unicode to a byte strings
#    """
#    if isinstance(inputStr, dict):
#        return {byteify(key): byteify(value)
#                for key, value in inputStr.items()}
#    elif isinstance(inputStr, list):
#        return [byteify(element) for element in inputStr]
#    elif isinstance(inputStr, str):
#        return inputStr.encode('utf-8')
#    else:
#        return inputStr

#####################################################################
def add_choices_to_docker_tag(proxy=""):
    """
    Get the available tags and add them to the DOCKER_TAG valid choices.
    Modifies StdConfigItems.
    """
    for std_item in STD_CONFIG_ITEMS:
        if std_item[1] =="DOCKER_TAG":
            tags = Docker(proxy=proxy).tags()
            for tag in tags:
                std_item[4].append(tag)

#####################################################################
def add_choices_to_git_branch(proxy=""):
    """
    Get the avaiable branches/releases and and them to the GIT_BRANCH valid choices.
    Modifies StdConfigItems.
    """
    for std_item in STD_CONFIG_ITEMS:
        if std_item[1] == "GIT_BRANCH":
          identifiers = ChordsGit(proxy=proxy).branches_and_releases()
          for identifier in identifiers:
              std_item[4].append(identifier)
          # Set the default to the most recent Release
          std_item[2] = std_item[4][-1]

#####################################################################
def check_file_exists(the_file, writeable=False, print_warning=True, exit_on_failure=True):
    """
    Make sure that the file is readable. If not, print error message
    and exit.
    """

    if not os.path.isfile(the_file) or not os.access(the_file, os.R_OK if not writeable else os.W_OK):
        if print_warning:
            print("File " + the_file +  " doesn't exist or isn't readable.")
        if exit_on_failure:
            exit(1)
        else:
            return False
    return True

#####################################################################
def grep(expression, the_file):
    """
    Find lines in a file containing the expression.
    Return as an array of strings. White space is
    removed.
    """
    our_file = open(the_file, "r")

    matches = []
    for line in our_file:
        if re.search(expression, line):
            matches.append(line.strip())

    return matches

#####################################################################
def renew_script(proxy):
    """
    Update this script and docker-compose configurations.
    A github branch will be requested.
    """
    script_name = 'chords_control'

    if not check_file_exists(script_name, writeable=True, print_warning=False, exit_on_failure=False):
        print("Either " + script_name + " isn't located in this directory, or it is not writeable.")
        exit(1)

    release_or_branch = choose_branch(script_name)

    files = [script_name, "docker-compose.yml", "docker-compose-dev.yml"]
    for our_file in files:
        create_backup_file(our_file)

    ChordsGit(proxy=proxy).fetch(release_or_branch=release_or_branch, files=files)

#####################################################################
def check_cmd_status(exit_status, cmd, err_msg):
    """
    Check the exit_status.
    If it is non-zero, the error message will be printed
    along with the command, and exit(1) will be called.
    cmd is a list, such as used with the subrocess command.
    """
    if exit_status != 0:
        print(err_msg + ' using "' + " ".join(cmd) + '"')
        exit(1)

#####################################################################
def choose_branch(the_file):
    """
    Prompt the user to choose which git branch to fetch the file from.
    """
    identifiers = ChordsGit().branches_and_releases()
    identifiers.sort()
    release_or_branch = ''
    while True:
        print("Which branch or release do you want to pull " + the_file + " from (usually the most recent release)\n(", end=' ')
        for identifier in identifiers:
            print('\''+identifier+'\'', end=' ')
        print(')? ', end=' ')
        sys.stdout.flush()
        release_or_branch = sys.stdin.readline().strip()
        if release_or_branch in identifiers:
            break
        else:
            print("The", release_or_branch, "branch is not available.")
    return release_or_branch

#####################################################################
def config_items(config_file, delimiter=":"):
    """
    Return a dictionary of pairs in a config file
    """
    items = dict()
    c_file = open(config_file)
    lines = c_file.readlines()
    c_file.close()
    i = 1
    for line in lines:
        line = line.strip()
        if line:
            if line[0] != "#":
                tokens = line.split(delimiter)
                if len(tokens) != 2:
                    print("Ambiguous value in line %d of %s" % (i, config_file))
                items[tokens[0].strip()] = tokens[1].strip()
        i = i + 1
    return items

#####################################################################
#####################################################################

if __name__ == '__main__':

    # Get the command line options
    OPTIONS = CommandArgs().get_options()
    VERBOSE = OPTIONS['verbose']

    # --- Command procesing ---

    # Run the CHORDS portal
    if OPTIONS["run"]:
        run()

    # Stop the currently running CHORDS portal
    if OPTIONS["stop"]:
        stop()

    # Restart the CHORDS portal
    if OPTIONS["restart"]:
        restart()

    # Show the current CHORDS status
    if OPTIONS["status"]:
        status()

    # Prompt and update the configuration
    if OPTIONS["config"]:
        add_choices_to_docker_tag()
        add_choices_to_git_branch()
        CONFIG = ChordsConfig(configfile=OPTIONS['file'])
        CONFIG.query_values(usedefault=OPTIONS['default'])
        CONFIG.write_yml_file(configfile=OPTIONS['file'])
        CONFIG.write_env_file()
        print()

        # Get items from git that are needed.
        # moved from --update
        ChordsGit(proxy=OPTIONS['proxy']).fetch(release_or_branch=CONFIG["GIT_BRANCH"], files="docker-compose.yml")

        # Pull the images.
        # moved from --update
        Docker(proxy=CONFIG['PROXY']).pull()

        #manage SSL configuration
        ssl(CONFIG)

        # print("*** Don't forget to run 'python chords_control --update' if you have changed the configuration.")

    # Create the .env file from the configuration file
    if OPTIONS["env"]:
        if not os.path.isfile(OPTIONS["file"]):
            print("The CHORDS configuration file " + OPTIONS['file'] + " does not exist. It can be created with ./chords_control --config --default [--file FILE]")
            exit(1)
        CONFIG = ChordsConfig(configfile=OPTIONS['file'])
        CONFIG.write_env_file()

    # Create the configuration file from the .env file
    if OPTIONS["backwards"]:
        create_config_file_from_env(OPTIONS)

    # Update the docker images
    if OPTIONS["update"]:
        if not os.path.isfile(OPTIONS["file"]):
            print("The CHORDS configuration file " + OPTIONS['file'] + " does not exist. It can be created with ./chords_control --config --default [--file FILE]")
            exit(1)
        # Get the system configuration.
        CONFIG = ChordsConfig(configfile=OPTIONS['file'])
        # Get items from git that are needed.
        ChordsGit(proxy=OPTIONS['proxy']).fetch(release_or_branch=CONFIG["GIT_BRANCH"], files="docker-compose.yml")
        # Pull the images.
        Docker(proxy=CONFIG['PROXY']).pull()

    # Replace this script with one from the repository
    if OPTIONS["renew"]:
        add_choices_to_docker_tag()
        add_choices_to_git_branch()
        renew_script(proxy=OPTIONS['proxy'])

    # Create a backup file
    if OPTIONS["backup"]:
        if platform.system() != "Windows":
            backup()
        else:
            print("Backup is not available on Windows.")

    # Load from a backup file
    if OPTIONS["restore"]:
        if platform.system() != "Windows":
            restore(OPTIONS['restore'])
        else:
            print("Restore is not available on Windows.")