k8.env

# CLUSTER_CONFIG values for setting environment variables
# this will eventually be ported to ansible group_vars

# deploy multiple clusters on a host by changing the K8_ENV name like:
# dev_k8.env  - export K8_ENV="dev"
# test_k8.env - export K8_ENV="test"
# prod_k8.env - export K8_ENV="prod"
export K8_ENV="dev"

# ensure debugging propagates as an env var
if [[ "${METAL_DEBUG}" == "1" ]]; then
    export METAL_DEBUG="1"
else
    export METAL_DEBUG="0"
fi

# set path to metalnetes repository or use the current directory:
# cannot exit here because on remote vms this path will not exist
# because the repository is not currently cloned locally
export REPO_BASE_DIR="${METAL_REPO_DIR}"
if [[ "${REPO_BASE_DIR}" == "" ]] || [[ ! -e "${REPO_BASE_DIR}" ]]; then
    # assume the command was ran from the base repo dir
    export REPO_BASE_DIR="$(pwd)"
    if [[ "${METAL_DEBUG}" == "1" ]]; then
        echo "$(date) - DEBUG - $(hostname) - using repository base directory: ${REPO_BASE_DIR}"
    fi
fi

# if "1" - destroy and rebuild the cluster on startup
# total data loss for anything not persisted outside
# of the cluster but great for testing...
# do not use in prod
export START_CLEAN="1"

# if "1" - start a private docker registry
# private docker registry for kubernetes
export START_REGISTRY="1"
export REGISTRY_USER="jay"
export REGISTRY_PASSWORD="123321"
export REGISTRY_VOLUME_BASE="/data/registry" # for persisting storage
export REGISTRY_STARTER="${REPO_BASE_DIR}/deploy-registry.sh"
export REGISTRY_SECRET="${REPO_BASE_DIR}/registry/secrets.yaml"
export REGISTRY_COMPOSE_FILE="${REPO_BASE_DIR}/registry/compose.yaml"
export REGISTRY_ADDRESS="${DOCKER_REGISTRY_FQDN}" # format: host:port
export REGISTRY_AUTH_DIR="${REGISTRY_VOLUME_BASE}/auth"
export REGISTRY_DATA_DIR="${REGISTRY_VOLUME_BASE}/data"

# if "1" - start helm and tiller
# helm
export START_HELM="1"
export HELM_STARTER="${REPO_BASE_DIR}/deploy-helm.sh"
export HELM_INSTALL_IF_NOT_FOUND_USING_CURL="1"
# tiller (needed by helm - uses START_HELM to trigger too)
export TILLER_STARTER="${REPO_BASE_DIR}/deploy-tiller.sh"
export TILLER_RBAC="${REPO_BASE_DIR}/tiller/rbac.yml"

# if "1" - start storage
# supported layers:
# rook-ceph - https://rook.io/docs/rook/master/helm-operator.html
export START_STORAGE="1"
export STORAGE_VALUES="${REPO_BASE_DIR}/rook-ceph/values.yaml"
export STORAGE_STARTER="${REPO_BASE_DIR}/deploy-rook-ceph.sh"
export STORAGE_OPERATOR="${REPO_BASE_DIR}/rook-ceph/run.sh"
export STORAGE_TYPE="rook-ceph-block"
export STORAGE_NAMESPACE="rook-ceph"
export STORAGE_DELETE_ON_REBOOT="0" # this can add 3 minutes on a server reboot by setting to "1"

# if "1" - start https://github.com/AlgoTraders/stock-analysis-engine/
export START_AE="0"
export AE_DEPLOY_DIR="${REPO_BASE_DIR}/ae"
export AE_VALUES="${AE_DEPLOY_DIR}/ae/values.yaml"
export AE_STARTER="${REPO_BASE_DIR}/deploy-ae.sh"
export AE_HELM_DEPLOY_TOOL="${AE_DEPLOY_DIR}/start.sh"
export AE_STOP="${AE_DEPLOY_DIR}/stop.sh"
export AE_RESTORE_LATEST="${AE_DEPLOY_DIR}/deploy-latest.sh"
export AE_RESTORE_DOWNLOAD_DIR="/data2/ae/" # large s3 files will be downloaded here with the ae-restore chart runs - should have more than 50 GB free to prevent filling up
export AE_CONTAINER_RESTORER="/opt/sa/analysis_engine/script/sa.py" # inside the docker container at this path
export AE_BACKUP_S3_BUCKET="ae-stock-datasets"
export AE_CHARTS="ae ae-grafana ae-jupyter ae-minio ae-prometheus ae-redis ae-restore"
export AE_CRON_STARTER="${AE_DEPLOY_DIR}/cron/run-job.sh"
export AE_INTRADAY_JOB="${AE_DEPLOY_DIR}/run-intraday-job.sh"
export AE_DAILY_JOB="${AE_DEPLOY_DIR}/run-daily-job.sh"
export AE_WEEKLY_JOB="${AE_DEPLOY_DIR}/run-weekly-job.sh"
export AE_BACKUP_JOB="${AE_DEPLOY_DIR}/run-backup-job.sh"
export AE_RESTORE_JOB="${AE_DEPLOY_DIR}/run-restore-job.sh"
export AE_RESTORE_ON_STARTUP="0"
export AE_JOB_LOG="/tmp/ae-job.log"
# path to a python 3 virtual environment for ae (by default '/opt/venv')
# install guide: https://github.com/AlgoTraders/stock-analysis-engine#running-on-ubuntu-and-centos
# required for importing backups from S3 and restoring to redis on startup
export AE_VENV="/opt/venv"

# if "1" - ingress will install
# https://github.com/nginxinc/kubernetes-ingress/
export START_INGRESS="1"
export INGRESS_TYPE="nginx"

# libvirt - fedora 29 kvm - use system scope
export LIBVIRT_DEFAULT_URI="qemu:///system"

# vm setup
export K8_VMS="m10 m11 m12"
export K8_DOMAIN="example.com"
export K8_INITIAL_MASTER="m10.${K8_DOMAIN}"
export K8_SECONDARY_MASTERS="m11.${K8_DOMAIN} m12.${K8_DOMAIN}"
export K8_DNS_SERVER_1="192.168.0.100"
export K8_GATEWAY="192.168.0.1"
export K8_VM_IP_1="192.168.0.110"
export K8_VM_IP_2="192.168.0.111"
export K8_VM_IP_3="192.168.0.112"
export K8_VM_MAC_1="00:10:01:9c:91:10"
export K8_VM_MAC_2="00:10:01:9c:91:11"
export K8_VM_MAC_3="00:10:01:9c:91:12"
export K8_VM_IPS="${K8_VM_IP_1} ${K8_VM_IP_2} ${K8_VM_IP_3}"
export K8_VM_MACS="${K8_VM_MAC_1} ${K8_VM_MAC_2} ${K8_VM_MAC_3}"
export K8_VM_SIZE="100" # vm's hdd size in GB
export K8_VM_BRIDGE="br0" # virbr0 is another one
export K8_VM_CPU="4" # number of cores per vm
export K8_VM_MEMORY="16960" # in MB
export K8_VM_USER="jay" # ssh user
export K8_VM_PASSWORD="123321" # ssh user's password
export K8_VM_TZ="US/Eastern" # timezone
if [[ -e $HOME/.ssh/id_rsa.pub ]]; then
    export K8_VM_SSH_KEY="$HOME/.ssh/id_rsa.pub"
else
    export K8_VM_SSH_KEY=""
fi
export K8_USER_DATA_SSH_ACCESS="1" # allow K8_VM_USER and root user ssh access into the vms (required for package installs during vm startup)
export K8_USER_DATA_STATIC_NETWORKING="1" # static networking="1", else dhcp from bridge device
export K8_IMAGES_DIR="/data/isos"   # store download OS images in this dir
export K8_VMS_DIR="/data/kvm/disks" # store kvm vm qcow2 disks in this dir
export K8_NODES="${K8_INITIAL_MASTER} ${K8_SECONDARY_MASTERS}"
export K8_LABELS="frontend=enabled backend=enabled datascience=enabled ceph=enabled minio=enabled monitoring=enabled splunk=disabled"
export K8_CNI_FLANNEL_VERSION="v0.11.0"
export K8_PASSWORD_FILE="${REPO_BASE_DIR}/pw/password" # used by sshpass -f K8_PASSWORD_FILE ssh-copy-id
export K8_CLEANER="${REPO_BASE_DIR}/clean.sh"
export K8_START="${REPO_BASE_DIR}/start.sh"
export K8_JOIN="${REPO_BASE_DIR}/join.sh"
export K8_VM_START="${REPO_BASE_DIR}/vms-start.sh"
export K8_VM_WAIT="${REPO_BASE_DIR}/vms-wait.sh"
export K8_CLEAN_CEPH="${REPO_BASE_DIR}/rook-ceph/patch-operator-teardown.sh"
export K8_CONFIG_DIR="/opt/k8/${K8_ENV}"
export K8_TOOLS_DIR="/opt/k8/tools"
export K8_SSH_KEY="/opt/k8/id_rsa"
export K8_SSH_KEY_PUB="/opt/k8/id_rsa.pub"
export LOCAL_OS_DIR="${REPO_BASE_DIR}/centos"
export LOCAL_VM_SRC_TOOLS="${REPO_BASE_DIR}/tools"
export LOCAL_SSH_KEY="${LOCAL_OS_DIR}/keys/id_rsa"
export LOCAL_SSH_KEY_PUB="${LOCAL_OS_DIR}/keys/id_rsa.pub"
export REMOTE_VM_NETWORK_INSTALLER="${LOCAL_OS_DIR}/install-network-device.sh"
export REMOTE_VM_INSTALLER="${K8_CONFIG_DIR}/vm-install.sh"
export REMOTE_VM_DOCKER_SERVICE="${K8_CONFIG_DIR}/docker.service"
export REMOTE_VM_KERNEL_MODULES="${K8_CONFIG_DIR}/kernel-modules.conf"
export KUBECONFIG="${K8_CONFIG_DIR}/admin_k8_cluster_${K8_ENV}.config"
export LOGIN_USER="root"
export DOCKER_DATA_DIR="/data/docker/*"
export DEPLOY_SSH_KEY="~/.ssh/id_rsa"
export TOOL_NODE_LABELER="${LOCAL_VM_SRC_TOOLS}/apply_labels.sh"
export TOOL_DNS_ETC_RESOLV="${LOCAL_VM_SRC_TOOLS}/install-etc-resolv-conf.sh"
export TOOL_UNLOCK_NODES="${LOCAL_VM_SRC_TOOLS}/unlock-all-nodes.sh"
export TOOL_DEPLOY_FILES="${LOCAL_VM_SRC_TOOLS}/deploy-files-to-nodes.sh"
export TOOL_CNI_STARTER="${LOCAL_VM_SRC_TOOLS}/start-cni-flannel.sh"
export REMOTE_CNI_DIR="/var/lib/cni/networks/cbr0" # on each vm
export REMOTE_TOOL_CNI_INSTALLER="${K8_TOOLS_DIR}/install-cni.sh"
export REMOTE_TOOL_HARD_RESET_VM="${K8_TOOLS_DIR}/reset-k8-and-docker-and-cni-on-vm.sh"
export REMOTE_TOOL_NODE_RESET="${K8_TOOLS_DIR}/reset-node.sh"
export REMOTE_TOOL_VM_PREPARE="${K8_TOOLS_DIR}/prepare.sh"
export REMOTE_TOOL_UPDATE_K8="${K8_TOOLS_DIR}/update-k8.sh"
export REMOTE_TOOL_INSTALL_GO="${K8_TOOLS_DIR}/install-go.sh"
export REMOTE_TOOL_INSTALL_HTOP="${K8_TOOLS_DIR}/install-htop.sh"
export REMOTE_TOOL_USER_INSTALL_KUBECONFIG="${K8_TOOLS_DIR}/user-install-kubeconfig.sh"
export REMOTE_TOOL_CLUSTER_JOINER="/root/k8join"
export PREPARE_MODE="fast"
export USE_LABELS="new-ceph"
export INSTALL_GO="0"
export INSTALL_HTOP="0"
export UPDATE_KUBE="1"
export APPLY_DNS="1"
export GO_VERSION="1.11.4"
export RUN_ON_CLUSTER_VM="export CLUSTER_CONFIG=${K8_CONFIG_DIR}/k8.env"
export RUN_CMD_ON_VM="export CLUSTER_CONFIG=${K8_CONFIG_DIR}/k8.env && source ${K8_CONFIG_DIR}/k8.env"

if [[ "${REGISTRY_ADDRESS}" == "" ]]; then
    test_reg_url=$(hostname | grep "${K8_DOMAIN}" | wc -l)
    if [[ "${test_reg_url}" == "0" ]]; then
        export REGISTRY_ADDRESS="$(hostname).${K8_DOMAIN}:5000"
    else
        export REGISTRY_ADDRESS="$(hostname):5000"
    fi
fi

# Additional external block device storage per vm in the cluster
export VM_DATA_DIR="/cephdata"
export VM_DISK_1_NAME="vdb"
export VM_DISK_1_SIZE="100G"
export VM_DISK_1_MOUNT_PATH="/var/lib/ceph"
export VM_DISK_2_NAME="vdc"
export VM_DISK_2_SIZE="20G"
export VM_DISK_2_MOUNT_PATH="/var/lib/rook"
export VM_DISK_3_NAME="vdd"
export VM_DISK_3_SIZE="150G"
export VM_DISK_3_MOUNT_PATH=""

# KVM new VM creation options
export KVM_USE_BASE_IMAGE="1"
export KVM_IMAGES_DIR="/data/isos"
export KVM_VMS_DIR="/data/kvm/disks"
export KVM_VM_SIZE="100"
export KVM_IMAGE_FILE="/data/isos/centos-7.iso"
export KVM_NAMED_ALLOW_QUERY_DNS_CIDR="192.168.0.0/24"
export KVM_BASE_NAME="metalbase"
export KVM_BASE_NODE="metalbase.example.com"
export KVM_BASE_IP="192.168.0.200"
export KVM_BASE_MAC="00:10:01:9c:02:00"
export KVM_STORAGE_TYPE="raw" # raw format is faster than qcow2
export KVM_SSH_INSTALL_TOOL="${REPO_BASE_DIR}/tools/install-ssh-keys.sh"
export KVM_BASE_BUILD_TOOL="${REPO_BASE_DIR}/kvm/build-k8-base-vm.sh"
export KVM_BASE_IMAGE_PATH="${KVM_VMS_DIR}/${KVM_BASE_NAME}/${KVM_BASE_NAME}.${KVM_STORAGE_TYPE}"

k8_wait_for_completed() {
    namespace="${1}"
    pod_name="${2}"
    sleep_interval="5"
    max_attempts="40"
    if [[ "${pod_name}" == "" ]]; then
        return
    fi
    if [[ "${3}" != "" ]]; then
        sleep_interval=${3}
    fi
    if [[ "${4}" != "" ]]; then
        max_attempts=${4}
    fi
    is_a_pod=$(kubectl get -n ${namespace} po | grep ${pod_name} | wc -l)
    if [[ "${is_a_pod}" == "0" ]]; then
        return
    fi
    not_done=$(kubectl get -n ${namespace} po | grep ${pod_name} | grep -i "completed" | wc -l)
    cur_attempt=1
    while [[ "${not_done}" == "0" ]]; do
        date_val=$(date -u +"%Y-%m-%d %H:%M:%S")
        inf "${date_val} - sleeping ${cur_attempt}/${max_attempts} - waiting for ${pod_name} to complete - seconds: ${sleep_interval}"
        sleep ${sleep_interval}
        kubectl get po | grep ${pod_name}
        not_done=$(kubectl get -n ${namespace} po | grep ${pod_name} | grep -i "completed" | wc -l)
        let "cur_attempt=cur_attempt+1"
        if [[ "${cur_attempt}" == "${max_attempts}" ]]; then
            err "stopping waiting for pod=${pod_name} in namespace=${namespace} after attempts=${cur_attempt}"
            not_done="1"
        fi
    done
    echo "done waiting for ${pod_name} to complete"
}

test_helm_installed() {
    test_helm=$(which helm | wc -l)
    if [[ "${test_helm}" == "0" ]]; then
        if [[ "${HELM_INSTALL_IF_NOT_FOUND_USING_CURL}" == "1" ]]; then
            echo ""
            anmt "detected helm is not installed - installing using the command:"
            inf "curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash"
            curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash 
            if [[ "$?" != "0" ]]; then
                helm_version="2.13.0"
                # https://github.com/helm/helm/releases/tag/v2.13.0
                wget https://storage.googleapis.com/kubernetes-helm/helm-v${helm_version}-linux-amd64.tar.gz -O /tmp/helm.tgz
                if [[ "$?" == "0" ]]; then
                    start_dir=$(pwd)
                    cd /tmp
                    tar xf /tmp/helm.tgz
                    if [[ "$?" != "0" ]]; then
                        inf ""
                        err "failed installing helm - please refer to the helm installation docs on:"
                        err "https://helm.sh/docs/using_helm/#from-script"
                        anmt "attempted:"
                        inf "curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash"
                        inf ""
                    else
                        sudo mv ./linux-amd64/helm /usr/local/bin/
                        rm -rf ./linux-amd64
                    fi
                    rm /tmp/helm.tgz
                    cd ${start_dir}
                else
                    inf ""
                    err "failed installing helm - please refer to the helm installation docs on:"
                    err "https://helm.sh/docs/using_helm/#from-script"
                    anmt "attempted:"
                    inf "curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash"
                    inf ""
                    exit 1
                fi
            fi
        else
            inf ""
            err "failed starting up:"
            inf ""
            err "please install helm before running using the install guide: "
            err "https://helm.sh/docs/using_helm/#from-script"
            err "or if you want to copy paste it (which may not be the latest way):"
            inf "curl https://raw.githubusercontent.com/helm/helm/master/scripts/get > get_helm.sh"
            inf "chmod 700 get_helm.sh"
            inf "./get_helm.sh"
            inf ""
            err "if helm is installed, please retry after exporting the PATH variable to include the path to helm with:"
            inf "export PATH=\$PATH:<path to helm>"
            exit 1
        fi
    fi
}

is_k8_ready() {
    nodes="${K8_INITIAL_MASTER} ${K8_SECONDARY_MASTERS}"
    num_k8_nodes_expected=$(echo "${nodes}" | sed -e 's/ /\n/g' | wc -l)
    num_k8_nodes_found=$(kubectl get nodes -o wide | grep Ready | wc -l)

    if [[ "${num_k8_nodes_expected}" == "${num_k8_nodes_found}" ]]; then
        echo "ONLINE"
    else
        echo "MISSING_NODES"
    fi
}

stop_if_not_ready() {
    k8_ready=$(is_k8_ready)
    if [[ "${k8_ready}" != "ONLINE" ]]; then
        err "kubernetes on ${K8_ENV} is not online with vms=${K8_VMS} nodes=${K8_NODES} KUBECONFIG=${KUBECONFIG}:"
        kubectl get nodes -o wide
        echo ""
        exit 1
    fi
}

is_tiller_ready() {
    tiller_running=$(kubectl get po -n kube-system | grep tiller-deploy | wc -l)
    if [[ "${tiller_running}" == "1" ]]; then
        echo "1"
    elif [[ "${tiller_running}" != "0" ]]; then
        # echo ""
        # echo "FOUND more than one tiller with:"
        # echo "kubectl get po -n kube-system | grep tiller-deploy | wc -l"
        # echo ""
        echo "1"
    else
        echo "starting tiller with: ${TILLER_STARTER}"
        ${TILLER_STARTER}
        if [[ "$?" != "0" ]]; then
            err "failed starting tiller with ${TILLER_STARTER}"
            exit 1
        fi
        tiller_running=$(kubectl get po -n kube-system | grep tiller-deploy | wc -l)
        if [[ "${tiller_running}" == "1" ]]; then
            k8_wait_for_completed kube-system tiller-deploy 5 30
            echo "1"
        else
            echo "0"
        fi
    fi
}

is_rook_ceph_ready() {
    pods_running=$(kubectl get po -n rook-ceph | grep Running | wc -l)
    pods_not_good=$(kubectl get po -n rook-ceph | grep -v -E "Completed|Running" | wc -l)
    if [[ "${pods_running}" != "0" ]] && [[ "${pods_not_good}" == "0" ]]; then
        echo "1"
    else
        echo "0"
    fi
}

wait_for_rook_ceph() {
    namespace="${STORAGE_NAMESPACE}"
    pod_name="rook-ceph-osd-id"
    sleep_interval="5"
    max_attempts="30"
    inf ""
    anmt "k8_wait_for_completed ${namespace} ${pod_name} ${sleep_interval} ${max_attempts}"
    k8_wait_for_completed ${namespace} ${pod_name} ${sleep_interval} ${max_attempts}
    anmt "done waiting for ceph to start:"
    kubectl -n ${namespace} get po
    inf ""
}

ensure_virtualenv_has_pip_or_exit() {
    venv_path="${1}"
    pipname="${2}"

    missing_venv="0"
    stop_now="0"
    if [[ ! -e ${use_venv} ]]; then
        missing_venv="1"
        stop_now="1"
    else
        source ${use_venv}/bin/activate
        test_pip_installed=$(pip list --format=columns | grep ${pipname} | wc -l)
        if [[ "${test_pip_installed}" == "0" ]]; then
            stop_now="1"
        fi
    fi

    test_pip_installed=$(pip list --format=columns | grep ${pipname} | wc -l)
    if [[ "${stop_now}" == "1" ]] || [[ "${test_pip_installed}" == "0" ]]; then
        echo "creating virtualenv: ${use_venv}"
        virtualenv -p python3 ${use_venv}
        if [[ "$?" != "0" ]]; then
            echo "failed creating virtualenv for python using:"
            echo "virtualenv -p python3 ${use_venv}"
            echo ""
            echo "Please refer to your install guide:"
            echo "Ubuntu/CentOS install steps:"
            echo "https://github.com/AlgoTraders/stock-analysis-engine#running-on-ubuntu-and-centos"
            echo "Mac OS X install steps:"
            echo "https://github.com/AlgoTraders/stock-analysis-engine#running-on-mac-os-x"
            echo ""
            exit 1
        fi
        source ${use_venv}/bin/activate
        if [[ "$?" != "0" ]]; then
            echo "failed activating virtualenv for python using:"
            echo "source ${use_venv}/bin/activate"
            echo ""
            echo "Please refer to your install guide:"
            echo "Ubuntu/CentOS install steps:"
            echo "https://github.com/AlgoTraders/stock-analysis-engine#running-on-ubuntu-and-centos"
            echo "Mac OS X install steps:"
            echo "https://github.com/AlgoTraders/stock-analysis-engine#running-on-mac-os-x"
            echo ""
            exit 1
        fi
        pip install --upgrade pip ${pipname}
        stop_now="0"
    fi

    test_pip_installed=$(pip list --format=columns | grep ${pipname} | wc -l)
    if [[ "${stop_now}" == "1" ]] || [[ "${test_pip_installed}" == "0" ]]; then
        err "Please install the ${pipname} pip into the virtualenv ${use_venv}"
        echo ""
        if [[ "${missing_venv}" == "1" ]]; then
            anmt "create the virtualenv for python and the ${pipname} with:"
            echo "virtualenv -p python3 ${use_venv}"
            echo "source ${use_venv}/bin/activate"
            echo ""
        fi
        anmt "install ${pipname} and upgrade pip manually with:"
        echo "pip install --upgrade pip ${pipname}"
        echo ""
        warn "if that fails please refer to the Ubuntu/CentOS steps:"
        echo "https://github.com/AlgoTraders/stock-analysis-engine#running-on-ubuntu-and-centos"
        warn "or the Mac OS X install steps:"
        echo "https://github.com/AlgoTraders/stock-analysis-engine#running-on-mac-os-x"
        echo ""
        exit 1
    fi
}

slp() {
    total_sleep=60
    sleep_interval=5
    msg=""
    if [[ "${1}" != "" ]]; then
        total_sleep="${1}"
    fi
    if [[ "${2}" != "" ]]; then
        sleep_interval="${2}"
    fi
    if [[ "${3}" != "" ]]; then
        msg="${3}"
    fi
    if [[ "${METAL_DEBUG}" == "1" ]]; then
        inf "slp total=${total_sleep} interval=${sleep_interval} msg=${msg}"
    fi
    sleep_left=${total_sleep}
    while [[ ${sleep_left} -gt 0 ]]; do
        if [[ "${msg}" == "" ]]; then
            anmt "$(date): sleeping for ${sleep_left} more seconds"
        else
            anmt "$(date): ${msg} - sleeping for ${sleep_left} more seconds"
        fi
        sleep $sleep_interval
        sleep_left="$((${sleep_left} - ${sleep_interval}))"
    done
}

# install kuberenetes config from the any cluster's master node
# on to the local host at the same path
metal() {
    anmt "${K8_ENV} - metal installing ${KUBECONFIG} from root@${K8_INITIAL_MASTER}"
    if [[ ! -e "$(dirname ${KUBECONFIG})" ]]; then
        mkdir -m -p 775 $(dirname ${KUBECONFIG})
        if [[ "$?" != "0" ]]; then
            sudo mkdir -m -p 775 $(dirname ${KUBECONFIG})
            if [[ "$?" != "0" ]]; then
                err "${K8_ENV} - failed to create KUBECONFIG=${KUBECONFIG} directory"
            fi
        fi
    fi
    good_to_deploy="1"
    scp -q root@${K8_INITIAL_MASTER}:${KUBECONFIG} ${KUBECONFIG}
    if [[ "$?" != "0" ]]; then
        echo "scp -q root@${K8_INITIAL_MASTER}:/etc/kubernetes/admin.conf ${KUBECONFIG}"
        scp -q root@${K8_INITIAL_MASTER}:/etc/kubernetes/admin.conf ${KUBECONFIG}
        if [[ "$?" != "0" ]]; then
            good_to_deploy="0"
            err "${K8_ENV} - failed to scp KUBECONFIG=${KUBECONFIG} from root@${K8_INITIAL_MASTER} using:"
            err "scp root@${K8_INITIAL_MASTER}:${KUBECONFIG} ${KUBECONFIG}"
            err "scp root@${K8_INITIAL_MASTER}:/etc/kubernetes/admin.conf ${KUBECONFIG}"
            inf "please confirm kubernetes is running on nodes: ${nodes}"
        fi
    else
        if [[ "${METAL_DEBUG}" == "1" ]]; then
            good "${K8_ENV} - installed KUBECONFIG=${KUBECONFIG}"
        fi
        export KUBECONFIG=${KUBECONFIG}
    fi

    metal_deploy_worked="${good_to_deploy}"
    if [[ "${good_to_deploy}" == "1" ]]; then
        deploy_user="root"
        if [[ "${METAL_DEBUG}" == "1" ]]; then
            anmt "${K8_ENV} - deploying local ${KUBECONFIG} using scp as ${deploy_user} to nodes=${K8_NODES}"
        fi
        for node in $K8_NODES; do
            if [[ "${METAL_DEBUG}" == "1" ]]; then
                echo "${K8_ENV} - creating dirs on ${node}:${KUBECONFIG}"
            fi
            ssh -o StrictHostKeyChecking=no ${deploy_user}@${node} "mkdir -p -m 775 ${K8_TOOLS_DIR}; mkdir -p -m 775 ${K8_CONFIG_DIR}; mkdir -p -m 775 ${K8_CONFIG_DIR}/../logs"
            if [[ "$?" != "0" ]]; then
                err "${K8_ENV} - failed to build dirs with:"
                err "ssh -o StrictHostKeyChecking=no ${deploy_user}@${node} \"mkdir -p -m 775 ${K8_TOOLS_DIR}; mkdir -p -m 775 ${K8_CONFIG_DIR}; mkdir -p -m 775 ${K8_CONFIG_DIR}/../logs\""
                metal_deploy_worked="0"
            else
                scp -q ${KUBECONFIG} ${deploy_user}@${node}:${KUBECONFIG}
                if [[ "$?" != "0" ]]; then
                    err "${K8_ENV} - failed to deploy ${KUBECONFIG} to ${node} as ${deploy_user}:"
                    err "scp -q ${KUBECONFIG} ${deploy_user}@${node}:${KUBECONFIG}"
                    metal_deploy_worked="0"
                fi
            fi
        done
    fi
    if [[ "${metal_deploy_worked}" == "0" ]]; then
        err "${K8_ENV} - metal failed KUBECONFIG=${KUBECONFIG} deployed to nodes=${K8_NODES}"
    else
        good "${K8_ENV} - metal success KUBECONFIG=${KUBECONFIG} deployed to nodes=${K8_NODES}"
    fi
}

# scripts need to start the logger by default disabled
start_logger() {
    if [[ "${LOG_FILE}" == "" ]]; then
        export LOG_FILE="./logs/${K8_ENV}.log"
    else
        # make sure it is installed
        export LOG_FILE="${LOG_FILE}"
    fi
    if [[ ! -e $(dirname ${LOG_FILE}) ]]; then
        mkdir -p -m 775 $(dirname ${LOG_FILE})
        if [[ "$?" != "0" ]]; then
            echo "failed to create log dir: $(dirname ${LOG_FILE})"
            echo "disabled logging to file: ${LOG_FILE}"
            export LOG_FILE=""
        fi
    fi
    if [[ "${LOG_FILE}" != "" ]]; then
        reload_logger
    fi
}

# show the deployment logs with:
logs() {
    if [[ "${LOG_FILE}" != "" ]]; then
        if [[ -e ${LOG_FILE} ]]; then
            echo "${LOG_FILE}"
            cat ${LOG_FILE}
        else
            echo "no log file found at path: ${LOG_FILE}"
        fi
    fi
}

reload_logger() {
    export PATH_TO_LOGGER=""
    if [[ -e ./tools/bash_colors.sh ]]; then
        export PATH_TO_LOGGER="./tools/bash_colors.sh"
    elif [[ -e ../tools/bash_colors.sh ]]; then
        export PATH_TO_LOGGER="../tools/bash_colors.sh"
    # deploy location on vms:
    elif [[ -e /opt/k8/tools/bash_colors.sh ]]; then
        export PATH_TO_LOGGER="/opt/k8/tools/bash_colors.sh"
        # detected we're not in the right spot
        if [[ "${STAY_IN_DIR}" != "1" ]]; then
            if [[ "${CLUSTER_CONFIG}" == "" ]]; then
                if [[ -e ${K8_CONFIG_DIR}/k8.env ]]; then
                    cd ${K8_CONFIG_DIR}
                fi
            else
                if [[ -e ${CLUSTER_CONFIG} ]]; then
                    parent_dir=$(dirname "${CLUSTER_CONFIG}")
                    cd ${parent_dir}
                else
                    if [[ -e ${K8_CONFIG_DIR}/k8.env ]]; then
                        cd ${K8_CONFIG_DIR}
                    fi
                fi
            fi
        fi
    elif [[ -e ../../tools/bash_colors.sh ]]; then
        export PATH_TO_LOGGER="../../tools/bash_colors.sh"
    elif [[ -e ../../../tools/bash_colors.sh ]]; then
        export PATH_TO_LOGGER="../../../tools/bash_colors.sh"
    elif [[ -e ../../../../tools/bash_colors.sh ]]; then
        export PATH_TO_LOGGER="../../../../tools/bash_colors.sh"
    fi

    if [[ -e ${PATH_TO_LOGGER} ]]; then
        source ${PATH_TO_LOGGER}
    fi
}

set_os_type() {
    test_fc=$(uname -a | grep fc29 | wc -l)
    if [[ "${test_fc}" == "1" ]]; then
        export OS="fc"
    else
        if [[ -e /etc/redhat-release ]]; then
            test_centos=$(cat /etc/redhat-release | grep -i centos)
            if [[ "${test_centos}" != "" ]]; then
                export OS="centos"
            else
                echo "$(date) - $(hostname) - unable to determine operating system from /etc/redhat-release: /etc/redhat-release"
                uname -a
                cat /etc/redhat-release
            fi
        else
            test_ubu=$(which lsb_release | wc -l)
            if [[ "${test_ubu}" != "0" ]]; then
                export OS="ubu"
            else
                echo "$(date) - $(hostname) - unable to determine operating system: $(pwd)"
                uname -a
            fi
        fi
    fi
}

function load_env() {
    # for kubectl:
    test_path=$(echo "${PATH}" | grep '/usr/bin:' | wc -l)
    if [[ "${test_path}" == "0" ]]; then
        export PATH=${PATH}:/usr/bin
    fi
    test_path=$(echo "${PATH}" | grep '/usr/local/bin:' | wc -l)
    if [[ "${test_path}" == "0" ]]; then
        export PATH=${PATH}:/usr/local/bin
    fi
    # for helm:
    test_path=$(echo "${PATH}" | grep '/snap/bin:' | wc -l)
    if [[ "${test_path}" == "0" ]]; then
        export PATH=${PATH}:/snap/bin
    fi

    set_os_type
    reload_logger

    # exit on major prerequisite-setup errors

    test_helm_installed
}

# load PATH and logging utils
load_env