Merge pull request #623 from nuagenetworks/dev

Release Candidate for v2.4

.gitignore

@@ -6,9 +6,11 @@
 *.txt
 *~
 *.txt
+*.md5
 group_vars
 host_vars
 reports
 .idea
 .metadata
 hosts
+.vscode

Documentation/CONTRIBUTING.md → CONTRIBUTING.md

@@ -15,7 +15,6 @@ Postdeploy | For component-level sanity validation.
 Health | For system-level sanity validation and monitoring.
 Destroy | For tear down of components and connections. This is one of two hypervisor-dependent roles (predeploy is the other). If you find yourself adding conditional execution based on the hypervisor anywhere else, it's probably a mistake.
 Upgrade | For upgrading components from one release to another.
-Rollback | For restoring components to their previous version if an upgrade fails.
 ## 1. Develop Code on a Fork
 1. Before you start developing code, create your own fork from the upstream MetroAG repo. [https://github.com/nuagenetworks/nuage-metro/](https://github.com/nuagenetworks/nuage-metro/)
 2. Clone your own fork on your machine and switch to the _dev_ branch.  
@@ -28,8 +27,8 @@ git checkout dev
 3. Develop and test all proposed contributions on the appropriate hypervisors in the `metro-fork` directory. If you choose not to provide support for one or more supported hypervisors, you must provide graceful error handling for those types.
 
 4. If you require any new User Input Variables:
-    * Extend the MetroAG variable files with sensible example values:<br> `build_vars.yml`, `upgrade_vars.yml`, and `user_creds.yml`.
-    * Ensure that the copies of the variable files in `roles/reset-build/files/` are identical to<br>`build_vars.yml`, `upgrade_vars.yml`, and `user_creds.yml`.
+    * Extend the MetroAG variable files with sensible example values:<br> `build_vars.yml` and `user_creds.yml`.
+    * Ensure that the copies of the variable files in `roles/reset-build/files/` are identical to<br>`build_vars.yml` and `user_creds.yml`.
     * Include comments with the variable specifications that explain the variable's purpose and acceptable values.
     * Variables that are almost never modified may be included in standard Ansible variable locations, e.g. `roles/<rolename>/vars/main.yml`.
 

Docker/Dockerfile

@@ -1,24 +1,28 @@
-FROM centos
-
-ENTRYPOINT ["/ecStart.sh"]
-CMD [""]
-
-RUN yum install -y epel-release && yum update -y && \
-    yum install -y python python-devel python-pip sshpass git gcc libssl2-dev libffi-devel python-devel openssl-devel python-netaddr python-dns && \
-    pip install ansible==2.3 netmiko pexpect && \
-    yum clean all && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-
-ADD nuage-metro/Docker/ssh_config /root/.ssh/config
-RUN chmod 600 /root/.ssh/config
-
-# ARG git_user=jbemmel
-# ARG git_password=xxxx
-
-# ADD vspk.tar.gz /tmp
-# RUN cd /tmp/vspk-5.0.2.32 && python ./setup.py install
-
-# RUN git clone https://$git_user:[email protected]/nuagenetworks/nuage-metro.git && \
-#    sed -i 's|build_vars.yml|/files/build_vars.yml|g' nuage-metro/build.yml
-ADD nuage-metro /nuage-metro
-
-ADD nuage-metro/Docker/ecStart.sh /
+FROM centos
+
+ENTRYPOINT ["/ecStart.sh"]
+CMD [""]
+
+RUN yum install -y epel-release && yum update -y && \
+    yum install -y python python-devel python-pip sshpass git gcc libssl2-dev libffi-devel python-devel openssl-devel python-netaddr python-dns python-jmespath unzip && \
+    curl -sL https://rpm.nodesource.com/setup_8.x | bash - && yum install -y nodejs && \ 
+    pip install ansible==2.4 netmiko netaddr pexpect vspk pyvmomi && \
+    mkdir -p /root/.ansible && cd /root/.ansible && \
+    npm i superagent agentkeepalive netmask express netmask body-parser multer && \
+    yum clean all && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+ADD nuage-metro/Docker/ssh_config /root/.ssh/config
+RUN chmod 600 /root/.ssh/config
+
+# ARG git_user=jbemmel
+# ARG git_password=xxxx
+
+# ADD vspk.tar.gz /tmp
+# RUN cd /tmp/vspk-5.0.2.32 && python ./setup.py install
+
+# RUN git clone https://$git_user:[email protected]/nuagenetworks/nuage-metro.git && \
+#    sed -i 's|build_vars.yml|/files/build_vars.yml|g' nuage-metro/build.yml
+ADD nuage-metro /nuage-metro
+ADD nuage-metro/Docker/ecStart.sh /
+
+RUN sed -i 's/{,3}(/{,3}>?(/g' /usr/lib/python2.7/site-packages/ansible/plugins/terminal/sros.py && rm -f /usr/lib/python2.7/site-packages/ansible/plugins/terminal/sros.pyc

Docker/README.md

@@ -0,0 +1,19 @@
+# Docker container for Nuage Metro
+## Overview
+Docker is a tool for capturing an entire deployment environment as a disk image, 
+to be easily downloaded and obtained from (public) repositories. These files can
+be used to create such an image for the Metro Ansible scripts and their dependencies.
+
+## Usage
+To use the public Docker container, simply install Docker and run 'docker run nuage/metro'.
+The output will explain what parameters are available; by bind-mounting the current
+working directory, the container copies the Metro scripts to the host such that they can
+be modified as needed. 
+
+## Build
+To build the Metro container image, simply run 'make'
+
+## TODO
+The container does not yet have the dependencies needed for vCenter deployments. It also
+does not come with 'upgrade' out of the box ( though upgrade scripts can be run 
+manually )

Docker/ecStart.sh

@@ -1,16 +1,20 @@
 #!/bin/bash
 
+# exit on non-zero return code
+set -e
+
 #
 # Start script for Nuage Metro container
 #
 
 function show_usage() {
  echo "Copyright (C) 2017 Nuage Networks, all rights reserved. Version 1.0 2017-06-14"
  echo "Usage: docker run -it --rm -v \`pwd\`:/files nuage/metro" 
- echo "  add 'destroy' to remove everything"
+ echo "  add 'health' to check, 'destroy' to remove everything"
  echo "  tip: You can add '--dns:x.x.x.x' to specify a DNS server for the Ansible host (this container) to use"
  echo "  You may have to 'ssh-copy-id -i id_rsa user@target_server' to the target servers"
  echo "  To deploy a subset of servers, you can add '--limit=vstats' ( or vsds, vscs, etc. ) at the end"
+ echo "  Also, you may use --tags xxxx to only execute certain tasks"
 
  exit 0
 }
@@ -22,177 +26,99 @@ fi
 # Always copy Ansible scripts, such that users can customize if needed
 cp -Rn nuage-metro /files
 
-# Generate sample 
-if [ ! -f /files/build_vars.yml ]; then
-
-cat > /files/metro_vsp_minimal_build_vars.yml << EOF
-nuage_zipped_files_dir: "/files"
-nuage_unzipped_files_dir: "/files/nuage-unpacked"
-
-###
-# Usernames
-# remote_user names for ansible to execute as on the target server (hypervisor)
-# and Ansible host. target_server_username is the remote_user for all hypervisors.
-# ansible_sudo_username is the sudo user for local actions.
-target_server_username: "root"
-ansible_sudo_username: "root"
-
-# Deploy everything on a single host, use it as default gateway and a VRS
-target_server_global: 10.0.0.10
-
-mgmt_net_global: 10.0.0
-mgmt_netmask_global: 255.255.255.0
-
-dns_server_list:
-  - 10.0.0.4
-  - 10.0.0.5
-  
-dns_domain: example.com
-
-ntp_server_list:
-  - 10.0.0.2
-  - 10.0.0.3
-
-vsd_sa_or_ha: sa
-vsd_fqdn_global: vsd1.example.com
-vsd_operations_list:
-  - install
-myvsds:
-  - { hostname: vsd1.{{ dns_domain }},
-      target_server_type: "kvm",
-      target_server: "{{ target_server_global }}",
-      mgmt_ip: "{{ mgmt_net_global }}.10",
-      mgmt_gateway: "{{ target_server_global }}",
-      mgmt_netmask: "{{ mgmt_netmask_global }}" }
-
-vsc_operations_list:
-  - install
-myvscs:
-  - { hostname: vsc1.{{ dns_domain }},
-      target_server_type: "kvm",
-      target_server: "{{ target_server_global }}",
-      mgmt_ip: "{{ mgmt_net_global }}.13",
-      mgmt_gateway: "{{ target_server_global }}",
-      mgmt_netmask_prefix: 24,
-      ctrl_ip: 192.168.0.13,
-      ctrl_netmask_prefix: 24,
-      ctrl_gateway: 192.168.0.1,
-      vsd_fqdn: "{{ vsd_fqdn_global }}",
-      system_ip: 1.1.1.1,
-      xmpp_username: vsc,
-      vsc_static_route_list: { 0.0.0.0/1,128.0.0.0/1 } }
-
-vrs_operations_list:
-  - install
-dockermon_install: false
-myvrss:
-  - { vrs_set_name: vrs_set_lab,
-      vrs_os_type: el7,
-      avrs: False,
-      active_controller_ip: 192.168.0.13,
-      standby_controller_ip: 0.0.0.0,
-      vrs_ip_list: [ "{{ target_server_global }}" ] }
-  
-ansible_deployment_host: 127.0.0.1
-mgmt_bridge: "br0"
-data_bridge: "br1"
-images_path: "/var/lib/libvirt/images/"
-
-## yum_proxy: http://xxxx
-## yum_update: no
-
-EOF
-
-cat > /files/metro_vsp_cluster_build_vars.yml << EOF
-nuage_zipped_files_dir: "/files"
-nuage_unzipped_files_dir: "/files/nuage-unpacked"
-
-###
-# Usernames
-# remote_user names for ansible to execute as on the target server (hypervisor)
-# and Ansible host. target_server_username is the remote_user for all hypervisors.
-# ansible_sudo_username is the sudo user for local actions.
-target_server_username: "root"
-ansible_sudo_username: "root"
-
-vsd_sa_or_ha: ha
-vsd_fqdn_global: xmpp.example.com
-vsd_operations_list:
-  - install
-myvsds:
-  - { hostname: vsd1.example.com,
-      target_server_type: "kvm",
-      target_server: 10.0.0.10,
-      mgmt_ip: 192.168.0.10,
-      mgmt_gateway: 192.168.0.1,
-      mgmt_netmask: 255.255.255.0 }
-  - { hostname: vsd2.example.com,
-      target_server_type: "kvm",
-      target_server: 10.0.0.11,
-      mgmt_ip: 192.168.0.11,
-      mgmt_gateway: 192.168.0.1,
-      mgmt_netmask: 255.255.255.0 }
-  - { hostname: vsd3.example.com,
-      target_server_type: "kvm",
-      target_server: 10.0.0.12,
-      mgmt_ip: 192.168.0.12,
-      mgmt_gateway: 192.168.0.1,
-      mgmt_netmask: 255.255.255.0 }
-ansible_deployment_host: 127.0.0.1
-mgmt_bridge: "br0"
-data_bridge: "br1"
-images_path: "/var/lib/libvirt/images/"
-ntp_server_list:
-  - 10.0.0.2
-  - 10.0.0.3
-dns_server_list:
-  - 10.0.0.4
-  - 10.0.0.5
-dns_domain: example.com
-
-## yum_proxy: http://xxxx
-## yum_update: no
-
-EOF
-
-echo "Sample config file created as '*build_vars.yml', copy one, edit it and then re-run this tool"
-exit 0
-fi
+# Copy sample to root dir, if not existing
+cp -n nuage-metro/build_vars.yml /files
 
 # Generate new host key if needed
 mkdir -p --mode=700 ~/.ssh 
 if [ ! -e /files/id_rsa ]; then
   ssh-keygen -h -f /files/id_rsa -N ''
 fi
 
-if [ ! -e /files/ansible.cfg ]; then
-cat > /files/ansible.cfg << EOF
-# Sample config file, modify to override settings
-[ssh_connection]
-# scp_if_ssh = True
-EOF
-fi
-
 # Always re-copy, even if already done
 # ssh-copy-id -i /files/id_rsa.pub [email protected] || exit 1
 cp /files/id_rsa* ~/.ssh/ && chmod 600 ~/.ssh/id_rsa*
 
+# Clear Ansible tmp directory
+[ -d /files/.ansible/tmp ] && rm -rf /files/.ansible/tmp/*
+
 # Run Ansible playbooks
 export ANSIBLE_HOST_KEY_CHECKING=False
-export ANSIBLE_CONFIG=/files/ansible.cfg
+export PARAMIKO_HOST_KEY_AUTO_ADD=True
+
 cp /files/build_vars.yml /files/nuage-metro/
 
+if [ "$1" == "health" ]; then
+  shift
+  cd /files/nuage-metro && \
+  ansible-playbook --key-file=/files/id_rsa build.yml && \
+  ansible-playbook -i hosts --key-file=/files/id_rsa nuage_health.yml $@
+  exit $?
+fi
+if [ "$1" == "upgrade-vsd" ]; then
+  shift
+  cd /files/nuage-metro
+  ansible-playbook --key-file=/files/id_rsa build.yml -v
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vsp_preupgrade_health.yml $@
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vsd_ha_upgrade_database_backup_and_decouple.yml $@
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vsd_ha_upgrade_shutdown_1_and_2.yml $@
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vsd_ha_upgrade_predeploy_1_and_2.yml $@
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vsd_ha_upgrade_deploy_1_and_2.yml $@
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vsd_ha_upgrade_shutdown_3.yml $@
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vsd_ha_upgrade_predeploy_3.yml $@
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vsd_ha_upgrade_deploy_3.yml $@
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vsd_upgrade_complete.yml $@
+  exit $?
+fi
+if [ "$1" == "upgrade-vsc" ]; then
+  shift
+  cd /files/nuage-metro
+  ansible-playbook --key-file=/files/id_rsa build.yml
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vsc_health.yml -e report_filename=vsc_preupgrade_health.txt $@
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vsc_ha_upgrade_backup_and_prep_1.yml $@
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vsc_ha_upgrade_deploy_1.yml $@
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vsc_ha_upgrade_postdeploy_1.yml $@
+  read -p 'Now upgrade *all* VRSs (--limit=vrss)... press any key to continue' -n1 -s
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vsc_ha_upgrade_backup_and_prep_2.yml $@
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vsc_ha_upgrade_deploy_2.yml $@
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vsc_ha_upgrade_postdeploy_2.yml $@
+  exit $?
+fi
+if [ "$1" == "upgrade-es" ]; then
+  shift
+  cd /files/nuage-metro && \
+  ansible-playbook --key-file=/files/id_rsa build.yml && \
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vstat_health.yml -e report_filename=vstat_preupgrade_health.txt $@
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vstat_upgrade_data_backup.yml $@
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vstat_destroy.yml $@
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vstat_predeploy.yml $@
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vstat_deploy.yml $@
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vstat_upgrade_data_migrate.yml $@
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vsp_upgrade_postdeploy.yml $@
+  ansible-playbook -i hosts --key-file=/files/id_rsa playbooks/vsp_postupgrade_health.yml $@
+  exit $?
+fi
 if [ "$1" == "destroy" ]; then
+  shift
   cd /files/nuage-metro && \
   ansible-playbook --key-file=/files/id_rsa build.yml && \
-  ansible-playbook -i hosts --key-file=/files/id_rsa destroy_everything.yml
+  ansible-playbook -i hosts --key-file=/files/id_rsa destroy_everything.yml $@
   exit $?
 fi
 
-if [ ! -d /files/nuage-unpacked ] || [ "$1" == "unpack" ]; then
+if [[ ! -d /files/nuage-unpacked || (( "$1" == "unpack" && shift )) ]]; then
 ansible-playbook /files/nuage-metro/nuage_unzip.yml $@
 fi
 
+if [ "$1" == "shell" ]; then
+   echo "This is a Docker shell. Use <CTRL>-(p + q) to exit while keeping the container running - alias 'deploy' is defined for your convenience"
+cat > /etc/profile.d/metro.sh << EOF
+alias deploy="cd /files/nuage-metro && ansible-playbook --key-file=/files/id_rsa build.yml && ansible-playbook -i hosts --key-file=/files/id_rsa install_everything.yml"
+EOF
+   /bin/bash
+   exit $?
+fi
+
 cd /files/nuage-metro && \
 ansible-playbook --key-file=/files/id_rsa build.yml && \
 ansible-playbook -i hosts --key-file=/files/id_rsa install_everything.yml $@