From 60791172f902cf194614abad607661affb241cbf Mon Sep 17 00:00:00 2001 From: Andrew Theurer Date: Thu, 5 Oct 2017 15:20:18 -0400 Subject: [PATCH] new VM automations scripts --- virt-add-vhostuser.sh | 104 ++++++++++++++++ virt-boot-vms.sh | 121 +++++++++++++++++++ virt-pin.sh | 249 +++++++++++++++++++++++++++++++++++++++ virt-remove-vhostuser.sh | 18 +++ virt-shutdown-all-vms.sh | 15 +++ 5 files changed, 507 insertions(+) create mode 100755 virt-add-vhostuser.sh create mode 100755 virt-boot-vms.sh create mode 100755 virt-pin.sh create mode 100755 virt-remove-vhostuser.sh create mode 100755 virt-shutdown-all-vms.sh diff --git a/virt-add-vhostuser.sh b/virt-add-vhostuser.sh new file mode 100755 index 0000000..efcce4c --- /dev/null +++ b/virt-add-vhostuser.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +function error_exit() { + echo "$1" + exit 1 +} + +function usage() { + echo "" + echo "usage:" + echo "virt-add-vhustuser [virtio-net driver options]" + echo "" + echo "--mode=[client|server] default is client" + echo "--nrq=[1-N] the number of queues" + echo "--rxqsz=[256-1024] the number of descriptors in the RX queue" + echo "--txqsz=[256-1024] the number of descriptors in the TX queue" + echo "--nomrg disable mergable buffers" + exit +} + +if [ -z $1 ]; then + error_exit "You must provide a VM name" +fi +vm_name=$1 +shift +source_opts="" +driver_opts="" +host_opts="" + +# Process options and arguments +opts=$(getopt -q -o i:c:t:r:m:p:M:S:C: --longoptions "help,mode:,rxqsz:,txqsz:,nomrg,nrq:" -n "getopt.sh" -- "$@") +if [ $? -ne 0 ]; then + printf -- "$*\n" + printf "\n" + printf "\t${benchmark}: you specified an invalid option\n\n" + usage + exit 1 +fi +eval set -- "$opts" +while true; do + case "$1" in + --help) + usage + exit + ;; + --mode) + shift + if [ -n "$1" ]; then + source_opts="$source_opts mode='$1'" + shift + fi + ;; + --nrq) + shift + if [ -n "$1" ]; then + driver_opts="$driver_opts queues='$1'" + shift + fi + ;; + --rxqsz) + shift + if [ -n "$1" ]; then + driver_opts="$driver_opts rx_queue_size='$1'" + shift + fi + ;; + --txqsz) + shift + if [ -n "$1" ]; then + driver_opts="$driver_opts tx_queue_size='$1'" + shift + fi + ;; + --nomrg) + shift + host_opts="$host_opts mrg_rxbuf='off'" + ;; + --) + shift + break + ;; + *) + error_exit "[$script_name] bad option, \"$1 $2\"" + ;; + esac +done + + +echo "vm_name is $vm_name" + +virsh list --name --all | grep -q -E \^$vm_name\$ || error_exit "$vm_name could not be found" + +vhu_count=`virsh domiflist $vm_name | grep -c vhostuser` +echo $source_opts | grep -q "mode=" || source_opts="$source_opts mode='client'" + +echo "\ + \ + \ + \ + \ + \ + " >/tmp/vhu.xml + +virsh attach-device $vm_name /tmp/vhu.xml --config diff --git a/virt-boot-vms.sh b/virt-boot-vms.sh new file mode 100755 index 0000000..db79a1a --- /dev/null +++ b/virt-boot-vms.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +# This script will boot VMs and provide a list like the following: +# +# waiting to vms to boot up +# vm-name hostname +# vm1 virbr0-122-11 +# vm4 virbr0-122-194 +# vm3 virbr0-122-209 +# +# There are a couple things required to make this work: +# +# 1) The VM config must have 2 serial ports configured in the XML, like: +# +# +# +# +# +# +# +# +# +# +# +# +# The first serial port is for the systems main console, +# which provides access to grub and access to log in +# without a graphical console or network, should you need +# to trouble-shoot something. +# +# The seconds serial port is used to log kernel & console +# message to a file on the kvm host. This script expects +# that file in /var/log/libvirt/qemu/.serial.log. +# +# 2) The VM OS must have both ttyS0 and ttyS1 configured: +# +# In the VM's /etc/default/grub, add these lines: +# GRUB_CMDLINE_LINUX_DEFAULT="console=ttyS1,115200n8 console=ttyS0,115200n8" +# GRUB_TERMINAL=serial +# GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1" +# +# And then on the VM run: +# +# grub2-mkconfig -o /boot/grub2/grub.cfg +# +# 3) The VM's login service needs to be configured for the serial consoles: +# +# Create /etc/systemd/system/ files: +# +# [Unit] +# Description=Serial Getty on %I +# Documentation=man:agetty(8) man:systemd-getty-generator(8) +# Documentation=http://0pointer.de/blog/projects/serial-console.html +# BindsTo=dev-%i.device +# After=dev-%i.device systemd-user-sessions.service plymouth-quit-wait.service +# After=rc-local.service +# After=network-online.target +# Wants=network-online.target +# +# Before=getty.target +# IgnoreOnIsolate=yes +# +# [Service] +# ExecStart=-/sbin/agetty --keep-baud %I 115200,38400,9600 +# Type=idle +# Restart=always +# RestartSec=0 +# UtmpIdentifier=%I +# TTYPath=/dev/%I +# TTYReset=yes +# TTYVHangup=yes +# KillMode=process +# IgnoreSIGPIPE=no +# SendSIGHUP=yes +# +# [Install] +# WantedBy=getty.target +# +# The "After/Wants=networ-online.target" will wait until the network +# device is online before the login prompt is available for the serial +# consoles. We wait for that because we want the login prompt to +# include the hostname, like: +# +# virbr0-122-195 login: +# +# This is what this script looks for in the /var/log/libvirt/qemu/.serial.log +# file in order to detect the VM's hostname. + + + + +vms=$1 #comma separated list (no spaces) of vms: vm1,vm2,vm4 +vms=`echo $vms | sed -e s/","/" "/g` +log_dir=/tmp + +# clear out the current serial log +for vm in $vms; do + if [ -f $log_dir/$vm.console ]; then + /bin/rm $log_dir/$vm.console + fi +done + +for vm in $vms; do + virsh start $vm +done +wait + +echo "waiting to vms to boot up" +printf "%20s%20s\n" "vm-name" "hostname" +while [ ! -z "$vms" ]; do + for vm in $vms; do + if grep -q "login:" "$log_dir/$vm.console"; then + vm_hostname=`grep "login:" "$log_dir/$vm.console" | awk '{print $1}'` + printf "%20s%20s\n" $vm $vm_hostname + vms="`echo $vms | sed -e s/"$vm"//`" + vms="`echo $vms | sed -e s/\s+/\s/`" + break + fi + sleep 5 + done +done diff --git a/virt-pin.sh b/virt-pin.sh new file mode 100755 index 0000000..2073076 --- /dev/null +++ b/virt-pin.sh @@ -0,0 +1,249 @@ +#!/bin/bash + +#defaults +cpu_usage_file="/var/log/isolated_cpu_usage.conf" +node=-1 +use_ht="n" +nrvcpus=3 + +function exit_error() { + local error_message=$1 + local error_code=$2 + if [ -z "$error_code"] ; then + error_code=1 + fi + echo "ERROR: $error_message" + exit $error_code +} + +function log_cpu_usage() { + # $1 = list of cpus, no spaces: 1,2,3 + local cpulist=$1 + local usage=$2 + if [ "$usage" == "" ]; then + exit_error "a string describing the usage must accompany the cpu list" + fi + for cpu in `echo $cpulist | sed -e 's/,/ /g'`; do + sed -i -e s/^$cpu:$/$cpu:$usage/ $cpu_usage_file || exit_error "cpu $cpu is already used or not an isolated cpu" + done +} + +function get_free_cpus { + local line + local cpu + local usage + local free_cpus + while read line; do + cpu=`echo $line | awk -F: '{print $1}'` + usage=`echo $line | awk -F: '{print $2}'` + if [ "$usage" == "" ]; then + free_cpus="$free_cpus,$cpu" + fi + done < $cpu_usage_file + free_cpus=`echo $free_cpus | sed -e s/^,//` + echo $free_cpus +} + +function usage() { + echo "" + echo "usage:" + echo "virt-pin [pin options]" + echo "" + echo "--node=[0-9]* the host node ID to use" + echo "--nrvcpus=[0-9]* number of vcpus to assign" + echo "--use-ht assigns pairs of vCPUs to host sibling CPU-threads" + exit +} + +function convert_number_range() { + # converts a range of cpus, like "1-3,5" to a list, like "1,2,3,5" + local cpu_range=$1 + local cpus_list="" + local cpus="" + for cpus in `echo "$cpu_range" | sed -e 's/,/ /g'`; do + if echo "$cpus" | grep -q -- "-"; then + cpus=`echo $cpus | sed -e 's/-/ /'` + cpus=`seq $cpus | sed -e 's/ /,/g'` + fi + for cpu in $cpus; do + cpus_list="$cpus_list,$cpu" + done + done + cpus_list=`echo $cpus_list | sed -e 's/^,//'` + echo "$cpus_list" +} + +function intersect_cpus() { + local cpus_a=$1 + local cpus_b=$2 + local cpu_set_a + local cpu_set_b + local intersect_cpu_list="" + # for easier manipulation, convert the cpu list strings to a associative array + for i in `echo $cpus_a | sed -e 's/,/ /g'`; do + cpu_set_a["$i"]=1 + done + for i in `echo $cpus_b | sed -e 's/,/ /g'`; do + cpu_set_b["$i"]=1 + done + for cpu in "${!cpu_set_a[@]}"; do + if [ "${cpu_set_b[$cpu]}" != "" ]; then + intersect_cpu_list="$intersect_cpu_list,$cpu" + fi + done + intersect_cpu_list=`echo $intersect_cpu_list | sed -e s/^,//` + echo "$intersect_cpu_list" +} + +function subtract_cpus() { + local current_cpus=$1 + local sub_cpus=$2 + local current_cpus_set + local count + local sub_cpu_list="" + # for easier manipulation, convert the current_cpus string to a associative array + for i in `echo $current_cpus | sed -e 's/,/ /g'`; do + current_cpus_set["$i"]=1 + done + for cpu in "${!current_cpus_set[@]}"; do + for sub_cpu in `echo $sub_cpus | sed -e 's/,/ /g'`; do + if [ "$sub_cpu" == "$cpu" ]; then + unset current_cpus_set[$sub_cpu] + break + fi + done + done + for cpu in "${!current_cpus_set[@]}"; do + sub_cpu_list="$sub_cpu_list,$cpu" + done + sub_cpu_list=`echo $sub_cpu_list | sed -e 's/^,//'` + echo "$sub_cpu_list" +} + +function get_vcpus() { + local avail_cpus=$1 + local nr_vcpus=$2 + local avail_cpus_set + local count + local vcpu_list="" + # for easier manipulation, convert the avail_cpus string to a associative array + for i in `echo $avail_cpus | sed -e 's/,/ /g'`; do + avail_cpus_set["$i"]=1 + done + set -x + if [ "$use_ht" == "n" ]; then + # when using 1 thread per core (higher per-PMD-thread throughput) + count=0 + for cpu in "${!avail_cpus_set[@]}"; do + vcpu_list="$vcpu_list,$cpu" + unset avail_cpus_set[$cpu] + ((count++)) + [ $count -ge $nr_vcpus ] && break + done + else + # when using 2 threads per core (higher throuhgput/core) + count=0 + for cpu in "${!avail_cpus_set[@]}"; do + vcpu_hyperthreads=`cat /sys/devices/system/cpu/cpu$cpu/topology/thread_siblings_list` + vcpu_hyperthreads=`convert_number_range $vcpu_hyperthreads` + for cpu_thread in `echo $vcpu_hyperthreads | sed -e 's/,/ /g'`; do + vcpu_list="$vcpu_list,$cpu_thread" + unset avail_cpus_set[$cpu_thread] + ((count++)) + done + [ $count -ge $nr_vcpus ] && break + done + fi + vcpu_list=`echo $vcpu_list | sed -e 's/^,//'` + echo "$vcpu_list" + set +x +} + +function get_cpumask() { + local cpu_list=$1 + local pmd_cpu_mask=0 + for cpu in `echo $cpu_list | sed -e 's/,/ /'g`; do + bc_math="$bc_math + 2^$cpu" + done + bc_math=`echo $bc_math | sed -e 's/\+//'` + pmd_cpu_mask=`echo "obase=16; $bc_math" | bc` + echo "$pmd_cpu_mask" +} +if [ -z $1 ]; then + exit_error "You must provide a VM name" +fi +vm_name=$1 +shift +if [ ! -e $cpu_usage_file ]; then + exit_error "The file, $cpu_usage_file, is missing. This must be created with start-vswitch.sh" +fi + +# Process options and arguments +opts=$(getopt -q -o i:c:t:r:m:p:M:S:C: --longoptions "help,use-ht,node:,nrvcpus:" -n "getopt.sh" -- "$@") +if [ $? -ne 0 ]; then + printf -- "$*\n" + printf "\n" + printf "\t${benchmark}: you specified an invalid option\n\n" + usage + exit 1 +fi +eval set -- "$opts" +while true; do + case "$1" in + --help) + usage + exit + ;; + --node) + shift + if [ -n "$1" ]; then + node=$1 + shift + fi + ;; + --nrvcpus) + shift + if [ -n "$1" ]; then + nrvcpus=$1 + shift + fi + ;; + --use-ht) + shift + use_ht="y" + ;; + --) + shift + break + ;; + *) + error_exit "[$script_name] bad option, \"$1 $2\"" + ;; + esac +done + +if [ $node -lt 0 ]; then + exit_error "you must specify a host node ID to use" +fi + +virsh list --name --all | grep -q -E \^$vm_name\$ || exit_error "The VM $vm_name could not be found. You must create with virt-install-vm.sh first" +virsh numatune $vm_name --mode=strict --nodeset=$node +all_cpus_range=`cat /sys/devices/system/cpu/online` +all_cpus_list=`convert_number_range $all_cpus_range` +node_cpus_range=`cat /sys/devices/system/node/node$node/cpulist` +# convert to a list with 1 entry per cpu and no "-" for ranges +node_cpus_list=`convert_number_range "$node_cpus_range"` +# remove the first cpu (and its sibling if present) because we want at least 1 cpu in the NUMA node +# for non-isolated work +node_first_cpu=`echo $node_cpus_list | awk -F, '{print $1}'` +node_first_cpu_threads_range=`cat /sys/devices/system/cpu/cpu$node_first_cpu/topology/thread_siblings_list` +node_first_cpu_threads_list=`convert_number_range $node_first_cpu_threads_range` +node_cpus_list=`subtract_cpus $node_cpus_list $node_first_cpu_threads_list` +free_cpus_list=`get_free_cpus` +avail_vcpus_list=`intersect_cpus $node_cpus_list $free_cpus_list` +vcpus_list=`get_vcpus "$avail_vcpus_list" "$nrvcpus"` +vcpu=0 +for cpu in `echo $vcpus_list | sed -e 's/,/ /g'`; do + virsh vcpupin $vm_name --vcpu $vcpu --cpulist $cpu --config + let vcpu=$vcpu+1 +done diff --git a/virt-remove-vhostuser.sh b/virt-remove-vhostuser.sh new file mode 100755 index 0000000..b0bb5d2 --- /dev/null +++ b/virt-remove-vhostuser.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +function error_exit() { + echo "$1" + exit 1 +} + +if [ -z $1 ]; then + error_exit "You must provide a VM name" +fi +vm_name=$1 + +virsh list --name --all | grep -q -E \^$vm_name\$ || error_exit "$vm_name could not be found" +virsh domiflist $vm_name | grep -q vhostuser || error_exit "$vm_name did not have any vhostuser devices" +for vhu_mac in `virsh domiflist $vm_name | grep vhostuser | awk '{print $5}'`; do + echo "detaching $vhu_mac" + virsh detach-interface $vm_name vhostuser --mac $vhu_mac --config +done diff --git a/virt-shutdown-all-vms.sh b/virt-shutdown-all-vms.sh new file mode 100755 index 0000000..5ca5e26 --- /dev/null +++ b/virt-shutdown-all-vms.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +for vm in `virsh list | grep running | awk '{print $2}'`; do + virsh shutdown $vm + vms="[$vm]$vms" +done +echo waiting for VMs to shut down +while [ ! -z "$vms" ]; do + sleep 5 + for vm in `echo $vms | sed -e 's/\]/ /g' -e 's/\[/ /g'`; do + virsh list | grep -q "$vm " || vms=`echo $vms | sed -e 's/\['$vm'\]//'` + #echo vms is now: $vms + done +done +echo all VMs have shut down