From 556b13793a87442c9e4d02cff3e75c7a5e0adfbc Mon Sep 17 00:00:00 2001 From: Anurag Porripireddi Date: Wed, 29 Sep 2021 09:57:49 -0700 Subject: [PATCH] Pythonify shunt setup methods and integrate into tests (#927) --- bin/shunt_functions | 8 +- shunt/__init__.py | 3 + shunt/shell_command_helper.py | 94 +++++++++++++++++ shunt/vxlan_over_ssh.py | 145 ++++++++++++++++++++++++++ testing/shunt/Dockerfile.shunthost | 9 +- testing/shunt/build_vxlan_ssh_conn.py | 41 ++++++++ testing/shunt/docker-compose.yaml | 12 ++- testing/shunt/start_shunt_host | 12 +-- testing/test_shunt.sh | 12 +-- 9 files changed, 313 insertions(+), 23 deletions(-) create mode 100644 shunt/__init__.py create mode 100644 shunt/shell_command_helper.py create mode 100644 shunt/vxlan_over_ssh.py create mode 100644 testing/shunt/build_vxlan_ssh_conn.py diff --git a/bin/shunt_functions b/bin/shunt_functions index 065e8a288..3ea17e64e 100644 --- a/bin/shunt_functions +++ b/bin/shunt_functions @@ -7,6 +7,7 @@ function build_vxlan_ssh_conn { SSH_IN_PORT=$1 SSH_OUT_PORT=$2 LOCAL_VXLAN_IP=${3:-"192.168.1.2"} + VNI=${4:-0} REMOTE_IP=${4:-"192.168.21.1"} LOCAL_IP=${5:-"192.168.21.2"} @@ -16,7 +17,7 @@ function build_vxlan_ssh_conn { sudo ip link set ep0 up # socat session to move packets from ssh tunnel to UDP port - socat -T15 tcp4-listen:30001,reuseaddr,fork UDP:localhost:4789 & + socat -T15 tcp4-listen:$SSH_IN_PORT,reuseaddr,fork UDP:localhost:4789 & # iptables command to change source IP from localhost sudo iptables -t nat -A POSTROUTING -p udp -d 127.0.0.1 --dport 4789 -j SNAT --to-source $REMOTE_IP:38000-45000 @@ -25,10 +26,10 @@ function build_vxlan_ssh_conn { sudo iptables -t nat -A OUTPUT -o ep0 -p udp --dport 4789 -j REDIRECT --to-ports 20000 # socat session to move packets from UDP port to ssh session - socat -T15 udp4-recvfrom:20000,reuseaddr,fork tcp:localhost:30000 & + socat -T15 udp4-recvfrom:20000,reuseaddr,fork tcp:localhost:$SSH_OUT_PORT & # create vxlan tunnel - sudo ip link add vxlan type vxlan id 0 remote $REMOTE_IP dstport 4789 srcport 4789 4789 nolearning + sudo ip link add vxlan type vxlan id $VNI remote $REMOTE_IP dstport 4789 srcport 4789 4789 nolearning sudo ip addr add $LOCAL_VXLAN_IP/24 dev vxlan sudo ip link set vxlan up } @@ -57,7 +58,6 @@ function build_ssh_tunnel { SSH_OUT_PORT=$2 REMOTE_HOST=$3 - sleep 5 ssh -o StrictHostKeyChecking=no -L $SSH_OUT_PORT:127.0.0.1:$SSH_IN_PORT -N -f $REMOTE_HOST ssh -o StrictHostKeyChecking=no -R $SSH_OUT_PORT:127.0.0.1:$SSH_IN_PORT -N -f $REMOTE_HOST } diff --git a/shunt/__init__.py b/shunt/__init__.py new file mode 100644 index 000000000..812208db5 --- /dev/null +++ b/shunt/__init__.py @@ -0,0 +1,3 @@ +"""Shunt module""" + +pass diff --git a/shunt/shell_command_helper.py b/shunt/shell_command_helper.py new file mode 100644 index 000000000..5ffd0498a --- /dev/null +++ b/shunt/shell_command_helper.py @@ -0,0 +1,94 @@ +"""Module to help run shell commands and retrieve output""" + +from __future__ import absolute_import +import subprocess +import multiprocessing + + +class ShellCommandHelper(): + """Hosts methods to run shell commands and retrieve output""" + + TEN_MIN_SEC = 10 * 60 + + def __init__(self): + pass + + def _run_process_command(self, command, capture=False): + """ + Args: + command: Takes an str line or list[str] to be run in shell + capture: Setting to True captures stdout, stderr and return code + returns: + Popen object + """ + command_list = command.split() if isinstance(command, str) else command + pipeout = subprocess.PIPE if capture else None + return subprocess.Popen(command_list, stdout=pipeout, stderr=pipeout) + + def _reap_process_command(self, process): + """ + Args: + process: Popen object to reap + returns: + return code of command, stdout, stderr + """ + process.wait(timeout=self.TEN_MIN_SEC) + stdout, stderr = process.communicate() + strout = str(stdout, 'utf-8') if stdout else None + strerr = str(stderr, 'utf-8') if stderr else None + return process.returncode, strout, strerr + + # pylint: disable=too-many-arguments + def run_cmd(self, cmd, arglist=None, strict=True, + capture=False, docker_container=None, detach=False): + """ + Args: + cmd: command to be executed + arglist: Additional arguments to add to command + strict: Raises exception if command execution fails + capture: Prints out and captures stdout and stderr output of command + docker_container: Set if command needs to be run within a docker container + returns: + return code of command, stdout, stderr + """ + command = ("docker exec %s " % docker_container) if docker_container else "" + command = command.split() + ([cmd] + arglist) if arglist else command + cmd + if detach: + self._run_process_command(command, capture=capture) + return None, None, None + retcode, out, err = self._reap_process_command( + self._run_process_command(command, capture=capture)) + if strict and retcode: + if capture: + print('stdout: \n%s' % out) + print('stderr: \n%s' % err) + raise Exception('Command execution failed: %s' % str(command)) + return retcode, out, err + + def parallelize(self, target, target_args, batch_size=200): + """ + Parallelizes multiple runs of a target method with multiprocessing. + Args: + target_args: List of tuples which serve as args for target. + List size determines number of jobs + target: Target method + batch_size: Thread pool size + """ + jobs = [] + for arg_tuple in target_args: + process = multiprocessing.Process(target=target, args=arg_tuple) + jobs.append(process) + + batch_start = 0 + while batch_start <= len(jobs): + batch_end = batch_start + batch_size + batch_jobs = jobs[batch_start:batch_end] + + for job in batch_jobs: + job.start() + + for job in batch_jobs: + job.join() + job.close() + + batch_start = batch_end diff --git a/shunt/vxlan_over_ssh.py b/shunt/vxlan_over_ssh.py new file mode 100644 index 000000000..455a3940f --- /dev/null +++ b/shunt/vxlan_over_ssh.py @@ -0,0 +1,145 @@ +"""Module to build VxLAN tunnels over SSH tunnels using shell commands""" + +from __future__ import absolute_import +from shunt.shell_command_helper import ShellCommandHelper + + +def build_ssh_tunnel(ssh_in_port, ssh_out_port, remote_host, reverse=False): + """ + Args: + ssh_in_port: Destination port for tunnel + ssh_out_port: Source port for tunnel + remote_host: Remote host IP + reverse: Boolean. Creates reverse forwarding tunnel if True. + Returns: None + """ + shellcmd = ShellCommandHelper() + direction = 'R' if reverse else 'L' + shellcmd.run_cmd("ssh -o StrictHostKeyChecking=no -%s %s:127.0.0.1:%s -N -f %s" + % (direction, ssh_out_port, ssh_in_port, remote_host)) + + +def build_bidirectional_ssh_tunnel(ssh_in_port, ssh_out_port, remote_host): + """ + Args: + ssh_in_port: Incoming traffice port + ssh_out_port: Outgoing traffic port + remote_host: Remote host IP + Returns: None + """ + build_ssh_tunnel(ssh_in_port, ssh_out_port, remote_host, reverse=False) + build_ssh_tunnel(ssh_in_port, ssh_out_port, remote_host, reverse=True) + + +def create_virtual_if(if_name, if_ip): + """ + Args: + if_name: interface name + if_ip: interface IP + Returns: None + """ + shellcmd = ShellCommandHelper() + shellcmd.run_cmd("sudo ip link add %s type dummy" % (if_name)) + shellcmd.run_cmd("sudo ip addr add %s/24 dev %s" % (if_ip, if_name)) + shellcmd.run_cmd("sudo ip link set %s up" % (if_name)) + + +def socat_tcp_to_udp(src_port, dst_port): + """ + Socat command to map TCP port to UDP port + Args: + src_port: Source TCP port + dst_port: Destination UDP port + Returns: None + """ + shellcmd = ShellCommandHelper() + shellcmd.run_cmd("socat -T15 tcp4-listen:%s,reuseaddr,fork udp:localhost:%s" + % (src_port, dst_port), detach=True) + + +def socat_udp_to_tcp(src_port, dst_port): + """ + Socat command to map UDP port to TCP port + Args: + src_port: Source UDP port + dst_port: Destincation TCP port + Returns: None + """ + shellcmd = ShellCommandHelper() + shellcmd.run_cmd("socat -T15 udp4-listen:%s,reuseaddr,fork tcp:localhost:%s" + % (src_port, dst_port), detach=True) + + +def iptables_udp_change_source(dst_ip, dst_port, src_ip_port): + """ + iptables command to change source IP/ports of UDP packets + Args: + dst_ip: Destination IP of packet to be changed + dst_port: Destination port of packet to be changed + src_ip_port: : e.g. 192.168.1.2:38000-45000 + Returns: None + """ + shellcmd = ShellCommandHelper() + shellcmd.run_cmd( + "sudo iptables -t nat -A POSTROUTING -p udp -d %s --dport %s -j SNAT --to-source %s" + % (dst_ip, dst_port, src_ip_port)) + + +def iptables_udp_divert_iface_traffic(iface, dst_port, target_dst_port): + """ + iptables command to divert udp traffic egressing from an interface to a different port + Args: + iface: Interface packets are egressign from + dst_port: Destination port of packets to be diverted + target_dst_port: Destination port packets are diverted to + Returns: None + """ + shellcmd = ShellCommandHelper() + shellcmd.run_cmd( + "sudo iptables -t nat -A OUTPUT -o %s -p udp --dport %s -j REDIRECT --to-ports %s" + % (iface, dst_port, target_dst_port)) + + +def create_vxlan_tunnel(vni, remote_ip, vxlan_port, vtep_ip): + """ + Method to create a VxLAN tunnel and assign an IP to the VTEP + Args: + vni: VNI of tunnel to be created + remote_ip: Remote IP address for underlay network + vxlan_port: Remote destination port for VxLAN tunnel + Returns: None + """ + shellcmd = ShellCommandHelper() + shellcmd.run_cmd( + "sudo ip link add vxlan type vxlan id %s remote %s dstport %s srcport %s %s nolearning" + % (vni, remote_ip, vxlan_port, vxlan_port, vxlan_port)) + shellcmd.run_cmd("sudo ip addr add %s/24 dev vxlan" % (vtep_ip)) + shellcmd.run_cmd("sudo ip link set vxlan up") + + +def build_vxlan_ssh_conn(conn_params): + """ + Args: + conn_params: JSON. Format:{ # TODO: Turn into a proto for easy config exchange. + virt_if_name: Interface name of virtual interface for underlay n/w + virt_if_ip: IP for virtual interface + ssh_in_port: Port for incoming traffic + ssh_out_port: Port for outgoing traffic + remote_ip: Remote IP of underlay n/w + vni: VxLAN identifier + vxlan_if_ip: VxLAN interface IP + } + Returns: None + """ + VXLAN_PORT = '4789' + LOCAL_HOST = '127.0.0.1' + VXLAN_SOURCE_PORT_RANGE = "38000-45000" + INTERMEDIATE_PORT = "20000" + create_virtual_if(conn_params['virt_if_name'], conn_params['virt_if_ip']) + socat_tcp_to_udp(conn_params['ssh_in_port'], VXLAN_PORT) + source_ip_port = conn_params['remote_ip'] + ":" + VXLAN_SOURCE_PORT_RANGE + iptables_udp_change_source(LOCAL_HOST, VXLAN_PORT, source_ip_port) + iptables_udp_divert_iface_traffic(conn_params['virt_if_name'], VXLAN_PORT, INTERMEDIATE_PORT) + socat_udp_to_tcp(INTERMEDIATE_PORT, conn_params['ssh_out_port']) + create_vxlan_tunnel( + conn_params['vni'], conn_params['remote_ip'], VXLAN_PORT, conn_params['vxlan_if_ip']) diff --git a/testing/shunt/Dockerfile.shunthost b/testing/shunt/Dockerfile.shunthost index 63f4a9856..d3e669e6a 100644 --- a/testing/shunt/Dockerfile.shunthost +++ b/testing/shunt/Dockerfile.shunthost @@ -5,13 +5,20 @@ FROM daqf/aardvark -RUN $AG update && $AG install openssh-server socat sudo iptables +RUN $AG update && $AG install openssh-server socat sudo iptables python3-dev COPY ./testing/shunt/id_rsa .ssh/ COPY ./testing/shunt/id_rsa.pub .ssh/ COPY ./testing/shunt/authorized_keys .ssh/ COPY ./testing/shunt/start_shunt_host ./ +RUN mkdir -p shunt +RUN mkdir -p testing +RUN mkdir -p testing/shunt + +COPY ./shunt shunt +COPY ./testing/shunt testing/shunt + RUN chmod -R 700 .ssh RUN service ssh start diff --git a/testing/shunt/build_vxlan_ssh_conn.py b/testing/shunt/build_vxlan_ssh_conn.py new file mode 100644 index 000000000..2d04bbb6d --- /dev/null +++ b/testing/shunt/build_vxlan_ssh_conn.py @@ -0,0 +1,41 @@ +"""Script to build vxlan over ssh for testing across docker hosts""" + + +from __future__ import absolute_import +import sys +from shunt.vxlan_over_ssh import build_vxlan_ssh_conn, build_bidirectional_ssh_tunnel + +CONN_PARAMS = { + 'virt_if_name': 'ep0', + 'ssh_in_port': '30001', + 'ssh_out_port': '30000', + 'vni': '0' +} + +CONN_PARAMS_CLIENT = { + 'virt_if_ip': '192.168.21.2', + 'remote_ip': '192.168.21.1', + 'vxlan_if_ip': '192.168.1.2' +} +CONN_PARAMS_CLIENT.update(CONN_PARAMS) + +CONN_PARAMS_SERVER = { + 'virt_if_ip': '192.168.21.1', + 'remote_ip': '192.168.21.2', + 'vxlan_if_ip': '192.168.1.1' +} +CONN_PARAMS_SERVER.update(CONN_PARAMS) + + +def main(): + """Build vxlan over ssh for client/server""" + if sys.argv[1] == 'client': + build_bidirectional_ssh_tunnel( + CONN_PARAMS['ssh_in_port'], CONN_PARAMS['ssh_out_port'], 'shunt_host_server_1') + build_vxlan_ssh_conn(CONN_PARAMS_CLIENT) + elif sys.argv[1] == 'server': + build_vxlan_ssh_conn(CONN_PARAMS_SERVER) + + +if __name__ == "__main__": + main() diff --git a/testing/shunt/docker-compose.yaml b/testing/shunt/docker-compose.yaml index c30e8d0d1..cd21cd634 100644 --- a/testing/shunt/docker-compose.yaml +++ b/testing/shunt/docker-compose.yaml @@ -1,17 +1,21 @@ version: '2' services: - host1: + host_client: build: context: ../.. dockerfile: ./testing/shunt/Dockerfile.shunthost - command: bash -c "./start_shunt_host 1" + depends_on: + - "host_server" + command: bash -c "./start_shunt_host client" + #command: bash -c "tail -f" cap_add: - ALL - host2: + host_server: build: context: ../.. dockerfile: ./testing/shunt/Dockerfile.shunthost - command: bash -c "./start_shunt_host 2" + command: bash -c "./start_shunt_host server" + #command: bash -c "tail -f" cap_add: - ALL diff --git a/testing/shunt/start_shunt_host b/testing/shunt/start_shunt_host index 5b49f04ed..ce0f05c9d 100755 --- a/testing/shunt/start_shunt_host +++ b/testing/shunt/start_shunt_host @@ -3,13 +3,9 @@ host=$1 service ssh start -source bin/shunt_functions - -if [ $host -eq 1 ]; then - build_ssh_tunnel 30001 30000 shunt_host2_1 - build_vxlan_ssh_conn 30001 30000 192.168.1.1 -else - build_vxlan_ssh_conn 30001 30000 192.168.1.2 -fi + +export PYTHONPATH=$PYTHONPATH:/root/shunt + +python3 testing/shunt/build_vxlan_ssh_conn.py $host tail -f /dev/null diff --git a/testing/test_shunt.sh b/testing/test_shunt.sh index 51ad1785a..dd2fa3563 100755 --- a/testing/test_shunt.sh +++ b/testing/test_shunt.sh @@ -13,8 +13,8 @@ echo Shunt Test > $TEST_RESULTS docker-compose -f ./testing/shunt/docker-compose.yaml up --build -d sleep 6 -docker exec shunt_host1_1 ps ax -docker exec shunt_host1_1 ping -c 1 192.168.1.2 +docker exec shunt_host_client_1 ps ax +docker exec shunt_host_client_1 ping -c 1 192.168.1.1 if [ $? -eq 0 ]; then echo Ping succesful | tee -a $TEST_RESULTS @@ -22,14 +22,14 @@ else echo Ping unsuccesful | tee -a $TEST_RESULTS fi -docker exec shunt_host2_1 bash -c "source bin/shunt_functions; clean_vxlan_ssh_conn" +docker exec shunt_host_server_1 bash -c "source bin/shunt_functions; clean_vxlan_ssh_conn" sleep 10 -docker logs shunt_host2_1 +docker logs shunt_host_server_1 -docker exec shunt_host2_1 ps ax -docker exec shunt_host1_1 ping -c 1 192.168.1.2 +docker exec shunt_host_server_1 ps ax +docker exec shunt_host_client_1 ping -c 1 192.168.1.1 if [ $? -eq 0 ]; then echo Ping succesful | tee -a $TEST_RESULTS