Allow more devices per gateway setup (#917)

.github/workflows/tests.yml

@@ -63,10 +63,10 @@ jobs:
           name: build_artifacts_base
           path: /tmp/build_artifacts_base
     - uses: actions/download-artifact@v2
+      if: matrix.test == 'aux'
       with:
           name: build_artifacts_subset
           path: /tmp/build_artifacts_subset
-      if: matrix.test == 'aux'
     - uses: actions/checkout@v2
       with:
         fetch-depth: 0
@@ -87,9 +87,9 @@ jobs:
       run: |
         bin/load_images faucet faux setup base
     - name: Loading subset docker images
+      if: matrix.test == 'aux'
       run: |
         bin/load_images subset
-      if: matrix.test == 'aux'
     - name: Running ${{ matrix.test }} test
       env:
         DOCKER_STARTUP_TIMEOUT_MS: 60000
@@ -176,7 +176,7 @@ jobs:
           bin/setup_remote daq
           bin/retry_cmd bin/build_docker controller
           bin/build_dts
-      - name: run fot integration tests
+      - name: run vxlan integration tests
         if: ${{ always() }}
         run: |
           cd forch

bin/catwrap

@@ -0,0 +1,9 @@
+#!/bin/bash
+#
+# Same problem as cited here:
+#   https://discuss.linuxcontainers.org/t/tcpdump-inside-a-container-failed/6637
+#
+# Really not sure what's going on, but piping the output through cat fixes the problem.
+#
+
+$@ 2>&1 | cat

bin/setup_dev

@@ -27,9 +27,6 @@ if [[ $PVERSION != $INSTALLED_PYTHON ]]; then
     false
 fi
 
-MININET=https://github.com/mininet/mininet
-MININETV=$(cat etc/MININET_VERSION)
-
 if [ -f .daq.local ]; then
     echo Loading config from .daq.local
     source .daq.local
@@ -73,39 +70,7 @@ $AG install \
     isc-dhcp-client network-manager netcat gnupg2 strace arp-scan libffi-dev \
     python python$PVERSION python3-pkg-resources python3-setuptools \
     python$PVERSION-dev python3-pip python$PVERSION-venv \
-    python3-distutils
-
-PYTHON2_BIN=$(readlink -f $(which python2))
-echo Python2 binary is $PYTHON2_BIN
-
-if [ -d mininet ]; then
-    echo Checking mininet version matches $MININETV...
-    targetrev=$(cd mininet; git rev-parse $MININETV)
-    instrev=$(cd mininet; git rev-parse HEAD)
-    if [ "$targetrev" != "$instrev" ]; then
-        echo Target mininet version $MININETV does not match installed version.
-        false
-    fi
-    if [ ! -f mininet/.the_house_that_daq_built ]; then
-        echo Mininet build seems to be incomplete. Try bin/clean_dev and try again.
-        false
-    fi
-else
-    echo Cloning $MININET $MININETV...
-    git clone $MININET
-    (
-        cd mininet
-        git reset --hard $MININETV
-        for i in ssh pep8 pyflakes python-pexpect pylint xterm ; do
-            perl -pi -e "s/${i}//g" util/install.sh ;
-        done
-        sed -i s/cgroup-bin/cgroup-tools/ util/install.sh
-        export PYTHON2_BIN
-        sed -i 's/sudo PYTHON=${PYTHON} make install/sudo PYTHON=${PYTHON2_BIN} make install/' util/install.sh
-        util/install.sh -n
-    )
-    touch mininet/.the_house_that_daq_built
-fi
+    python3-distutils golang-go protobuf-compiler
 
 # Can't use venv inside of containers because of absolute paths.
 if [ -n "$CI" ]; then
@@ -142,30 +107,31 @@ $PIP freeze
 echo Resetting .cache directory permissions...
 test -n "$USER" && sudo chown $USER -R $HOME/.cache
 
+bin/setup_mininet
+
 if [ -n "$SIMPLE" ]; then
     echo Finished with simple setup.
     exit 0
 fi
 
+rm -rf protoc-gen-doc
+git clone https://github.com/pseudomuto/protoc-gen-doc.git
+
 bin/setup_remote faucet
 bin/setup_remote forch
 bin/setup_remote udmi
 
-echo -n "DAQ commit "
-git log -n 1 --pretty=format:"%h - %an, %ar : %s" || true
-echo
-
 docker --version
-
 if ! docker images > /dev/null; then
     echo
     echo Docker execution failed, is the docker group setup?
     echo If this is the first time, try logging out and log back in again.
     false
 fi
 
-rm -rf protoc-gen-doc
-git clone https://github.com/pseudomuto/protoc-gen-doc.git
+echo -n "DAQ commit "
+git log -n 1 --pretty=format:"%h - %an, %ar : %s" || true
+echo
 
 echo
 echo Setup completed successfully.

bin/setup_mininet

@@ -0,0 +1,58 @@
+#!/bin/bash -e
+
+ROOT=$(dirname $0)/..
+cd $ROOT
+
+MININET=https://github.com/mininet/mininet
+MININETV=$(cat etc/MININET_VERSION)
+
+source venv/bin/activate
+
+PYTHON2_BIN=$(readlink -f $(which python2))
+echo Python2 binary is $PYTHON2_BIN
+
+#
+# If there's trouble installing this because of setuptools, then some
+# manual trickery needs to be done to install setuptools for python2.
+# This just involves downloading it manually for python2, and then
+# running pip for python2 with setuptools. This link should help:
+#
+# https://stackoverflow.com/a/66719099
+#
+# And then ultimately run the command:
+#
+#   python -m pip install setuptools
+#
+# This isn't automated because it's only required on some legacy systems,
+# and might involve a security hole (downloading packages directly).
+#
+
+if [ -d mininet ]; then
+    echo Checking mininet version matches $MININETV...
+    targetrev=$(cd mininet; git rev-parse $MININETV)
+    instrev=$(cd mininet; git rev-parse HEAD)
+    if [ "$targetrev" != "$instrev" ]; then
+        echo Target mininet version $MININETV does not match installed version.
+        false
+    fi
+    if [ ! -f mininet/.the_house_that_daq_built ]; then
+        echo Mininet build seems to be incomplete. Try bin/clean_dev and try again.
+        false
+    fi
+else
+    echo Cloning $MININET $MININETV...
+    git clone $MININET
+    (
+        cd mininet
+        git reset --hard $MININETV
+        for i in ssh pep8 pyflakes python-pexpect pylint xterm ; do
+            perl -pi -e "s/${i}//g" util/install.sh ;
+        done
+        sed -i s/cgroup-bin/cgroup-tools/ util/install.sh
+        export PYTHON2_BIN
+        sed -i 's/sudo PYTHON=${PYTHON} make install/sudo PYTHON=${PYTHON2_BIN} make install/' util/install.sh
+        util/install.sh -n
+    )
+    touch mininet/.the_house_that_daq_built
+fi
+

config/modules/ata.conf

@@ -0,0 +1,9 @@
+# Testing include files for ATA runs.
+
+# Tests to build.
+build ${DAQ_LIB}/docker/modules
+
+add pass first
+add fail first
+add ping first
+add nmap

config/modules/dts.conf

@@ -1,4 +1,4 @@
-# Baseline test file for officially included tests.
+# Baseline test file for DTS hosted tests.
 
 # Tests to build.
 build ${DAQ_LIB}/docker/modules

config/system/ata.yaml

@@ -6,6 +6,9 @@ include: ${DAQ_LIB}/config/system/default.yaml
 # Description for dashboard.
 site_description: "ATA Proxy Configuration"
 
+# Configure with proper set of tests.
+host_tests: config/modules/ata.conf
+
 # For large-scale network, no room for sleeping.
 settle_sec: 0
 
@@ -19,7 +22,10 @@ switch_setup:
 
 run_trigger:
   native_vlan: 122
-  max_hosts: 3
+  max_hosts: 10
+  device_block_sec: 3600
+  retain_results: True
+  arp_scan_sec: 600
 
 internal_subnet:
   subnet: 10.21.0.0/16

daq/base_gateway.py

@@ -16,15 +16,14 @@
 
 LOGGER = logger.get_logger('gateway')
 
+DEFAULT_TEST_HOSTS = 8
 
 class BaseGateway(ABC):
     """Gateway collection class for managing testing services"""
 
     GATEWAY_OFFSET = 0
     FAKE_HOST_OFFSET = 1
     TEST_OFFSET_START = 2
-    NUM_SET_PORTS = 6
-    SET_SPACING = 10
     _PING_RETRY_COUNT = 5
 
     TEST_IP_FORMAT = '192.168.84.%d'
@@ -45,6 +44,10 @@ def __init__(self, runner, name, port_set, env_params=None):
         self.activated = False
         self.result_linger = False
         self.dhcp_monitor = None
+        is_native = bool(self.runner.run_trigger.get('native_vlan'))
+        max_hosts = self.runner.run_trigger.get('max_hosts') or DEFAULT_TEST_HOSTS
+        num_host_ports = max_hosts if is_native else DEFAULT_TEST_HOSTS
+        self._gw_set_size = num_host_ports + self.TEST_OFFSET_START
         self._env_params = env_params
         self._ext_intf = env_params.get('ext_intf') if env_params else None
         self._is_native = bool(self._ext_intf)
@@ -136,7 +139,7 @@ def allocate_test_port(self):
         test_port = self._switch_port(self.TEST_OFFSET_START)
         while test_port in self.test_ports:
             test_port = test_port + 1
-        limit_port = self._switch_port(self.NUM_SET_PORTS)
+        limit_port = self._switch_port(self._gw_set_size)
         assert test_port < limit_port, 'no test ports available'
         self.test_ports.add(test_port)
         return test_port
@@ -147,15 +150,10 @@ def release_test_port(self, test_port):
         self.test_ports.remove(test_port)
 
     def _switch_port(self, offset):
-        return self.port_set * self.SET_SPACING + offset
+        return self.port_set * self._gw_set_size + offset
 
     def _is_target_expected(self, target):
-        if not target:
-            return False
-        target_mac = target['mac']
-        if target_mac in self.targets:
-            return True
-        return False
+        return target and target['mac'] in self.targets
 
     def _dhcp_callback(self, state, target, exception=None):
         if exception:
@@ -175,16 +173,16 @@ def _setup_tmpdir(self, base_name):
     def attach_target(self, device):
         """Attach the given target to this gateway; return number of attached targets."""
         assert device.mac not in self.targets, 'target %s already attached to gw' % device
-        LOGGER.info('Attaching target %s to gateway group %s',
-                    device, self.name)
+        LOGGER.info('Attaching target %s to gateway %s group %s',
+                    device, self.port_set, self.name)
         self.targets[device.mac] = device
         return len(self.targets)
 
     def detach_target(self, device):
         """Detach the given target from this gateway; return number of remaining targets."""
         assert device.mac in self.targets, 'target %s not attached to gw' % device
-        LOGGER.info('Detach target %s from gateway group %s: %s',
-                    device, self.name, list(self.targets.keys()))
+        LOGGER.info('Detaching target %s from gateway %s group %s: %s',
+                    device, self.port_set, self.name, list(self.targets.keys()))
         del self.targets[device.mac]
         return len(self.targets)
 
@@ -200,10 +198,16 @@ def get_targets(self):
         """Return the host targets associated with this gateway"""
         return self.targets.values()
 
+    def get_all_gw_ports(self):
+        """Return all ports associated with gateway"""
+        base_port = self._switch_port(0)
+        limit_port = self._switch_port(self._gw_set_size)
+        return list(range(base_port, limit_port))
+
     def get_possible_test_ports(self):
         """Return test ports associated with gateway"""
         test_port = self._switch_port(self.TEST_OFFSET_START)
-        limit_port = self._switch_port(self.NUM_SET_PORTS)
+        limit_port = self._switch_port(self._gw_set_size)
         return list(range(test_port, limit_port))
 
     def terminate(self):
@@ -233,7 +237,7 @@ def terminate(self):
     def _get_scan_interface(self):
         return self.host, self.host_intf
 
-    def _discover_host_hangup_callback(self, mac, log_fd, log_file, callback):
+    def _discover_host_hangup_callback(self, mac, log_fd, log_file, scan_callback):
         def process_line(line):
             sections = [section for section in line.split('\t') if section]
             if len(sections) >= 2:
@@ -252,38 +256,55 @@ def process_line(line):
                 if device_ip:
                     LOGGER.info('Host discovery for %s completed. Found ip %s.',
                                 mac, device_ip)
-                    return callback(device_ip)
+                    return scan_callback(device_ip)
         LOGGER.info('Host discovery for %s completed. Found no ip.', mac)
-        callback(None)
+        scan_callback(None)
 
-    def discover_host(self, mac: str, subnets: List[ip_network], callback: Callable):
+    def discover_host(self, mac: str, all_subnets: List[ip_network], host_callback: Callable):
         """Discovers a host using arp-scan in a list of subnets."""
         cmd = 'arp-scan --retry=2 --bandwidth=512K --interface=%s --destaddr=%s -s %s %s'
         host, intf = self._get_scan_interface()
+        scans = self.runner.run_trigger.get('arp_scan_count') or 2
+        if scans <= 0:
+            return
+        subnets = list(all_subnets)
         LOGGER.info('Starting host discovery for %s', mac)
 
-        def hangup_callback_callback(device_ip):
+        def scan_callback(device_ip):
+            nonlocal scans
             if device_ip:
-                callback(device_ip)
+                scans = 0
+                host_callback(device_ip)
             else:
                 recursive_discover()
 
         def recursive_discover():
+            nonlocal subnets, scans
             if not subnets:
-                callback(None)
-                return
+                if scans > 0:
+                    scans -= 1
+                    subnets = list(all_subnets)
+                else:
+                    host_callback(None)
+                    return
             subnet = subnets.pop(0)
-            address = next(subnet.hosts())
+            hosts = subnet.hosts()
+            address = next(hosts)
+            # This handles the case where the local IP conflicts with the remote.
+            # Not really a good general solution, but works in many cases.
+            # TODO: Generalize a bit more to (e.g.) pick a random src IP.
+            if scans % 2:
+                address = next(hosts)
             log_file = os.path.join(self.tmpdir, str(subnet).replace('/', '_'))
             log_fd = open(log_file, 'w')
-            LOGGER.info('Scanning subnet %s from %s for %s', subnet, address, mac)
+            LOGGER.info('Scanning %s subnet %s from %s for %s', scans, subnet, address, mac)
             host.cmd('ip addr add %s/%s dev %s' % (str(address), subnet.prefixlen, intf))
             full_cmd = cmd % (intf, mac, str(address), str(subnet))
-            LOGGER.info('arp-scan command: %s', full_cmd)
             active_pipe = host.popen(full_cmd, stdin=DEVNULL, stdout=PIPE, env=os.environ)
             self.runner.monitor_stream(self.name, active_pipe.stdout, copy_to=log_fd,
                                        hangup=lambda: self._discover_host_hangup_callback(
-                                           mac, log_fd, log_file, hangup_callback_callback))
+                                           mac, log_fd, log_file, scan_callback))
+
         recursive_discover()
 
     def _ping_test(self, src, dst, src_addr=None):