integration-test

Having a test suite that covers the code from an out-of-process perspective helps
in testing specific scenarios, e.g. when the concurrency of multiple program instances
occurs. Also, using test doubles instead of real SR-IOV capable NICs makes tests fast
and portable, in a way that can be run at every commit.

Add a test suite that invokes a system-mocked version of the sriov CNI binary, in which
the `/sys` filesystem content is defined at code, and the network interfaces are of type dummy.

Add `test-integration` make target to invoke the test suite.

Signed-off-by: Andrea Panattoni <[email protected]>

.github/workflows/buildtest.yml

@@ -33,6 +33,12 @@ jobs:
         if: ${{ matrix.goarch }} == "amd64"
         run: sudo make test-race # sudo needed for netns change in test
 
+      - name: Integration test for ${{ matrix.goarch }}
+        env:
+          GOARCH: ${{ matrix.goarch }}
+          GOOS: ${{ matrix.goos }}
+        run: sudo make test-integration
+
   coverage:
     runs-on: ubuntu-latest
     needs: build-test

Makefile

@@ -105,6 +105,14 @@ image: ; $(info Building Docker image...) @ ## Build SR-IOV CNI docker image
 test-image: image
 	$Q $(IMAGEDIR)/image_test.sh $(IMAGE_BUILDER) $(TAG)
 
+BASH_UNIT=$(BINDIR)/bash_unit
+$(BASH_UNIT): $(BINDIR)
+	curl -L https://github.com/pgrange/bash_unit/raw/refs/tags/v2.3.2/bash_unit > bin/bash_unit
+	chmod a+x bin/bash_unit
+
+test-integration: $(BASH_UNIT)
+	$(BASH_UNIT) test/integration/test_*.sh
+
 # Misc
 .PHONY: deps-update
 deps-update: ; $(info  Updating dependencies...) @ ## Update dependencies

pkg/utils/testing.go

@@ -7,6 +7,9 @@
 package utils
 
 import (
+	"fmt"
+	"log"
+	"net"
 	"os"
 	"path/filepath"
 	"syscall"
@@ -154,3 +157,154 @@ func (l *FakeLink) Attrs() *netlink.LinkAttrs {
 func (l *FakeLink) Type() string {
 	return "FakeLink"
 }
+
+
+func MockNetlinkLib(methodCallRecordingDir string) (func(), error) {
+	var err error
+	oldnetlinkLib := netLinkLib
+	// see `ts` variable in this file
+	// "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:00.1/sriov_numvfs": []byte("2"),
+	netLinkLib, err = newPFMockNetlinkLib(methodCallRecordingDir, "enp175s0f1", 2)
+
+	return func() {
+		netLinkLib = oldnetlinkLib
+	}, err
+}
+
+// pfMockNetlinkLib creates dummy interfaces for Physical and Virtual functions, recording method calls on a log file in the form
+// <method_name> <arg1> <arg2> ...
+type pfMockNetlinkLib struct {
+	pf                           netlink.Link
+	methodCallsRecordingFilePath string
+}
+
+func newPFMockNetlinkLib(recordDir, pfName string, numvfs int) (*pfMockNetlinkLib, error) {
+	ret := &pfMockNetlinkLib{
+		pf: &netlink.Dummy{
+			LinkAttrs: netlink.LinkAttrs{
+				Name: pfName,
+				Vfs: []netlink.VfInfo{},
+			},
+		},
+	}
+
+	for i := 0; i<numvfs; i++ {
+		ret.pf.Attrs().Vfs = append(ret.pf.Attrs().Vfs, netlink.VfInfo{
+			ID: i,
+			Mac: mustParseMAC(fmt.Sprintf("ab:cd:ef:ab:cd:%02x", i)),
+		})
+	}
+
+	ret.methodCallsRecordingFilePath = filepath.Join(recordDir, pfName+".calls")
+
+	ret.recordMethodCall("---")
+
+	return ret, nil
+}
+
+func (p *pfMockNetlinkLib) LinkByName(name string) (netlink.Link, error) {
+	p.recordMethodCall("LinkByName %s", name)
+	if name == p.pf.Attrs().Name {
+		return p.pf, nil
+	}
+	return netlink.LinkByName(name)
+}
+
+func (p *pfMockNetlinkLib) LinkSetVfVlanQosProto(link netlink.Link, vfIndex int, vlan int, vlanQos int, vlanProto int) error {
+	p.recordMethodCall("LinkSetVfVlanQosProto %s %d %d %d %d", link.Attrs().Name, vfIndex, vlan, vlanQos, vlanProto)
+	return nil
+}
+
+func (p *pfMockNetlinkLib) LinkSetVfHardwareAddr(pfLink netlink.Link, vfIndex int, hwaddr net.HardwareAddr) error {
+	p.recordMethodCall("LinkSetVfHardwareAddr %s %d %s", pfLink.Attrs().Name, vfIndex, hwaddr.String())
+	pfLink.Attrs().Vfs[vfIndex].Mac = hwaddr
+	return nil
+}
+
+func (p *pfMockNetlinkLib) LinkSetHardwareAddr(link netlink.Link, hwaddr net.HardwareAddr) error {
+	p.recordMethodCall("LinkSetHardwareAddr %s %s", link.Attrs().Name, hwaddr.String())
+	return netlink.LinkSetHardwareAddr(link, hwaddr)
+}
+
+func (p *pfMockNetlinkLib) LinkSetUp(link netlink.Link) error {
+	p.recordMethodCall("LinkSetUp %s", link.Attrs().Name)
+	return netlink.LinkSetUp(link)
+}
+
+func (p *pfMockNetlinkLib) LinkSetDown(link netlink.Link) error {
+	p.recordMethodCall("LinkSetDown %s", link.Attrs().Name)
+	return netlink.LinkSetDown(link)
+}
+
+func (p *pfMockNetlinkLib) LinkSetNsFd(link netlink.Link, nsFd int) error {
+	p.recordMethodCall("LinkSetNsFd %s %d", link.Attrs().Name, nsFd)
+	return netlink.LinkSetNsFd(link, nsFd)
+}
+
+func (p *pfMockNetlinkLib) LinkSetName(link netlink.Link, name string) error {
+	p.recordMethodCall("LinkSetName %s %s", link.Attrs().Name, name)
+	link.Attrs().Name = name
+	return netlink.LinkSetName(link, name)
+}
+
+func (p *pfMockNetlinkLib) LinkSetVfRate(pfLink netlink.Link, vfIndex int, minRate int, maxRate int) error {
+	p.recordMethodCall("LinkSetVfRate %s %d %d %d", pfLink.Attrs().Name, vfIndex, minRate, maxRate)
+	pfLink.Attrs().Vfs[vfIndex].MaxTxRate = uint32(maxRate)
+	pfLink.Attrs().Vfs[vfIndex].MinTxRate = uint32(minRate)
+	return nil
+}
+
+func (p *pfMockNetlinkLib) LinkSetVfSpoofchk(pfLink netlink.Link, vfIndex int, spoofChk bool) error {
+	p.recordMethodCall("LinkSetVfRate %s %d %t", pfLink.Attrs().Name, vfIndex, spoofChk)
+	pfLink.Attrs().Vfs[vfIndex].Spoofchk = spoofChk
+	return nil
+}
+
+func (p *pfMockNetlinkLib) LinkSetVfTrust(pfLink netlink.Link, vfIndex int, trust bool) error {
+	p.recordMethodCall("LinkSetVfTrust %s %d %d", pfLink.Attrs().Name, vfIndex, trust)
+	if trust {
+		pfLink.Attrs().Vfs[vfIndex].Trust = 1
+	} else {
+		pfLink.Attrs().Vfs[vfIndex].Trust = 0
+	}
+
+	return nil
+}
+
+func (p *pfMockNetlinkLib) LinkSetVfState(pfLink netlink.Link, vfIndex int, state uint32) error {
+	p.recordMethodCall("LinkSetVfState %s %d %d", pfLink.Attrs().Name, vfIndex, state)
+	pfLink.Attrs().Vfs[vfIndex].LinkState = state
+	return nil
+}
+
+func (p *pfMockNetlinkLib) LinkSetMTU(link netlink.Link, mtu int) error {
+	p.recordMethodCall("LinkSetMTU %s %d", link.Attrs().Name, mtu)
+	return netlink.LinkSetMTU(link, mtu)
+}
+
+
+func (p *pfMockNetlinkLib) LinkDelAltName(link netlink.Link, name string) error {
+	p.recordMethodCall("LinkDelAltName %s %s", link.Attrs().Name, name)
+	return netlink.LinkDelAltName(link, name)
+}
+
+func (p *pfMockNetlinkLib) recordMethodCall(format string, a ...any) {
+	f, err := os.OpenFile(p.methodCallsRecordingFilePath,
+		os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		log.Println(err)
+		return
+	}
+	defer f.Close()
+	if _, err := f.WriteString(fmt.Sprintf(format+"\n", a...)); err != nil {
+		log.Println(err)
+	}
+}
+
+func mustParseMAC(x string) net.HardwareAddr {
+	ret, err := net.ParseMAC(x)
+	if err != nil {
+		panic(err)
+	}
+	return ret
+}

test/integration/README.md

@@ -0,0 +1,73 @@
+# Integration tests
+
+This folder contains tests and related tools to run integration test suite. These tests leverages a mocked version of the cni, which runs with a 
+fake, programmed version of the `/sys` filesystem and a mocked version of `NetlinkLib`. Both are implemented in `pkg/utils/testing.go`.
+
+The following diagram describes the interactions between the component. 
+
+```mermaid
+graph TD
+
+    subgraph "pkg/utils/testing.go"
+        CreateTmpSysFs
+        MockNetlinkLib
+    end
+  
+    sriovmocked["test/integration/sriov-mocked.go"]
+    subgraph "sriov CNI"
+
+        sriov["cmd/sriov/main.go"]
+        cnicommands_pkg["CmdAdd | CmdDel"]
+        sriov --- cnicommands_pkg
+        
+    end
+
+    sriovmocked --- cnicommands_pkg
+    sriovmocked -.setup.- CreateTmpSysFs
+    sriovmocked -.setup.- MockNetlinkLib
+
+    subgraph "System"graph TD
+  
+    subgraph "Go files"
+        sriovmocked["test/integration/<b>sriov-mocked.go</b>"]
+        sriov["cmd/sriov/<b>main.go</b>"]
+        cnicommands_pkg["CmdAdd | CmdDel"]
+        sriov --- cnicommands_pkg
+        subgraph "pkg/utils/testing.go"
+            CreateTmpSysFs
+            MockNetlinkLib
+        end
+
+        netlinkLib["<small><< lib >></small><br>github.com/vishvananda/netlink"]
+
+    end
+
+    sriovmocked --- cnicommands_pkg
+
+    subgraph "Test Harness"
+        calls_file[(<small><< file >><br>/tmp/x/< pf_name >.calls)]
+        PF{{<small><< dummy >></small><br>PF}}
+        VF1{{<small><< dummy >></small><br>VF1}}
+        VF2{{<small><< dummy >></small><br>VF2}}
+    end
+
+    test_sriov_cni.sh
+    test_sriov_cni.sh --> sriovmocked 
+  
+    cnicommands_pkg --> CreateTmpSysFs
+    cnicommands_pkg --> MockNetlinkLib
+
+    MockNetlinkLib -.write.- calls_file
+    MockNetlinkLib -..- netlinkLib
+    netlinkLib -..- PF
+    netlinkLib -..- VF1
+    netlinkLib -..- VF2
+
+    test_sriov_cni.sh -.read.- calls_file
+    test_sriov_cni.sh -.assert.- PF
+    test_sriov_cni.sh -.assert.- VF1
+    test_sriov_cni.sh -.assert.- VF2
+
+    linkStyle default stroke-width:2px
+    linkStyle 1,2,3,4 stroke:green,stroke-width:4px
+```

test/integration/sriov-mocked.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"os"
+	"runtime"
+
+	"github.com/containernetworking/cni/pkg/skel"
+	"github.com/containernetworking/cni/pkg/version"
+	"github.com/k8snetworkplumbingwg/sriov-cni/pkg/cnicommands"
+	"github.com/k8snetworkplumbingwg/sriov-cni/pkg/config"
+	"github.com/k8snetworkplumbingwg/sriov-cni/pkg/utils"
+)
+
+func init() {
+	// this ensures that main runs only on main thread (thread group leader).
+	// since namespace ops (unshare, setns) are done for a single thread, we
+	// must ensure that the goroutine does not jump from OS thread to thread
+	runtime.LockOSThread()
+}
+
+func main() {
+	customCNIDir, ok := os.LookupEnv("DEFAULT_CNI_DIR")
+	if ok {
+		config.DefaultCNIDir = customCNIDir
+	}
+
+	err := utils.CreateTmpSysFs()
+	if err != nil {
+		panic(err)
+	}
+
+	defer func() {
+		err := utils.RemoveTmpSysFs()
+		if err != nil {
+			panic(err)
+		}
+	}()
+
+	cancel, err := utils.MockNetlinkLib(config.DefaultCNIDir)
+	if err != nil {
+		panic(err)
+	}
+	defer cancel()
+
+
+	cniFuncs := skel.CNIFuncs{
+		Add:   cnicommands.CmdAdd,
+		Del:   cnicommands.CmdDel,
+		Check: cnicommands.CmdCheck,
+	}
+	skel.PluginMainFuncs(cniFuncs, version.All, "")
+}

test/integration/test-ipam-cni

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+if [[ -n "${IPAM_MOCK_SLEEP}" ]]; then
+  sleep "${IPAM_MOCK_SLEEP}"
+fi
+
+cat << EOF
+{
+  "cniVersion": "0.3.1",
+  "interfaces": [{
+    "name": "${CNI_IFNAME}"
+  }],
+  "ips": [{
+    "name": "${CNI_IFNAME}",
+    "address": "192.0.2.1/24"
+  }]
+}
+EOF