diff --git a/integration/integration_linux_test.go b/integration/integration_linux_test.go index 97e73a1ee..32d4ccce7 100644 --- a/integration/integration_linux_test.go +++ b/integration/integration_linux_test.go @@ -17,7 +17,6 @@ import ( "bytes" "fmt" "io" - "log" "math/rand" "net" "os" @@ -61,13 +60,6 @@ var _ = Describe("Basic PTP using cnitool", func() { netConfPath, err := filepath.Abs("./testdata") Expect(err).NotTo(HaveOccurred()) - // Flush ipam stores to avoid conflicts - err = os.RemoveAll("/tmp/chained-ptp-bandwidth-test") - Expect(err).NotTo(HaveOccurred()) - - err = os.RemoveAll("/tmp/basic-ptp-test") - Expect(err).NotTo(HaveOccurred()) - env = TestEnv([]string{ "CNI_PATH=" + cniPath, "NETCONFPATH=" + netConfPath, @@ -90,7 +82,6 @@ var _ = Describe("Basic PTP using cnitool", func() { env.runInNS(hostNS, cnitoolBin, "add", netName, contNS.LongName()) addrOutput := env.runInNS(contNS, "ip", "addr") - Expect(addrOutput).To(ContainSubstring(expectedIPPrefix)) env.runInNS(hostNS, cnitoolBin, "del", netName, contNS.LongName()) @@ -154,13 +145,9 @@ var _ = Describe("Basic PTP using cnitool", func() { chainedBridgeBandwidthEnv.runInNS(hostNS, cnitoolBin, "del", "network-chain-test", contNS1.LongName()) basicBridgeEnv.runInNS(hostNS, cnitoolBin, "del", "network-chain-test", contNS2.LongName()) - - contNS1.Del() - contNS2.Del() - hostNS.Del() }) - It("limits traffic only on the restricted bandwidth veth device", func() { + Measure("limits traffic only on the restricted bandwidth veth device", func(b Benchmarker) { ipRegexp := regexp.MustCompile(`10\.1[12]\.2\.\d{1,3}`) By(fmt.Sprintf("adding %s to %s\n\n", "chained-bridge-bandwidth", contNS1.ShortName())) @@ -181,23 +168,21 @@ var _ = Describe("Basic PTP using cnitool", func() { By(fmt.Sprintf("starting echo server in %s\n\n", contNS2.ShortName())) basicBridgePort, basicBridgeSession = startEchoServerInNamespace(contNS2) - packetInBytes := 3000 + packetInBytes := 20000 // The shaper needs to 'warm'. Send enough to cause it to throttle, + // balanced by run time. By(fmt.Sprintf("sending tcp traffic to the chained, bridged, traffic shaped container on ip address '%s:%d'\n\n", chainedBridgeIP, chainedBridgeBandwidthPort)) - start := time.Now() - makeTCPClientInNS(hostNS.ShortName(), chainedBridgeIP, chainedBridgeBandwidthPort, packetInBytes) - runtimeWithLimit := time.Since(start) - - log.Printf("Runtime with qos limit %.2f seconds", runtimeWithLimit.Seconds()) + runtimeWithLimit := b.Time("with chained bridge and bandwidth plugins", func() { + makeTCPClientInNS(hostNS.ShortName(), chainedBridgeIP, chainedBridgeBandwidthPort, packetInBytes) + }) By(fmt.Sprintf("sending tcp traffic to the basic bridged container on ip address '%s:%d'\n\n", basicBridgeIP, basicBridgePort)) - start = time.Now() - makeTCPClientInNS(hostNS.ShortName(), basicBridgeIP, basicBridgePort, packetInBytes) - runtimeWithoutLimit := time.Since(start) - log.Printf("Runtime without qos limit %.2f seconds", runtimeWithLimit.Seconds()) + runtimeWithoutLimit := b.Time("with basic bridged plugin", func() { + makeTCPClientInNS(hostNS.ShortName(), basicBridgeIP, basicBridgePort, packetInBytes) + }) Expect(runtimeWithLimit).To(BeNumerically(">", runtimeWithoutLimit+1000*time.Millisecond)) - }) + }, 1) }) }) diff --git a/integration/testdata/basic-ptp.json b/integration/testdata/basic-ptp.json index dc04c5301..11dc3aa61 100644 --- a/integration/testdata/basic-ptp.json +++ b/integration/testdata/basic-ptp.json @@ -6,7 +6,6 @@ "mtu": 512, "ipam": { "type": "host-local", - "subnet": "10.1.2.0/24", - "dataDir": "/tmp/basic-ptp-test" + "subnet": "10.1.2.0/24" } } diff --git a/integration/testdata/chained-ptp-bandwidth.conflist b/integration/testdata/chained-ptp-bandwidth.conflist index b7b3d60a7..2510ba882 100644 --- a/integration/testdata/chained-ptp-bandwidth.conflist +++ b/integration/testdata/chained-ptp-bandwidth.conflist @@ -8,8 +8,7 @@ "mtu": 512, "ipam": { "type": "host-local", - "subnet": "10.9.2.0/24", - "dataDir": "/tmp/chained-ptp-bandwidth-test" + "subnet": "10.9.2.0/24" } }, { diff --git a/plugins/meta/bandwidth/bandwidth_config_test.go b/plugins/meta/bandwidth/bandwidth_config_test.go deleted file mode 100644 index 5a9c1dd3d..000000000 --- a/plugins/meta/bandwidth/bandwidth_config_test.go +++ /dev/null @@ -1,563 +0,0 @@ -// Copyright 2023 CNI authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "fmt" - "math" - "net" - "syscall" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/vishvananda/netlink" - - "github.com/containernetworking/cni/pkg/skel" - types100 "github.com/containernetworking/cni/pkg/types/100" - "github.com/containernetworking/plugins/pkg/ns" - "github.com/containernetworking/plugins/pkg/testutils" -) - -var _ = Describe("bandwidth config test", func() { - var ( - hostNs ns.NetNS - containerNs ns.NetNS - ifbDeviceName string - hostIfname string - containerIfname string - hostIP net.IP - containerIP net.IP - hostIfaceMTU int - ) - - BeforeEach(func() { - var err error - - hostIfname = "host-veth" - containerIfname = "container-veth" - - hostNs, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) - - containerNs, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) - - hostIP = net.IP{169, 254, 0, 1} - containerIP = net.IP{10, 254, 0, 1} - hostIfaceMTU = 1024 - ifbDeviceName = "bwpa8eda89404b7" - - createVeth(hostNs, hostIfname, containerNs, containerIfname, hostIP, containerIP, hostIfaceMTU) - }) - - AfterEach(func() { - Expect(containerNs.Close()).To(Succeed()) - Expect(testutils.UnmountNS(containerNs)).To(Succeed()) - Expect(hostNs.Close()).To(Succeed()) - Expect(testutils.UnmountNS(hostNs)).To(Succeed()) - }) - - // Bandwidth requires host-side interface info, and thus only - // supports 0.3.0 and later CNI versions - for _, ver := range []string{"0.3.0", "0.3.1", "0.4.0", "1.0.0"} { - // Redefine ver inside for scope so real value is picked up by each dynamically defined It() - // See Gingkgo's "Patterns for dynamically generating tests" documentation. - ver := ver - - Describe("cmdADD", func() { - It(fmt.Sprintf("[%s] fails with invalid UnshapedSubnets", ver), func() { - conf := fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "cni-plugin-bandwidth-test", - "type": "bandwidth", - "ingressRate": 123, - "ingressBurst": 123, - "egressRate": 123, - "egressBurst": 123, - "unshapedSubnets": ["10.0.0.0/8", "hello"], - "prevResult": { - "interfaces": [ - { - "name": "%s", - "sandbox": "" - }, - { - "name": "%s", - "sandbox": "%s" - } - ], - "ips": [ - { - "version": "4", - "address": "%s/24", - "gateway": "10.0.0.1", - "interface": 1 - } - ], - "routes": [] - } - }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) - - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: containerNs.Path(), - IfName: "eth0", - StdinData: []byte(conf), - } - - Expect(hostNs.Do(func(netNS ns.NetNS) error { - defer GinkgoRecover() - - _, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) - Expect(err).To(MatchError("bad subnet \"hello\" provided, details invalid CIDR address: hello")) - return nil - })).To(Succeed()) - }) - - It(fmt.Sprintf("[%s] fails with invalid ShapedSubnets", ver), func() { - conf := fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "cni-plugin-bandwidth-test", - "type": "bandwidth", - "ingressRate": 123, - "ingressBurst": 123, - "egressRate": 123, - "egressBurst": 123, - "ShapedSubnets": ["10.0.0.0/8", "hello"], - "prevResult": { - "interfaces": [ - { - "name": "%s", - "sandbox": "" - }, - { - "name": "%s", - "sandbox": "%s" - } - ], - "ips": [ - { - "version": "4", - "address": "%s/24", - "gateway": "10.0.0.1", - "interface": 1 - } - ], - "routes": [] - } - }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) - - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: containerNs.Path(), - IfName: "eth0", - StdinData: []byte(conf), - } - - Expect(hostNs.Do(func(netNS ns.NetNS) error { - defer GinkgoRecover() - - _, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) - Expect(err).To(MatchError("bad subnet \"hello\" provided, details invalid CIDR address: hello")) - return nil - })).To(Succeed()) - }) - - It(fmt.Sprintf("[%s] fails with both ShapedSubnets and UnshapedSubnets specified", ver), func() { - conf := fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "cni-plugin-bandwidth-test", - "type": "bandwidth", - "ingressRate": 123, - "ingressBurst": 123, - "egressRate": 123, - "egressBurst": 123, - "shapedSubnets": ["10.0.0.0/8"], - "unshapedSubnets": ["192.168.0.0/16"], - "prevResult": { - "interfaces": [ - { - "name": "%s", - "sandbox": "" - }, - { - "name": "%s", - "sandbox": "%s" - } - ], - "ips": [ - { - "version": "4", - "address": "%s/24", - "gateway": "10.0.0.1", - "interface": 1 - } - ], - "routes": [] - } - }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) - - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: containerNs.Path(), - IfName: "eth0", - StdinData: []byte(conf), - } - - Expect(hostNs.Do(func(netNS ns.NetNS) error { - defer GinkgoRecover() - - _, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) - Expect(err).To(MatchError("unshapedSubnets and shapedSubnets cannot be both specified, one of them should be discarded")) - return nil - })).To(Succeed()) - }) - - It(fmt.Sprintf("[%s] fails an invalid ingress config", ver), func() { - conf := fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "cni-plugin-bandwidth-test", - "type": "bandwidth", - "ingressRate": 0, - "ingressBurst": 123, - "egressRate": 123, - "egressBurst": 123, - "prevResult": { - "interfaces": [ - { - "name": "%s", - "sandbox": "" - }, - { - "name": "%s", - "sandbox": "%s" - } - ], - "ips": [ - { - "version": "4", - "address": "%s/24", - "gateway": "10.0.0.1", - "interface": 1 - } - ], - "routes": [] - } - }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) - - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: containerNs.Path(), - IfName: "eth0", - StdinData: []byte(conf), - } - - Expect(hostNs.Do(func(netNS ns.NetNS) error { - defer GinkgoRecover() - - _, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) - Expect(err).To(MatchError("if burst is set, rate must also be set")) - return nil - })).To(Succeed()) - }) - - It(fmt.Sprintf("[%s] fails an invalid egress config", ver), func() { - conf := fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "cni-plugin-bandwidth-test", - "type": "bandwidth", - "ingressRate": 123, - "ingressBurst": 123, - "egressRate": 0, - "egressBurst": 123, - "prevResult": { - "interfaces": [ - { - "name": "%s", - "sandbox": "" - }, - { - "name": "%s", - "sandbox": "%s" - } - ], - "ips": [ - { - "version": "4", - "address": "%s/24", - "gateway": "10.0.0.1", - "interface": 1 - } - ], - "routes": [] - } - }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) - - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: containerNs.Path(), - IfName: "eth0", - StdinData: []byte(conf), - } - - Expect(hostNs.Do(func(netNS ns.NetNS) error { - defer GinkgoRecover() - - _, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) - Expect(err).To(MatchError("if burst is set, rate must also be set")) - return nil - })).To(Succeed()) - }) - - // Runtime config parameters are expected to be preempted by the global config ones whenever specified - It(fmt.Sprintf("[%s] should apply static config when both static config and runtime config exist", ver), func() { - conf := fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "cni-plugin-bandwidth-test", - "type": "bandwidth", - "ingressRate": 0, - "ingressBurst": 0, - "egressRate": 123, - "egressBurst": 123, - "unshapedSubnets": ["192.168.0.0/24"], - "runtimeConfig": { - "bandWidth": { - "ingressRate": 8, - "ingressBurst": 8, - "egressRate": 16, - "egressBurst": 9, - "unshapedSubnets": ["10.0.0.0/8", "fd00:db8:abcd:1234:e000::/68"] - } - }, - "prevResult": { - "interfaces": [ - { - "name": "%s", - "sandbox": "" - }, - { - "name": "%s", - "sandbox": "%s" - } - ], - "ips": [ - { - "version": "4", - "address": "%s/24", - "gateway": "10.0.0.1", - "interface": 1 - } - ], - "routes": [] - } - }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) - - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: containerNs.Path(), - IfName: containerIfname, - StdinData: []byte(conf), - } - - Expect(hostNs.Do(func(netNS ns.NetNS) error { - defer GinkgoRecover() - r, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) - Expect(err).NotTo(HaveOccurred(), string(out)) - result, err := types100.GetResult(r) - Expect(err).NotTo(HaveOccurred()) - - Expect(result.Interfaces).To(HaveLen(3)) - Expect(result.Interfaces[2].Name).To(Equal(ifbDeviceName)) - Expect(result.Interfaces[2].Sandbox).To(Equal("")) - - ifbLink, err := netlink.LinkByName(ifbDeviceName) - Expect(err).NotTo(HaveOccurred()) - Expect(ifbLink.Attrs().MTU).To(Equal(hostIfaceMTU)) - - qdiscs, err := netlink.QdiscList(ifbLink) - Expect(err).NotTo(HaveOccurred()) - - Expect(qdiscs).To(HaveLen(1)) - Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) - Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{})) - Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID))) - - classes, err := netlink.ClassList(ifbLink, qdiscs[0].Attrs().Handle) - - Expect(err).NotTo(HaveOccurred()) - Expect(classes).To(HaveLen(2)) - - // Uncapped class - Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1))) - Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0))) - Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - // Class with traffic shapping settings - Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls)))) - Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(15))) - // Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(7812500))) - Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(30))) - // Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - filters, err := netlink.FilterList(ifbLink, qdiscs[0].Attrs().Handle) - Expect(err).NotTo(HaveOccurred()) - Expect(filters).To(HaveLen(1)) - - // traffic to 192.168.0.0/24 redirected to uncapped class - Expect(filters[0]).To(BeAssignableToTypeOf(&netlink.U32{})) - Expect(filters[0].(*netlink.U32).Actions).To(BeEmpty()) - Expect(filters[0].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IP))) - Expect(filters[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) - Expect(filters[0].Attrs().Priority).To(Equal(uint16(16))) - Expect(filters[0].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle)) - Expect(filters[0].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1))) - - filterSel := filters[0].(*netlink.U32).Sel - Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{})) - Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL))) - Expect(filterSel.Keys).To(HaveLen(1)) - Expect(filterSel.Nkeys).To(Equal(uint8(1))) - - // The filter should match to 192.168.0.0/24 dst address in other words it should be: - // match c0a80000/ffffff00 at 16 - selKey := filterSel.Keys[0] - Expect(selKey.Val).To(Equal(uint32(192*math.Pow(256, 3) + 168*math.Pow(256, 2)))) - Expect(selKey.Off).To(Equal(int32(16))) - Expect(selKey.Mask).To(Equal(uint32(255*math.Pow(256, 3) + 255*math.Pow(256, 2) + 255*256))) - - hostVethLink, err := netlink.LinkByName(hostIfname) - Expect(err).NotTo(HaveOccurred()) - - qdiscFilters, err := netlink.FilterList(hostVethLink, netlink.MakeHandle(0xffff, 0)) - Expect(err).NotTo(HaveOccurred()) - - Expect(qdiscFilters).To(HaveLen(1)) - Expect(qdiscFilters[0].(*netlink.U32).Actions[0].(*netlink.MirredAction).Ifindex).To(Equal(ifbLink.Attrs().Index)) - - return nil - })).To(Succeed()) - - // Container ingress (host egress) - Expect(hostNs.Do(func(n ns.NetNS) error { - defer GinkgoRecover() - - vethLink, err := netlink.LinkByName(hostIfname) - Expect(err).NotTo(HaveOccurred()) - - qdiscs, err := netlink.QdiscList(vethLink) - Expect(err).NotTo(HaveOccurred()) - - // No ingress QoS just mirroring - Expect(qdiscs).To(HaveLen(2)) - Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index)) - Expect(qdiscs[0]).NotTo(BeAssignableToTypeOf(&netlink.Htb{})) - Expect(qdiscs[1]).NotTo(BeAssignableToTypeOf(&netlink.Htb{})) - - return nil - })).To(Succeed()) - }) - - It(fmt.Sprintf("[%s] should apply static config when both static config and runtime config exist (bad config example)", ver), func() { - conf := fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "cni-plugin-bandwidth-test", - "type": "bandwidth", - "ingressRate": 0, - "ingressBurst": 123, - "egressRate": 123, - "egressBurst": 123, - "runtimeConfig": { - "bandWidth": { - "ingressRate": 8, - "ingressBurst": 8, - "egressRate": 16, - "egressBurst": 9 - } - }, - "prevResult": { - "interfaces": [ - { - "name": "%s", - "sandbox": "" - }, - { - "name": "%s", - "sandbox": "%s" - } - ], - "ips": [ - { - "version": "4", - "address": "%s/24", - "gateway": "10.0.0.1", - "interface": 1 - } - ], - "routes": [] - } - }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) - - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: containerNs.Path(), - IfName: "eth0", - StdinData: []byte(conf), - } - - Expect(hostNs.Do(func(netNS ns.NetNS) error { - defer GinkgoRecover() - - _, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) - Expect(err).To(MatchError("if burst is set, rate must also be set")) - return nil - })).To(Succeed()) - }) - }) - } - - Describe("Validating input", func() { - It("Should allow only 4GB burst rate", func() { - err := validateRateAndBurst(5000, 4*1024*1024*1024*8-16) // 2 bytes less than the max should pass - Expect(err).NotTo(HaveOccurred()) - err = validateRateAndBurst(5000, 4*1024*1024*1024*8) // we're 1 bit above MaxUint32 - Expect(err).To(HaveOccurred()) - err = validateRateAndBurst(0, 1) - Expect(err).To(HaveOccurred()) - err = validateRateAndBurst(1, 0) - Expect(err).To(HaveOccurred()) - err = validateRateAndBurst(0, 0) - Expect(err).NotTo(HaveOccurred()) - }) - - It("Should fail if both ShapedSubnets and UnshapedSubnets are specified", func() { - err := validateSubnets([]string{"10.0.0.0/8"}, []string{"192.168.0.0/24"}) - Expect(err).To(HaveOccurred()) - }) - - It("Should fail if specified UnshapedSubnets are not valid CIDRs", func() { - err := validateSubnets([]string{"10.0.0.0/8", "hello"}, []string{}) - Expect(err).To(HaveOccurred()) - }) - - It("Should fail if specified ShapedSubnets are not valid CIDRs", func() { - err := validateSubnets([]string{}, []string{"10.0.0.0/8", "hello"}) - Expect(err).To(HaveOccurred()) - }) - }) -}) diff --git a/plugins/meta/bandwidth/bandwidth_linux_test.go b/plugins/meta/bandwidth/bandwidth_linux_test.go index e6dc2796a..9d2aa9cbc 100644 --- a/plugins/meta/bandwidth/bandwidth_linux_test.go +++ b/plugins/meta/bandwidth/bandwidth_linux_test.go @@ -15,21 +15,70 @@ package main import ( + "context" + "encoding/json" "fmt" - "math" "net" - "syscall" + "os" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" "github.com/vishvananda/netlink" + "github.com/containernetworking/cni/pkg/invoke" "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" types100 "github.com/containernetworking/cni/pkg/types/100" "github.com/containernetworking/plugins/pkg/ns" "github.com/containernetworking/plugins/pkg/testutils" ) +func buildOneConfig(name, cniVersion string, orig *PluginConf, prevResult types.Result) ([]byte, error) { + var err error + + inject := map[string]interface{}{ + "name": name, + "cniVersion": cniVersion, + } + // Add previous plugin result + if prevResult != nil { + r, err := prevResult.GetAsVersion(cniVersion) + Expect(err).NotTo(HaveOccurred()) + inject["prevResult"] = r + } + + // Ensure every config uses the same name and version + config := make(map[string]interface{}) + + confBytes, err := json.Marshal(orig) + if err != nil { + return nil, err + } + + err = json.Unmarshal(confBytes, &config) + if err != nil { + return nil, fmt.Errorf("unmarshal existing network bytes: %s", err) + } + + for key, value := range inject { + config[key] = value + } + + newBytes, err := json.Marshal(config) + if err != nil { + return nil, err + } + + conf := &PluginConf{} + if err := json.Unmarshal(newBytes, &conf); err != nil { + return nil, fmt.Errorf("error parsing configuration: %s", err) + } + + return newBytes, nil +} + var _ = Describe("bandwidth test", func() { var ( hostNs ns.NetNS @@ -77,7 +126,7 @@ var _ = Describe("bandwidth test", func() { ver := ver Describe("cmdADD", func() { - It(fmt.Sprintf("[%s] works with a Veth pair without any unbounded traffic", ver), func() { + It(fmt.Sprintf("[%s] works with a Veth pair", ver), func() { conf := fmt.Sprintf(`{ "cniVersion": "%s", "name": "cni-plugin-bandwidth-test", @@ -85,7 +134,7 @@ var _ = Describe("bandwidth test", func() { "ingressRate": 8, "ingressBurst": 8, "egressRate": 16, - "egressBurst": 12, + "egressBurst": 8, "prevResult": { "interfaces": [ { @@ -116,7 +165,6 @@ var _ = Describe("bandwidth test", func() { StdinData: []byte(conf), } - // Container egress (host ingress) Expect(hostNs.Do(func(_ ns.NetNS) error { defer GinkgoRecover() r, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) @@ -137,34 +185,10 @@ var _ = Describe("bandwidth test", func() { Expect(qdiscs).To(HaveLen(1)) Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) - Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{})) - Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID))) - - classes, err := netlink.ClassList(ifbLink, qdiscs[0].Attrs().Handle) - Expect(err).NotTo(HaveOccurred()) - Expect(classes).To(HaveLen(2)) - - // Uncapped class - Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1))) - Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0))) - Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - // Class with traffic shapping settings - Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls)))) - Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(2))) - // Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(7812500))) - Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(4))) - // Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - // Since we do not exclude anything from egress traffic shapping, we should not find any filter - filters, err := netlink.FilterList(ifbLink, qdiscs[0].Attrs().Handle) - Expect(err).NotTo(HaveOccurred()) - Expect(filters).To(BeEmpty()) + Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Tbf{})) + Expect(qdiscs[0].(*netlink.Tbf).Rate).To(Equal(uint64(2))) + Expect(qdiscs[0].(*netlink.Tbf).Limit).To(Equal(uint32(1))) hostVethLink, err := netlink.LinkByName(hostIfname) Expect(err).NotTo(HaveOccurred()) @@ -178,85 +202,56 @@ var _ = Describe("bandwidth test", func() { return nil })).To(Succeed()) - // Container ingress (host egress) - Expect(hostNs.Do(func(n ns.NetNS) error { + Expect(hostNs.Do(func(_ ns.NetNS) error { defer GinkgoRecover() - vethLink, err := netlink.LinkByName(hostIfname) + ifbLink, err := netlink.LinkByName(hostIfname) Expect(err).NotTo(HaveOccurred()) - qdiscs, err := netlink.QdiscList(vethLink) + qdiscs, err := netlink.QdiscList(ifbLink) Expect(err).NotTo(HaveOccurred()) Expect(qdiscs).To(HaveLen(2)) - Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index)) - Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{})) - Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID))) - - classes, err := netlink.ClassList(vethLink, qdiscs[0].Attrs().Handle) + Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) - Expect(err).NotTo(HaveOccurred()) - Expect(classes).To(HaveLen(2)) - - // Uncapped class - Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1))) - Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0))) - Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - // Class with traffic shapping settings - Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls)))) - Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(1))) - // Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(15625000))) - Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(2))) - // Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - // Since we do not exclude anything from ingress traffic shapping, we should not find any filter - filters, err := netlink.FilterList(vethLink, qdiscs[0].Attrs().Handle) - Expect(err).NotTo(HaveOccurred()) - Expect(filters).To(BeEmpty()) + Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Tbf{})) + Expect(qdiscs[0].(*netlink.Tbf).Rate).To(Equal(uint64(1))) + Expect(qdiscs[0].(*netlink.Tbf).Limit).To(Equal(uint32(1))) return nil })).To(Succeed()) }) - It(fmt.Sprintf("[%s] works with a Veth pair with some ipv4 and ipv6 unbounded traffic", ver), func() { + It(fmt.Sprintf("[%s] does not apply ingress when disabled", ver), func() { conf := fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "cni-plugin-bandwidth-test", - "type": "bandwidth", - "ingressRate": 8, - "ingressBurst": 8, - "egressRate": 16, - "egressBurst": 12, - "unshapedSubnets": [ - "10.0.0.0/8", - "fd00:db8:abcd:1234:e000::/68" - ], - "prevResult": { - "interfaces": [ - { - "name": "%s", - "sandbox": "" - }, - { - "name": "%s", - "sandbox": "%s" - } - ], - "ips": [ - { - "version": "4", - "address": "%s/24", - "gateway": "10.0.0.1", - "interface": 1 - } - ], - "routes": [] - } - }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) + "cniVersion": "%s", + "name": "cni-plugin-bandwidth-test", + "type": "bandwidth", + "ingressRate": 0, + "ingressBurst": 0, + "egressRate": 8000, + "egressBurst": 80, + "prevResult": { + "interfaces": [ + { + "name": "%s", + "sandbox": "" + }, + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "%s/24", + "gateway": "10.0.0.1", + "interface": 1 + } + ], + "routes": [] + } + }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) args := &skel.CmdArgs{ ContainerID: "dummy", @@ -265,948 +260,267 @@ var _ = Describe("bandwidth test", func() { StdinData: []byte(conf), } - // Container egress (host ingress) Expect(hostNs.Do(func(_ ns.NetNS) error { defer GinkgoRecover() - r, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) - Expect(err).NotTo(HaveOccurred(), string(out)) - result, err := types100.GetResult(r) - Expect(err).NotTo(HaveOccurred()) - - Expect(result.Interfaces).To(HaveLen(3)) - Expect(result.Interfaces[2].Name).To(Equal(ifbDeviceName)) - Expect(result.Interfaces[2].Sandbox).To(Equal("")) - ifbLink, err := netlink.LinkByName(ifbDeviceName) - Expect(err).NotTo(HaveOccurred()) - Expect(ifbLink.Attrs().MTU).To(Equal(hostIfaceMTU)) - - qdiscs, err := netlink.QdiscList(ifbLink) - Expect(err).NotTo(HaveOccurred()) - - Expect(qdiscs).To(HaveLen(1)) - Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) - Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{})) - Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID))) - - classes, err := netlink.ClassList(ifbLink, qdiscs[0].Attrs().Handle) - - Expect(err).NotTo(HaveOccurred()) - Expect(classes).To(HaveLen(2)) - - // Uncapped class - Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1))) - Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0))) - Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - // Class with traffic shapping settings - Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls)))) - Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(2))) - // Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(7812500))) - Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(4))) - // Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - filters, err := netlink.FilterList(ifbLink, qdiscs[0].Attrs().Handle) - Expect(err).NotTo(HaveOccurred()) - Expect(filters).To(HaveLen(2)) - - // traffic to fd00:db8:abcd:1234:e000::/68 redirected to uncapped class - Expect(filters[0]).To(BeAssignableToTypeOf(&netlink.U32{})) - Expect(filters[0].(*netlink.U32).Actions).To(BeEmpty()) - Expect(filters[0].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IPV6))) - Expect(filters[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) - Expect(filters[0].Attrs().Priority).To(Equal(uint16(15))) - Expect(filters[0].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle)) - Expect(filters[0].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1))) - - filterSel := filters[0].(*netlink.U32).Sel - Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{})) - Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL))) - Expect(filterSel.Keys).To(HaveLen(3)) - Expect(filterSel.Nkeys).To(Equal(uint8(3))) - - // The filter should match to fd00:db8:abcd:1234:e000::/68 dst address in other words it should be: - // match 0xfd000db8/0xffffffff at 24 - // match 0xabcd1234/0xffffffff at 28 - // match 0xe0000000/0xf0000000 at 32 - // (and last match discarded because it would be equivalent to a matchall/true condition at 36) - Expect(filterSel.Keys[0].Off).To(Equal(int32(24))) - Expect(filterSel.Keys[0].Val).To(Equal(uint32(4244639160))) - Expect(filterSel.Keys[0].Mask).To(Equal(uint32(4294967295))) - - Expect(filterSel.Keys[1].Off).To(Equal(int32(28))) - Expect(filterSel.Keys[1].Val).To(Equal(uint32(2882343476))) - Expect(filterSel.Keys[1].Mask).To(Equal(uint32(4294967295))) - - Expect(filterSel.Keys[2].Off).To(Equal(int32(32))) - Expect(filterSel.Keys[2].Val).To(Equal(uint32(3758096384))) - Expect(filterSel.Keys[2].Mask).To(Equal(uint32(4026531840))) - - // traffic to 10.0.0.0/8 redirected to uncapped class - Expect(filters[1]).To(BeAssignableToTypeOf(&netlink.U32{})) - Expect(filters[1].(*netlink.U32).Actions).To(BeEmpty()) - Expect(filters[1].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IP))) - Expect(filters[1].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) - Expect(filters[1].Attrs().Priority).To(Equal(uint16(16))) - Expect(filters[1].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle)) - Expect(filters[1].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1))) - - filterSel = filters[1].(*netlink.U32).Sel - Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{})) - Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL))) - Expect(filterSel.Keys).To(HaveLen(1)) - Expect(filterSel.Nkeys).To(Equal(uint8(1))) - - // The filter should match to 10.0.0.0/8 dst address in other words it should be: - // match 0a000000/ff000000 at 16 - selKey := filterSel.Keys[0] - Expect(selKey.Val).To(Equal(uint32(10 * math.Pow(256, 3)))) - Expect(selKey.Off).To(Equal(int32(16))) - Expect(selKey.Mask).To(Equal(uint32(255 * math.Pow(256, 3)))) - - hostVethLink, err := netlink.LinkByName(hostIfname) - Expect(err).NotTo(HaveOccurred()) + _, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, ifbDeviceName, []byte(conf), func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred(), string(out)) - qdiscFilters, err := netlink.FilterList(hostVethLink, netlink.MakeHandle(0xffff, 0)) + _, err = netlink.LinkByName(ifbDeviceName) Expect(err).NotTo(HaveOccurred()) - - Expect(qdiscFilters).To(HaveLen(1)) - Expect(qdiscFilters[0].(*netlink.U32).Actions[0].(*netlink.MirredAction).Ifindex).To(Equal(ifbLink.Attrs().Index)) - return nil })).To(Succeed()) - // Container ingress (host egress) - Expect(hostNs.Do(func(n ns.NetNS) error { + Expect(hostNs.Do(func(_ ns.NetNS) error { defer GinkgoRecover() - vethLink, err := netlink.LinkByName(hostIfname) + containerIfLink, err := netlink.LinkByName(hostIfname) Expect(err).NotTo(HaveOccurred()) - qdiscs, err := netlink.QdiscList(vethLink) + qdiscs, err := netlink.QdiscList(containerIfLink) Expect(err).NotTo(HaveOccurred()) Expect(qdiscs).To(HaveLen(2)) - Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index)) - Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{})) - Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID))) - - classes, err := netlink.ClassList(vethLink, qdiscs[0].Attrs().Handle) - - Expect(err).NotTo(HaveOccurred()) - Expect(classes).To(HaveLen(2)) - - // Uncapped class - Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1))) - Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0))) - Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - // Class with traffic shapping settings - Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls)))) - Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(1))) - // Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(15625000))) - Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(2))) - // Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - filters, err := netlink.FilterList(vethLink, qdiscs[0].Attrs().Handle) - Expect(err).NotTo(HaveOccurred()) - Expect(filters).To(HaveLen(2)) - - // traffic to fd00:db8:abcd:1234:e000::/68 redirected to uncapped class - Expect(filters[0]).To(BeAssignableToTypeOf(&netlink.U32{})) - Expect(filters[0].(*netlink.U32).Actions).To(BeEmpty()) - Expect(filters[0].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IPV6))) - Expect(filters[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index)) - Expect(filters[0].Attrs().Priority).To(Equal(uint16(15))) - Expect(filters[0].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle)) - Expect(filters[0].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1))) - - filterSel := filters[0].(*netlink.U32).Sel - Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{})) - Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL))) - Expect(filterSel.Keys).To(HaveLen(3)) - Expect(filterSel.Nkeys).To(Equal(uint8(3))) - - // The filter should match to fd00:db8:abcd:1234:e000::/68 dst address in other words it should be: - // match 0xfd000db8/0xffffffff at 24 - // match 0xabcd1234/0xffffffff at 28 - // match 0xe0000000/0xf0000000 at 32 - // (and last match discarded because it would be equivalent to a matchall/true condition at 36) - Expect(filterSel.Keys[0].Off).To(Equal(int32(24))) - Expect(filterSel.Keys[0].Val).To(Equal(uint32(4244639160))) - Expect(filterSel.Keys[0].Mask).To(Equal(uint32(4294967295))) - - Expect(filterSel.Keys[1].Off).To(Equal(int32(28))) - Expect(filterSel.Keys[1].Val).To(Equal(uint32(2882343476))) - Expect(filterSel.Keys[1].Mask).To(Equal(uint32(4294967295))) - - Expect(filterSel.Keys[2].Off).To(Equal(int32(32))) - Expect(filterSel.Keys[2].Val).To(Equal(uint32(3758096384))) - Expect(filterSel.Keys[2].Mask).To(Equal(uint32(4026531840))) - - // traffic to 10.0.0.0/8 redirected to uncapped class - Expect(filters[1]).To(BeAssignableToTypeOf(&netlink.U32{})) - Expect(filters[1].(*netlink.U32).Actions).To(BeEmpty()) - Expect(filters[1].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IP))) - Expect(filters[1].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index)) - Expect(filters[1].Attrs().Priority).To(Equal(uint16(16))) - Expect(filters[1].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle)) - Expect(filters[1].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1))) - - filterSel = filters[1].(*netlink.U32).Sel - Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{})) - Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL))) - Expect(filterSel.Keys).To(HaveLen(1)) - Expect(filterSel.Nkeys).To(Equal(uint8(1))) - - // The filter should match to 10.0.0.0/8 dst address in other words it should be: - // match 0a000000/ff000000 at 16 - selKey := filterSel.Keys[0] - Expect(selKey.Val).To(Equal(uint32(10 * math.Pow(256, 3)))) - Expect(selKey.Off).To(Equal(int32(16))) - Expect(selKey.Mask).To(Equal(uint32(255 * math.Pow(256, 3)))) + Expect(qdiscs[0]).NotTo(BeAssignableToTypeOf(&netlink.Tbf{})) + Expect(qdiscs[1]).NotTo(BeAssignableToTypeOf(&netlink.Tbf{})) return nil })).To(Succeed()) }) - }) - It(fmt.Sprintf("[%s] works with a Veth pair with some ipv4 and ipv6 shaped traffic for specific subnets", ver), func() { - conf := fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "cni-plugin-bandwidth-test", - "type": "bandwidth", - "ingressRate": 8, - "ingressBurst": 8, - "egressRate": 16, - "egressBurst": 12, - "shapedSubnets": [ - "10.0.0.0/8", - "fd00:db8:abcd:1234:e000::/68" - ], - "prevResult": { - "interfaces": [ - { - "name": "%s", - "sandbox": "" - }, - { - "name": "%s", - "sandbox": "%s" - } - ], - "ips": [ - { - "version": "4", - "address": "%s/24", - "gateway": "10.0.0.1", - "interface": 1 + It(fmt.Sprintf("[%s] does not apply egress when disabled", ver), func() { + conf := fmt.Sprintf(`{ + "cniVersion": "%s", + "name": "cni-plugin-bandwidth-test", + "type": "bandwidth", + "egressRate": 0, + "egressBurst": 0, + "ingressRate": 8000, + "ingressBurst": 80, + "prevResult": { + "interfaces": [ + { + "name": "%s", + "sandbox": "" + }, + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "%s/24", + "gateway": "10.0.0.1", + "interface": 1 + } + ], + "routes": [] } - ], - "routes": [] - } - }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) - - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: containerNs.Path(), - IfName: containerIfname, - StdinData: []byte(conf), - } - - // Container egress (host ingress) - Expect(hostNs.Do(func(_ ns.NetNS) error { - defer GinkgoRecover() - r, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) - Expect(err).NotTo(HaveOccurred(), string(out)) - result, err := types100.GetResult(r) - Expect(err).NotTo(HaveOccurred()) - - Expect(result.Interfaces).To(HaveLen(3)) - Expect(result.Interfaces[2].Name).To(Equal(ifbDeviceName)) - Expect(result.Interfaces[2].Sandbox).To(Equal("")) - - ifbLink, err := netlink.LinkByName(ifbDeviceName) - Expect(err).NotTo(HaveOccurred()) - Expect(ifbLink.Attrs().MTU).To(Equal(hostIfaceMTU)) - - qdiscs, err := netlink.QdiscList(ifbLink) - Expect(err).NotTo(HaveOccurred()) - - Expect(qdiscs).To(HaveLen(1)) - Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) - Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{})) - Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(UnShapedClassMinorID))) - - classes, err := netlink.ClassList(ifbLink, qdiscs[0].Attrs().Handle) - - Expect(err).NotTo(HaveOccurred()) - Expect(classes).To(HaveLen(2)) - - // Uncapped class - Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, UnShapedClassMinorID))) - Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0))) - Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - // Class with traffic shapping settings - Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, ShapedClassMinorID))) - Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(2))) - // Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(7812500))) - Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(4))) - // Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - filters, err := netlink.FilterList(ifbLink, qdiscs[0].Attrs().Handle) - Expect(err).NotTo(HaveOccurred()) - Expect(filters).To(HaveLen(2)) - - // traffic to fd00:db8:abcd:1234:e000::/68 redirected to uncapped class - Expect(filters[0]).To(BeAssignableToTypeOf(&netlink.U32{})) - Expect(filters[0].(*netlink.U32).Actions).To(BeEmpty()) - Expect(filters[0].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IPV6))) - Expect(filters[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) - Expect(filters[0].Attrs().Priority).To(Equal(uint16(15))) - Expect(filters[0].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle)) - Expect(filters[0].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, ShapedClassMinorID))) - - filterSel := filters[0].(*netlink.U32).Sel - Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{})) - Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL))) - Expect(filterSel.Keys).To(HaveLen(3)) - Expect(filterSel.Nkeys).To(Equal(uint8(3))) - - // The filter should match to fd00:db8:abcd:1234:e000::/68 dst address in other words it should be: - // match 0xfd000db8/0xffffffff at 24 - // match 0xabcd1234/0xffffffff at 28 - // match 0xe0000000/0xf0000000 at 32 - // (and last match discarded because it would be equivalent to a matchall/true condition at 36) - Expect(filterSel.Keys[0].Off).To(Equal(int32(24))) - Expect(filterSel.Keys[0].Val).To(Equal(uint32(4244639160))) - Expect(filterSel.Keys[0].Mask).To(Equal(uint32(4294967295))) - - Expect(filterSel.Keys[1].Off).To(Equal(int32(28))) - Expect(filterSel.Keys[1].Val).To(Equal(uint32(2882343476))) - Expect(filterSel.Keys[1].Mask).To(Equal(uint32(4294967295))) - - Expect(filterSel.Keys[2].Off).To(Equal(int32(32))) - Expect(filterSel.Keys[2].Val).To(Equal(uint32(3758096384))) - Expect(filterSel.Keys[2].Mask).To(Equal(uint32(4026531840))) - - // traffic to 10.0.0.0/8 redirected to uncapped class - Expect(filters[1]).To(BeAssignableToTypeOf(&netlink.U32{})) - Expect(filters[1].(*netlink.U32).Actions).To(BeEmpty()) - Expect(filters[1].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IP))) - Expect(filters[1].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) - Expect(filters[1].Attrs().Priority).To(Equal(uint16(16))) - Expect(filters[1].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle)) - Expect(filters[1].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, ShapedClassMinorID))) - - filterSel = filters[1].(*netlink.U32).Sel - Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{})) - Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL))) - Expect(filterSel.Keys).To(HaveLen(1)) - Expect(filterSel.Nkeys).To(Equal(uint8(1))) - - // The filter should match to 10.0.0.0/8 dst address in other words it should be: - // match 0a000000/ff000000 at 16 - selKey := filterSel.Keys[0] - Expect(selKey.Val).To(Equal(uint32(10 * math.Pow(256, 3)))) - Expect(selKey.Off).To(Equal(int32(16))) - Expect(selKey.Mask).To(Equal(uint32(255 * math.Pow(256, 3)))) - - hostVethLink, err := netlink.LinkByName(hostIfname) - Expect(err).NotTo(HaveOccurred()) + }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) - qdiscFilters, err := netlink.FilterList(hostVethLink, netlink.MakeHandle(0xffff, 0)) - Expect(err).NotTo(HaveOccurred()) + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: containerNs.Path(), + IfName: containerIfname, + StdinData: []byte(conf), + } - Expect(qdiscFilters).To(HaveLen(1)) - Expect(qdiscFilters[0].(*netlink.U32).Actions[0].(*netlink.MirredAction).Ifindex).To(Equal(ifbLink.Attrs().Index)) + Expect(hostNs.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() - return nil - })).To(Succeed()) + _, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, ifbDeviceName, []byte(conf), func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred(), string(out)) - // Container ingress (host egress) - Expect(hostNs.Do(func(n ns.NetNS) error { - defer GinkgoRecover() + _, err = netlink.LinkByName(ifbDeviceName) + Expect(err).To(HaveOccurred()) + return nil + })).To(Succeed()) - vethLink, err := netlink.LinkByName(hostIfname) - Expect(err).NotTo(HaveOccurred()) + Expect(hostNs.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() - qdiscs, err := netlink.QdiscList(vethLink) - Expect(err).NotTo(HaveOccurred()) + containerIfLink, err := netlink.LinkByName(hostIfname) + Expect(err).NotTo(HaveOccurred()) - Expect(qdiscs).To(HaveLen(2)) - Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index)) - Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{})) - Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(UnShapedClassMinorID))) + qdiscs, err := netlink.QdiscList(containerIfLink) + Expect(err).NotTo(HaveOccurred()) - classes, err := netlink.ClassList(vethLink, qdiscs[0].Attrs().Handle) + Expect(qdiscs).To(HaveLen(1)) + Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(containerIfLink.Attrs().Index)) - Expect(err).NotTo(HaveOccurred()) - Expect(classes).To(HaveLen(2)) - - // Uncapped class - Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, UnShapedClassMinorID))) - Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0))) - Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - // Class with traffic shapping settings - Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, ShapedClassMinorID))) - Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(1))) - // Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(15625000))) - Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(2))) - // Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - filters, err := netlink.FilterList(vethLink, qdiscs[0].Attrs().Handle) - Expect(err).NotTo(HaveOccurred()) - Expect(filters).To(HaveLen(2)) - - // traffic to fd00:db8:abcd:1234:e000::/68 redirected to uncapped class - Expect(filters[0]).To(BeAssignableToTypeOf(&netlink.U32{})) - Expect(filters[0].(*netlink.U32).Actions).To(BeEmpty()) - Expect(filters[0].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IPV6))) - Expect(filters[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index)) - Expect(filters[0].Attrs().Priority).To(Equal(uint16(15))) - Expect(filters[0].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle)) - Expect(filters[0].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, ShapedClassMinorID))) - - filterSel := filters[0].(*netlink.U32).Sel - Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{})) - Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL))) - Expect(filterSel.Keys).To(HaveLen(3)) - Expect(filterSel.Nkeys).To(Equal(uint8(3))) - - // The filter should match to fd00:db8:abcd:1234:e000::/68 dst address in other words it should be: - // match 0xfd000db8/0xffffffff at 24 - // match 0xabcd1234/0xffffffff at 28 - // match 0xe0000000/0xf0000000 at 32 - // (and last match discarded because it would be equivalent to a matchall/true condition at 36) - Expect(filterSel.Keys[0].Off).To(Equal(int32(24))) - Expect(filterSel.Keys[0].Val).To(Equal(uint32(4244639160))) - Expect(filterSel.Keys[0].Mask).To(Equal(uint32(4294967295))) - - Expect(filterSel.Keys[1].Off).To(Equal(int32(28))) - Expect(filterSel.Keys[1].Val).To(Equal(uint32(2882343476))) - Expect(filterSel.Keys[1].Mask).To(Equal(uint32(4294967295))) - - Expect(filterSel.Keys[2].Off).To(Equal(int32(32))) - Expect(filterSel.Keys[2].Val).To(Equal(uint32(3758096384))) - Expect(filterSel.Keys[2].Mask).To(Equal(uint32(4026531840))) - - // traffic to 10.0.0.0/8 redirected to uncapped class - Expect(filters[1]).To(BeAssignableToTypeOf(&netlink.U32{})) - Expect(filters[1].(*netlink.U32).Actions).To(BeEmpty()) - Expect(filters[1].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IP))) - Expect(filters[1].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index)) - Expect(filters[1].Attrs().Priority).To(Equal(uint16(16))) - Expect(filters[1].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle)) - Expect(filters[1].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, ShapedClassMinorID))) - - filterSel = filters[1].(*netlink.U32).Sel - Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{})) - Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL))) - Expect(filterSel.Keys).To(HaveLen(1)) - Expect(filterSel.Nkeys).To(Equal(uint8(1))) - - // The filter should match to 10.0.0.0/8 dst address in other words it should be: - // match 0a000000/ff000000 at 16 - selKey := filterSel.Keys[0] - Expect(selKey.Val).To(Equal(uint32(10 * math.Pow(256, 3)))) - Expect(selKey.Off).To(Equal(int32(16))) - Expect(selKey.Mask).To(Equal(uint32(255 * math.Pow(256, 3)))) - - return nil - })).To(Succeed()) - }) + Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Tbf{})) + Expect(qdiscs[0].(*netlink.Tbf).Rate).To(Equal(uint64(1000))) + Expect(qdiscs[0].(*netlink.Tbf).Limit).To(Equal(uint32(35))) + return nil + })).To(Succeed()) + }) - It(fmt.Sprintf("[%s] does not apply ingress when disabled", ver), func() { - conf := fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "cni-plugin-bandwidth-test", - "type": "bandwidth", - "ingressRate": 0, - "ingressBurst": 0, - "egressRate": 8000, - "egressBurst": 80, - "unshapedSubnets": [ - "10.0.0.0/8", - "fd00:db8:abcd:1234:e000::/68" - ], - "prevResult": { - "interfaces": [ - { - "name": "%s", - "sandbox": "" - }, - { - "name": "%s", - "sandbox": "%s" - } - ], - "ips": [ - { - "version": "4", - "address": "%s/24", - "gateway": "10.0.0.1", - "interface": 1 + It(fmt.Sprintf("[%s] fails an invalid ingress config", ver), func() { + conf := fmt.Sprintf(`{ + "cniVersion": "%s", + "name": "cni-plugin-bandwidth-test", + "type": "bandwidth", + "ingressRate": 0, + "ingressBurst": 123, + "egressRate": 123, + "egressBurst": 123, + "prevResult": { + "interfaces": [ + { + "name": "%s", + "sandbox": "" + }, + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "%s/24", + "gateway": "10.0.0.1", + "interface": 1 + } + ], + "routes": [] } - ], - "routes": [] - } - }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) - - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: containerNs.Path(), - IfName: containerIfname, - StdinData: []byte(conf), - } - - // check container egress side / host ingress side, we expect to get some QoS setup there - Expect(hostNs.Do(func(_ ns.NetNS) error { - defer GinkgoRecover() - - _, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, ifbDeviceName, []byte(conf), func() error { return cmdAdd(args) }) - Expect(err).NotTo(HaveOccurred(), string(out)) - - ifbLink, err := netlink.LinkByName(ifbDeviceName) - Expect(err).NotTo(HaveOccurred()) - - qdiscs, err := netlink.QdiscList(ifbLink) - Expect(err).NotTo(HaveOccurred()) - - Expect(qdiscs).To(HaveLen(1)) - Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) - Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{})) - Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID))) - classes, err := netlink.ClassList(ifbLink, qdiscs[0].Attrs().Handle) - Expect(err).NotTo(HaveOccurred()) - Expect(classes).To(HaveLen(2)) - - // Uncapped class - Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1))) - Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0))) - Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - // Class with traffic shapping settings - Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, ShapedClassMinorID))) - Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(1000))) - // Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(7812500))) - Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(2000))) - // Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - filters, err := netlink.FilterList(ifbLink, qdiscs[0].Attrs().Handle) - Expect(err).NotTo(HaveOccurred()) - Expect(filters).To(HaveLen(2)) - - // traffic to fd00:db8:abcd:1234:e000::/68 redirected to uncapped class - Expect(filters[0]).To(BeAssignableToTypeOf(&netlink.U32{})) - Expect(filters[0].(*netlink.U32).Actions).To(BeEmpty()) - Expect(filters[0].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IPV6))) - Expect(filters[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) - Expect(filters[0].Attrs().Priority).To(Equal(uint16(15))) - Expect(filters[0].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle)) - Expect(filters[0].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1))) - - filterSel := filters[0].(*netlink.U32).Sel - Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{})) - Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL))) - Expect(filterSel.Keys).To(HaveLen(3)) - Expect(filterSel.Nkeys).To(Equal(uint8(3))) - - // The filter should match to fd00:db8:abcd:1234:e000::/68 dst address in other words it should be: - // match 0xfd000db8/0xffffffff at 24 - // match 0xabcd1234/0xffffffff at 28 - // match 0xe0000000/0xf0000000 at 32 - // (and last match discarded because it would be equivalent to a matchall/true condition at 36) - Expect(filterSel.Keys[0].Off).To(Equal(int32(24))) - Expect(filterSel.Keys[0].Val).To(Equal(uint32(4244639160))) - Expect(filterSel.Keys[0].Mask).To(Equal(uint32(4294967295))) - - Expect(filterSel.Keys[1].Off).To(Equal(int32(28))) - Expect(filterSel.Keys[1].Val).To(Equal(uint32(2882343476))) - Expect(filterSel.Keys[1].Mask).To(Equal(uint32(4294967295))) - - Expect(filterSel.Keys[2].Off).To(Equal(int32(32))) - Expect(filterSel.Keys[2].Val).To(Equal(uint32(3758096384))) - Expect(filterSel.Keys[2].Mask).To(Equal(uint32(4026531840))) - - // traffic to 10.0.0.0/8 redirected to uncapped class - Expect(filters[1]).To(BeAssignableToTypeOf(&netlink.U32{})) - Expect(filters[1].(*netlink.U32).Actions).To(BeEmpty()) - Expect(filters[1].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IP))) - Expect(filters[1].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) - Expect(filters[1].Attrs().Priority).To(Equal(uint16(16))) - Expect(filters[1].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle)) - Expect(filters[1].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1))) - - filterSel = filters[1].(*netlink.U32).Sel - Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{})) - Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL))) - Expect(filterSel.Keys).To(HaveLen(1)) - Expect(filterSel.Nkeys).To(Equal(uint8(1))) - - // The filter should match to 10.0.0.0/8 dst address in other words it should be: - // match 0a000000/ff000000 at 16 - selKey := filterSel.Keys[0] - Expect(selKey.Val).To(Equal(uint32(10 * math.Pow(256, 3)))) - Expect(selKey.Off).To(Equal(int32(16))) - Expect(selKey.Mask).To(Equal(uint32(255 * math.Pow(256, 3)))) - - // check traffic mirroring from veth to ifb - hostVethLink, err := netlink.LinkByName(hostIfname) - Expect(err).NotTo(HaveOccurred()) - - qdiscFilters, err := netlink.FilterList(hostVethLink, netlink.MakeHandle(0xffff, 0)) - Expect(err).NotTo(HaveOccurred()) - - Expect(qdiscFilters).To(HaveLen(1)) - Expect(qdiscFilters[0].(*netlink.U32).Actions[0].(*netlink.MirredAction).Ifindex).To(Equal(ifbLink.Attrs().Index)) - - return nil - })).To(Succeed()) - - // check container ingress side / host egress side, we should not have any htb qdisc/classes/filters defined for the host veth - // only the qdisc ingress + a noqueue qdisc - Expect(hostNs.Do(func(n ns.NetNS) error { - defer GinkgoRecover() - - containerIfLink, err := netlink.LinkByName(hostIfname) - Expect(err).NotTo(HaveOccurred()) - - qdiscs, err := netlink.QdiscList(containerIfLink) - Expect(err).NotTo(HaveOccurred()) - - Expect(qdiscs).To(HaveLen(2)) - Expect(qdiscs[0]).NotTo(BeAssignableToTypeOf(&netlink.Htb{})) - Expect(qdiscs[1]).NotTo(BeAssignableToTypeOf(&netlink.Htb{})) - - return nil - })).To(Succeed()) - }) + }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) - It(fmt.Sprintf("[%s] does not apply egress when disabled", ver), func() { - conf := fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "cni-plugin-bandwidth-test", - "type": "bandwidth", - "egressRate": 0, - "egressBurst": 0, - "ingressRate": 8000, - "ingressBurst": 80, - "prevResult": { - "interfaces": [ - { - "name": "%s", - "sandbox": "" - }, - { - "name": "%s", - "sandbox": "%s" - } - ], - "ips": [ - { - "version": "4", - "address": "%s/24", - "gateway": "10.0.0.1", - "interface": 1 - } - ], - "routes": [] + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: containerNs.Path(), + IfName: "eth0", + StdinData: []byte(conf), } - }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) - - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: containerNs.Path(), - IfName: containerIfname, - StdinData: []byte(conf), - } - - Expect(hostNs.Do(func(_ ns.NetNS) error { - defer GinkgoRecover() - - _, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, ifbDeviceName, []byte(conf), func() error { return cmdAdd(args) }) - Expect(err).NotTo(HaveOccurred(), string(out)) - - // Since we do not setup any egress QoS, no ifb interface should be created at all - _, err = netlink.LinkByName(ifbDeviceName) - Expect(err).To(HaveOccurred()) - - return nil - })).To(Succeed()) - Expect(hostNs.Do(func(n ns.NetNS) error { - defer GinkgoRecover() - - containerIfLink, err := netlink.LinkByName(hostIfname) - Expect(err).NotTo(HaveOccurred()) - - // Only one qdisc should be found this time, no ingress qdisc should be there - qdiscs, err := netlink.QdiscList(containerIfLink) - Expect(err).NotTo(HaveOccurred()) - - Expect(qdiscs).To(HaveLen(1)) - Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(containerIfLink.Attrs().Index)) - Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{})) - Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID))) - - classes, err := netlink.ClassList(containerIfLink, qdiscs[0].Attrs().Handle) - - Expect(err).NotTo(HaveOccurred()) - Expect(classes).To(HaveLen(2)) - - // Uncapped class - Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1))) - Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0))) - Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - // Class with traffic shapping settings - Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls)))) - Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(1000))) - // Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(15625000))) - Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(2000))) - // Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - // No subnets are exluded in this example so we should not get any filter - filters, err := netlink.FilterList(containerIfLink, qdiscs[0].Attrs().Handle) - Expect(err).NotTo(HaveOccurred()) - Expect(filters).To(BeEmpty()) + Expect(hostNs.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() - // Just check no mirroring is setup - qdiscFilters, err := netlink.FilterList(containerIfLink, netlink.MakeHandle(0xffff, 0)) - Expect(err).NotTo(HaveOccurred()) - Expect(qdiscFilters).To(BeEmpty()) - return nil - })).To(Succeed()) - }) + _, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) + Expect(err).To(MatchError("if burst is set, rate must also be set")) + return nil + })).To(Succeed()) + }) - It(fmt.Sprintf("[%s] works with a Veth pair using runtime config", ver), func() { - conf := fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "cni-plugin-bandwidth-test", - "type": "bandwidth", - "runtimeConfig": { - "bandWidth": { - "ingressRate": 8, - "ingressBurst": 8, - "egressRate": 16, - "egressBurst": 9, - "unshapedSubnets": ["192.168.0.0/24"] - } - }, - "prevResult": { - "interfaces": [ - { - "name": "%s", - "sandbox": "" + It(fmt.Sprintf("[%s] works with a Veth pair using runtime config", ver), func() { + conf := fmt.Sprintf(`{ + "cniVersion": "%s", + "name": "cni-plugin-bandwidth-test", + "type": "bandwidth", + "runtimeConfig": { + "bandWidth": { + "ingressRate": 8, + "ingressBurst": 8, + "egressRate": 16, + "egressBurst": 9 + } }, - { - "name": "%s", - "sandbox": "%s" - } - ], - "ips": [ - { - "version": "4", - "address": "%s/24", - "gateway": "10.0.0.1", - "interface": 1 + "prevResult": { + "interfaces": [ + { + "name": "%s", + "sandbox": "" + }, + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "%s/24", + "gateway": "10.0.0.1", + "interface": 1 + } + ], + "routes": [] } - ], - "routes": [] - } - }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) - - args := &skel.CmdArgs{ - ContainerID: "dummy", - Netns: containerNs.Path(), - IfName: containerIfname, - StdinData: []byte(conf), - } - - Expect(hostNs.Do(func(_ ns.NetNS) error { - defer GinkgoRecover() - r, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) - Expect(err).NotTo(HaveOccurred(), string(out)) - result, err := types100.GetResult(r) - Expect(err).NotTo(HaveOccurred()) + }`, ver, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) + + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: containerNs.Path(), + IfName: containerIfname, + StdinData: []byte(conf), + } - Expect(result.Interfaces).To(HaveLen(3)) - Expect(result.Interfaces[2].Name).To(Equal(ifbDeviceName)) - Expect(result.Interfaces[2].Sandbox).To(Equal("")) + Expect(hostNs.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() + r, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred(), string(out)) + result, err := types100.GetResult(r) + Expect(err).NotTo(HaveOccurred()) - ifbLink, err := netlink.LinkByName(ifbDeviceName) - Expect(err).NotTo(HaveOccurred()) - Expect(ifbLink.Attrs().MTU).To(Equal(hostIfaceMTU)) + Expect(result.Interfaces).To(HaveLen(3)) + Expect(result.Interfaces[2].Name).To(Equal(ifbDeviceName)) + Expect(result.Interfaces[2].Sandbox).To(Equal("")) - qdiscs, err := netlink.QdiscList(ifbLink) - Expect(err).NotTo(HaveOccurred()) + ifbLink, err := netlink.LinkByName(ifbDeviceName) + Expect(err).NotTo(HaveOccurred()) + Expect(ifbLink.Attrs().MTU).To(Equal(hostIfaceMTU)) - Expect(qdiscs).To(HaveLen(1)) - Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) - Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{})) - Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID))) + qdiscs, err := netlink.QdiscList(ifbLink) + Expect(err).NotTo(HaveOccurred()) - classes, err := netlink.ClassList(ifbLink, qdiscs[0].Attrs().Handle) + Expect(qdiscs).To(HaveLen(1)) + Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) - Expect(err).NotTo(HaveOccurred()) - Expect(classes).To(HaveLen(2)) - - // Uncapped class - Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1))) - Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0))) - Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - // Class with traffic shapping settings - Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls)))) - Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(2))) - // Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(7812500))) - Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(4))) - // Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - filters, err := netlink.FilterList(ifbLink, qdiscs[0].Attrs().Handle) - Expect(err).NotTo(HaveOccurred()) - Expect(filters).To(HaveLen(1)) - - // traffic to 192.168.0.0/24 redirected to uncapped class - Expect(filters[0]).To(BeAssignableToTypeOf(&netlink.U32{})) - Expect(filters[0].(*netlink.U32).Actions).To(BeEmpty()) - Expect(filters[0].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IP))) - Expect(filters[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) - Expect(filters[0].Attrs().Priority).To(Equal(uint16(16))) - Expect(filters[0].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle)) - Expect(filters[0].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1))) - - filterSel := filters[0].(*netlink.U32).Sel - Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{})) - Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL))) - Expect(filterSel.Keys).To(HaveLen(1)) - Expect(filterSel.Nkeys).To(Equal(uint8(1))) - - // The filter should match to 192.168.0.0/24 dst address in other words it should be: - // match c0a80000/ffffff00 at 16 - selKey := filterSel.Keys[0] - Expect(selKey.Val).To(Equal(uint32(192*math.Pow(256, 3) + 168*math.Pow(256, 2)))) - Expect(selKey.Off).To(Equal(int32(16))) - Expect(selKey.Mask).To(Equal(uint32(255*math.Pow(256, 3) + 255*math.Pow(256, 2) + 255*256))) - - hostVethLink, err := netlink.LinkByName(hostIfname) - Expect(err).NotTo(HaveOccurred()) + Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Tbf{})) + Expect(qdiscs[0].(*netlink.Tbf).Rate).To(Equal(uint64(2))) + Expect(qdiscs[0].(*netlink.Tbf).Limit).To(Equal(uint32(1))) - qdiscFilters, err := netlink.FilterList(hostVethLink, netlink.MakeHandle(0xffff, 0)) - Expect(err).NotTo(HaveOccurred()) + hostVethLink, err := netlink.LinkByName(hostIfname) + Expect(err).NotTo(HaveOccurred()) - Expect(qdiscFilters).To(HaveLen(1)) - Expect(qdiscFilters[0].(*netlink.U32).Actions[0].(*netlink.MirredAction).Ifindex).To(Equal(ifbLink.Attrs().Index)) + qdiscFilters, err := netlink.FilterList(hostVethLink, netlink.MakeHandle(0xffff, 0)) + Expect(err).NotTo(HaveOccurred()) - return nil - })).To(Succeed()) + Expect(qdiscFilters).To(HaveLen(1)) + Expect(qdiscFilters[0].(*netlink.U32).Actions[0].(*netlink.MirredAction).Ifindex).To(Equal(ifbLink.Attrs().Index)) - // Container ingress (host egress) - Expect(hostNs.Do(func(n ns.NetNS) error { - defer GinkgoRecover() + return nil + })).To(Succeed()) - vethLink, err := netlink.LinkByName(hostIfname) - Expect(err).NotTo(HaveOccurred()) + Expect(hostNs.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() - qdiscs, err := netlink.QdiscList(vethLink) - Expect(err).NotTo(HaveOccurred()) + ifbLink, err := netlink.LinkByName(hostIfname) + Expect(err).NotTo(HaveOccurred()) - Expect(qdiscs).To(HaveLen(2)) - Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index)) - Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{})) - Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID))) + qdiscs, err := netlink.QdiscList(ifbLink) + Expect(err).NotTo(HaveOccurred()) - classes, err := netlink.ClassList(vethLink, qdiscs[0].Attrs().Handle) + Expect(qdiscs).To(HaveLen(2)) + Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) - Expect(err).NotTo(HaveOccurred()) - Expect(classes).To(HaveLen(2)) - - // Uncapped class - Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1))) - Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0))) - Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - // Class with traffic shapping settings - Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls)))) - Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(1))) - // Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(15625000))) - Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(2))) - // Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - filters, err := netlink.FilterList(vethLink, qdiscs[0].Attrs().Handle) - Expect(err).NotTo(HaveOccurred()) - Expect(filters).To(HaveLen(1)) - - // traffic to 192.168.0.0/24 redirected to uncapped class - Expect(filters[0]).To(BeAssignableToTypeOf(&netlink.U32{})) - Expect(filters[0].(*netlink.U32).Actions).To(BeEmpty()) - Expect(filters[0].Attrs().Protocol).To(Equal(uint16(syscall.ETH_P_IP))) - Expect(filters[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index)) - Expect(filters[0].Attrs().Priority).To(Equal(uint16(16))) - Expect(filters[0].Attrs().Parent).To(Equal(qdiscs[0].Attrs().Handle)) - Expect(filters[0].(*netlink.U32).ClassId).To(Equal(netlink.MakeHandle(1, 1))) - - filterSel := filters[0].(*netlink.U32).Sel - Expect(filterSel).To(BeAssignableToTypeOf(&netlink.TcU32Sel{})) - Expect(filterSel.Flags).To(Equal(uint8(netlink.TC_U32_TERMINAL))) - Expect(filterSel.Keys).To(HaveLen(1)) - Expect(filterSel.Nkeys).To(Equal(uint8(1))) - - // The filter should match to 192.168.0.0/24 dst address in other words it should be: - // match c0a80000/ffffff00 at 16 - selKey := filterSel.Keys[0] - Expect(selKey.Val).To(Equal(uint32(192*math.Pow(256, 3) + 168*math.Pow(256, 2)))) - Expect(selKey.Off).To(Equal(int32(16))) - Expect(selKey.Mask).To(Equal(uint32(255*math.Pow(256, 3) + 255*math.Pow(256, 2) + 255*256))) - return nil - })).To(Succeed()) - }) + Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Tbf{})) + Expect(qdiscs[0].(*netlink.Tbf).Rate).To(Equal(uint64(1))) + Expect(qdiscs[0].(*netlink.Tbf).Limit).To(Equal(uint32(1))) + return nil + })).To(Succeed()) + }) - Describe("cmdDEL", func() { - It(fmt.Sprintf("[%s] works with a Veth pair using 0.3.0 config", ver), func() { + It(fmt.Sprintf("[%s] should apply static config when both static config and runtime config exist", ver), func() { conf := fmt.Sprintf(`{ "cniVersion": "%s", "name": "cni-plugin-bandwidth-test", "type": "bandwidth", - "ingressRate": 8, - "ingressBurst": 8, - "egressRate": 9, - "egressBurst": 9, + "ingressRate": 0, + "ingressBurst": 123, + "egressRate": 123, + "egressBurst": 123, + "runtimeConfig": { + "bandWidth": { + "ingressRate": 8, + "ingressBurst": 8, + "egressRate": 16, + "egressBurst": 9 + } + }, "prevResult": { "interfaces": [ { @@ -1233,38 +547,22 @@ var _ = Describe("bandwidth test", func() { args := &skel.CmdArgs{ ContainerID: "dummy", Netns: containerNs.Path(), - IfName: containerIfname, + IfName: "eth0", StdinData: []byte(conf), } Expect(hostNs.Do(func(_ ns.NetNS) error { defer GinkgoRecover() - _, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) - Expect(err).NotTo(HaveOccurred(), string(out)) - - _, err = netlink.LinkByName(hostIfname) - Expect(err).NotTo(HaveOccurred()) - - _, err = netlink.LinkByName(ifbDeviceName) - Expect(err).NotTo(HaveOccurred()) - - err = testutils.CmdDel(containerNs.Path(), args.ContainerID, "", func() error { return cmdDel(args) }) - Expect(err).NotTo(HaveOccurred(), string(out)) - - _, err = netlink.LinkByName(ifbDeviceName) - Expect(err).To(HaveOccurred()) - - // The host veth peer should remain as it has not be created by this plugin - _, err = netlink.LinkByName(hostIfname) - Expect(err).NotTo(HaveOccurred()) + _, _, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) + Expect(err).To(MatchError("if burst is set, rate must also be set")) return nil })).To(Succeed()) }) }) - Describe("cmdCHECK", func() { - It(fmt.Sprintf("[%s] works with a Veth pair", ver), func() { + Describe("cmdDEL", func() { + It(fmt.Sprintf("[%s] works with a Veth pair using 0.3.0 config", ver), func() { conf := fmt.Sprintf(`{ "cniVersion": "%s", "name": "cni-plugin-bandwidth-test", @@ -1308,29 +606,12 @@ var _ = Describe("bandwidth test", func() { _, out, err := testutils.CmdAdd(containerNs.Path(), args.ContainerID, "", []byte(conf), func() error { return cmdAdd(args) }) Expect(err).NotTo(HaveOccurred(), string(out)) - _, err = netlink.LinkByName(hostIfname) - Expect(err).NotTo(HaveOccurred()) - - _, err = netlink.LinkByName(ifbDeviceName) - Expect(err).NotTo(HaveOccurred()) - - if testutils.SpecVersionHasCHECK(ver) { - // Do CNI Check - - err = testutils.CmdCheck(containerNs.Path(), args.ContainerID, "", func() error { return cmdCheck(args) }) - Expect(err).NotTo(HaveOccurred()) - } - err = testutils.CmdDel(containerNs.Path(), args.ContainerID, "", func() error { return cmdDel(args) }) Expect(err).NotTo(HaveOccurred(), string(out)) _, err = netlink.LinkByName(ifbDeviceName) Expect(err).To(HaveOccurred()) - // The host veth peer should remain as it has not be created by this plugin - _, err = netlink.LinkByName(hostIfname) - Expect(err).NotTo(HaveOccurred()) - return nil })).To(Succeed()) }) @@ -1408,34 +689,10 @@ var _ = Describe("bandwidth test", func() { Expect(qdiscs).To(HaveLen(1)) Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) - Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{})) - Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID))) - classes, err := netlink.ClassList(ifbLink, qdiscs[0].Attrs().Handle) - - Expect(err).NotTo(HaveOccurred()) - Expect(classes).To(HaveLen(2)) - - // Uncapped class - Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1))) - Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0))) - Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - // Class with traffic shapping settings - Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls)))) - Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(2))) - // Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(7812500))) - Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(4))) - // Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - // Since we do not exclude anything from egress traffic shapping, we should not find any filter - filters, err := netlink.FilterList(ifbLink, qdiscs[0].Attrs().Handle) - Expect(err).NotTo(HaveOccurred()) - Expect(filters).To(BeEmpty()) + Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Tbf{})) + Expect(qdiscs[0].(*netlink.Tbf).Rate).To(Equal(uint64(2))) + Expect(qdiscs[0].(*netlink.Tbf).Limit).To(Equal(uint32(1))) hostVethLink, err := netlink.LinkByName(hostIfname) Expect(err).NotTo(HaveOccurred()) @@ -1452,43 +709,18 @@ var _ = Describe("bandwidth test", func() { Expect(hostNs.Do(func(_ ns.NetNS) error { defer GinkgoRecover() - vethLink, err := netlink.LinkByName(hostIfname) + ifbLink, err := netlink.LinkByName(hostIfname) Expect(err).NotTo(HaveOccurred()) - qdiscs, err := netlink.QdiscList(vethLink) + qdiscs, err := netlink.QdiscList(ifbLink) Expect(err).NotTo(HaveOccurred()) Expect(qdiscs).To(HaveLen(2)) - Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(vethLink.Attrs().Index)) - Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Htb{})) - Expect(qdiscs[0].(*netlink.Htb).Defcls).To(Equal(uint32(ShapedClassMinorID))) - - classes, err := netlink.ClassList(vethLink, qdiscs[0].Attrs().Handle) - - Expect(err).NotTo(HaveOccurred()) - Expect(classes).To(HaveLen(2)) - - // Uncapped class - Expect(classes[0]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[0].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, 1))) - Expect(classes[0].(*netlink.HtbClass).Rate).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Buffer).To(Equal(uint32(0))) - Expect(classes[0].(*netlink.HtbClass).Ceil).To(Equal(UncappedRate)) - Expect(classes[0].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - // Class with traffic shapping settings - Expect(classes[1]).To(BeAssignableToTypeOf(&netlink.HtbClass{})) - Expect(classes[1].(*netlink.HtbClass).Handle).To(Equal(netlink.MakeHandle(1, uint16(qdiscs[0].(*netlink.Htb).Defcls)))) - Expect(classes[1].(*netlink.HtbClass).Rate).To(Equal(uint64(1))) - // Expect(classes[1].(*netlink.HtbClass).Buffer).To(Equal(uint32(15625000))) - Expect(classes[1].(*netlink.HtbClass).Ceil).To(Equal(uint64(2))) - // Expect(classes[1].(*netlink.HtbClass).Cbuffer).To(Equal(uint32(0))) - - // Since we do not exclude anything from ingress traffic shapping, we should not find any filter - filters, err := netlink.FilterList(vethLink, qdiscs[0].Attrs().Handle) - Expect(err).NotTo(HaveOccurred()) - Expect(filters).To(BeEmpty()) + Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) + Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Tbf{})) + Expect(qdiscs[0].(*netlink.Tbf).Rate).To(Equal(uint64(1))) + Expect(qdiscs[0].(*netlink.Tbf).Limit).To(Equal(uint32(1))) return nil })).To(Succeed()) }) @@ -1631,5 +863,198 @@ var _ = Describe("bandwidth test", func() { })).To(Succeed()) }) }) + + Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP", ver), func() { + var ptpConf string + var rateInBits uint64 + var burstInBits uint64 + var packetInBytes int + var containerWithoutTbfNS ns.NetNS + var containerWithTbfNS ns.NetNS + var portServerWithTbf int + var portServerWithoutTbf int + + var containerWithTbfRes types.Result + var containerWithoutTbfRes types.Result + var echoServerWithTbf *gexec.Session + var echoServerWithoutTbf *gexec.Session + var dataDir string + + BeforeEach(func() { + rateInBytes := 1000 + rateInBits = uint64(rateInBytes * 8) + burstInBits = rateInBits * 2 + packetInBytes = rateInBytes * 25 + + var err error + dataDir, err = os.MkdirTemp("", "bandwidth_linux_test") + Expect(err).NotTo(HaveOccurred()) + + ptpConf = fmt.Sprintf(`{ + "cniVersion": "%s", + "name": "myBWnet", + "type": "ptp", + "ipMasq": true, + "mtu": 512, + "ipam": { + "type": "host-local", + "subnet": "10.1.2.0/24", + "dataDir": "%s" + } + }`, ver, dataDir) + + const ( + containerWithTbfIFName = "ptp0" + containerWithoutTbfIFName = "ptp1" + ) + + containerWithTbfNS, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + + containerWithoutTbfNS, err = testutils.NewNS() + Expect(err).NotTo(HaveOccurred()) + + By("create two containers, and use the bandwidth plugin on one of them") + Expect(hostNs.Do(func(ns.NetNS) error { + defer GinkgoRecover() + + containerWithTbfRes, _, err = testutils.CmdAdd(containerWithTbfNS.Path(), "dummy", containerWithTbfIFName, []byte(ptpConf), func() error { + r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) + Expect(err).NotTo(HaveOccurred()) + Expect(r.Print()).To(Succeed()) + + return err + }) + Expect(err).NotTo(HaveOccurred()) + + containerWithoutTbfRes, _, err = testutils.CmdAdd(containerWithoutTbfNS.Path(), "dummy2", containerWithoutTbfIFName, []byte(ptpConf), func() error { + r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) + Expect(err).NotTo(HaveOccurred()) + Expect(r.Print()).To(Succeed()) + + return err + }) + Expect(err).NotTo(HaveOccurred()) + + containerWithTbfResult, err := types100.GetResult(containerWithTbfRes) + Expect(err).NotTo(HaveOccurred()) + + tbfPluginConf := &PluginConf{} + err = json.Unmarshal([]byte(ptpConf), &tbfPluginConf) + Expect(err).NotTo(HaveOccurred()) + + tbfPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ + IngressBurst: burstInBits, + IngressRate: rateInBits, + EgressBurst: burstInBits, + EgressRate: rateInBits, + } + tbfPluginConf.Type = "bandwidth" + newConfBytes, err := buildOneConfig("myBWnet", ver, tbfPluginConf, containerWithTbfResult) + Expect(err).NotTo(HaveOccurred()) + + args := &skel.CmdArgs{ + ContainerID: "dummy3", + Netns: containerWithTbfNS.Path(), + IfName: containerWithTbfIFName, + StdinData: newConfBytes, + } + + result, out, err := testutils.CmdAdd(containerWithTbfNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred(), string(out)) + + if testutils.SpecVersionHasCHECK(ver) { + // Do CNI Check + checkConf := &PluginConf{} + err = json.Unmarshal([]byte(ptpConf), &checkConf) + Expect(err).NotTo(HaveOccurred()) + + checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ + IngressBurst: burstInBits, + IngressRate: rateInBits, + EgressBurst: burstInBits, + EgressRate: rateInBits, + } + checkConf.Type = "bandwidth" + + newCheckBytes, err := buildOneConfig("myBWnet", ver, checkConf, result) + Expect(err).NotTo(HaveOccurred()) + + args = &skel.CmdArgs{ + ContainerID: "dummy3", + Netns: containerWithTbfNS.Path(), + IfName: containerWithTbfIFName, + StdinData: newCheckBytes, + } + + err = testutils.CmdCheck(containerWithTbfNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) }) + Expect(err).NotTo(HaveOccurred()) + } + + return nil + })).To(Succeed()) + + By("starting a tcp server on both containers") + portServerWithTbf, echoServerWithTbf = startEchoServerInNamespace(containerWithTbfNS) + portServerWithoutTbf, echoServerWithoutTbf = startEchoServerInNamespace(containerWithoutTbfNS) + }) + + AfterEach(func() { + Expect(os.RemoveAll(dataDir)).To(Succeed()) + + Expect(containerWithTbfNS.Close()).To(Succeed()) + Expect(testutils.UnmountNS(containerWithTbfNS)).To(Succeed()) + Expect(containerWithoutTbfNS.Close()).To(Succeed()) + Expect(testutils.UnmountNS(containerWithoutTbfNS)).To(Succeed()) + + if echoServerWithoutTbf != nil { + echoServerWithoutTbf.Kill() + } + if echoServerWithTbf != nil { + echoServerWithTbf.Kill() + } + }) + + Measure("limits ingress traffic on veth device", func(b Benchmarker) { + var runtimeWithLimit time.Duration + var runtimeWithoutLimit time.Duration + + By("gather timing statistics about both containers") + By("sending tcp traffic to the container that has traffic shaped", func() { + runtimeWithLimit = b.Time("with tbf", func() { + result, err := types100.GetResult(containerWithTbfRes) + Expect(err).NotTo(HaveOccurred()) + + makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithTbf, packetInBytes) + }) + }) + + By("sending tcp traffic to the container that does not have traffic shaped", func() { + runtimeWithoutLimit = b.Time("without tbf", func() { + result, err := types100.GetResult(containerWithoutTbfRes) + Expect(err).NotTo(HaveOccurred()) + + makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutTbf, packetInBytes) + }) + }) + + Expect(runtimeWithLimit).To(BeNumerically(">", runtimeWithoutLimit+1000*time.Millisecond)) + }, 1) + }) } + + Describe("Validating input", func() { + It("Should allow only 4GB burst rate", func() { + err := validateRateAndBurst(5000, 4*1024*1024*1024*8-16) // 2 bytes less than the max should pass + Expect(err).NotTo(HaveOccurred()) + err = validateRateAndBurst(5000, 4*1024*1024*1024*8) // we're 1 bit above MaxUint32 + Expect(err).To(HaveOccurred()) + err = validateRateAndBurst(0, 1) + Expect(err).To(HaveOccurred()) + err = validateRateAndBurst(1, 0) + Expect(err).To(HaveOccurred()) + err = validateRateAndBurst(0, 0) + Expect(err).NotTo(HaveOccurred()) + }) + }) }) diff --git a/plugins/meta/bandwidth/bandwidth_measure_linux_test.go b/plugins/meta/bandwidth/bandwidth_measure_linux_test.go deleted file mode 100644 index 79c30d33c..000000000 --- a/plugins/meta/bandwidth/bandwidth_measure_linux_test.go +++ /dev/null @@ -1,824 +0,0 @@ -// Copyright 2023 CNI authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "context" - "encoding/json" - "fmt" - "log" - "net" - "os" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "github.com/onsi/gomega/gexec" - - "github.com/containernetworking/cni/pkg/invoke" - "github.com/containernetworking/cni/pkg/skel" - "github.com/containernetworking/cni/pkg/types" - types100 "github.com/containernetworking/cni/pkg/types/100" - "github.com/containernetworking/plugins/pkg/ns" - "github.com/containernetworking/plugins/pkg/testutils" -) - -var _ = Describe("bandwidth measure test", func() { - var ( - hostNs ns.NetNS - containerNs ns.NetNS - hostIfname string - containerIfname string - hostIP net.IP - containerIP net.IP - hostIfaceMTU int - ) - - BeforeEach(func() { - var err error - - hostIfname = "host-veth" - containerIfname = "container-veth" - - hostNs, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) - - containerNs, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) - - hostIP = net.IP{169, 254, 0, 1} - containerIP = net.IP{10, 254, 0, 1} - hostIfaceMTU = 1024 - - createVeth(hostNs, hostIfname, containerNs, containerIfname, hostIP, containerIP, hostIfaceMTU) - }) - - AfterEach(func() { - Expect(containerNs.Close()).To(Succeed()) - Expect(testutils.UnmountNS(containerNs)).To(Succeed()) - Expect(hostNs.Close()).To(Succeed()) - Expect(testutils.UnmountNS(hostNs)).To(Succeed()) - }) - - // Bandwidth requires host-side interface info, and thus only - // supports 0.3.0 and later CNI versions - for _, ver := range []string{"0.3.0", "0.3.1", "0.4.0", "1.0.0"} { - Describe(fmt.Sprintf("[%s] QoS effective", ver), func() { - Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP", ver), func() { - var ptpConf string - var rateInBits uint64 - var burstInBits uint64 - var packetInBytes int - var containerWithoutQoSNS ns.NetNS - var containerWithQoSNS ns.NetNS - var portServerWithQoS int - var portServerWithoutQoS int - - var containerWithQoSRes types.Result - var containerWithoutQoSRes types.Result - var echoServerWithQoS *gexec.Session - var echoServerWithoutQoS *gexec.Session - var dataDir string - - BeforeEach(func() { - rateInBytes := 1000 - rateInBits = uint64(rateInBytes * 8) - burstInBits = rateInBits * 2 - - // NOTE: Traffic shapping is not that precise at low rates, would be better to use higher rates + simple time+netcat for data transfer, rather than the provided - // client/server bin (limited to small amount of data) - packetInBytes = rateInBytes * 3 - - var err error - dataDir, err = os.MkdirTemp("", "bandwidth_linux_test") - Expect(err).NotTo(HaveOccurred()) - - ptpConf = fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "myBWnet", - "type": "ptp", - "ipMasq": true, - "mtu": 512, - "ipam": { - "type": "host-local", - "subnet": "10.1.2.0/24", - "dataDir": "%s" - } - }`, ver, dataDir) - - const ( - containerWithQoSIFName = "ptp0" - containerWithoutQoSIFName = "ptp1" - ) - - containerWithQoSNS, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) - - containerWithoutQoSNS, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) - - By("create two containers, and use the bandwidth plugin on one of them") - Expect(hostNs.Do(func(ns.NetNS) error { - defer GinkgoRecover() - - containerWithQoSRes, _, err = testutils.CmdAdd(containerWithQoSNS.Path(), "dummy", containerWithQoSIFName, []byte(ptpConf), func() error { - r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) - Expect(err).NotTo(HaveOccurred()) - Expect(r.Print()).To(Succeed()) - - return err - }) - Expect(err).NotTo(HaveOccurred()) - - containerWithoutQoSRes, _, err = testutils.CmdAdd(containerWithoutQoSNS.Path(), "dummy2", containerWithoutQoSIFName, []byte(ptpConf), func() error { - r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) - Expect(err).NotTo(HaveOccurred()) - Expect(r.Print()).To(Succeed()) - - return err - }) - Expect(err).NotTo(HaveOccurred()) - - containerWithQoSResult, err := types100.GetResult(containerWithQoSRes) - Expect(err).NotTo(HaveOccurred()) - - bandwidthPluginConf := &PluginConf{} - err = json.Unmarshal([]byte(ptpConf), &bandwidthPluginConf) - Expect(err).NotTo(HaveOccurred()) - - bandwidthPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ - IngressBurst: burstInBits, - IngressRate: rateInBits, - EgressBurst: burstInBits, - EgressRate: rateInBits, - } - bandwidthPluginConf.Type = "bandwidth" - newConfBytes, err := buildOneConfig(ver, bandwidthPluginConf, containerWithQoSResult) - Expect(err).NotTo(HaveOccurred()) - - args := &skel.CmdArgs{ - ContainerID: "dummy3", - Netns: containerWithQoSNS.Path(), - IfName: containerWithQoSIFName, - StdinData: newConfBytes, - } - - result, out, err := testutils.CmdAdd(containerWithQoSNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) }) - Expect(err).NotTo(HaveOccurred(), string(out)) - - if testutils.SpecVersionHasCHECK(ver) { - // Do CNI Check - checkConf := &PluginConf{} - err = json.Unmarshal([]byte(ptpConf), &checkConf) - Expect(err).NotTo(HaveOccurred()) - - checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ - IngressBurst: burstInBits, - IngressRate: rateInBits, - EgressBurst: burstInBits, - EgressRate: rateInBits, - } - checkConf.Type = "bandwidth" - - newCheckBytes, err := buildOneConfig(ver, checkConf, result) - Expect(err).NotTo(HaveOccurred()) - - args = &skel.CmdArgs{ - ContainerID: "dummy3", - Netns: containerWithQoSNS.Path(), - IfName: containerWithQoSIFName, - StdinData: newCheckBytes, - } - - err = testutils.CmdCheck(containerWithQoSNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) }) - Expect(err).NotTo(HaveOccurred()) - } - - return nil - })).To(Succeed()) - - By("starting a tcp server on both containers") - portServerWithQoS, echoServerWithQoS = startEchoServerInNamespace(containerWithQoSNS) - portServerWithoutQoS, echoServerWithoutQoS = startEchoServerInNamespace(containerWithoutQoSNS) - }) - - AfterEach(func() { - Expect(os.RemoveAll(dataDir)).To(Succeed()) - - Expect(containerWithQoSNS.Close()).To(Succeed()) - Expect(testutils.UnmountNS(containerWithQoSNS)).To(Succeed()) - Expect(containerWithoutQoSNS.Close()).To(Succeed()) - Expect(testutils.UnmountNS(containerWithoutQoSNS)).To(Succeed()) - - if echoServerWithoutQoS != nil { - echoServerWithoutQoS.Kill() - } - if echoServerWithQoS != nil { - echoServerWithQoS.Kill() - } - }) - - It("limits ingress traffic on veth device", func() { - var runtimeWithLimit time.Duration - var runtimeWithoutLimit time.Duration - - By("gather timing statistics about both containers") - - By("sending tcp traffic to the container that has traffic shaped", func() { - start := time.Now() - result, err := types100.GetResult(containerWithQoSRes) - Expect(err).NotTo(HaveOccurred()) - makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithQoS, packetInBytes) - end := time.Now() - runtimeWithLimit = end.Sub(start) - log.Printf("Elapsed with qos %.2f", runtimeWithLimit.Seconds()) - }) - - By("sending tcp traffic to the container that does not have traffic shaped", func() { - start := time.Now() - result, err := types100.GetResult(containerWithoutQoSRes) - Expect(err).NotTo(HaveOccurred()) - makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutQoS, packetInBytes) - end := time.Now() - runtimeWithoutLimit = end.Sub(start) - log.Printf("Elapsed without qos %.2f", runtimeWithoutLimit.Seconds()) - }) - - Expect(runtimeWithLimit).To(BeNumerically(">", runtimeWithoutLimit+1000*time.Millisecond)) - }) - }) - }) - - Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP and excluding specific subnets from traffic", ver), func() { - var ptpConf string - var rateInBits uint64 - var burstInBits uint64 - var packetInBytes int - var containerWithoutQoSNS ns.NetNS - var containerWithQoSNS ns.NetNS - var portServerWithQoS int - var portServerWithoutQoS int - - var containerWithQoSRes types.Result - var containerWithoutQoSRes types.Result - var echoServerWithQoS *gexec.Session - var echoServerWithoutQoS *gexec.Session - var dataDir string - - BeforeEach(func() { - rateInBytes := 1000 - rateInBits = uint64(rateInBytes * 8) - burstInBits = rateInBits * 2 - unshapedSubnets := []string{"10.1.2.0/24"} - // NOTE: Traffic shapping is not that precise at low rates, would be better to use higher rates + simple time+netcat for data transfer, rather than the provided - // client/server bin (limited to small amount of data) - packetInBytes = rateInBytes * 3 - - var err error - dataDir, err = os.MkdirTemp("", "bandwidth_linux_test") - Expect(err).NotTo(HaveOccurred()) - - ptpConf = fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "myBWnet", - "type": "ptp", - "ipMasq": true, - "mtu": 512, - "ipam": { - "type": "host-local", - "subnet": "10.1.2.0/24", - "dataDir": "%s" - } - }`, ver, dataDir) - - const ( - containerWithQoSIFName = "ptp0" - containerWithoutQoSIFName = "ptp1" - ) - - containerWithQoSNS, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) - - containerWithoutQoSNS, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) - - By("create two containers, and use the bandwidth plugin on one of them") - Expect(hostNs.Do(func(ns.NetNS) error { - defer GinkgoRecover() - - containerWithQoSRes, _, err = testutils.CmdAdd(containerWithQoSNS.Path(), "dummy", containerWithQoSIFName, []byte(ptpConf), func() error { - r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) - Expect(err).NotTo(HaveOccurred()) - Expect(r.Print()).To(Succeed()) - - return err - }) - Expect(err).NotTo(HaveOccurred()) - - containerWithoutQoSRes, _, err = testutils.CmdAdd(containerWithoutQoSNS.Path(), "dummy2", containerWithoutQoSIFName, []byte(ptpConf), func() error { - r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) - Expect(err).NotTo(HaveOccurred()) - Expect(r.Print()).To(Succeed()) - - return err - }) - Expect(err).NotTo(HaveOccurred()) - - containerWithQoSResult, err := types100.GetResult(containerWithQoSRes) - Expect(err).NotTo(HaveOccurred()) - - bandwidthPluginConf := &PluginConf{} - err = json.Unmarshal([]byte(ptpConf), &bandwidthPluginConf) - Expect(err).NotTo(HaveOccurred()) - - bandwidthPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ - IngressBurst: burstInBits, - IngressRate: rateInBits, - EgressBurst: burstInBits, - EgressRate: rateInBits, - UnshapedSubnets: unshapedSubnets, - } - bandwidthPluginConf.Type = "bandwidth" - newConfBytes, err := buildOneConfig(ver, bandwidthPluginConf, containerWithQoSResult) - Expect(err).NotTo(HaveOccurred()) - - args := &skel.CmdArgs{ - ContainerID: "dummy3", - Netns: containerWithQoSNS.Path(), - IfName: containerWithQoSIFName, - StdinData: newConfBytes, - } - - result, out, err := testutils.CmdAdd(containerWithQoSNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) }) - Expect(err).NotTo(HaveOccurred(), string(out)) - - if testutils.SpecVersionHasCHECK(ver) { - // Do CNI Check - checkConf := &PluginConf{} - err = json.Unmarshal([]byte(ptpConf), &checkConf) - Expect(err).NotTo(HaveOccurred()) - - checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ - IngressBurst: burstInBits, - IngressRate: rateInBits, - EgressBurst: burstInBits, - EgressRate: rateInBits, - UnshapedSubnets: unshapedSubnets, - } - checkConf.Type = "bandwidth" - - newCheckBytes, err := buildOneConfig(ver, checkConf, result) - Expect(err).NotTo(HaveOccurred()) - - args = &skel.CmdArgs{ - ContainerID: "dummy3", - Netns: containerWithQoSNS.Path(), - IfName: containerWithQoSIFName, - StdinData: newCheckBytes, - } - - err = testutils.CmdCheck(containerWithQoSNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) }) - Expect(err).NotTo(HaveOccurred()) - } - - return nil - })).To(Succeed()) - - By("starting a tcp server on both containers") - portServerWithQoS, echoServerWithQoS = startEchoServerInNamespace(containerWithQoSNS) - portServerWithoutQoS, echoServerWithoutQoS = startEchoServerInNamespace(containerWithoutQoSNS) - }) - - AfterEach(func() { - Expect(os.RemoveAll(dataDir)).To(Succeed()) - - Expect(containerWithQoSNS.Close()).To(Succeed()) - Expect(testutils.UnmountNS(containerWithQoSNS)).To(Succeed()) - Expect(containerWithoutQoSNS.Close()).To(Succeed()) - Expect(testutils.UnmountNS(containerWithoutQoSNS)).To(Succeed()) - - if echoServerWithoutQoS != nil { - echoServerWithoutQoS.Kill() - } - if echoServerWithQoS != nil { - echoServerWithQoS.Kill() - } - }) - - It("does not limits ingress traffic on veth device coming from 10.1.2.0/24", func() { - var runtimeWithLimit time.Duration - var runtimeWithoutLimit time.Duration - - By("gather timing statistics about both containers") - - By("sending tcp traffic to the container that has traffic shaped", func() { - start := time.Now() - result, err := types100.GetResult(containerWithQoSRes) - Expect(err).NotTo(HaveOccurred()) - makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithQoS, packetInBytes) - end := time.Now() - runtimeWithLimit = end.Sub(start) - log.Printf("Elapsed with qos %.2f", runtimeWithLimit.Seconds()) - }) - - By("sending tcp traffic to the container that does not have traffic shaped", func() { - start := time.Now() - result, err := types100.GetResult(containerWithoutQoSRes) - Expect(err).NotTo(HaveOccurred()) - makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutQoS, packetInBytes) - end := time.Now() - runtimeWithoutLimit = end.Sub(start) - log.Printf("Elapsed without qos %.2f", runtimeWithoutLimit.Seconds()) - }) - - Expect(runtimeWithLimit - runtimeWithoutLimit).To(BeNumerically("<", 100*time.Millisecond)) - }) - }) - - Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP and only including specific subnets in traffic shapping (not including the main ns one)", ver), func() { - var ptpConf string - var rateInBits uint64 - var burstInBits uint64 - var packetInBytes int - var containerWithoutQoSNS ns.NetNS - var containerWithQoSNS ns.NetNS - var portServerWithQoS int - var portServerWithoutQoS int - - var containerWithQoSRes types.Result - var containerWithoutQoSRes types.Result - var echoServerWithQoS *gexec.Session - var echoServerWithoutQoS *gexec.Session - var dataDir string - - BeforeEach(func() { - rateInBytes := 1000 - rateInBits = uint64(rateInBytes * 8) - burstInBits = rateInBits * 2 - shapedSubnets := []string{"10.2.2.0/24"} - // NOTE: Traffic shapping is not that precise at low rates, would be better to use higher rates + simple time+netcat for data transfer, rather than the provided - // client/server bin (limited to small amount of data) - packetInBytes = rateInBytes * 3 - - var err error - dataDir, err = os.MkdirTemp("", "bandwidth_linux_test") - Expect(err).NotTo(HaveOccurred()) - - ptpConf = fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "myBWnet", - "type": "ptp", - "ipMasq": true, - "mtu": 512, - "ipam": { - "type": "host-local", - "subnet": "10.1.2.0/24", - "dataDir": "%s" - } - }`, ver, dataDir) - - const ( - containerWithQoSIFName = "ptp0" - containerWithoutQoSIFName = "ptp1" - ) - - containerWithQoSNS, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) - - containerWithoutQoSNS, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) - - By("create two containers, and use the bandwidth plugin on one of them") - - Expect(hostNs.Do(func(ns.NetNS) error { - defer GinkgoRecover() - - containerWithQoSRes, _, err = testutils.CmdAdd(containerWithQoSNS.Path(), "dummy", containerWithQoSIFName, []byte(ptpConf), func() error { - r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) - Expect(err).NotTo(HaveOccurred()) - Expect(r.Print()).To(Succeed()) - - return err - }) - Expect(err).NotTo(HaveOccurred()) - - containerWithoutQoSRes, _, err = testutils.CmdAdd(containerWithoutQoSNS.Path(), "dummy2", containerWithoutQoSIFName, []byte(ptpConf), func() error { - r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) - Expect(err).NotTo(HaveOccurred()) - Expect(r.Print()).To(Succeed()) - - return err - }) - Expect(err).NotTo(HaveOccurred()) - - containerWithQoSResult, err := types100.GetResult(containerWithQoSRes) - Expect(err).NotTo(HaveOccurred()) - - bandwidthPluginConf := &PluginConf{} - err = json.Unmarshal([]byte(ptpConf), &bandwidthPluginConf) - Expect(err).NotTo(HaveOccurred()) - - bandwidthPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ - IngressBurst: burstInBits, - IngressRate: rateInBits, - EgressBurst: burstInBits, - EgressRate: rateInBits, - ShapedSubnets: shapedSubnets, - } - bandwidthPluginConf.Type = "bandwidth" - newConfBytes, err := buildOneConfig(ver, bandwidthPluginConf, containerWithQoSResult) - Expect(err).NotTo(HaveOccurred()) - - args := &skel.CmdArgs{ - ContainerID: "dummy3", - Netns: containerWithQoSNS.Path(), - IfName: containerWithQoSIFName, - StdinData: newConfBytes, - } - - result, out, err := testutils.CmdAdd(containerWithQoSNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) }) - Expect(err).NotTo(HaveOccurred(), string(out)) - - if testutils.SpecVersionHasCHECK(ver) { - // Do CNI Check - checkConf := &PluginConf{} - err = json.Unmarshal([]byte(ptpConf), &checkConf) - Expect(err).NotTo(HaveOccurred()) - - checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ - IngressBurst: burstInBits, - IngressRate: rateInBits, - EgressBurst: burstInBits, - EgressRate: rateInBits, - ShapedSubnets: shapedSubnets, - } - checkConf.Type = "bandwidth" - - newCheckBytes, err := buildOneConfig(ver, checkConf, result) - Expect(err).NotTo(HaveOccurred()) - - args = &skel.CmdArgs{ - ContainerID: "dummy3", - Netns: containerWithQoSNS.Path(), - IfName: containerWithQoSIFName, - StdinData: newCheckBytes, - } - - err = testutils.CmdCheck(containerWithQoSNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) }) - Expect(err).NotTo(HaveOccurred()) - } - - return nil - })).To(Succeed()) - - By("starting a tcp server on both containers") - portServerWithQoS, echoServerWithQoS = startEchoServerInNamespace(containerWithQoSNS) - portServerWithoutQoS, echoServerWithoutQoS = startEchoServerInNamespace(containerWithoutQoSNS) - }) - - AfterEach(func() { - Expect(os.RemoveAll(dataDir)).To(Succeed()) - - Expect(containerWithQoSNS.Close()).To(Succeed()) - Expect(testutils.UnmountNS(containerWithQoSNS)).To(Succeed()) - Expect(containerWithoutQoSNS.Close()).To(Succeed()) - Expect(testutils.UnmountNS(containerWithoutQoSNS)).To(Succeed()) - - if echoServerWithoutQoS != nil { - echoServerWithoutQoS.Kill() - } - if echoServerWithQoS != nil { - echoServerWithQoS.Kill() - } - }) - - It("does not limit ingress traffic on veth device coming from non included subnets", func() { - var runtimeWithLimit time.Duration - var runtimeWithoutLimit time.Duration - - By("gather timing statistics about both containers") - - By("sending tcp traffic to the container that has traffic shaped", func() { - start := time.Now() - result, err := types100.GetResult(containerWithQoSRes) - Expect(err).NotTo(HaveOccurred()) - makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithQoS, packetInBytes) - end := time.Now() - runtimeWithLimit = end.Sub(start) - log.Printf("Elapsed with qos %.2f", runtimeWithLimit.Seconds()) - }) - - By("sending tcp traffic to the container that does not have traffic shaped", func() { - start := time.Now() - result, err := types100.GetResult(containerWithoutQoSRes) - Expect(err).NotTo(HaveOccurred()) - makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutQoS, packetInBytes) - end := time.Now() - runtimeWithoutLimit = end.Sub(start) - log.Printf("Elapsed without qos %.2f", runtimeWithoutLimit.Seconds()) - }) - - Expect(runtimeWithLimit - runtimeWithoutLimit).To(BeNumerically("<", 100*time.Millisecond)) - }) - }) - - Context(fmt.Sprintf("[%s] when chaining bandwidth plugin with PTP and only including specific subnets in traffic shapping (including the main ns one)", ver), func() { - var ptpConf string - var rateInBits uint64 - var burstInBits uint64 - var packetInBytes int - var containerWithoutQoSNS ns.NetNS - var containerWithQoSNS ns.NetNS - var portServerWithQoS int - var portServerWithoutQoS int - - var containerWithQoSRes types.Result - var containerWithoutQoSRes types.Result - var echoServerWithQoS *gexec.Session - var echoServerWithoutQoS *gexec.Session - var dataDir string - - BeforeEach(func() { - rateInBytes := 1000 - rateInBits = uint64(rateInBytes * 8) - burstInBits = rateInBits * 2 - shapedSubnets := []string{"10.1.2.1/32"} - // NOTE: Traffic shapping is not that precise at low rates, would be better to use higher rates + simple time+netcat for data transfer, rather than the provided - // client/server bin (limited to small amount of data) - packetInBytes = rateInBytes * 3 - - var err error - dataDir, err = os.MkdirTemp("", "bandwidth_linux_test") - Expect(err).NotTo(HaveOccurred()) - - ptpConf = fmt.Sprintf(`{ - "cniVersion": "%s", - "name": "myBWnet", - "type": "ptp", - "ipMasq": true, - "mtu": 512, - "ipam": { - "type": "host-local", - "subnet": "10.1.2.0/24", - "dataDir": "%s" - } - }`, ver, dataDir) - - const ( - containerWithQoSIFName = "ptp0" - containerWithoutQoSIFName = "ptp1" - ) - - containerWithQoSNS, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) - - containerWithoutQoSNS, err = testutils.NewNS() - Expect(err).NotTo(HaveOccurred()) - - By("create two containers, and use the bandwidth plugin on one of them") - - Expect(hostNs.Do(func(ns.NetNS) error { - defer GinkgoRecover() - - containerWithQoSRes, _, err = testutils.CmdAdd(containerWithQoSNS.Path(), "dummy", containerWithQoSIFName, []byte(ptpConf), func() error { - r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) - Expect(err).NotTo(HaveOccurred()) - Expect(r.Print()).To(Succeed()) - - return err - }) - Expect(err).NotTo(HaveOccurred()) - - containerWithoutQoSRes, _, err = testutils.CmdAdd(containerWithoutQoSNS.Path(), "dummy2", containerWithoutQoSIFName, []byte(ptpConf), func() error { - r, err := invoke.DelegateAdd(context.TODO(), "ptp", []byte(ptpConf), nil) - Expect(err).NotTo(HaveOccurred()) - Expect(r.Print()).To(Succeed()) - - return err - }) - Expect(err).NotTo(HaveOccurred()) - - containerWithQoSResult, err := types100.GetResult(containerWithQoSRes) - Expect(err).NotTo(HaveOccurred()) - - bandwidthPluginConf := &PluginConf{} - err = json.Unmarshal([]byte(ptpConf), &bandwidthPluginConf) - Expect(err).NotTo(HaveOccurred()) - - bandwidthPluginConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ - IngressBurst: burstInBits, - IngressRate: rateInBits, - EgressBurst: burstInBits, - EgressRate: rateInBits, - ShapedSubnets: shapedSubnets, - } - bandwidthPluginConf.Type = "bandwidth" - newConfBytes, err := buildOneConfig(ver, bandwidthPluginConf, containerWithQoSResult) - Expect(err).NotTo(HaveOccurred()) - - args := &skel.CmdArgs{ - ContainerID: "dummy3", - Netns: containerWithQoSNS.Path(), - IfName: containerWithQoSIFName, - StdinData: newConfBytes, - } - - result, out, err := testutils.CmdAdd(containerWithQoSNS.Path(), args.ContainerID, "", newConfBytes, func() error { return cmdAdd(args) }) - Expect(err).NotTo(HaveOccurred(), string(out)) - - if testutils.SpecVersionHasCHECK(ver) { - // Do CNI Check - checkConf := &PluginConf{} - err = json.Unmarshal([]byte(ptpConf), &checkConf) - Expect(err).NotTo(HaveOccurred()) - - checkConf.RuntimeConfig.Bandwidth = &BandwidthEntry{ - IngressBurst: burstInBits, - IngressRate: rateInBits, - EgressBurst: burstInBits, - EgressRate: rateInBits, - ShapedSubnets: shapedSubnets, - } - checkConf.Type = "bandwidth" - - newCheckBytes, err := buildOneConfig(ver, checkConf, result) - Expect(err).NotTo(HaveOccurred()) - - args = &skel.CmdArgs{ - ContainerID: "dummy3", - Netns: containerWithQoSNS.Path(), - IfName: containerWithQoSIFName, - StdinData: newCheckBytes, - } - - err = testutils.CmdCheck(containerWithQoSNS.Path(), args.ContainerID, "", func() error { return cmdCheck(args) }) - Expect(err).NotTo(HaveOccurred()) - } - - return nil - })).To(Succeed()) - - By("starting a tcp server on both containers") - portServerWithQoS, echoServerWithQoS = startEchoServerInNamespace(containerWithQoSNS) - portServerWithoutQoS, echoServerWithoutQoS = startEchoServerInNamespace(containerWithoutQoSNS) - }) - - AfterEach(func() { - Expect(os.RemoveAll(dataDir)).To(Succeed()) - - Expect(containerWithQoSNS.Close()).To(Succeed()) - Expect(testutils.UnmountNS(containerWithQoSNS)).To(Succeed()) - Expect(containerWithoutQoSNS.Close()).To(Succeed()) - Expect(testutils.UnmountNS(containerWithoutQoSNS)).To(Succeed()) - - if echoServerWithoutQoS != nil { - echoServerWithoutQoS.Kill() - } - if echoServerWithQoS != nil { - echoServerWithQoS.Kill() - } - }) - - It("limits ingress traffic on veth device coming from included subnets", func() { - var runtimeWithLimit time.Duration - var runtimeWithoutLimit time.Duration - - By("gather timing statistics about both containers") - - By("sending tcp traffic to the container that has traffic shaped", func() { - start := time.Now() - result, err := types100.GetResult(containerWithQoSRes) - Expect(err).NotTo(HaveOccurred()) - makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithQoS, packetInBytes) - end := time.Now() - runtimeWithLimit = end.Sub(start) - log.Printf("Elapsed with qos %.2f", runtimeWithLimit.Seconds()) - }) - - By("sending tcp traffic to the container that does not have traffic shaped", func() { - start := time.Now() - result, err := types100.GetResult(containerWithoutQoSRes) - Expect(err).NotTo(HaveOccurred()) - makeTCPClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutQoS, packetInBytes) - end := time.Now() - runtimeWithoutLimit = end.Sub(start) - log.Printf("Elapsed without qos %.2f", runtimeWithoutLimit.Seconds()) - }) - - Expect(runtimeWithLimit).To(BeNumerically(">", runtimeWithoutLimit+1000*time.Millisecond)) - }) - }) - } -}) diff --git a/plugins/meta/bandwidth/bandwidth_suite_test.go b/plugins/meta/bandwidth/bandwidth_suite_test.go index ce7a94ce4..a6f5edd90 100644 --- a/plugins/meta/bandwidth/bandwidth_suite_test.go +++ b/plugins/meta/bandwidth/bandwidth_suite_test.go @@ -15,7 +15,6 @@ package main import ( "bytes" - "encoding/json" "fmt" "io" "net" @@ -31,11 +30,10 @@ import ( "github.com/onsi/gomega/gexec" "github.com/vishvananda/netlink" - "github.com/containernetworking/cni/pkg/types" "github.com/containernetworking/plugins/pkg/ns" ) -func TestHTB(t *testing.T) { +func TestTBF(t *testing.T) { RegisterFailHandler(Fail) RunSpecs(t, "plugins/meta/bandwidth") } @@ -245,47 +243,3 @@ func createMacvlan(netNS ns.NetNS, master, macvlanName string) { }) Expect(err).NotTo(HaveOccurred()) } - -func buildOneConfig(cniVersion string, orig *PluginConf, prevResult types.Result) ([]byte, error) { - var err error - - inject := map[string]interface{}{ - "name": "myBWnet", - "cniVersion": cniVersion, - } - // Add previous plugin result - if prevResult != nil { - r, err := prevResult.GetAsVersion(cniVersion) - Expect(err).NotTo(HaveOccurred()) - inject["prevResult"] = r - } - - // Ensure every config uses the same name and version - config := make(map[string]interface{}) - - confBytes, err := json.Marshal(orig) - if err != nil { - return nil, err - } - - err = json.Unmarshal(confBytes, &config) - if err != nil { - return nil, fmt.Errorf("unmarshal existing network bytes: %s", err) - } - - for key, value := range inject { - config[key] = value - } - - newBytes, err := json.Marshal(config) - if err != nil { - return nil, err - } - - conf := &PluginConf{} - if err := json.Unmarshal(newBytes, &conf); err != nil { - return nil, fmt.Errorf("error parsing configuration: %s", err) - } - - return newBytes, nil -} diff --git a/plugins/meta/bandwidth/ifb_creator.go b/plugins/meta/bandwidth/ifb_creator.go index 88b8549e9..58d76add9 100644 --- a/plugins/meta/bandwidth/ifb_creator.go +++ b/plugins/meta/bandwidth/ifb_creator.go @@ -15,8 +15,6 @@ package main import ( - "bytes" - "encoding/binary" "fmt" "net" "syscall" @@ -26,24 +24,14 @@ import ( "github.com/containernetworking/plugins/pkg/ip" ) -const ( - latencyInMillis = 25 - UncappedRate uint64 = 100_000_000_000 - ShapedClassMinorID uint16 = 48 - UnShapedClassMinorID uint16 = 1 -) - -func CreateIfb(ifbDeviceName string, mtu int, qlen int) error { - if qlen < 1000 { - qlen = 1000 - } +const latencyInMillis = 25 +func CreateIfb(ifbDeviceName string, mtu int) error { err := netlink.LinkAdd(&netlink.Ifb{ LinkAttrs: netlink.LinkAttrs{ - Name: ifbDeviceName, - Flags: net.FlagUp, - MTU: mtu, - TxQLen: qlen, + Name: ifbDeviceName, + Flags: net.FlagUp, + MTU: mtu, }, }) if err != nil { @@ -61,24 +49,15 @@ func TeardownIfb(deviceName string) error { return err } -func CreateIngressQdisc(rateInBits, burstInBits uint64, excludeSubnets []string, includeSubnets []string, hostDeviceName string) error { +func CreateIngressQdisc(rateInBits, burstInBits uint64, hostDeviceName string) error { hostDevice, err := netlink.LinkByName(hostDeviceName) if err != nil { return fmt.Errorf("get host device: %s", err) } - - subnets := includeSubnets - exclude := false - - if len(excludeSubnets) > 0 { - subnets = excludeSubnets - exclude = true - } - - return createHTB(rateInBits, burstInBits, hostDevice.Attrs().Index, subnets, exclude) + return createTBF(rateInBits, burstInBits, hostDevice.Attrs().Index) } -func CreateEgressQdisc(rateInBits, burstInBits uint64, excludeSubnets []string, includeSubnets []string, hostDeviceName string, ifbDeviceName string) error { +func CreateEgressQdisc(rateInBits, burstInBits uint64, hostDeviceName string, ifbDeviceName string) error { ifbDevice, err := netlink.LinkByName(ifbDeviceName) if err != nil { return fmt.Errorf("get ifb device: %s", err) @@ -125,216 +104,44 @@ func CreateEgressQdisc(rateInBits, burstInBits uint64, excludeSubnets []string, return fmt.Errorf("add filter: %s", err) } - subnets := excludeSubnets - exclude := true - - if len(includeSubnets) > 0 { - subnets = includeSubnets - exclude = false - } - // throttle traffic on ifb device - err = createHTB(rateInBits, burstInBits, ifbDevice.Attrs().Index, subnets, exclude) + err = createTBF(rateInBits, burstInBits, ifbDevice.Attrs().Index) if err != nil { - // egress from the container/netns pov = ingress from the main netns/host pov - return fmt.Errorf("create htb container egress qos rules: %s", err) + return fmt.Errorf("create ifb qdisc: %s", err) } return nil } -func createHTB(rateInBits, burstInBits uint64, linkIndex int, subnets []string, excludeSubnets bool) error { - // Netlink struct fields are not clear, let's use shell - - defaultClassID := UnShapedClassMinorID - // If no subnets are specified, then shaping should apply to everything - if len(subnets) == 0 || excludeSubnets { - defaultClassID = ShapedClassMinorID +func createTBF(rateInBits, burstInBits uint64, linkIndex int) error { + // Equivalent to + // tc qdisc add dev link root tbf + // rate netConf.BandwidthLimits.Rate + // burst netConf.BandwidthLimits.Burst + if rateInBits <= 0 { + return fmt.Errorf("invalid rate: %d", rateInBits) } - - // Step 1 qdisc - // cmd := exec.Command("/usr/sbin/tc", "qdisc", "add", "dev", interfaceName, "root", "handle", "1:", "htb", "default", "30") - qdisc := &netlink.Htb{ - QdiscAttrs: netlink.QdiscAttrs{ - LinkIndex: linkIndex, - Handle: netlink.MakeHandle(1, 0), - Parent: netlink.HANDLE_ROOT, - }, - Defcls: uint32(defaultClassID), - // No idea what these are so let's keep the default values from source code... - Version: 3, - Rate2Quantum: 10, + if burstInBits <= 0 { + return fmt.Errorf("invalid burst: %d", burstInBits) } - err := netlink.QdiscAdd(qdisc) - if err != nil { - return fmt.Errorf("error while creating qdisc: %s", err) - } - - // Step 2 classes - rateInBytes := rateInBits / 8 burstInBytes := burstInBits / 8 bufferInBytes := buffer(rateInBytes, uint32(burstInBytes)) + latency := latencyInUsec(latencyInMillis) + limitInBytes := limit(rateInBytes, latency, uint32(burstInBytes)) - // The capped class for shaped traffic (included subnets or all but excluded subnets) - // cmd = exec.Command("/usr/sbin/tc", "class", "add", "dev", interfaceName, "parent", "1:", "classid", "1:30", "htb", "rate", - // fmt.Sprintf("%d", rateInBits), "burst", fmt.Sprintf("%d", burstInBits)) - shapedClass := &netlink.HtbClass{ - ClassAttrs: netlink.ClassAttrs{ + qdisc := &netlink.Tbf{ + QdiscAttrs: netlink.QdiscAttrs{ LinkIndex: linkIndex, - Handle: netlink.MakeHandle(1, ShapedClassMinorID), - Parent: netlink.MakeHandle(1, 0), + Handle: netlink.MakeHandle(1, 0), + Parent: netlink.HANDLE_ROOT, }, + Limit: limitInBytes, Rate: rateInBytes, Buffer: bufferInBytes, - // Let's set up the "burst" rate to twice the specified rate - Ceil: 2 * rateInBytes, - Cbuffer: bufferInBytes, - } - - err = netlink.ClassAdd(shapedClass) - if err != nil { - return fmt.Errorf("error while creating htb default class: %s", err) } - - // The uncapped class for non shaped traffic (either all but included subnets or excluded subnets only) - // cmd = exec.Command("/usr/sbin/tc", "class", "add", "dev", interfaceName, "parent", "1:", "classid", "1:1", "htb", - // "rate", "100000000000") - bigRate := UncappedRate - unshapedClass := &netlink.HtbClass{ - ClassAttrs: netlink.ClassAttrs{ - LinkIndex: linkIndex, - Handle: netlink.MakeHandle(1, UnShapedClassMinorID), - Parent: qdisc.Handle, - }, - Rate: bigRate, - Ceil: bigRate, - // No need for any burst, the minimum buffer size in q_htb.c should be enough to handle the rate which - // is already more than enough - } - err = netlink.ClassAdd(unshapedClass) + err := netlink.QdiscAdd(qdisc) if err != nil { - return fmt.Errorf("error while creating htb uncapped class: %s", err) - } - - // Now add filters to redirect subnets to the class 1 if excluded instead of the default one (30) - for _, subnet := range subnets { - - // cmd = exec.Command("/usr/sbin/tc", "filter", "add", "dev", interfaceName, "parent", "1:", "protocol", protocol, - // "prio", "16", "u32", "match", "ip", "dst", subnet, "flowid", "1:1") - _, nw, err := net.ParseCIDR(subnet) - if err != nil { - return fmt.Errorf("bad subnet %s: %s", subnet, err) - } - var maskBytes []byte = nw.Mask - var subnetBytes []byte = nw.IP - - if len(maskBytes) != len(subnetBytes) { - return fmt.Errorf("error using net lib for subnet %s len(maskBytes) != len(subnetBytes) "+ - "(%d != %d) should not happen", subnet, len(maskBytes), len(subnetBytes)) - } - - isIpv4 := nw.IP.To4() != nil - protocol := syscall.ETH_P_IPV6 - var prio uint16 = 15 - var offset int32 = 24 - keepBytes := 16 - if isIpv4 { - protocol = syscall.ETH_P_IP - offset = 16 - keepBytes = 4 - // prio/pref needs to be changed if we change the protocol, looks like we cannot mix protocols with the same pref - prio = 16 - } - - if len(maskBytes) < keepBytes { - return fmt.Errorf("error with net lib, unexpected count of bytes for ipv4 mask (%d < %d)", - len(maskBytes), keepBytes) - } - if len(subnetBytes) < keepBytes { - return fmt.Errorf("error with net lib, unexpected count of bytes for ipv4 subnet (%d < %d)", - len(subnetBytes), keepBytes) - } - maskBytes = maskBytes[len(maskBytes)-keepBytes:] - subnetBytes = subnetBytes[len(subnetBytes)-keepBytes:] - - // For ipv4 we should have at most 1 key, for ipv6 at most 4 - keys := make([]netlink.TcU32Key, 0, 4) - - for i := 0; i < len(maskBytes); i += 4 { - var mask, subnetI uint32 - buf := bytes.NewReader(maskBytes[i : i+4]) - err = binary.Read(buf, binary.BigEndian, &mask) - if err != nil { - return fmt.Errorf("error, htb filter, unable to build mask match filter, iter %d for subnet %s", - i, subnet) - } - - if mask != 0 { - // If mask == 0, any value on this section will be a match and we do not need a filter for this - buf = bytes.NewReader(subnetBytes[i : i+4]) - err = binary.Read(buf, binary.BigEndian, &subnetI) - if err != nil { - return fmt.Errorf("error, htb filter, unable to build subnet match filter, iter %d for subnet %s", - i, subnet) - } - keys = append(keys, netlink.TcU32Key{ - Mask: mask, - Val: subnetI, - Off: offset, - OffMask: 0, - }) - } - - offset += 4 - } - - if len(keys) != cap(keys) { - shrinkedKeys := make([]netlink.TcU32Key, len(keys)) - copied := copy(shrinkedKeys, keys) - if copied != len(keys) { - return fmt.Errorf("copy tc u32 keys error, for subnet %s copied %d != keys %d", subnet, copied, len(keys)) - } - keys = shrinkedKeys - } - - if isIpv4 && len(keys) > 1 { - return fmt.Errorf("error, htb ipv4 filter, unexpected rule length (%d > 1), for subnet %s", - len(keys), subnet) - } else if len(keys) > 4 { - return fmt.Errorf("error, htb ipv6 filter, unexpected rule length (%d > 4), for subnet %s", - len(keys), subnet) - } - - // If len(keys) == 0, it means that we want to wildcard all traffic on the non default/uncapped class - var selector *netlink.TcU32Sel - if len(keys) > 0 { - selector = &netlink.TcU32Sel{ - Nkeys: uint8(len(keys)), - Flags: netlink.TC_U32_TERMINAL, - Keys: keys, - } - } - - classID := shapedClass.Handle - if excludeSubnets { - classID = unshapedClass.Handle - } - - tcFilter := netlink.U32{ - FilterAttrs: netlink.FilterAttrs{ - LinkIndex: linkIndex, - Parent: qdisc.Handle, - Priority: prio, - Protocol: uint16(protocol), - }, - ClassId: classID, - Sel: selector, - } - - err = netlink.FilterAdd(&tcFilter) - if err != nil { - return fmt.Errorf("error, unable to create htb filter, details %s", err) - } + return fmt.Errorf("create qdisc: %s", err) } return nil } @@ -346,3 +153,11 @@ func time2Tick(time uint32) uint32 { func buffer(rate uint64, burst uint32) uint32 { return time2Tick(uint32(float64(burst) * float64(netlink.TIME_UNITS_PER_SEC) / float64(rate))) } + +func limit(rate uint64, latency float64, buffer uint32) uint32 { + return uint32(float64(rate)*latency/float64(netlink.TIME_UNITS_PER_SEC)) + buffer +} + +func latencyInUsec(latencyInMillis float64) float64 { + return float64(netlink.TIME_UNITS_PER_SEC) * (latencyInMillis / 1000.0) +} diff --git a/plugins/meta/bandwidth/main.go b/plugins/meta/bandwidth/main.go index 0e4a963a8..27de7c86d 100644 --- a/plugins/meta/bandwidth/main.go +++ b/plugins/meta/bandwidth/main.go @@ -18,7 +18,6 @@ import ( "encoding/json" "fmt" "math" - "net" "github.com/vishvananda/netlink" @@ -40,12 +39,11 @@ const ( // BandwidthEntry corresponds to a single entry in the bandwidth argument, // see CONVENTIONS.md type BandwidthEntry struct { - UnshapedSubnets []string `json:"unshapedSubnets"` // Ipv4/ipv6 subnets to be excluded from traffic shaping. UnshapedSubnets and ShapedSubnets parameters are mutually exlusive - ShapedSubnets []string `json:"shapedSubnets"` // Ipv4/ipv6 subnets to be included in traffic shaping. UnshapedSubnets and ShapedSubnets parameters are mutually exlusive - IngressRate uint64 `json:"ingressRate"` // Bandwidth rate in bps for traffic through container. 0 for no limit. If ingressRate is set, ingressBurst must also be set - IngressBurst uint64 `json:"ingressBurst"` // Bandwidth burst in bits for traffic through container. 0 for no limit. If ingressBurst is set, ingressRate must also be set - EgressRate uint64 `json:"egressRate"` // Bandwidth rate in bps for traffic through container. 0 for no limit. If egressRate is set, egressBurst must also be set - EgressBurst uint64 `json:"egressBurst"` // Bandwidth burst in bits for traffic through container. 0 for no limit. If egressBurst is set, egressRate must also be set + IngressRate uint64 `json:"ingressRate"` // Bandwidth rate in bps for traffic through container. 0 for no limit. If ingressRate is set, ingressBurst must also be set + IngressBurst uint64 `json:"ingressBurst"` // Bandwidth burst in bits for traffic through container. 0 for no limit. If ingressBurst is set, ingressRate must also be set + + EgressRate uint64 `json:"egressRate"` // Bandwidth rate in bps for traffic through container. 0 for no limit. If egressRate is set, egressBurst must also be set + EgressBurst uint64 `json:"egressBurst"` // Bandwidth burst in bits for traffic through container. 0 for no limit. If egressBurst is set, egressRate must also be set } func (bw *BandwidthEntry) isZero() bool { @@ -98,21 +96,10 @@ func parseConfig(stdin []byte) (*PluginConf, error) { } func getBandwidth(conf *PluginConf) *BandwidthEntry { - bw := conf.BandwidthEntry - if bw == nil && conf.RuntimeConfig.Bandwidth != nil { - bw = conf.RuntimeConfig.Bandwidth - } - - if bw != nil { - if bw.UnshapedSubnets == nil { - bw.UnshapedSubnets = make([]string, 0) - } - if bw.ShapedSubnets == nil { - bw.ShapedSubnets = make([]string, 0) - } + if conf.BandwidthEntry == nil && conf.RuntimeConfig.Bandwidth != nil { + return conf.RuntimeConfig.Bandwidth } - - return bw + return conf.BandwidthEntry } func validateRateAndBurst(rate, burst uint64) error { @@ -132,13 +119,13 @@ func getIfbDeviceName(networkName string, containerID string) string { return utils.MustFormatHashWithPrefix(maxIfbDeviceLength, ifbDevicePrefix, networkName+containerID) } -func getMTUAndQLen(deviceName string) (int, int, error) { +func getMTU(deviceName string) (int, error) { link, err := netlink.LinkByName(deviceName) if err != nil { - return -1, -1, err + return -1, err } - return link.Attrs().MTU, link.Attrs().TxQLen, nil + return link.Attrs().MTU, nil } // get the veth peer of container interface in host namespace @@ -172,28 +159,6 @@ func getHostInterface(interfaces []*current.Interface, containerIfName string, n return nil, fmt.Errorf("no veth peer of container interface found in host ns") } -func validateSubnets(unshapedSubnets []string, shapedSubnets []string) error { - if len(unshapedSubnets) > 0 && len(shapedSubnets) > 0 { - return fmt.Errorf("unshapedSubnets and shapedSubnets cannot be both specified, one of them should be discarded") - } - - for _, subnet := range unshapedSubnets { - _, _, err := net.ParseCIDR(subnet) - if err != nil { - return fmt.Errorf("bad subnet %q provided, details %s", subnet, err) - } - } - - for _, subnet := range shapedSubnets { - _, _, err := net.ParseCIDR(subnet) - if err != nil { - return fmt.Errorf("bad subnet %q provided, details %s", subnet, err) - } - } - - return nil -} - func cmdAdd(args *skel.CmdArgs) error { conf, err := parseConfig(args.StdinData) if err != nil { @@ -205,10 +170,6 @@ func cmdAdd(args *skel.CmdArgs) error { return types.PrintResult(conf.PrevResult, conf.CNIVersion) } - if err = validateSubnets(bandwidth.UnshapedSubnets, bandwidth.ShapedSubnets); err != nil { - return err - } - if conf.PrevResult == nil { return fmt.Errorf("must be called as chained plugin") } @@ -230,22 +191,21 @@ func cmdAdd(args *skel.CmdArgs) error { } if bandwidth.IngressRate > 0 && bandwidth.IngressBurst > 0 { - err = CreateIngressQdisc(bandwidth.IngressRate, bandwidth.IngressBurst, - bandwidth.UnshapedSubnets, bandwidth.ShapedSubnets, hostInterface.Name) + err = CreateIngressQdisc(bandwidth.IngressRate, bandwidth.IngressBurst, hostInterface.Name) if err != nil { return err } } if bandwidth.EgressRate > 0 && bandwidth.EgressBurst > 0 { - mtu, qlen, err := getMTUAndQLen(hostInterface.Name) + mtu, err := getMTU(hostInterface.Name) if err != nil { return err } ifbDeviceName := getIfbDeviceName(conf.Name, args.ContainerID) - err = CreateIfb(ifbDeviceName, mtu, qlen) + err = CreateIfb(ifbDeviceName, mtu) if err != nil { return err } @@ -259,9 +219,7 @@ func cmdAdd(args *skel.CmdArgs) error { Name: ifbDeviceName, Mac: ifbDevice.Attrs().HardwareAddr.String(), }) - err = CreateEgressQdisc(bandwidth.EgressRate, bandwidth.EgressBurst, - bandwidth.UnshapedSubnets, bandwidth.ShapedSubnets, hostInterface.Name, - ifbDeviceName) + err = CreateEgressQdisc(bandwidth.EgressRate, bandwidth.EgressBurst, hostInterface.Name, ifbDeviceName) if err != nil { return err } @@ -340,99 +298,75 @@ func cmdCheck(args *skel.CmdArgs) error { bandwidth := getBandwidth(bwConf) - if err = validateSubnets(bandwidth.UnshapedSubnets, bandwidth.ShapedSubnets); err != nil { - return fmt.Errorf("failed to check subnets, details %s", err) - } - if bandwidth.IngressRate > 0 && bandwidth.IngressBurst > 0 { rateInBytes := bandwidth.IngressRate / 8 burstInBytes := bandwidth.IngressBurst / 8 bufferInBytes := buffer(rateInBytes, uint32(burstInBytes)) - err = checkHTB(link, rateInBytes, bufferInBytes, bandwidth.ShapedSubnets) + latency := latencyInUsec(latencyInMillis) + limitInBytes := limit(rateInBytes, latency, uint32(burstInBytes)) + + qdiscs, err := SafeQdiscList(link) if err != nil { return err } + if len(qdiscs) == 0 { + return fmt.Errorf("Failed to find qdisc") + } + + for _, qdisc := range qdiscs { + tbf, isTbf := qdisc.(*netlink.Tbf) + if !isTbf { + break + } + if tbf.Rate != rateInBytes { + return fmt.Errorf("Rate doesn't match") + } + if tbf.Limit != limitInBytes { + return fmt.Errorf("Limit doesn't match") + } + if tbf.Buffer != bufferInBytes { + return fmt.Errorf("Buffer doesn't match") + } + } } + if bandwidth.EgressRate > 0 && bandwidth.EgressBurst > 0 { rateInBytes := bandwidth.EgressRate / 8 burstInBytes := bandwidth.EgressBurst / 8 bufferInBytes := buffer(rateInBytes, uint32(burstInBytes)) + latency := latencyInUsec(latencyInMillis) + limitInBytes := limit(rateInBytes, latency, uint32(burstInBytes)) + ifbDeviceName := getIfbDeviceName(bwConf.Name, args.ContainerID) + ifbDevice, err := netlink.LinkByName(ifbDeviceName) if err != nil { return fmt.Errorf("get ifb device: %s", err) } - err = checkHTB(ifbDevice, rateInBytes, bufferInBytes, bandwidth.ShapedSubnets) - if err != nil { - return err - } - } - return nil -} -func checkHTB(link netlink.Link, rateInBytes uint64, bufferInBytes uint32, shapedSubnets []string) error { - qdiscs, err := SafeQdiscList(link) - if err != nil { - return err - } - if len(qdiscs) == 0 { - return fmt.Errorf("Failed to find qdisc") - } - foundHTB := false - for _, qdisc := range qdiscs { - htb, isHtb := qdisc.(*netlink.Htb) - if !isHtb { - continue - } - - if foundHTB { - return fmt.Errorf("Several htb qdisc found for device %s", link.Attrs().Name) - } - - foundHTB = true - defaultClassMinorID := ShapedClassMinorID - if len(shapedSubnets) > 0 { - defaultClassMinorID = UnShapedClassMinorID - } - - if htb.Defcls != uint32(defaultClassMinorID) { - return fmt.Errorf("Default class does not match") - } - - classes, err := netlink.ClassList(link, htb.Handle) + qdiscs, err := SafeQdiscList(ifbDevice) if err != nil { - return fmt.Errorf("Unable to list classes bound to htb qdisc for device %s. Details %s", - link.Attrs().Name, err) + return err } - if len(classes) != 2 { - return fmt.Errorf("Number of htb classes does not match for device %s (%d != 2)", - link.Attrs().Name, len(classes)) + if len(qdiscs) == 0 { + return fmt.Errorf("Failed to find qdisc") } - for _, c := range classes { - htbClass, isHtb := c.(*netlink.HtbClass) - if !isHtb { - return fmt.Errorf("Unexpected class for parent htb qdisc bound to device %s", link.Attrs().Name) + for _, qdisc := range qdiscs { + tbf, isTbf := qdisc.(*netlink.Tbf) + if !isTbf { + break } - if htbClass.Handle == htb.Defcls { - if htbClass.Rate != rateInBytes { - return fmt.Errorf("Rate does not match for the default class for device %s (%d != %d)", - link.Attrs().Name, htbClass.Rate, rateInBytes) - } - - if htbClass.Buffer != bufferInBytes { - return fmt.Errorf("Burst buffer size does not match for the default class for device %s (%d != %d)", - link.Attrs().Name, htbClass.Buffer, bufferInBytes) - } - } else if htbClass.Handle == netlink.MakeHandle(1, 1) { - if htbClass.Rate != UncappedRate { - return fmt.Errorf("Rate does not match for the uncapped class for device %s (%d != %d)", - link.Attrs().Name, htbClass.Rate, UncappedRate) - } + if tbf.Rate != rateInBytes { + return fmt.Errorf("Rate doesn't match") + } + if tbf.Limit != limitInBytes { + return fmt.Errorf("Limit doesn't match") + } + if tbf.Buffer != bufferInBytes { + return fmt.Errorf("Buffer doesn't match") } } - - // TODO: check subnet filters } return nil