Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bandwidth: introduce configurable queue length #1025

Closed
wants to merge 1 commit into from
Closed

bandwidth: introduce configurable queue length #1025

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions plugins/meta/bandwidth/bandwidth_linux_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,8 +295,10 @@ var _ = Describe("bandwidth test", func() {
"type": "bandwidth",
"egressRate": 0,
"egressBurst": 0,
"egressLatency": 0,
"ingressRate": 8000,
"ingressBurst": 80,
"ingressLatency": 25000,
"prevResult": {
"interfaces": [
{
Expand Down
24 changes: 9 additions & 15 deletions plugins/meta/bandwidth/ifb_creator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,6 @@ import (
"github.com/containernetworking/plugins/pkg/ip"
)

const latencyInMillis = 25

func CreateIfb(ifbDeviceName string, mtu int) error {
err := netlink.LinkAdd(&netlink.Ifb{
LinkAttrs: netlink.LinkAttrs{
Expand All @@ -49,15 +47,15 @@ func TeardownIfb(deviceName string) error {
return err
}

func CreateIngressQdisc(rateInBits, burstInBits uint64, hostDeviceName string) error {
func CreateIngressQdisc(rateInBits, burstInBits uint64, latencyInUsec float64, hostDeviceName string) error {
hostDevice, err := netlink.LinkByName(hostDeviceName)
if err != nil {
return fmt.Errorf("get host device: %s", err)
}
return createTBF(rateInBits, burstInBits, hostDevice.Attrs().Index)
return createTBF(rateInBits, burstInBits, latencyInUsec, hostDevice.Attrs().Index)
}

func CreateEgressQdisc(rateInBits, burstInBits uint64, hostDeviceName string, ifbDeviceName string) error {
func CreateEgressQdisc(rateInBits, burstInBits uint64, latencyInUsec float64, hostDeviceName string, ifbDeviceName string) error {
ifbDevice, err := netlink.LinkByName(ifbDeviceName)
if err != nil {
return fmt.Errorf("get ifb device: %s", err)
Expand Down Expand Up @@ -105,14 +103,14 @@ func CreateEgressQdisc(rateInBits, burstInBits uint64, hostDeviceName string, if
}

// throttle traffic on ifb device
err = createTBF(rateInBits, burstInBits, ifbDevice.Attrs().Index)
err = createTBF(rateInBits, burstInBits, latencyInUsec, ifbDevice.Attrs().Index)
if err != nil {
return fmt.Errorf("create ifb qdisc: %s", err)
}
return nil
}

func createTBF(rateInBits, burstInBits uint64, linkIndex int) error {
func createTBF(rateInBits, burstInBits uint64, latencyInUsec float64, linkIndex int) error {
// Equivalent to
// tc qdisc add dev link root tbf
// rate netConf.BandwidthLimits.Rate
Expand All @@ -126,8 +124,7 @@ func createTBF(rateInBits, burstInBits uint64, linkIndex int) error {
rateInBytes := rateInBits / 8
burstInBytes := burstInBits / 8
bufferInBytes := buffer(rateInBytes, uint32(burstInBytes))
latency := latencyInUsec(latencyInMillis)
limitInBytes := limit(rateInBytes, latency, uint32(burstInBytes))
limitInBytes := limit(rateInBytes, latencyInUsec, uint32(burstInBytes))

qdisc := &netlink.Tbf{
QdiscAttrs: netlink.QdiscAttrs{
Expand All @@ -154,10 +151,7 @@ func buffer(rate uint64, burst uint32) uint32 {
return time2Tick(uint32(float64(burst) * float64(netlink.TIME_UNITS_PER_SEC) / float64(rate)))
}

func limit(rate uint64, latency float64, buffer uint32) uint32 {
return uint32(float64(rate)*latency/float64(netlink.TIME_UNITS_PER_SEC)) + buffer
}

func latencyInUsec(latencyInMillis float64) float64 {
return float64(netlink.TIME_UNITS_PER_SEC) * (latencyInMillis / 1000.0)
func limit(rate uint64, latency float64, burst uint32) uint32 {
// Only when rate * latency > 1000000, the result will be greater than burst
return uint32(float64(rate)*latency/float64(netlink.TIME_UNITS_PER_SEC)) + burst
}
20 changes: 11 additions & 9 deletions plugins/meta/bandwidth/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,17 @@ const (
// BandwidthEntry corresponds to a single entry in the bandwidth argument,
// see CONVENTIONS.md
type BandwidthEntry struct {
IngressRate uint64 `json:"ingressRate"` // Bandwidth rate in bps for traffic through container. 0 for no limit. If ingressRate is set, ingressBurst must also be set
IngressBurst uint64 `json:"ingressBurst"` // Bandwidth burst in bits for traffic through container. 0 for no limit. If ingressBurst is set, ingressRate must also be set
IngressRate uint64 `json:"ingressRate"` // Bandwidth rate in bps for traffic through container. 0 for no limit. If ingressRate is set, ingressBurst must also be set
IngressBurst uint64 `json:"ingressBurst"` // Bandwidth burst in bits for traffic through container. 0 for no limit. If ingressBurst is set, ingressRate must also be set
IngressLatency uint64 `json:"ingressLatency"` // Bandwidth latency in microseconds for traffic through container. 0 for no limit. If ingressLatency is set, ingressRate must also be set

EgressRate uint64 `json:"egressRate"` // Bandwidth rate in bps for traffic through container. 0 for no limit. If egressRate is set, egressBurst must also be set
EgressBurst uint64 `json:"egressBurst"` // Bandwidth burst in bits for traffic through container. 0 for no limit. If egressBurst is set, egressRate must also be set
EgressRate uint64 `json:"egressRate"` // Bandwidth rate in bps for traffic through container. 0 for no limit. If egressRate is set, egressBurst must also be set
EgressBurst uint64 `json:"egressBurst"` // Bandwidth burst in bits for traffic through container. 0 for no limit. If egressBurst is set, egressRate must also be set
EgressLatency uint64 `json:"egressLatency"` // Bandwidth latency in microseconds for traffic through container, 0 by default.
}

func (bw *BandwidthEntry) isZero() bool {
return bw.IngressBurst == 0 && bw.IngressRate == 0 && bw.EgressBurst == 0 && bw.EgressRate == 0
return bw.IngressBurst == 0 && bw.IngressRate == 0 && bw.IngressLatency == 0 && bw.EgressBurst == 0 && bw.EgressRate == 0 && bw.EgressLatency == 0
}

type PluginConf struct {
Expand Down Expand Up @@ -191,7 +193,7 @@ func cmdAdd(args *skel.CmdArgs) error {
}

if bandwidth.IngressRate > 0 && bandwidth.IngressBurst > 0 {
err = CreateIngressQdisc(bandwidth.IngressRate, bandwidth.IngressBurst, hostInterface.Name)
err = CreateIngressQdisc(bandwidth.IngressRate, bandwidth.IngressBurst, float64(bandwidth.IngressLatency), hostInterface.Name)
if err != nil {
return err
}
Expand Down Expand Up @@ -219,7 +221,7 @@ func cmdAdd(args *skel.CmdArgs) error {
Name: ifbDeviceName,
Mac: ifbDevice.Attrs().HardwareAddr.String(),
})
err = CreateEgressQdisc(bandwidth.EgressRate, bandwidth.EgressBurst, hostInterface.Name, ifbDeviceName)
err = CreateEgressQdisc(bandwidth.EgressRate, bandwidth.EgressBurst, float64(bandwidth.EgressLatency), hostInterface.Name, ifbDeviceName)
if err != nil {
return err
}
Expand Down Expand Up @@ -296,7 +298,7 @@ func cmdCheck(args *skel.CmdArgs) error {
rateInBytes := bandwidth.IngressRate / 8
burstInBytes := bandwidth.IngressBurst / 8
bufferInBytes := buffer(rateInBytes, uint32(burstInBytes))
latency := latencyInUsec(latencyInMillis)
latency := float64(bandwidth.IngressLatency)
limitInBytes := limit(rateInBytes, latency, uint32(burstInBytes))

qdiscs, err := SafeQdiscList(link)
Expand Down Expand Up @@ -328,7 +330,7 @@ func cmdCheck(args *skel.CmdArgs) error {
rateInBytes := bandwidth.EgressRate / 8
burstInBytes := bandwidth.EgressBurst / 8
bufferInBytes := buffer(rateInBytes, uint32(burstInBytes))
latency := latencyInUsec(latencyInMillis)
latency := float64(bandwidth.EgressLatency)
limitInBytes := limit(rateInBytes, latency, uint32(burstInBytes))

ifbDeviceName := getIfbDeviceName(bwConf.Name, args.ContainerID)
Expand Down