Make IP ranges for veth devices and routable uVM IPs configurable

Signed-off-by: Leonid Kondrashov <[email protected]>

ctriface/orch.go

@@ -23,7 +23,6 @@
 package ctriface
 
 import (
-	"github.com/vhive-serverless/vhive/devmapper"
 	"os"
 	"os/signal"
 	"path/filepath"
@@ -32,6 +31,8 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/vhive-serverless/vhive/devmapper"
+
 	log "github.com/sirupsen/logrus"
 
 	"github.com/containerd/containerd"
@@ -91,6 +92,9 @@ type Orchestrator struct {
 	isMetricsMode    bool
 	netPoolSize      int
 
+	vethPrefix  string
+	clonePrefix string
+
 	memoryManager *manager.MemoryManager
 }
 
@@ -108,7 +112,7 @@ func NewOrchestrator(snapshotter, hostIface string, opts ...OrchestratorOption)
 		opt(o)
 	}
 
-	o.vmPool = misc.NewVMPool(hostIface, o.netPoolSize)
+	o.vmPool = misc.NewVMPool(hostIface, o.netPoolSize, o.vethPrefix, o.clonePrefix)
 
 	if _, err := os.Stat(o.snapshotsDir); err != nil {
 		if !os.IsNotExist(err) {

ctriface/orch_options.go

@@ -78,3 +78,15 @@ func WithNetPoolSize(netPoolSize int) OrchestratorOption {
 		o.netPoolSize = netPoolSize
 	}
 }
+
+func WithVethPrefix(vethPrefix string) OrchestratorOption {
+	return func(o *Orchestrator) {
+		o.vethPrefix = vethPrefix
+	}
+}
+
+func WithClonePrefix(clonePrefix string) OrchestratorOption {
+	return func(o *Orchestrator) {
+		o.clonePrefix = clonePrefix
+	}
+}

misc/misc_test.go

@@ -47,7 +47,7 @@ func TestMain(m *testing.M) {
 }
 
 func TestAllocateFreeVMs(t *testing.T) {
-	vmPool := NewVMPool("", 10)
+	vmPool := NewVMPool("", 10, "172.17", "172.18")
 
 	vmIDs := [2]string{"test1", "test2"}
 
@@ -67,7 +67,7 @@ func TestAllocateFreeVMs(t *testing.T) {
 func TestAllocateFreeVMsParallel(t *testing.T) {
 	vmNum := 100
 
-	vmPool := NewVMPool("", 10)
+	vmPool := NewVMPool("", 10, "172.17", "172.18")
 
 	var vmGroup sync.WaitGroup
 	for i := 0; i < vmNum; i++ {

misc/vm_pool.go

@@ -29,9 +29,9 @@ import (
 )
 
 // NewVMPool Initializes a pool of VMs
-func NewVMPool(hostIfaceName string, netPoolSize int) *VMPool {
+func NewVMPool(hostIfaceName string, netPoolSize int, vethPrefix, clonePrefix string) *VMPool {
 	p := new(VMPool)
-	networkManager, err := networking.NewNetworkManager(hostIfaceName, netPoolSize)
+	networkManager, err := networking.NewNetworkManager(hostIfaceName, netPoolSize, vethPrefix, clonePrefix)
 	if err != nil {
 		log.Println(err)
 	}

networking/networkManager.go

@@ -24,8 +24,9 @@
 package networking
 
 import (
-	log "github.com/sirupsen/logrus"
 	"sync"
+
+	log "github.com/sirupsen/logrus"
 )
 
 // NetworkManager manages the in use network configurations along with a pool of free network configurations
@@ -34,6 +35,8 @@ type NetworkManager struct {
 	sync.Mutex
 	nextID        int
 	hostIfaceName string
+	vethPrefix    string
+	clonePrefix   string
 
 	// Pool of free network configs
 	networkPool []*NetworkConfig
@@ -51,7 +54,7 @@ type NetworkManager struct {
 // using the supplied interface. If no interface is supplied, the default interface is used. To take the network
 // setup of the critical path of a function creation, the network manager tries to maintain a pool of ready to use
 // network configurations of size at least poolSize.
-func NewNetworkManager(hostIfaceName string, poolSize int) (*NetworkManager, error) {
+func NewNetworkManager(hostIfaceName string, poolSize int, vethPrefix, clonePrefix string) (*NetworkManager, error) {
 	manager := new(NetworkManager)
 
 	manager.hostIfaceName = hostIfaceName
@@ -77,6 +80,8 @@ func NewNetworkManager(hostIfaceName string, poolSize int) (*NetworkManager, err
 	manager.poolCond = sync.NewCond(new(sync.Mutex))
 	manager.initConfigPool(poolSize)
 	manager.poolSize = poolSize
+	manager.vethPrefix = vethPrefix
+	manager.clonePrefix = clonePrefix
 
 	return manager, nil
 }
@@ -107,7 +112,7 @@ func (mgr *NetworkManager) addNetConfig() {
 	mgr.inCreation.Add(1)
 	mgr.Unlock()
 
-	netCfg := NewNetworkConfig(id, mgr.hostIfaceName)
+	netCfg := NewNetworkConfig(id, mgr.hostIfaceName, mgr.vethPrefix, mgr.vethPrefix)
 	if err := netCfg.CreateNetwork(); err != nil {
 		log.Errorf("failed to create network %s:", err)
 	}

networking/networkconfig.go

@@ -24,18 +24,19 @@ package networking
 
 import (
 	"fmt"
+	"net"
+	"runtime"
+
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/vishvananda/netns"
-	"net"
-	"runtime"
 )
 
 const (
 	defaultContainerCIDR = "172.16.0.2/24"
 	defaultGatewayCIDR   = "172.16.0.1/24"
 	defaultContainerTap  = "tap0"
-	defaultContainerMac  = "AA:FC:00:00:00:01"
+	defaultContainerMac  = "06:00:AC:10:00:02"
 )
 
 // NetworkConfig represents the network devices, IPs, namespaces, routes and filter rules to connect a uVM
@@ -48,17 +49,23 @@ type NetworkConfig struct {
 	containerTap  string // Container tap name
 	containerMac  string // Container Mac address
 	hostIfaceName string // Host network interface name
+
+	vethPrefix  string // Prefix for IP addresses of veth devices
+	clonePrefix string // Prefix for IP addresses of clone devices
 }
 
 // NewNetworkConfig creates a new network config with a given id and default host interface
-func NewNetworkConfig(id int, hostIfaceName string) *NetworkConfig {
+func NewNetworkConfig(id int, hostIfaceName string, vethPrefix, clonePrefix string) *NetworkConfig {
 	return &NetworkConfig{
 		id:            id,
 		containerCIDR: defaultContainerCIDR,
 		gatewayCIDR:   defaultGatewayCIDR,
 		containerTap:  defaultContainerTap,
 		containerMac:  defaultContainerMac,
 		hostIfaceName: hostIfaceName,
+
+		vethPrefix:  vethPrefix,
+		clonePrefix: clonePrefix,
 	}
 }
 
@@ -79,7 +86,7 @@ func (cfg *NetworkConfig) getVeth0Name() string {
 
 // getVeth0CIDR returns the IP address for the veth device at the side of the uVM in CIDR notation
 func (cfg *NetworkConfig) getVeth0CIDR() string {
-	return fmt.Sprintf("172.17.%d.%d/30", (4*cfg.id)/256, ((4*cfg.id)+2)%256)
+	return fmt.Sprintf("%s.%d.%d/30", cfg.vethPrefix, (4*cfg.id)/256, ((4*cfg.id)+2)%256)
 }
 
 // getVeth1Name returns the name for the veth device at the side of the host
@@ -89,12 +96,12 @@ func (cfg *NetworkConfig) getVeth1Name() string {
 
 // getVeth1Name returns the IP address for the veth device at the side of the host in CIDR notation
 func (cfg *NetworkConfig) getVeth1CIDR() string {
-	return fmt.Sprintf("172.17.%d.%d/30", (4*cfg.id)/256, ((4*cfg.id)+1)%256)
+	return fmt.Sprintf("%s.%d.%d/30", cfg.vethPrefix, (4*cfg.id)/256, ((4*cfg.id)+1)%256)
 }
 
 // GetCloneIP returns the IP address the uVM is reachable at from the host
 func (cfg *NetworkConfig) GetCloneIP() string {
-	return fmt.Sprintf("172.18.%d.%d", cfg.id/254, 1+(cfg.id%254))
+	return fmt.Sprintf("%s.%d.%d", cfg.clonePrefix, cfg.id/254, 1+(cfg.id%254))
 }
 
 // GetContainerCIDR returns the internal IP of the uVM in CIDR notation

networking/networking_test.go

@@ -53,7 +53,7 @@ func TestCreateCleanManager(t *testing.T) {
 	poolSize := []int{1, 5, 20}
 
 	for _, n := range poolSize {
-		mgr, createErr := NewNetworkManager("", n)
+		mgr, createErr := NewNetworkManager("", n, "172.17", "172.18")
 		require.NoError(t, createErr, "Network manager creation returned error")
 
 		cleanErr := mgr.Cleanup()
@@ -64,7 +64,7 @@ func TestCreateCleanManager(t *testing.T) {
 func TestCreateRemoveNetworkParallel(t *testing.T) {
 	netNum := []int{50, 200}
 
-	mgr, err := NewNetworkManager("", 10)
+	mgr, err := NewNetworkManager("", 10, "172.17", "172.18")
 	require.NoError(t, err, "Network manager creation returned error")
 	defer func() { _ = mgr.Cleanup() }()
 
@@ -94,7 +94,7 @@ func TestCreateRemoveNetworkParallel(t *testing.T) {
 func TestCreateRemoveNetworkSerial(t *testing.T) {
 	netNum := 50
 
-	mgr, err := NewNetworkManager("", 50)
+	mgr, err := NewNetworkManager("", 50, "172.17", "172.18")
 	require.NoError(t, err, "Network manager creation returned error")
 	defer func() { _ = mgr.Cleanup() }()
 

vhive.go

@@ -84,6 +84,8 @@ func main() {
 	hostIface = flag.String("hostIface", "", "Host net-interface for the VMs to bind to for internet access")
 	netPoolSize = flag.Int("netPoolSize", 10, "Amount of network configs to preallocate in a pool")
 	sandbox := flag.String("sandbox", "firecracker", "Sandbox tech to use, valid options: firecracker, gvisor")
+	vethPrefix := flag.String("vethPrefix", "172.17", "Prefix for IP addresses of veth devices, expected subnet is /16")
+	clonePrefix := flag.String("clonePrefix", "172.18", "Prefix for node-accessible IP addresses of uVMs, expected subnet is /16")
 	flag.Parse()
 
 	if *sandbox != "firecracker" && *sandbox != "gvisor" {
@@ -142,6 +144,8 @@ func main() {
 			ctriface.WithMetricsMode(*isMetricsMode),
 			ctriface.WithLazyMode(*isLazyMode),
 			ctriface.WithNetPoolSize(*netPoolSize),
+			ctriface.WithVethPrefix(*vethPrefix),
+			ctriface.WithClonePrefix(*clonePrefix),
 		)
 		funcPool = NewFuncPool(*isSaveMemory, *servedThreshold, *pinnedFuncNum, testModeOn)
 		go setupFirecrackerCRI()