From 0d5deddfaef5676ae79cf3699d9e5c2fc9be6f0a Mon Sep 17 00:00:00 2001 From: Leonid Kondrashov Date: Wed, 19 Feb 2025 18:31:24 +0800 Subject: [PATCH] Make IP ranges for veth devices and routable uVM IPs configurable Signed-off-by: Leonid Kondrashov --- ctriface/orch.go | 8 ++++++-- ctriface/orch_options.go | 12 ++++++++++++ go.work.sum | 2 ++ misc/misc_test.go | 4 ++-- misc/vm_pool.go | 4 ++-- networking/networkManager.go | 11 ++++++++--- networking/networkconfig.go | 21 ++++++++++++++------- networking/networking_test.go | 6 +++--- vhive.go | 4 ++++ 9 files changed, 53 insertions(+), 19 deletions(-) diff --git a/ctriface/orch.go b/ctriface/orch.go index d7dc5df7b..07a4bf433 100644 --- a/ctriface/orch.go +++ b/ctriface/orch.go @@ -23,7 +23,6 @@ package ctriface import ( - "github.com/vhive-serverless/vhive/devmapper" "os" "os/signal" "path/filepath" @@ -32,6 +31,8 @@ import ( "syscall" "time" + "github.com/vhive-serverless/vhive/devmapper" + log "github.com/sirupsen/logrus" "github.com/containerd/containerd" @@ -91,6 +92,9 @@ type Orchestrator struct { isMetricsMode bool netPoolSize int + vethPrefix string + clonePrefix string + memoryManager *manager.MemoryManager } @@ -108,7 +112,7 @@ func NewOrchestrator(snapshotter, hostIface string, opts ...OrchestratorOption) opt(o) } - o.vmPool = misc.NewVMPool(hostIface, o.netPoolSize) + o.vmPool = misc.NewVMPool(hostIface, o.netPoolSize, o.vethPrefix, o.clonePrefix) if _, err := os.Stat(o.snapshotsDir); err != nil { if !os.IsNotExist(err) { diff --git a/ctriface/orch_options.go b/ctriface/orch_options.go index 8e9896d5f..09a2cba7a 100644 --- a/ctriface/orch_options.go +++ b/ctriface/orch_options.go @@ -78,3 +78,15 @@ func WithNetPoolSize(netPoolSize int) OrchestratorOption { o.netPoolSize = netPoolSize } } + +func WithVethPrefix(vethPrefix string) OrchestratorOption { + return func(o *Orchestrator) { + o.vethPrefix = vethPrefix + } +} + +func WithClonePrefix(clonePrefix string) OrchestratorOption { + return func(o *Orchestrator) { + o.clonePrefix = clonePrefix + } +} diff --git a/go.work.sum b/go.work.sum index aa5238c23..4df8fb226 100644 --- a/go.work.sum +++ b/go.work.sum @@ -219,6 +219,7 @@ golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp/shiny v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:UH99kUObWAZkDnWqppdQe5ZhPYESUw8I0zVV1uWBR+0= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -270,6 +271,7 @@ golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/misc/misc_test.go b/misc/misc_test.go index 493c6d4cb..1fcd0a04b 100644 --- a/misc/misc_test.go +++ b/misc/misc_test.go @@ -47,7 +47,7 @@ func TestMain(m *testing.M) { } func TestAllocateFreeVMs(t *testing.T) { - vmPool := NewVMPool("", 10) + vmPool := NewVMPool("", 10, "172.17", "172.18") vmIDs := [2]string{"test1", "test2"} @@ -67,7 +67,7 @@ func TestAllocateFreeVMs(t *testing.T) { func TestAllocateFreeVMsParallel(t *testing.T) { vmNum := 100 - vmPool := NewVMPool("", 10) + vmPool := NewVMPool("", 10, "172.17", "172.18") var vmGroup sync.WaitGroup for i := 0; i < vmNum; i++ { diff --git a/misc/vm_pool.go b/misc/vm_pool.go index 9270a6265..d3e8df03c 100644 --- a/misc/vm_pool.go +++ b/misc/vm_pool.go @@ -29,9 +29,9 @@ import ( ) // NewVMPool Initializes a pool of VMs -func NewVMPool(hostIfaceName string, netPoolSize int) *VMPool { +func NewVMPool(hostIfaceName string, netPoolSize int, vethPrefix, clonePrefix string) *VMPool { p := new(VMPool) - networkManager, err := networking.NewNetworkManager(hostIfaceName, netPoolSize) + networkManager, err := networking.NewNetworkManager(hostIfaceName, netPoolSize, vethPrefix, clonePrefix) if err != nil { log.Println(err) } diff --git a/networking/networkManager.go b/networking/networkManager.go index a4515b603..f6ad0581a 100644 --- a/networking/networkManager.go +++ b/networking/networkManager.go @@ -24,8 +24,9 @@ package networking import ( - log "github.com/sirupsen/logrus" "sync" + + log "github.com/sirupsen/logrus" ) // NetworkManager manages the in use network configurations along with a pool of free network configurations @@ -34,6 +35,8 @@ type NetworkManager struct { sync.Mutex nextID int hostIfaceName string + vethPrefix string + clonePrefix string // Pool of free network configs networkPool []*NetworkConfig @@ -51,7 +54,7 @@ type NetworkManager struct { // using the supplied interface. If no interface is supplied, the default interface is used. To take the network // setup of the critical path of a function creation, the network manager tries to maintain a pool of ready to use // network configurations of size at least poolSize. -func NewNetworkManager(hostIfaceName string, poolSize int) (*NetworkManager, error) { +func NewNetworkManager(hostIfaceName string, poolSize int, vethPrefix, clonePrefix string) (*NetworkManager, error) { manager := new(NetworkManager) manager.hostIfaceName = hostIfaceName @@ -75,6 +78,8 @@ func NewNetworkManager(hostIfaceName string, poolSize int) (*NetworkManager, err } manager.poolCond = sync.NewCond(new(sync.Mutex)) + manager.vethPrefix = vethPrefix + manager.clonePrefix = clonePrefix manager.initConfigPool(poolSize) manager.poolSize = poolSize @@ -107,7 +112,7 @@ func (mgr *NetworkManager) addNetConfig() { mgr.inCreation.Add(1) mgr.Unlock() - netCfg := NewNetworkConfig(id, mgr.hostIfaceName) + netCfg := NewNetworkConfig(id, mgr.hostIfaceName, mgr.vethPrefix, mgr.clonePrefix) if err := netCfg.CreateNetwork(); err != nil { log.Errorf("failed to create network %s:", err) } diff --git a/networking/networkconfig.go b/networking/networkconfig.go index a14525df6..7a7cda594 100644 --- a/networking/networkconfig.go +++ b/networking/networkconfig.go @@ -24,18 +24,19 @@ package networking import ( "fmt" + "net" + "runtime" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/vishvananda/netns" - "net" - "runtime" ) const ( defaultContainerCIDR = "172.16.0.2/24" defaultGatewayCIDR = "172.16.0.1/24" defaultContainerTap = "tap0" - defaultContainerMac = "AA:FC:00:00:00:01" + defaultContainerMac = "06:00:AC:10:00:02" ) // NetworkConfig represents the network devices, IPs, namespaces, routes and filter rules to connect a uVM @@ -48,10 +49,13 @@ type NetworkConfig struct { containerTap string // Container tap name containerMac string // Container Mac address hostIfaceName string // Host network interface name + + vethPrefix string // Prefix for IP addresses of veth devices + clonePrefix string // Prefix for IP addresses of clone devices } // NewNetworkConfig creates a new network config with a given id and default host interface -func NewNetworkConfig(id int, hostIfaceName string) *NetworkConfig { +func NewNetworkConfig(id int, hostIfaceName string, vethPrefix, clonePrefix string) *NetworkConfig { return &NetworkConfig{ id: id, containerCIDR: defaultContainerCIDR, @@ -59,6 +63,9 @@ func NewNetworkConfig(id int, hostIfaceName string) *NetworkConfig { containerTap: defaultContainerTap, containerMac: defaultContainerMac, hostIfaceName: hostIfaceName, + + vethPrefix: vethPrefix, + clonePrefix: clonePrefix, } } @@ -79,7 +86,7 @@ func (cfg *NetworkConfig) getVeth0Name() string { // getVeth0CIDR returns the IP address for the veth device at the side of the uVM in CIDR notation func (cfg *NetworkConfig) getVeth0CIDR() string { - return fmt.Sprintf("172.17.%d.%d/30", (4*cfg.id)/256, ((4*cfg.id)+2)%256) + return fmt.Sprintf("%s.%d.%d/30", cfg.vethPrefix, (4*cfg.id)/256, ((4*cfg.id)+2)%256) } // getVeth1Name returns the name for the veth device at the side of the host @@ -89,12 +96,12 @@ func (cfg *NetworkConfig) getVeth1Name() string { // getVeth1Name returns the IP address for the veth device at the side of the host in CIDR notation func (cfg *NetworkConfig) getVeth1CIDR() string { - return fmt.Sprintf("172.17.%d.%d/30", (4*cfg.id)/256, ((4*cfg.id)+1)%256) + return fmt.Sprintf("%s.%d.%d/30", cfg.vethPrefix, (4*cfg.id)/256, ((4*cfg.id)+1)%256) } // GetCloneIP returns the IP address the uVM is reachable at from the host func (cfg *NetworkConfig) GetCloneIP() string { - return fmt.Sprintf("172.18.%d.%d", cfg.id/254, 1+(cfg.id%254)) + return fmt.Sprintf("%s.%d.%d", cfg.clonePrefix, cfg.id/254, 1+(cfg.id%254)) } // GetContainerCIDR returns the internal IP of the uVM in CIDR notation diff --git a/networking/networking_test.go b/networking/networking_test.go index 9725136a1..40a73c642 100644 --- a/networking/networking_test.go +++ b/networking/networking_test.go @@ -53,7 +53,7 @@ func TestCreateCleanManager(t *testing.T) { poolSize := []int{1, 5, 20} for _, n := range poolSize { - mgr, createErr := NewNetworkManager("", n) + mgr, createErr := NewNetworkManager("", n, "172.17", "172.18") require.NoError(t, createErr, "Network manager creation returned error") cleanErr := mgr.Cleanup() @@ -64,7 +64,7 @@ func TestCreateCleanManager(t *testing.T) { func TestCreateRemoveNetworkParallel(t *testing.T) { netNum := []int{50, 200} - mgr, err := NewNetworkManager("", 10) + mgr, err := NewNetworkManager("", 10, "172.17", "172.18") require.NoError(t, err, "Network manager creation returned error") defer func() { _ = mgr.Cleanup() }() @@ -94,7 +94,7 @@ func TestCreateRemoveNetworkParallel(t *testing.T) { func TestCreateRemoveNetworkSerial(t *testing.T) { netNum := 50 - mgr, err := NewNetworkManager("", 50) + mgr, err := NewNetworkManager("", 50, "172.17", "172.18") require.NoError(t, err, "Network manager creation returned error") defer func() { _ = mgr.Cleanup() }() diff --git a/vhive.go b/vhive.go index 829b0eed0..22c262be2 100644 --- a/vhive.go +++ b/vhive.go @@ -84,6 +84,8 @@ func main() { hostIface = flag.String("hostIface", "", "Host net-interface for the VMs to bind to for internet access") netPoolSize = flag.Int("netPoolSize", 10, "Amount of network configs to preallocate in a pool") sandbox := flag.String("sandbox", "firecracker", "Sandbox tech to use, valid options: firecracker, gvisor") + vethPrefix := flag.String("vethPrefix", "172.17", "Prefix for IP addresses of veth devices, expected subnet is /16") + clonePrefix := flag.String("clonePrefix", "172.18", "Prefix for node-accessible IP addresses of uVMs, expected subnet is /16") flag.Parse() if *sandbox != "firecracker" && *sandbox != "gvisor" { @@ -142,6 +144,8 @@ func main() { ctriface.WithMetricsMode(*isMetricsMode), ctriface.WithLazyMode(*isLazyMode), ctriface.WithNetPoolSize(*netPoolSize), + ctriface.WithVethPrefix(*vethPrefix), + ctriface.WithClonePrefix(*clonePrefix), ) funcPool = NewFuncPool(*isSaveMemory, *servedThreshold, *pinnedFuncNum, testModeOn) go setupFirecrackerCRI()