diff --git a/Makefile b/Makefile index 8e4f6f655..8ed8d005f 100644 --- a/Makefile +++ b/Makefile @@ -24,11 +24,8 @@ SUBDIRS:=ctriface taps misc profile EXTRAGOARGS:=-v -race -cover EXTRAGOARGS_NORACE:=-v EXTRATESTFILES:=vhive_test.go stats.go vhive.go functions.go -# User-level page faults are temporarily disabled (gh-807) -# WITHUPF:=-upfTest -# WITHLAZY:=-lazyTest -WITHUPF:= -WITHLAZY:= +WITHUPF:=-upfTest +WITHLAZY:=-lazyTest WITHSNAPSHOTS:=-snapshotsTest CTRDLOGDIR:=/tmp/ctrd-logs diff --git a/bin/containerd-shim-aws-firecracker b/bin/containerd-shim-aws-firecracker index 9aec8ccaa..108a88b8e 100755 --- a/bin/containerd-shim-aws-firecracker +++ b/bin/containerd-shim-aws-firecracker @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:4208028fa44c5897563f67b4f1c56efd1400d9a145826b2556773c9d1876bd93 -size 33776240 +oid sha256:299c9623ed3262dede9e37aa52a76c84c3342cdcadc1571a7997c3c160ff9cc0 +size 36354520 diff --git a/bin/default-rootfs.img b/bin/default-rootfs.img index 2a4d5e66a..1b2764cfd 100644 --- a/bin/default-rootfs.img +++ b/bin/default-rootfs.img @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:58d378a908efd9b604da4659c64c81cb835932a00d382c10b0e2d0d8770fc7d7 -size 64409600 +oid sha256:ba09eb5181dd977c16af4a1890333dc61fccb01dc4a7c0b11ffc229863e5462e +size 73318400 diff --git a/bin/firecracker b/bin/firecracker index 750f24cfe..639384a49 100755 --- a/bin/firecracker +++ b/bin/firecracker @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8060c35d1669a57197985e4589b3e98f4a221b334c6d1f102aee62a3f77822cd +oid sha256:c44d9ea84a0ff0c5315ed0d3672494f77bafed6f6edaaf6b050a4b5e3425ebe1 size 10012224 diff --git a/bin/firecracker-containerd b/bin/firecracker-containerd index 7cb01c83b..2334c5121 100755 --- a/bin/firecracker-containerd +++ b/bin/firecracker-containerd @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1959d901c4a8a6bdf8394628d5c62c0d6eba23a976acfdf8f9173fc21bc26e68 -size 69041344 +oid sha256:c4450b3c8e9cb2db1a193cd2b9594eb90054cdbdae1d837a7f426b4b0d83950f +size 72445304 diff --git a/bin/firecracker-ctr b/bin/firecracker-ctr index 40d64e16f..3bfb99c22 100755 --- a/bin/firecracker-ctr +++ b/bin/firecracker-ctr @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bfc28b8f8092d10190a4f7fc74cea36be12a9ef9f4ec81a3b7b46cde7d7ea857 -size 33034096 +oid sha256:37487275ed6a08f4e759d9847435537bf52f20c207087e3e4226e599de8bae72 +size 35276648 diff --git a/ctriface/Makefile b/ctriface/Makefile index bc8f6fcd3..6d2811be7 100644 --- a/ctriface/Makefile +++ b/ctriface/Makefile @@ -23,11 +23,8 @@ EXTRAGOARGS:=-v -race -cover EXTRATESTFILES:=iface_test.go iface.go orch_options.go orch.go BENCHFILES:=bench_test.go iface.go orch_options.go orch.go -# User-level page faults are temporarily disabled (gh-807) -# WITHUPF:=-upf -# WITHLAZY:=-lazy -WITHUPF:= -WITHLAZY:= +WITHUPF:=-upf +WITHLAZY:=-lazy GOBENCH:=-v -timeout 1500s CTRDLOGDIR:=/tmp/ctrd-logs diff --git a/ctriface/iface.go b/ctriface/iface.go index d6e79a114..b016cae91 100644 --- a/ctriface/iface.go +++ b/ctriface/iface.go @@ -64,6 +64,8 @@ type StartVMResponse struct { const ( testImageName = "ghcr.io/ease-lab/helloworld:var_workload" + fileBackend = "File" + uffdBackend = "Uffd" ) // StartVM Boots a VM if it does not exist @@ -104,7 +106,7 @@ func (o *Orchestrator) StartVMWithEnvironment(ctx context.Context, vmID, imageNa tStart = time.Now() conf := o.getVMConfig(vm) - _, err = o.fcClient.CreateVM(ctx, conf) + resp, err := o.fcClient.CreateVM(ctx, conf) startVMMetric.MetricMap[metrics.FcCreateVM] = metrics.ToUS(time.Since(tStart)) if err != nil { return nil, nil, errors.Wrap(err, "failed to create the microVM in firecracker-containerd") @@ -206,15 +208,14 @@ func (o *Orchestrator) StartVMWithEnvironment(ctx context.Context, vmID, imageNa logger.Debug("Registering VM with the memory manager") stateCfg := manager.SnapshotStateCfg{ - VMID: vmID, - GuestMemPath: o.getMemoryFile(vmID), - BaseDir: o.getVMBaseDir(vmID), - GuestMemSize: int(conf.MachineCfg.MemSizeMib) * 1024 * 1024, - IsLazyMode: o.isLazyMode, - VMMStatePath: o.getSnapshotFile(vmID), - WorkingSetPath: o.getWorkingSetFile(vmID), - // FIXME (gh-807) - //InstanceSockAddr: resp.UPFSockPath, + VMID: vmID, + GuestMemPath: o.getMemoryFile(vmID), + BaseDir: o.getVMBaseDir(vmID), + GuestMemSize: int(conf.MachineCfg.MemSizeMib) * 1024 * 1024, + IsLazyMode: o.isLazyMode, + VMMStatePath: o.getSnapshotFile(vmID), + WorkingSetPath: o.getWorkingSetFile(vmID), + InstanceSockAddr: resp.GetSocketPath(), } if err := o.memoryManager.RegisterVM(stateCfg); err != nil { return nil, nil, errors.Wrap(err, "failed to register VM with memory manager") @@ -494,13 +495,20 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, vmID string, snap *snap conf := o.getVMConfig(vm) conf.LoadSnapshot = true conf.SnapshotPath = snap.GetSnapshotFilePath() - conf.MemFilePath = snap.GetMemFilePath() conf.ContainerSnapshotPath = containerSnap.GetDevicePath() if o.GetUPFEnabled() { + conf.MemBackend.BackendType = uffdBackend + conf.MemBackend.BackendPath, err = o.memoryManager.GetUPFSockPath(vmID) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to get UPF socket path for uffd backend") + } + if err := o.memoryManager.FetchState(vmID); err != nil { return nil, nil, err } + } else { + conf.MemFilePath = snap.GetMemFilePath() } tStart = time.Now() diff --git a/ctriface/iface_test.go b/ctriface/iface_test.go index 7e678c6eb..2f1526c9a 100644 --- a/ctriface/iface_test.go +++ b/ctriface/iface_test.go @@ -47,12 +47,6 @@ var ( func TestMain(m *testing.M) { flag.Parse() - - if *isUPFEnabled { - log.Error("User-level page faults are temporarily disabled (gh-807)") - os.Exit(-1) - } - os.Exit(m.Run()) } diff --git a/go.mod b/go.mod index 09fb95f9e..cd66b4676 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ replace ( ) replace ( - github.com/firecracker-microvm/firecracker-containerd => github.com/vhive-serverless/firecracker-containerd v0.0.0-20230912063208-ad6383f05e45 + github.com/firecracker-microvm/firecracker-containerd => github.com/char-1ee/firecracker-containerd v0.0.0-20231018191519-49cac5eea134 github.com/vhive-serverless/vhive/examples/protobuf/helloworld => ./examples/protobuf/helloworld ) diff --git a/go.sum b/go.sum index c5ed066b7..fd6b5eec0 100644 --- a/go.sum +++ b/go.sum @@ -147,6 +147,8 @@ github.com/cespare/prettybench v0.0.0-20150116022406-03b8cfe5406c/go.mod h1:Xe6Z github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= +github.com/char-1ee/firecracker-containerd v0.0.0-20231018191519-49cac5eea134 h1:InrwKCxhDU1PJTNJ0wOHM/PvsIruaz2HriViJ5swrX4= +github.com/char-1ee/firecracker-containerd v0.0.0-20231018191519-49cac5eea134/go.mod h1:XC5a/4PWbzipD5Ron745odZxoVy/J6d8xFldwTZJbSU= github.com/checkpoint-restore/go-criu v0.0.0-20190109184317-bdb7599cd87b/go.mod h1:TrMrLQfeENAPYPRsJuq3jsqdlRh3lvi6trTZJG8+tho= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= @@ -1022,8 +1024,6 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s= github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= -github.com/vhive-serverless/firecracker-containerd v0.0.0-20230912063208-ad6383f05e45 h1:B+2NmtrRoWgfYkaqqG9Dyqud5HRjfibFpB8wbqER/PQ= -github.com/vhive-serverless/firecracker-containerd v0.0.0-20230912063208-ad6383f05e45/go.mod h1:XC5a/4PWbzipD5Ron745odZxoVy/J6d8xFldwTZJbSU= github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= diff --git a/memory/manager/manager.go b/memory/manager/manager.go index c1f9464aa..1c9a755fe 100644 --- a/memory/manager/manager.go +++ b/memory/manager/manager.go @@ -351,6 +351,35 @@ func (m *MemoryManager) GetUPFLatencyStats(vmID string) ([]*metrics.Metric, erro return state.latencyMetrics, nil } +func (m *MemoryManager) GetUPFSockPath(vmID string) (string, error) { + logger := log.WithFields(log.Fields{"vmID": vmID}) + + logger.Debug("Get the path of firecracker unix domain socket") + + m.Lock() + + state, ok := m.instances[vmID] + if !ok { + m.Unlock() + logger.Error("VM not registered with the memory manager") + return "", errors.New("VM not registered with the memory manager") + } + + m.Unlock() + + if state.isActive { + logger.Error("Cannot get stats while VM is active") + return "", errors.New("Cannot get stats while VM is active") + } + + if !m.MetricsModeOn || !state.metricsModeOn { + logger.Error("Metrics mode is not on") + return "", errors.New("Metrics mode is not on") + } + + return m.instances[vmID].SnapshotStateCfg.InstanceSockAddr, nil +} + func getLazyHeaderStats(state *SnapshotState, functionName string) ([]string, []string) { header := []string{ "FuncName", diff --git a/vhive.go b/vhive.go index 829b0eed0..d829f63be 100644 --- a/vhive.go +++ b/vhive.go @@ -91,11 +91,6 @@ func main() { return } - if *isUPFEnabled { - log.Error("User-level page faults are temporarily disabled (gh-807)") - return - } - if *isUPFEnabled && !*isSnapshotsEnabled { log.Error("User-level page faults are not supported without snapshots") return diff --git a/vhive_test.go b/vhive_test.go index 9a7947f84..abf45e327 100644 --- a/vhive_test.go +++ b/vhive_test.go @@ -65,11 +65,6 @@ func TestMain(m *testing.M) { flag.Parse() - if *isUPFEnabledTest { - log.Error("User-level page faults are temporarily disabled (gh-807)") - os.Exit(-1) - } - log.Infof("Orchestrator snapshots enabled: %t", *isSnapshotsEnabledTest) log.Infof("Orchestrator UPF enabled: %t", *isUPFEnabledTest) log.Infof("Orchestrator lazy serving mode enabled: %t", *isLazyModeTest)