Skip to content

Commit

Permalink
Update ctriface modue
Browse files Browse the repository at this point in the history 
Signed-off-by: char-1ee <[email protected]>
  • Loading branch information
@char-1ee
char-1ee committed Feb 1, 2024
1 parent 2ab9059 commit 66c5f0c
Show file tree
Hide file tree
Showing 10 changed files with 193 additions and 118 deletions.
6 changes: 3 additions & 3 deletions cri/firecracker/coordinator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,14 @@ import (
"context"
"errors"
"fmt"
"github.com/google/uuid"
"github.com/vhive-serverless/vhive/snapshotting"
"strconv"
"sync"
"sync/atomic"
"time"

"github.com/google/uuid"
"github.com/vhive-serverless/vhive/snapshotting"

log "github.com/sirupsen/logrus"
"github.com/vhive-serverless/vhive/ctriface"
)
Expand Down Expand Up @@ -181,7 +182,6 @@ func (c *coordinator) orchLoadInstance(ctx context.Context, snap *snapshotting.S
ctxTimeout, cancel := context.WithTimeout(ctx, time.Second*30)
defer cancel()

logger.Debug("FIXME: temp pass same lastVmID")
resp, _, err := c.orch.LoadSnapshot(ctxTimeout, vmID, vmID, snap)
if err != nil {
logger.WithError(err).Error("failed to load VM")
Expand Down
6 changes: 6 additions & 0 deletions ctriface/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,12 @@ WITHLAZY:=-lazy
GOBENCH:=-v -timeout 1500s
CTRDLOGDIR:=/tmp/ctrd-logs

debug:
./../scripts/clean_fcctr.sh
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log.out 2>$(CTRDLOGDIR)/ctriface_log.err &
sudo env "PATH=$(PATH)" go test $(EXTRATESTFILES) $(EXTRAGOARGS) -args $(WITHUPF)
./../scripts/clean_fcctr.sh

test:
./../scripts/clean_fcctr.sh
sudo mkdir -m777 -p $(CTRDLOGDIR) && sudo env "PATH=$(PATH)" /usr/local/bin/firecracker-containerd --config /etc/firecracker-containerd/config.toml 1>$(CTRDLOGDIR)/ctriface_log.out 2>$(CTRDLOGDIR)/ctriface_log.err &
Expand Down
79 changes: 33 additions & 46 deletions ctriface/failing_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,62 +22,49 @@

package ctriface

import (
"context"
"os"
"testing"
"time"
// func TestStartSnapStop(t *testing.T) {
// // BROKEN BECAUSE StopVM does not work yet.
// // t.Skip("skipping failing test")
// log.SetFormatter(&log.TextFormatter{
// TimestampFormat: ctrdlog.RFC3339NanoFixed,
// FullTimestamp: true,
// })
// //log.SetReportCaller(true) // FIXME: make sure it's false unless debugging

ctrdlog "github.com/containerd/containerd/log"
"github.com/containerd/containerd/namespaces"
log "github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
"github.com/vhive-serverless/vhive/snapshotting"
)
// log.SetOutput(os.Stdout)

func TestStartSnapStop(t *testing.T) {
// BROKEN BECAUSE StopVM does not work yet.
t.Skip("skipping failing test")
log.SetFormatter(&log.TextFormatter{
TimestampFormat: ctrdlog.RFC3339NanoFixed,
FullTimestamp: true,
})
//log.SetReportCaller(true) // FIXME: make sure it's false unless debugging
// log.SetLevel(log.DebugLevel)

log.SetOutput(os.Stdout)
// testTimeout := 120 * time.Second
// ctx, cancel := context.WithTimeout(namespaces.WithNamespace(context.Background(), namespaceName), testTimeout)
// defer cancel()

log.SetLevel(log.DebugLevel)
// orch := NewOrchestrator("devmapper", "", WithTestModeOn(true))

testTimeout := 120 * time.Second
ctx, cancel := context.WithTimeout(namespaces.WithNamespace(context.Background(), namespaceName), testTimeout)
defer cancel()
// vmID := "2"

orch := NewOrchestrator("devmapper", "", WithTestModeOn(true))
// _, _, err := orch.StartVM(ctx, vmID, testImageName)
// require.NoError(t, err, "Failed to start VM")

vmID := "2"
// err = orch.PauseVM(ctx, vmID)
// require.NoError(t, err, "Failed to pause VM")

_, _, err := orch.StartVM(ctx, vmID, testImageName)
require.NoError(t, err, "Failed to start VM")
// snap := snapshotting.NewSnapshot(vmID, "/fccd/snapshots", testImageName)
// err = orch.CreateSnapshot(ctx, vmID, snap)
// require.NoError(t, err, "Failed to create snapshot of VM")

err = orch.PauseVM(ctx, vmID)
require.NoError(t, err, "Failed to pause VM")
// err = orch.StopSingleVM(ctx, vmID)
// require.NoError(t, err, "Failed to stop VM")

snap := snapshotting.NewSnapshot(vmID, "/fccd/snapshots", testImageName)
err = orch.CreateSnapshot(ctx, vmID, snap)
require.NoError(t, err, "Failed to create snapshot of VM")
// _, _, err = orch.LoadSnapshot(ctx, "1", vmID, snap)
// require.NoError(t, err, "Failed to load snapshot of VM")

err = orch.StopSingleVM(ctx, vmID)
require.NoError(t, err, "Failed to stop VM")
// _, err = orch.ResumeVM(ctx, vmID)
// require.NoError(t, err, "Failed to resume VM")

_, _, err = orch.LoadSnapshot(ctx, "1", vmID, snap)
require.NoError(t, err, "Failed to load snapshot of VM")
// err = orch.StopSingleVM(ctx, vmID)
// require.NoError(t, err, "Failed to stop VM")

_, err = orch.ResumeVM(ctx, vmID)
require.NoError(t, err, "Failed to resume VM")

err = orch.StopSingleVM(ctx, vmID)
require.NoError(t, err, "Failed to stop VM")

_ = snap.Cleanup()
orch.Cleanup()
}
// _ = snap.Cleanup()
// orch.Cleanup()
// }
29 changes: 13 additions & 16 deletions ctriface/iface.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ package ctriface

import (
"context"
"github.com/vhive-serverless/vhive/snapshotting"
"os"
"os/exec"
"path/filepath"
Expand All @@ -33,6 +32,8 @@ import (
"syscall"
"time"

"github.com/vhive-serverless/vhive/snapshotting"

log "github.com/sirupsen/logrus"

"github.com/containerd/containerd"

Check failure on line 39 in ctriface/iface.go

View workflow job for this annotation

GitHub Actions / Unit test (misc) 

github.com/firecracker-microvm/[email protected] (replaced by ../firecracker-containerd): reading ../firecracker-containerd/go.mod: open /home/runner/work/vHive/firecracker-containerd/go.mod: no such file or directory

Check failure on line 39 in ctriface/iface.go

View workflow job for this annotation

GitHub Actions / Unit test (networking) 

github.com/firecracker-microvm/[email protected] (replaced by ../firecracker-containerd): reading ../firecracker-containerd/go.mod: open /home/runner/work/vHive/firecracker-containerd/go.mod: no such file or directory

Check failure on line 39 in ctriface/iface.go

View workflow job for this annotation

GitHub Actions / Unit test (snapshotting) 

github.com/firecracker-microvm/[email protected] (replaced by ../firecracker-containerd): reading ../firecracker-containerd/go.mod: open /home/runner/work/vHive/firecracker-containerd/go.mod: no such file or directory

Check failure on line 39 in ctriface/iface.go

View workflow job for this annotation

GitHub Actions / Unit test: profile unit test (profile) 

github.com/firecracker-microvm/[email protected] (replaced by ../firecracker-containerd): reading ../firecracker-containerd/go.mod: open /root/actions-runner/_work/vHive/firecracker-containerd/go.mod: no such file or directory

Check failure on line 39 in ctriface/iface.go

View workflow job for this annotation

GitHub Actions / Integration tests (test) 

github.com/firecracker-microvm/[email protected] (replaced by ../firecracker-containerd): reading ../firecracker-containerd/go.mod: open /root/actions-runner/_work/vHive/firecracker-containerd/go.mod: no such file or directory

Check failure on line 39 in ctriface/iface.go

View workflow job for this annotation

GitHub Actions / Integration tests (test-man-bench) 

github.com/firecracker-microvm/[email protected] (replaced by ../firecracker-containerd): reading ../firecracker-containerd/go.mod: open /root/actions-runner/_work/vHive/firecracker-containerd/go.mod: no such file or directory

Check failure on line 39 in ctriface/iface.go

View workflow job for this annotation

GitHub Actions / Unit tests: Firecracker-containerd interface (ctriface) 

github.com/firecracker-microvm/[email protected] (replaced by ../firecracker-containerd): reading ../firecracker-containerd/go.mod: open /root/actions-runner/_work/vHive/firecracker-containerd/go.mod: no such file or directory

Check failure on line 39 in ctriface/iface.go

View workflow job for this annotation

GitHub Actions / Unit tests: Firecracker-containerd interface (ctriface/image) 

github.com/firecracker-microvm/[email protected] (replaced by ../firecracker-containerd): reading ../firecracker-containerd/go.mod: open /root/actions-runner/_work/vHive/firecracker-containerd/go.mod: no such file or directory

Check failure on line 39 in ctriface/iface.go

View workflow job for this annotation

GitHub Actions / Unit tests: Firecracker-containerd interface (devmapper) 

github.com/firecracker-microvm/[email protected] (replaced by ../firecracker-containerd): reading ../firecracker-containerd/go.mod: open /root/actions-runner/_work/vHive/firecracker-containerd/go.mod: no such file or directory
Expand Down Expand Up @@ -106,7 +107,7 @@ func (o *Orchestrator) StartVMWithEnvironment(ctx context.Context, vmID, imageNa

tStart = time.Now()
conf := o.getVMConfig(vm)
resp, err := o.fcClient.CreateVM(ctx, conf)
_, err = o.fcClient.CreateVM(ctx, conf)
startVMMetric.MetricMap[metrics.FcCreateVM] = metrics.ToUS(time.Since(tStart))
if err != nil {
return nil, nil, errors.Wrap(err, "failed to create the microVM in firecracker-containerd")
Expand Down Expand Up @@ -216,11 +217,10 @@ func (o *Orchestrator) StartVMWithEnvironment(ctx context.Context, vmID, imageNa
IsLazyMode: o.isLazyMode,
VMMStatePath: o.getSnapshotFile(vmID),
WorkingSetPath: o.getWorkingSetFile(vmID),
InstanceSockAddr: resp.GetSocketPath(),
InstanceSockAddr: o.uffdSockAddr,
}
logger.Debugf("TEST: show to-reg snapStat: %+v", stateCfg)
logger.Debugf("TEST: show socket path: %s", resp.GetSocketPath())
if err := o.memoryManager.RegisterVM(stateCfg, false, ""); err != nil {
if err := o.memoryManager.RegisterVM(stateCfg); err != nil {
return nil, nil, errors.Wrap(err, "failed to register VM with memory manager")
// NOTE (Plamen): Potentially need a defer(DeregisteVM) here if RegisterVM is not last to execute
}
Expand Down Expand Up @@ -451,7 +451,7 @@ func (o *Orchestrator) CreateSnapshot(ctx context.Context, vmID string, snap *sn
}

// LoadSnapshot Loads a snapshot of a VM
func (o *Orchestrator) LoadSnapshot(ctx context.Context, lastVmID string, vmID string, snap *snapshotting.Snapshot) (_ *StartVMResponse, _ *metrics.Metric, retErr error) {
func (o *Orchestrator) LoadSnapshot(ctx context.Context, originVmID string, vmID string, snap *snapshotting.Snapshot) (_ *StartVMResponse, _ *metrics.Metric, retErr error) {
var (
loadSnapshotMetric *metrics.Metric = metrics.NewMetric()
tStart time.Time
Expand Down Expand Up @@ -499,32 +499,31 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, lastVmID string, vmID s
conf.LoadSnapshot = true
conf.SnapshotPath = snap.GetSnapshotFilePath()
conf.ContainerSnapshotPath = containerSnap.GetDevicePath()
conf.MemBackend = &proto.MemoryBackend {
conf.MemBackend = &proto.MemoryBackend{
BackendType: fileBackend,
BackendPath: snap.GetMemFilePath(),
}

if o.GetUPFEnabled() {
logger.Debug("TEST: UPF is enabled")
conf.MemBackend.BackendType = uffdBackend
conf.MemBackend.BackendPath, err = o.memoryManager.GetUPFSockPath(lastVmID, true)
conf.MemBackend.BackendPath = o.uffdSockAddr
logger.Debugf("TEST: the upf socket: %s", conf.MemBackend.BackendPath)
if err != nil {
return nil, nil, errors.Wrapf(err, "failed to get UPF socket path for uffd backend")
}

if err := o.memoryManager.FetchState(lastVmID); err != nil {
if err := o.memoryManager.FetchState(originVmID); err != nil {
return nil, nil, err
}
}

tStart = time.Now()
newUPFSockPath := ""

go func() {
defer close(loadDone)

resp, loadErr := o.fcClient.CreateVM(ctx, conf)
_, loadErr := o.fcClient.CreateVM(ctx, conf)
if loadErr != nil {
logger.Error("Failed to load snapshot of the VM: ", loadErr)
logger.Errorf("snapFilePath: %s, memFilePath: %s, newSnapshotPath: %s", snap.GetSnapshotFilePath(), snap.GetMemFilePath(), containerSnap.GetDevicePath())
Expand All @@ -551,12 +550,11 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, lastVmID string, vmID s
}
logger.Error(snapFiles)
}
newUPFSockPath = resp.GetSocketPath()
}()

logger.Debug("TEST: CreatVM request sent")
if o.GetUPFEnabled() {

logger.Debug("TEST: Registering VM with the memory manager")

stateCfg := manager.SnapshotStateCfg{
Expand All @@ -567,13 +565,12 @@ func (o *Orchestrator) LoadSnapshot(ctx context.Context, lastVmID string, vmID s
IsLazyMode: o.isLazyMode,
VMMStatePath: o.getSnapshotFile(vmID),
WorkingSetPath: o.getWorkingSetFile(vmID),
InstanceSockAddr: newUPFSockPath,
InstanceSockAddr: o.uffdSockAddr,
}
if err := o.memoryManager.RegisterVM(stateCfg, true, vmID); err != nil {
if err := o.memoryManager.RegisterVM(stateCfg); err != nil {
logger.Error(err, "failed to register new VM with memory manager")
}


if activateErr = o.memoryManager.Activate(vmID); activateErr != nil {
logger.Warn("Failed to activate VM in the memory manager", activateErr)
}
Expand Down
50 changes: 50 additions & 0 deletions ctriface/iface_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,56 @@ func TestMain(m *testing.M) {
os.Exit(m.Run())
}

// Test for ctriface uffd feature
func TestStartSnapStop(t *testing.T) {
// BROKEN BECAUSE StopVM does not work yet.
// t.Skip("skipping failing test")
log.SetFormatter(&log.TextFormatter{
TimestampFormat: ctrdlog.RFC3339NanoFixed,
FullTimestamp: true,
})
//log.SetReportCaller(true) // FIXME: make sure it's false unless debugging

log.SetOutput(os.Stdout)

log.SetLevel(log.DebugLevel)

testTimeout := 120 * time.Second
ctx, cancel := context.WithTimeout(namespaces.WithNamespace(context.Background(), namespaceName), testTimeout)
defer cancel()

orch := NewOrchestrator("devmapper", "", WithTestModeOn(true))

vmID := "2"

_, _, err := orch.StartVM(ctx, vmID, testImageName)
require.NoError(t, err, "Failed to start VM")

err = orch.PauseVM(ctx, vmID)
require.NoError(t, err, "Failed to pause VM")

snap := snapshotting.NewSnapshot(vmID, "/fccd/snapshots", testImageName)
err = orch.CreateSnapshot(ctx, vmID, snap)
require.NoError(t, err, "Failed to create snapshot of VM")

err = orch.StopSingleVM(ctx, vmID)
require.NoError(t, err, "Failed to stop VM")

_, _, err = orch.LoadSnapshot(ctx, "1", vmID, snap)
require.NoError(t, err, "Failed to load snapshot of VM")

_, err = orch.ResumeVM(ctx, vmID)
require.NoError(t, err, "Failed to resume VM")

time.Sleep(30 * time.Second)

err = orch.StopSingleVM(ctx, vmID)
require.NoError(t, err, "Failed to stop VM")

_ = snap.Cleanup()
orch.Cleanup()
}

func TestPauseSnapResume(t *testing.T) {
log.SetFormatter(&log.TextFormatter{
TimestampFormat: ctrdlog.RFC3339NanoFixed,
Expand Down
8 changes: 4 additions & 4 deletions ctriface/manual_cleanup_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,11 +234,11 @@ func TestParallelSnapLoad(t *testing.T) {
err = orch.StopSingleVM(ctx, vmID)
require.NoError(t, err, "Failed to offload VM, "+vmID)

lastVmID := vmID
originVmID := vmID
vmIDInt, _ := strconv.Atoi(vmID)
vmID = strconv.Itoa(vmIDInt + 1)

_, _, err = orch.LoadSnapshot(ctx, lastVmID, vmID, snap)
_, _, err = orch.LoadSnapshot(ctx, originVmID, vmID, snap)
require.NoError(t, err, "Failed to load snapshot of VM, "+vmID)

_, err = orch.ResumeVM(ctx, vmID)
Expand Down Expand Up @@ -358,10 +358,10 @@ func TestParallelPhasedSnapLoad(t *testing.T) {
defer vmGroup.Done()
vmID := fmt.Sprintf("%d", i+vmIDBase)
snap := snapshotting.NewSnapshot(vmID, "/fccd/snapshots", testImageName)
lastVmID := vmID
originVmID := vmID
vmIDInt, _ := strconv.Atoi(vmID)
vmID = strconv.Itoa(vmIDInt + 1)
_, _, err := orch.LoadSnapshot(ctx, lastVmID, vmID, snap)
_, _, err := orch.LoadSnapshot(ctx, originVmID, vmID, snap)
require.NoError(t, err, "Failed to load snapshot of VM, "+vmID)
}(i)
}
Expand Down
14 changes: 14 additions & 0 deletions ctriface/orch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ type Orchestrator struct {
isUPFEnabled bool
isLazyMode bool
snapshotsDir string
uffdSockAddr string
isMetricsMode bool
netPoolSize int

Expand Down Expand Up @@ -121,8 +122,16 @@ func NewOrchestrator(snapshotter, hostIface string, opts ...OrchestratorOption)
}

if o.GetUPFEnabled() {
// o.uffdSockAddr = "/tmp/uffd.sock" // "/tmp/uffd/firecracker-containerd#3-0/uffd.sock"
_, err = os.Create("/tmp/uffd.sock")

if err != nil {
log.Fatal("TEST: failed to create uffd sock", err)
}

managerCfg := manager.MemoryManagerCfg{
MetricsModeOn: o.isMetricsMode,
UffdSockAddr: o.uffdSockAddr,
}
o.memoryManager = manager.NewMemoryManager(managerCfg)
}
Expand Down Expand Up @@ -208,6 +217,11 @@ func (o *Orchestrator) GetSnapshotsDir() string {
return o.snapshotsDir
}

// TODO: /tmp/uffd/firecracker-containerd#3-0/uffd.sock
func (o *Orchestrator) getUffdSockAddr(vmID string) string {
return filepath.Join(o.getVMBaseDir(vmID), "uffd.sock")
}

func (o *Orchestrator) getSnapshotFile(vmID string) string {
return filepath.Join(o.getVMBaseDir(vmID), "snap_file")
}
Expand Down
Loading

0 comments on commit 66c5f0c

Please sign in to comment.