feat: mount OCI-SIF overlays and support --writable

When an OCI-SIF contains an ext3 overlay layer, created with
`singularity overlay create`, it is now mounted when the container is
run.

By default, an embedded overlay is mounted read-only. In OCI-Mode we
have a `--writable-tmpfs` in place by default, so changes can still be
made to the container at runtime, but they are made in the ephemeral
tmpfs, and do not write into the overlay.

To write changes into the embedded overlay, the container must be
started with the `--writable` flag, which is now supported in OCI-Mode.

Embedded overlay layers are handled by extending the code that
previously dealt with user specified `--overlay`s.

Note that fuse2fs >=1.46.6 is required to use an embedded overlay, as
older versions do not support the `-o` (offset) flag.

Closes #2868
Closes #2869

CHANGELOG.md

@@ -20,6 +20,10 @@
   image to be pushed to `library://` and `docker://` registries in `squashfs`
   (default) or `tar` format. Images pushed with `--layer-format tar` can be
   pulled and run by other OCI runtimes.
+- A writable overlay can be added to an OCI-SIF file with the `singularity
+  overlay create` command. The overlay will be applied read-only, by default,
+  when executing the OCI-SIF. To write changes to the container into the overlay,
+  use the `--writable` flag.
 
 ### Bug Fixes
 

e2e/overlay/overlay.go

@@ -189,8 +189,10 @@ func (c ctx) testOverlayCreate(t *testing.T) {
 	}
 }
 
-func (c ctx) testOverlayCreateOCI(t *testing.T) {
-	require.Filesystem(t, "overlay")
+func (c ctx) testOverlayOCI(t *testing.T) {
+	require.Command(t, "fuse2fs")
+	require.Command(t, "fuse-overlayfs")
+	require.Command(t, "fusermount")
 	require.MkfsExt3(t)
 	e2e.EnsureOCISIF(t, c.env)
 
@@ -240,6 +242,13 @@ func (c ctx) testOverlayCreateOCI(t *testing.T) {
 	// native SIF tests above. Same code path for OCI-SIF. We don't need to
 	// repeat them here.
 	tests := []test{
+		{
+			name:    "create fail signed",
+			profile: e2e.UserProfile,
+			command: "overlay",
+			args:    []string{"create", ocisifSigned},
+			exit:    255,
+		},
 		{
 			name:    "create",
 			profile: e2e.UserProfile,
@@ -254,12 +263,81 @@ func (c ctx) testOverlayCreateOCI(t *testing.T) {
 			args:    []string{"create", ocisif},
 			exit:    255,
 		},
+		// Add a file without `--writable` - should go into ephemeral tmpfs, not the overlay.
 		{
-			name:    "create fail signed",
-			profile: e2e.UserProfile,
-			command: "overlay",
-			args:    []string{"create", ocisifSigned},
-			exit:    255,
+			name:    "tmpfs touch",
+			profile: e2e.OCIUserProfile,
+			command: "exec",
+			args:    []string{ocisif, "touch", "/in-overlay"},
+			exit:    0,
+		},
+		{
+			name:    "tmpfs check",
+			profile: e2e.OCIUserProfile,
+			command: "exec",
+			args:    []string{ocisif, "ls", "/in-overlay"},
+			exit:    1,
+		},
+
+		// Add a file to the overlay with `--writable` and check that it exists on re-run.
+		{
+			name:    "writable touch",
+			profile: e2e.OCIUserProfile,
+			command: "exec",
+			args:    []string{"--writable", ocisif, "touch", "/in-overlay"},
+			exit:    0,
+		},
+		{
+			name:    "writable touch check",
+			profile: e2e.OCIUserProfile,
+			command: "exec",
+			args:    []string{ocisif, "ls", "/in-overlay"},
+			exit:    0,
+		},
+		// Remove file without `--writable` - should be an ephemeral change, file still in overlay.
+		{
+			name:    "tmpfs rm",
+			profile: e2e.OCIUserProfile,
+			command: "exec",
+			args:    []string{ocisif, "rm", "/in-overlay"},
+			exit:    0,
+		},
+		{
+			name:    "tmpfs rm check",
+			profile: e2e.OCIUserProfile,
+			command: "exec",
+			args:    []string{ocisif, "ls", "/in-overlay"},
+			exit:    0,
+		},
+		// Remove file with `--writable` - file gone from overlay.
+		{
+			name:    "writable rm",
+			profile: e2e.OCIUserProfile,
+			command: "exec",
+			args:    []string{"--writable", ocisif, "rm", "/in-overlay"},
+			exit:    0,
+		},
+		{
+			name:    "writable rm check",
+			profile: e2e.OCIUserProfile,
+			command: "exec",
+			args:    []string{ocisif, "ls", "/in-overlay"},
+			exit:    1,
+		},
+		// Touch file without `--writable` and no tmpfs (via --no-compat)... should fail
+		{
+			name:    "readonly touch",
+			profile: e2e.OCIUserProfile,
+			command: "exec",
+			args:    []string{"--no-compat", ocisif, "touch", "/in-overlay"},
+			exit:    1,
+		},
+		{
+			name:    "readonly touch check",
+			profile: e2e.OCIUserProfile,
+			command: "exec",
+			args:    []string{ocisif, "ls", "/in-overlay"},
+			exit:    1,
 		},
 	}
 
@@ -282,7 +360,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests {
 	}
 
 	return testhelper.Tests{
-		"create":    c.testOverlayCreate,
-		"createOCI": c.testOverlayCreateOCI,
+		"create": c.testOverlayCreate,
+		"OCI":    c.testOverlayOCI,
 	}
 }

internal/app/singularity/overlay_create.go

@@ -92,7 +92,7 @@ func canAddOverlay(img *image.Image) (bool, error) {
 			return false, errOverlaySigned
 		}
 
-		hasOverlay, err := ocisif.HasOverlay(img.Path)
+		hasOverlay, _, err := ocisif.HasOverlay(img.Path)
 		if err != nil {
 			return false, fmt.Errorf("while checking for overlays: %s", err)
 		} else if hasOverlay {

internal/pkg/ocisif/overlay.go

@@ -22,47 +22,62 @@ import (
 var Ext3LayerMediaType types.MediaType = "application/vnd.sylabs.image.layer.v1.ext3"
 
 // HasOverlay returns whether the OCI-SIF at imgPath has an ext3 writable final
-// layer - an 'overlay'.
-func HasOverlay(imagePath string) (bool, error) {
+// layer - an 'overlay'. If present, the offset of the overlay data in the
+// OCI-SIF file is also returned.
+func HasOverlay(imagePath string) (bool, int64, error) {
 	fi, err := sif.LoadContainerFromPath(imagePath,
 		sif.OptLoadWithFlag(os.O_RDONLY),
 	)
 	if err != nil {
-		return false, err
+		return false, 0, err
 	}
 	defer fi.UnloadContainer()
 
 	ii, err := ocitsif.ImageIndexFromFileImage(fi)
 	if err != nil {
-		return false, fmt.Errorf("while obtaining image index: %w", err)
+		return false, 0, fmt.Errorf("while obtaining image index: %w", err)
 	}
 	ix, err := ii.IndexManifest()
 	if err != nil {
-		return false, fmt.Errorf("while obtaining index manifest: %w", err)
+		return false, 0, fmt.Errorf("while obtaining index manifest: %w", err)
 	}
 
 	// One image only.
 	if len(ix.Manifests) != 1 {
-		return false, fmt.Errorf("only single image data containers are supported, found %d images", len(ix.Manifests))
+		return false, 0, fmt.Errorf("only single image data containers are supported, found %d images", len(ix.Manifests))
 	}
 	imageDigest := ix.Manifests[0].Digest
 	img, err := ii.Image(imageDigest)
 	if err != nil {
-		return false, fmt.Errorf("while initializing image: %w", err)
+		return false, 0, fmt.Errorf("while initializing image: %w", err)
 	}
 
 	layers, err := img.Layers()
 	if err != nil {
-		return false, fmt.Errorf("while getting image layers: %w", err)
+		return false, 0, fmt.Errorf("while getting image layers: %w", err)
 	}
 	if len(layers) < 1 {
-		return false, fmt.Errorf("image has no layers")
+		return false, 0, fmt.Errorf("image has no layers")
 	}
 	mt, err := layers[len(layers)-1].MediaType()
 	if err != nil {
-		return false, fmt.Errorf("while getting layer mediatype: %w", err)
+		return false, 0, fmt.Errorf("while getting layer mediatype: %w", err)
 	}
-	return mt == Ext3LayerMediaType, nil
+	// Not an overlay as last layer
+	if mt != Ext3LayerMediaType {
+		return false, 0, nil
+	}
+
+	// Overlay as last layer, get offset
+	ld, err := layers[len(layers)-1].Digest()
+	if err != nil {
+		return false, 0, fmt.Errorf("while getting layer digest: %w", err)
+	}
+	desc, err := fi.GetDescriptor(sif.WithOCIBlobDigest(ld))
+	if err != nil {
+		return false, 0, fmt.Errorf("while getting layer descriptor: %w", err)
+	}
+	return true, desc.Offset(), nil
 }
 
 // AddOverlay adds the provided ext3 overlay file at overlayPath to the OCI-SIF

internal/pkg/ocisif/overlay_test.go

@@ -65,7 +65,7 @@ func TestHasOverlay(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			got, err := HasOverlay(imgFile)
+			got, _, err := HasOverlay(imgFile)
 
 			if got != tt.want {
 				t.Errorf("Expected %v, got %v", tt.want, got)
@@ -132,7 +132,7 @@ func TestAddOverlay(t *testing.T) {
 				t.Error("Expected error, but no error returned.")
 			}
 
-			hasOverlay, err := HasOverlay(imgFile)
+			hasOverlay, _, err := HasOverlay(imgFile)
 			if err != nil {
 				t.Fatal(err)
 			}

internal/pkg/runtime/launcher/oci/launcher_linux.go

@@ -38,7 +38,7 @@ import (
 	imgutil "github.com/sylabs/singularity/v4/pkg/image"
 	"github.com/sylabs/singularity/v4/pkg/ocibundle"
 	"github.com/sylabs/singularity/v4/pkg/ocibundle/native"
-	"github.com/sylabs/singularity/v4/pkg/ocibundle/ocisif"
+	ocisifbundle "github.com/sylabs/singularity/v4/pkg/ocibundle/ocisif"
 	sifbundle "github.com/sylabs/singularity/v4/pkg/ocibundle/sif"
 	"github.com/sylabs/singularity/v4/pkg/ocibundle/tools"
 	"github.com/sylabs/singularity/v4/pkg/sylog"
@@ -117,6 +117,10 @@ func NewLauncher(opts ...launcher.Option) (*Launcher, error) {
 	if !lo.NoCompat || lo.WritableTmpfs {
 		lo.WritableTmpfs = true
 	}
+	// Explicit writable (overlay) request means no WritableTmpfs
+	if lo.Writable {
+		lo.WritableTmpfs = false
+	}
 
 	return &Launcher{
 		cfg:                     lo,
@@ -133,10 +137,6 @@ func NewLauncher(opts ...launcher.Option) (*Launcher, error) {
 func checkOpts(lo launcher.Options) error {
 	badOpt := []string{}
 
-	if lo.Writable {
-		badOpt = append(badOpt, "Writable")
-	}
-
 	if len(lo.FuseMount) > 0 {
 		badOpt = append(badOpt, "FuseMount")
 	}
@@ -252,10 +252,10 @@ func (l *Launcher) createSpec() (spec *specs.Spec, err error) {
 	ms := minimalSpec()
 	spec = &ms
 
-	// The OCI mode always wraps the rootfs in a tmpfs.
-	// Whether we  make it writable inside the container depends on a request for `--writable-tmpfs`.
-	// Note that --writable-tmpfs is inferred by default in OCI mode. See NewLauncher().
-	spec.Root.Readonly = !l.cfg.WritableTmpfs
+	// Rootfs is writable if there is a writable tmpfs in place, or --writable
+	// is requested with an overlay in the image. Note that --writable-tmpfs is
+	// inferred by default in OCI mode. See NewLauncher().
+	spec.Root.Readonly = !l.cfg.WritableTmpfs && !l.cfg.Writable
 
 	err = addNamespaces(spec, l.cfg.Namespaces)
 	if err != nil {
@@ -722,9 +722,9 @@ func (l *Launcher) Exec(ctx context.Context, ep launcher.ExecParams) error {
 	var b ocibundle.Bundle
 	switch {
 	case strings.HasPrefix(image, "oci-sif:"):
-		b, err = ocisif.New(
-			ocisif.OptBundlePath(bundleDir),
-			ocisif.OptImageRef(image),
+		b, err = ocisifbundle.New(
+			ocisifbundle.OptBundlePath(bundleDir),
+			ocisifbundle.OptImageRef(image),
 		)
 	case strings.HasPrefix(image, "sif:"):
 		sylog.Infof("Running a non-OCI SIF in OCI mode. See user guide for compatibility information.")
@@ -832,11 +832,7 @@ func (l *Launcher) RunWrapped(ctx context.Context, containerID, bundlePath, pidF
 		return err
 	}
 
-	if len(l.cfg.OverlayPaths) > 0 {
-		return WrapWithOverlays(ctx, runFunc, absBundle, l.cfg.OverlayPaths, l.cfg.AllowSUID)
-	}
-
-	return WrapWithWritableTmpFs(ctx, runFunc, absBundle, l.cfg.AllowSUID)
+	return l.WrapWithOverlays(ctx, runFunc, absBundle)
 }
 
 // getCgroup will return a cgroup path and resources for the runtime to create.