diff --git a/config/config.go b/config/config.go index 34d9d6bf23..932bafe3a8 100644 --- a/config/config.go +++ b/config/config.go @@ -107,11 +107,16 @@ const ( ) type Experimental struct { - EnableStargz bool `toml:"enable_stargz"` - EnableReferrerDetect bool `toml:"enable_referrer_detect"` - EnableTarfs bool `toml:"enable_tarfs"` - TarfsHint bool `toml:"tarfs_hint"` - TarfsMaxConcurrentProc int `toml:"tarfs_max_concurrent_proc"` + EnableStargz bool `toml:"enable_stargz"` + EnableReferrerDetect bool `toml:"enable_referrer_detect"` + TarfsConfig TarfsConfig `toml:"tarfs"` +} + +type TarfsConfig struct { + EnableTarfs bool `toml:"enable_tarfs"` + TarfsHint bool `toml:"tarfs_hint"` + MaxConcurrentProc int `toml:"max_concurrent_proc"` + ExportMode string `toml:"export_mode"` } type CgroupConfig struct { diff --git a/config/global.go b/config/global.go index 56c7ce294f..8b7cd220f5 100644 --- a/config/global.go +++ b/config/global.go @@ -111,6 +111,44 @@ func GetDaemonProfileCPUDuration() int64 { return globalConfig.origin.SystemControllerConfig.DebugConfig.ProfileDuration } +func GetTarfsExportEnabled() bool { + switch globalConfig.origin.Experimental.TarfsConfig.ExportMode { + case "layer_verity_only": + return true + case "image_verity_only": + return true + case "layer_block": + return true + case "image_block": + return true + case "layer_block_with_verity": + return true + case "image_block_with_verity": + return true + default: + return false + } +} + +func GetTarfsExportFlags() (bool, bool, bool) { + switch globalConfig.origin.Experimental.TarfsConfig.ExportMode { + case "layer_verity_only": + return false, false, true + case "image_verity_only": + return true, false, true + case "layer_block": + return false, true, false + case "image_block": + return true, true, false + case "layer_block_with_verity": + return false, true, true + case "image_block_with_verity": + return true, true, true + default: + return false, false, false + } +} + func ProcessConfigurations(c *SnapshotterConfig) error { if c.LoggingConfig.LogDir == "" { c.LoggingConfig.LogDir = filepath.Join(c.Root, logging.DefaultLogDirName) diff --git a/misc/snapshotter/config.toml b/misc/snapshotter/config.toml index dbc9728830..cc40152d8f 100644 --- a/misc/snapshotter/config.toml +++ b/misc/snapshotter/config.toml @@ -104,3 +104,19 @@ enable_stargz = false # The option enables trying to fetch the Nydus image associated with the OCI image and run it. # Also see https://github.com/opencontainers/distribution-spec/blob/main/spec.md#listing-referrers enable_referrer_detect = false +[experimental.tarfs] +# Whether to enable nydus tarfs mode +enable_tarfs = false +# Only enable nydus tarfs mode for images with `tarfs hint` label when true +tarfs_hint = false +# Maximum of concurrence to converting OCIv1 images to tarfs, 0 means default +max_concurrent_proc = 0 +# Mode to export tarfs images: +# - "none"/"" +# - "layer_verity_only" +# - "image_verity_only" +# - "layer_block" +# - "image_block" +# - "layer_block_with_verity" +# - "image_block_with_verity" +export_mode = "" diff --git a/pkg/cache/manager.go b/pkg/cache/manager.go index ca055bfdd8..33b221a296 100644 --- a/pkg/cache/manager.go +++ b/pkg/cache/manager.go @@ -21,8 +21,10 @@ import ( ) const ( - chunkMapFileSuffix = ".chunk_map" - metaFileSuffix = ".blob.meta" + imageDiskFileSuffix = ".image.disk" + layerDiskFileSuffix = ".layer.disk" + chunkMapFileSuffix = ".chunk_map" + metaFileSuffix = ".blob.meta" // Blob cache is suffixed after nydus v2.1 dataFileSuffix = ".blob.data" ) @@ -72,8 +74,10 @@ func (m *Manager) CacheUsage(ctx context.Context, blobID string) (snapshots.Usag blobCacheSuffixedPath := path.Join(m.cacheDir, blobID+dataFileSuffix) blobChunkMap := path.Join(m.cacheDir, blobID+chunkMapFileSuffix) blobMeta := path.Join(m.cacheDir, blobID+metaFileSuffix) + imageDisk := path.Join(m.cacheDir, blobID+imageDiskFileSuffix) + layerDisk := path.Join(m.cacheDir, blobID+layerDiskFileSuffix) - stuffs := []string{blobCachePath, blobCacheSuffixedPath, blobChunkMap, blobMeta} + stuffs := []string{blobCachePath, blobCacheSuffixedPath, blobChunkMap, blobMeta, imageDisk, layerDisk} for _, f := range stuffs { du, err := fs.DiskUsage(ctx, f) @@ -95,9 +99,11 @@ func (m *Manager) RemoveBlobCache(blobID string) error { blobCacheSuffixedPath := path.Join(m.cacheDir, blobID+dataFileSuffix) blobChunkMap := path.Join(m.cacheDir, blobID+chunkMapFileSuffix) blobMeta := path.Join(m.cacheDir, blobID+metaFileSuffix) + imageDisk := path.Join(m.cacheDir, blobID+imageDiskFileSuffix) + layerDisk := path.Join(m.cacheDir, blobID+layerDiskFileSuffix) // NOTE: Delete chunk bitmap file before data blob - stuffs := []string{blobChunkMap, blobMeta, blobCachePath, blobCacheSuffixedPath} + stuffs := []string{blobChunkMap, blobMeta, blobCachePath, blobCacheSuffixedPath, imageDisk, layerDisk} for _, f := range stuffs { err := os.Remove(f) diff --git a/pkg/filesystem/tarfs_adaptor.go b/pkg/filesystem/tarfs_adaptor.go index 86824e408f..45f3a328f9 100755 --- a/pkg/filesystem/tarfs_adaptor.go +++ b/pkg/filesystem/tarfs_adaptor.go @@ -49,18 +49,21 @@ func (fs *Filesystem) PrepareTarfsLayer(ctx context.Context, labels map[string]s } } - go func() { - if err := fs.tarfsMgr.PrepareLayer(snapshotID, ref, manifestDigest, layerDigest, upperDirPath); err != nil { - log.L.WithError(err).Errorf("async prepare tarfs layer of snapshot ID %s", snapshotID) - } - if limiter != nil { - limiter.Release(1) - } - }() + if err := fs.tarfsMgr.PrepareLayer(snapshotID, ref, manifestDigest, layerDigest, upperDirPath); err != nil { + log.L.WithError(err).Errorf("async prepare tarfs layer of snapshot ID %s", snapshotID) + } + if limiter != nil { + limiter.Release(1) + } return nil } +func (fs *Filesystem) ExportBlockData(s storage.Snapshot, perLayer bool, labels map[string]string, + storageLocater func(string) string) ([]string, error) { + return fs.tarfsMgr.ExportBlockData(s, perLayer, labels, storageLocater) +} + func (fs *Filesystem) MergeTarfsLayers(s storage.Snapshot, storageLocater func(string) string) error { return fs.tarfsMgr.MergeLayers(s, storageLocater) } diff --git a/pkg/label/label.go b/pkg/label/label.go index ffe960be19..b533c2ab32 100644 --- a/pkg/label/label.go +++ b/pkg/label/label.go @@ -42,6 +42,10 @@ const ( NydusImagePullUsername = "containerd.io/snapshot/pullusername" // A bool flag to enable integrity verification of meta data blob NydusSignature = "containerd.io/snapshot/nydus-signature" + // Information for image block device + NydusImageBlockInfo = "containerd.io/snapshot/nydus-image-block" + // Information for layer block device + NydusLayerBlockInfo = "containerd.io/snapshot/nydus-layer-block" // A bool flag to mark the blob as a estargz data blob, set by the snapshotter. StargzLayer = "containerd.io/snapshot/stargz" diff --git a/pkg/tarfs/tarfs.go b/pkg/tarfs/tarfs.go index 4e95d98868..f42a87b805 100755 --- a/pkg/tarfs/tarfs.go +++ b/pkg/tarfs/tarfs.go @@ -10,11 +10,13 @@ import ( "bytes" "context" "encoding/json" + "fmt" "io" "os" "os/exec" "path" "path/filepath" + "strconv" "strings" "sync" "syscall" @@ -22,6 +24,7 @@ import ( "github.com/containerd/containerd/archive/compression" "github.com/containerd/containerd/log" "github.com/containerd/containerd/snapshots/storage" + "github.com/containerd/nydus-snapshotter/config" "github.com/containerd/nydus-snapshotter/pkg/auth" "github.com/containerd/nydus-snapshotter/pkg/daemon" "github.com/containerd/nydus-snapshotter/pkg/label" @@ -61,9 +64,11 @@ const ( ) const ( - MaxManifestConfigSize = 0x100000 - TarfsLayerBootstapName = "layer.boot" - TarfsMeragedBootstapName = "image.boot" + MaxManifestConfigSize = 0x100000 + TarfsLayerBootstrapName = "layer.boot" + TarfsImageBootstrapName = "image.boot" + TarfsLayerDiskName = "layer.disk" + TarfsImageDiskName = "image.disk" ) var ErrEmptyBlob = errors.New("empty blob") @@ -337,7 +342,6 @@ func (t *Manager) PrepareLayer(snapshotID, ref string, manifestDigest, layerDige } wg := &sync.WaitGroup{} wg.Add(1) - defer wg.Done() ctx, cancel := context.WithCancel(context.Background()) t.snapshotMap[snapshotID] = &snapshotStatus{ @@ -347,36 +351,43 @@ func (t *Manager) PrepareLayer(snapshotID, ref string, manifestDigest, layerDige } t.mutex.Unlock() - layerBlobID := layerDigest.Hex() - err := t.blobProcess(ctx, snapshotID, ref, manifestDigest, layerDigest, layerBlobID, upperDirPath) + go func() { + defer wg.Done() - st, err1 := t.getSnapshotStatus(snapshotID, true) - if err1 != nil { - return errors.Errorf("can not found status object for snapshot %s after prepare", snapshotID) - } - defer st.mutex.Unlock() + layerBlobID := layerDigest.Hex() + err := t.blobProcess(ctx, snapshotID, ref, manifestDigest, layerDigest, layerBlobID, upperDirPath) - st.blobID = layerBlobID - st.blobTarFilePath = t.layerTarFilePath(layerBlobID) - if err != nil { - if errors.Is(err, ErrEmptyBlob) { - st.isEmptyBlob = true - st.status = TarfsStatusReady - err = nil + st, err1 := t.getSnapshotStatus(snapshotID, true) + if err1 != nil { + // return errors.Errorf("can not found status object for snapshot %s after prepare", snapshotID) + err1 = errors.Wrapf(err1, "can not found status object for snapshot %s after prepare", snapshotID) + log.L.WithError(err1).Errorf("async prepare tarfs layer of snapshot ID %s", snapshotID) + return + } + defer st.mutex.Unlock() + + st.blobID = layerBlobID + st.blobTarFilePath = t.layerTarFilePath(layerBlobID) + if err != nil { + if errors.Is(err, ErrEmptyBlob) { + st.isEmptyBlob = true + st.status = TarfsStatusReady + } else { + log.L.WithError(err).Errorf("failed to convert OCI image to tarfs") + st.status = TarfsStatusFailed + } } else { - st.status = TarfsStatusFailed + st.isEmptyBlob = false + st.status = TarfsStatusReady } - } else { - st.isEmptyBlob = false - st.status = TarfsStatusReady - } - log.L.Debugf("finish converting snapshot %s to tarfs, status %d, empty blob %v", snapshotID, st.status, st.isEmptyBlob) + log.L.Debugf("finish converting snapshot %s to tarfs, status %d, empty blob %v", snapshotID, st.status, st.isEmptyBlob) + }() - return err + return nil } func (t *Manager) MergeLayers(s storage.Snapshot, storageLocater func(string) string) error { - mergedBootstrap := t.mergedMetaFilePath(storageLocater(s.ParentIDs[0])) + mergedBootstrap := t.imageMetaFilePath(storageLocater(s.ParentIDs[0])) if _, err := os.Stat(mergedBootstrap); err == nil { log.L.Debugf("tarfs snapshot %s already has merged bootstrap %s", s.ParentIDs[0], mergedBootstrap) return nil @@ -429,6 +440,102 @@ func (t *Manager) MergeLayers(s storage.Snapshot, storageLocater func(string) st return nil } +func (t *Manager) ExportBlockData(s storage.Snapshot, perLayer bool, labels map[string]string, storageLocater func(string) string) ([]string, error) { + updateFields := []string{} + + wholeImage, exportDisk, withVerity := config.GetTarfsExportFlags() + // Nothing to do for this case, all needed datum are ready. + if !exportDisk && !withVerity { + return updateFields, nil + } else if !wholeImage != perLayer { + return updateFields, nil + } + + var snapshotID string + if perLayer { + snapshotID = s.ID + } else { + if len(s.ParentIDs) == 0 { + return updateFields, errors.Errorf("snapshot %s has parent", s.ID) + } + snapshotID = s.ParentIDs[0] + } + err := t.waitLayerReady(snapshotID) + if err != nil { + return updateFields, errors.Wrapf(err, "wait for tarfs snapshot %s to get ready", snapshotID) + } + st, err := t.getSnapshotStatus(snapshotID, false) + if err != nil { + return updateFields, err + } + if st.status != TarfsStatusReady { + return updateFields, errors.Errorf("tarfs snapshot %s is not ready, %d", snapshotID, st.status) + } + + var metaFileName, diskFileName string + if wholeImage { + metaFileName = t.imageMetaFilePath(storageLocater(snapshotID)) + diskFileName = t.imageDiskFilePath(st.blobID) + } else { + metaFileName = t.layerMetaFilePath(storageLocater(snapshotID)) + diskFileName = t.layerDiskFilePath(st.blobID) + } + + // Do not regenerate if the disk image already exists. + if _, err := os.Stat(diskFileName); err == nil { + return updateFields, nil + } + diskFileNameTmp := diskFileName + ".tarfs.tmp" + defer os.Remove(diskFileNameTmp) + + options := []string{ + "export", + "--block", + "--localfs-dir", t.cacheDirPath, + "--bootstrap", metaFileName, + "--output", diskFileNameTmp, + } + if withVerity { + options = append(options, "--verity") + } + log.L.Warnf("nydus image command %v", options) + cmd := exec.Command(t.nydusImagePath, options...) + var errb, outb bytes.Buffer + cmd.Stderr = &errb + cmd.Stdout = &outb + err = cmd.Run() + if err != nil { + return updateFields, errors.Wrap(err, "merge tarfs image layers") + } + log.L.Debugf("nydus image export command, stdout: %s, stderr: %s", &outb, &errb) + + if withVerity { + pattern := "dm-verity options: --no-superblock --format=1 -s \"\" --hash=sha256 --data-block-size=512 --hash-block-size=4096 --data-blocks %d --hash-offset %d %s\n" + var dataBlobks, hashOffset uint64 + var rootHash string + if count, err := fmt.Sscanf(outb.String(), pattern, &dataBlobks, &hashOffset, &rootHash); err != nil || count != 3 { + return updateFields, errors.Errorf("failed to parse dm-verity options from nydus image output: %s", outb.String()) + } + + blockInfo := strconv.FormatUint(dataBlobks, 10) + "," + strconv.FormatUint(hashOffset, 10) + "," + "sha256:" + rootHash + if wholeImage { + labels[label.NydusImageBlockInfo] = blockInfo + updateFields = append(updateFields, "labels."+label.NydusImageBlockInfo) + } else { + labels[label.NydusLayerBlockInfo] = blockInfo + updateFields = append(updateFields, "labels."+label.NydusLayerBlockInfo) + } + log.L.Warnf("export block labels %v", labels) + } + + err = os.Rename(diskFileNameTmp, diskFileName) + if err != nil { + return updateFields, errors.Wrap(err, "rename disk image file") + } + + return updateFields, nil +} + func (t *Manager) attachLoopdev(blob string) (*losetup.Device, error) { // losetup.Attach() is not thread-safe hold lock here t.mutexLoopDev.Lock() @@ -493,7 +600,7 @@ func (t *Manager) MountTarErofs(snapshotID string, s *storage.Snapshot, rafs *da if st.metaLoopdev == nil { upperDirPath := path.Join(rafs.GetSnapshotDir(), "fs") - mergedBootstrap := t.mergedMetaFilePath(upperDirPath) + mergedBootstrap := t.imageMetaFilePath(upperDirPath) loopdev, err := t.attachLoopdev(mergedBootstrap) if err != nil { return errors.Wrapf(err, "attach merged bootstrap %s to loopdev", mergedBootstrap) @@ -541,6 +648,9 @@ func (t *Manager) waitLayerReady(snapshotID string) error { log.L.Debugf("wait tarfs conversion task for snapshot %s", snapshotID) } st.wg.Wait() + if st.status != TarfsStatusReady { + return errors.Errorf("snapshot %s is in state %d instead of ready state", snapshotID, st.status) + } return nil } @@ -670,10 +780,18 @@ func (t *Manager) layerTarFilePath(blobID string) string { return filepath.Join(t.cacheDirPath, blobID) } +func (t *Manager) layerDiskFilePath(blobID string) string { + return filepath.Join(t.cacheDirPath, blobID+"."+TarfsLayerDiskName) +} + +func (t *Manager) imageDiskFilePath(blobID string) string { + return filepath.Join(t.cacheDirPath, blobID+"."+TarfsImageDiskName) +} + func (t *Manager) layerMetaFilePath(upperDirPath string) string { - return filepath.Join(upperDirPath, "image", TarfsLayerBootstapName) + return filepath.Join(upperDirPath, "image", TarfsLayerBootstrapName) } -func (t *Manager) mergedMetaFilePath(upperDirPath string) string { - return filepath.Join(upperDirPath, "image", TarfsMeragedBootstapName) +func (t *Manager) imageMetaFilePath(upperDirPath string) string { + return filepath.Join(upperDirPath, "image", TarfsImageBootstrapName) } diff --git a/snapshot/process.go b/snapshot/process.go index 60c76c3325..56075d0143 100644 --- a/snapshot/process.go +++ b/snapshot/process.go @@ -16,6 +16,7 @@ import ( "github.com/containerd/containerd/mount" snpkg "github.com/containerd/containerd/pkg/snapshotters" "github.com/containerd/containerd/snapshots/storage" + "github.com/containerd/nydus-snapshotter/config" "github.com/containerd/nydus-snapshotter/pkg/label" "github.com/containerd/nydus-snapshotter/pkg/snapshot" ) @@ -93,6 +94,12 @@ func chooseProcessor(ctx context.Context, logger *logrus.Entry, logger.Warnf("snapshot ID %s can't be converted into tarfs, fallback to containerd, err: %v", s.ID, err) } else { logger.Debugf("convert OCIv1 layer to tarfs") + if config.GetTarfsExportEnabled() { + _, err = sn.fs.ExportBlockData(s, true, labels, func(id string) string { return sn.upperPath(id) }) + if err != nil { + return nil, "", errors.Wrap(err, "export layer as tarfs block device") + } + } labels[label.NydusTarfsLayer] = "true" handler = skipHandler } @@ -151,6 +158,12 @@ func chooseProcessor(ctx context.Context, logger *logrus.Entry, if err != nil { return nil, "", errors.Wrap(err, "merge tarfs layers") } + if config.GetTarfsExportEnabled() { + _, err = sn.fs.ExportBlockData(s, false, labels, func(id string) string { return sn.upperPath(id) }) + if err != nil { + return nil, "", errors.Wrap(err, "export image as tarfs block device") + } + } handler = remoteHandler(id, pInfo.Labels) } } diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index 90b73b6ace..cd2a22504f 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -205,12 +205,12 @@ func NewSnapshotter(ctx context.Context, cfg *config.SnapshotterConfig) (snapsho opts = append(opts, filesystem.WithReferrerManager(referrerMgr)) } - if cfg.Experimental.EnableTarfs { + if cfg.Experimental.TarfsConfig.EnableTarfs { // FIXME: get the insecure option from nydusd config. _, backendConfig := daemonConfig.StorageBackend() - tarfsMgr := tarfs.NewManager(backendConfig.SkipVerify, cfg.Experimental.TarfsHint, + tarfsMgr := tarfs.NewManager(backendConfig.SkipVerify, cfg.Experimental.TarfsConfig.TarfsHint, cacheConfig.CacheDir, cfg.DaemonConfig.NydusImagePath, - int64(cfg.Experimental.TarfsMaxConcurrentProc)) + int64(cfg.Experimental.TarfsConfig.MaxConcurrentProc)) opts = append(opts, filesystem.WithTarfsManager(tarfsMgr)) } @@ -344,6 +344,8 @@ func (o *snapshotter) Usage(ctx context.Context, key string) (snapshots.Usage, e usage.Add(cacheUsage) } } + case snapshots.KindUnknown: + case snapshots.KindView: } return usage, nil @@ -406,6 +408,8 @@ func (o *snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, er return nil, errors.Wrapf(err, "get parent snapshot info, parent key=%q", pKey) } } + case snapshots.KindCommitted: + case snapshots.KindUnknown: } if o.fs.ReferrerDetectEnabled() && !needRemoteMounts { @@ -508,6 +512,18 @@ func (o *snapshotter) View(ctx context.Context, key, parent string, opts ...snap if err := o.fs.MergeTarfsLayers(s, func(id string) string { return o.upperPath(id) }); err != nil { return nil, errors.Wrapf(err, "tarfs merge fail %s", pID) } + if config.GetTarfsExportEnabled() { + updateFields, err := o.fs.ExportBlockData(s, false, pInfo.Labels, func(id string) string { return o.upperPath(id) }) + if err != nil { + return nil, errors.Wrap(err, "export tarfs as block image") + } + if len(updateFields) > 0 { + _, err = o.Update(ctx, pInfo, updateFields...) + if err != nil { + return nil, errors.Wrapf(err, "update snapshot label information") + } + } + } if err := o.fs.Mount(pID, pInfo.Labels, &s); err != nil { return nil, errors.Wrapf(err, "mount tarfs, snapshot id %s", pID) }