Merge pull request #41 from AkihiroSuda/dev

add HTTP cache (~/Library/Caches/lima)

Author: AkihiroSuda

cmd/limactl/main.go

@@ -46,6 +46,7 @@ func newApp() *cli.App {
 		shellCommand,
 		lsCommand,
 		deleteCommand,
+		pruneCommand,
 		completionCommand,
 	}
 	return app

cmd/limactl/prune.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+var pruneCommand = &cli.Command{
+	Name:   "prune",
+	Usage:  "Prune garbage objects",
+	Action: pruneAction,
+}
+
+func pruneAction(clicontext *cli.Context) error {
+	ucd, err := os.UserCacheDir()
+	if err != nil {
+		return err
+	}
+	cacheDir := filepath.Join(ucd, "lima")
+	logrus.Infof("Pruning %q", cacheDir)
+	return os.RemoveAll(cacheDir)
+}

docs/internal.md

@@ -13,3 +13,11 @@ An instance directory contains the following files:
 - `ga.sock`: Forwarded to `/run/user/$UID/lima-guestagent.sock`
 - `serial.log`: QEMU serial log, for debugging
 - `serial.sock`: QEMU serial socket, for debugging (Usage: `socat -,echo=0,icanon=0 unix-connect:serial.sock`)
+
+
+## Cache directory (`~/Library/Caches/lima/download/by-url-sha256/<SHA256_OF_URL>`)
+
+The directory contains the following files:
+
+- `url`: raw url text, without "\n"
+- `data`: data

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/AkihiroSuda/sshocker v0.1.1-0.20210510144941-56aa3c7472b0
 	github.com/alessio/shellescape v1.4.1
 	github.com/containerd/containerd v1.5.0
+	github.com/containerd/continuity v0.1.0
 	github.com/diskfs/go-diskfs v1.1.2-0.20210512141858-8a6b8b88d14a
 	github.com/docker/go-units v0.4.0
 	github.com/gorilla/mux v1.8.0

go.sum

@@ -164,6 +164,7 @@ github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL
 github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo=
 github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y=
 github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
+github.com/containerd/continuity v0.1.0 h1:UFRRY5JemiAhPZrr/uE0n8fMTLcZsUvySPr1+D7pgr8=
 github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=
 github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
 github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
@@ -524,6 +525,7 @@ github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1
 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
@@ -812,6 +814,7 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

pkg/cidata/cidata.go

@@ -2,20 +2,26 @@ package cidata
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"io/fs"
+	"io/ioutil"
 	"os"
 	"os/user"
 	"path/filepath"
 	"strconv"
 
+	"github.com/AkihiroSuda/lima/pkg/downloader"
 	"github.com/AkihiroSuda/lima/pkg/iso9660util"
 	"github.com/AkihiroSuda/lima/pkg/limayaml"
 	"github.com/AkihiroSuda/lima/pkg/localpathutil"
 	"github.com/AkihiroSuda/lima/pkg/sshutil"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
+const NerdctlVersion = "0.8.3"
+
 func GenerateISO9660(isoPath, name string, y *limayaml.LimaYAML) error {
 	if err := limayaml.ValidateRaw(*y); err != nil {
 		return err
@@ -81,6 +87,50 @@ func GenerateISO9660(isoPath, name string, y *limayaml.LimaYAML) error {
 		})
 	}
 
+	if args.Containerd.System || args.Containerd.User {
+		var nftgzBase string
+		switch y.Arch {
+		case limayaml.X8664:
+			nftgzBase = fmt.Sprintf("nerdctl-full-%s-linux-amd64.tar.gz", NerdctlVersion)
+		case limayaml.AARCH64:
+			nftgzBase = fmt.Sprintf("nerdctl-full-%s-linux-arm64.tar.gz", NerdctlVersion)
+		default:
+			return errors.Errorf("unexpected arch %q", y.Arch)
+		}
+		td, err := ioutil.TempDir("", "lima-download-nerdctl")
+		if err != nil {
+			return err
+		}
+		defer os.RemoveAll(td)
+		nftgzLocal := filepath.Join(td, nftgzBase)
+		nftgzURL := fmt.Sprintf("https://github.com/containerd/nerdctl/releases/download/v%s/%s",
+			NerdctlVersion, nftgzBase)
+		logrus.Infof("Downloading %q", nftgzURL)
+		res, err := downloader.Download(nftgzLocal, nftgzURL, downloader.WithCache())
+		if err != nil {
+			return errors.Wrapf(err, "failed to download %q", nftgzURL)
+		}
+		switch res.Status {
+		case downloader.StatusDownloaded:
+			logrus.Infof("Downloaded %q", nftgzBase)
+		case downloader.StatusUsedCache:
+			logrus.Infof("Using cache %q", res.CachePath)
+		default:
+			logrus.Warnf("Unexpected result from downloader.Download(): %+v", res)
+		}
+		// TODO: verify sha256
+		nftgzR, err := os.Open(nftgzLocal)
+		if err != nil {
+			return err
+		}
+		defer nftgzR.Close()
+		layout = append(layout, iso9660util.Entry{
+			// ISO9660 requires len(Path) <= 30
+			Path:   "nerdctl-full.tgz",
+			Reader: nftgzR,
+		})
+	}
+
 	return iso9660util.Write(isoPath, "cidata", layout)
 }
 

pkg/cidata/user-data.TEMPLATE

@@ -101,12 +101,10 @@ write_files:
       #!/bin/bash
       set -eux -o pipefail
       if [ ! -x /usr/local/bin/nerdctl ]; then
-        version="0.8.3"
-        goarch="amd64"
-        if [ "$(uname -m )" = "aarch64" ]; then
-          goarch="arm64"
-        fi
-        curl -fsSL https://github.com/containerd/nerdctl/releases/download/v${version}/nerdctl-full-${version}-linux-${goarch}.tar.gz | tar Cxz /usr/local
+        mkdir -p -m 600 /mnt/lima-cidata
+        mount -t iso9660 -o ro /dev/disk/by-label/cidata /mnt/lima-cidata
+        tar Cxzf /usr/local /mnt/lima-cidata/nerdctl-full.tgz
+        umount /mnt/lima-cidata
       fi
       {{- if .Containerd.System}}
       cat >"/etc/containerd/config.toml" <<EOF

pkg/downloader/downloader.go

@@ -0,0 +1,191 @@
+package downloader
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+
+	"github.com/AkihiroSuda/lima/pkg/localpathutil"
+	"github.com/containerd/continuity/fs"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+type Status = string
+
+const (
+	StatusUnknown    Status = ""
+	StatusDownloaded Status = "downloaded"
+	StatusSkipped    Status = "skipped"
+	StatusUsedCache  Status = "used-cache"
+)
+
+type Result struct {
+	Status    Status
+	CachePath string // "/Users/foo/Library/Caches/lima/download/by-url-sha256/<SHA256_OF_URL>/data"
+}
+
+type options struct {
+	cacheDir string // default: empty (disables caching)
+}
+
+type Opt func(*options) error
+
+// WithCache enables caching using filepath.Join(os.UserCacheDir(), "lima") as the cache dir.
+func WithCache() Opt {
+	return func(o *options) error {
+		ucd, err := os.UserCacheDir()
+		if err != nil {
+			return err
+		}
+		cacheDir := filepath.Join(ucd, "lima")
+		return WithCacheDir(cacheDir)(o)
+	}
+}
+
+// WithCacheDir enables caching using the specified dir.
+// Empty value disables caching.
+func WithCacheDir(cacheDir string) Opt {
+	return func(o *options) error {
+		o.cacheDir = cacheDir
+		return nil
+	}
+}
+
+func Download(local, remote string, opts ...Opt) (*Result, error) {
+	var o options
+	for _, f := range opts {
+		if err := f(&o); err != nil {
+			return nil, err
+		}
+	}
+	localPath, err := localPath(local)
+	if err != nil {
+		return nil, err
+	}
+	if _, err := os.Stat(localPath); err == nil {
+		logrus.Debugf("file %q already exists, skipping downloading from %q", localPath, remote)
+		res := &Result{
+			Status: StatusSkipped,
+		}
+		return res, nil
+	} else if !errors.Is(err, os.ErrNotExist) {
+		return nil, err
+	}
+
+	localPathDir := filepath.Dir(localPath)
+	if err := os.MkdirAll(localPathDir, 0755); err != nil {
+		return nil, err
+	}
+
+	if isLocal(remote) {
+		if err := copyLocal(localPath, remote); err != nil {
+			return nil, err
+		}
+		res := &Result{
+			Status: StatusDownloaded,
+		}
+		return res, nil
+	}
+
+	if o.cacheDir == "" {
+		if err := downloadHTTP(localPath, remote); err != nil {
+			return nil, err
+		}
+		res := &Result{
+			Status: StatusDownloaded,
+		}
+		return res, nil
+	}
+
+	shad := filepath.Join(o.cacheDir, "download", "by-url-sha256", fmt.Sprintf("%x", sha256.Sum256([]byte(remote))))
+	shadData := filepath.Join(shad, "data")
+	if _, err := os.Stat(shadData); err == nil {
+		logrus.Debugf("file %q is cached as %q", localPath, shadData)
+		if err := copyLocal(localPath, shadData); err != nil {
+			return nil, err
+		}
+		res := &Result{
+			Status:    StatusUsedCache,
+			CachePath: shadData,
+		}
+		return res, nil
+	}
+	if err := os.RemoveAll(shad); err != nil {
+		return nil, err
+	}
+	if err := os.MkdirAll(shad, 0700); err != nil {
+		return nil, err
+	}
+	shadURL := filepath.Join(shad, "url")
+	if err := os.WriteFile(shadURL, []byte(remote), 0644); err != nil {
+		return nil, err
+	}
+	if err := downloadHTTP(shadData, remote); err != nil {
+		return nil, err
+	}
+	if err := copyLocal(localPath, shadData); err != nil {
+		return nil, err
+	}
+
+	res := &Result{
+		Status:    StatusDownloaded,
+		CachePath: shadData,
+	}
+	return res, nil
+}
+
+func isLocal(s string) bool {
+	return !strings.Contains(s, "://") || strings.HasPrefix(s, "file://")
+}
+
+func localPath(s string) (string, error) {
+	if !isLocal(s) {
+		return "", errors.Errorf("got non-local path: %q", s)
+	}
+	if strings.HasPrefix(s, "file://") {
+		res := strings.TrimPrefix(s, "file://")
+		if !filepath.IsAbs(res) {
+			return "", errors.Errorf("got non-absolute path %q", res)
+		}
+		return res, nil
+	}
+	return localpathutil.Expand(s)
+}
+
+func copyLocal(dst, src string) error {
+	srcPath, err := localPath(src)
+	if err != nil {
+		return err
+	}
+	dstPath, err := localPath(dst)
+	if err != nil {
+		return err
+	}
+	return fs.CopyFile(dstPath, srcPath)
+}
+
+func downloadHTTP(localPath, url string) error {
+	logrus.Debugf("downloading %q into %q", url, localPath)
+	localPathTmp := localPath + ".tmp"
+	if err := os.RemoveAll(localPathTmp); err != nil {
+		return err
+	}
+	// use curl for printing progress
+	cmd := exec.Command("curl", "-fSL", "-o", localPathTmp, url)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return errors.Wrapf(err, "failed to run %v", cmd.Args)
+	}
+	if err := os.RemoveAll(localPath); err != nil {
+		return err
+	}
+	if err := os.Rename(localPathTmp, localPath); err != nil {
+		return err
+	}
+	return nil
+}