From 8bbcd58a5f2240f5fd7f61e30be9938ad49802da Mon Sep 17 00:00:00 2001 From: Jernej Kos Date: Thu, 3 Oct 2024 13:05:37 +0200 Subject: [PATCH] feat(cmd/rofl): Add support for building TDX ROFL apps --- cmd/rofl/build/artifacts.go | 304 ++++++++++++++++++++++++++++++ cmd/rofl/build/artifacts_other.go | 17 ++ cmd/rofl/build/artifacts_unix.go | 30 +++ cmd/rofl/build/build.go | 1 + cmd/rofl/build/tdx.go | 253 +++++++++++++++++++++++++ cmd/rofl/identity.go | 11 +- go.mod | 2 +- 7 files changed, 614 insertions(+), 4 deletions(-) create mode 100644 cmd/rofl/build/artifacts.go create mode 100644 cmd/rofl/build/artifacts_other.go create mode 100644 cmd/rofl/build/artifacts_unix.go create mode 100644 cmd/rofl/build/tdx.go diff --git a/cmd/rofl/build/artifacts.go b/cmd/rofl/build/artifacts.go new file mode 100644 index 0000000..9d8e9cf --- /dev/null +++ b/cmd/rofl/build/artifacts.go @@ -0,0 +1,304 @@ +package build + +import ( + "archive/tar" + "compress/bzip2" + "crypto/sha256" + "errors" + "fmt" + "io" + "io/fs" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/adrg/xdg" + "github.com/spf13/cobra" + + "github.com/oasisprotocol/oasis-core/go/common/crypto/hash" +) + +const artifactCacheDir = "build_cache" + +// maybeDownloadArtifact downloads the given artifact and optionally verifies its integrity against +// the provided hash. +func maybeDownloadArtifact(kind, uri, knownHash string) string { + fmt.Printf("Downloading %s artifact...\n", kind) + fmt.Printf(" URI: %s\n", uri) + if knownHash != "" { + fmt.Printf(" Hash: %s\n", knownHash) + } + + url, err := url.Parse(uri) + if err != nil { + cobra.CheckErr(fmt.Errorf("failed to parse %s artifact URL: %w", kind, err)) + } + + // In case the URI represents a local file, just return it. + if url.Host == "" { + return url.Path + } + + // TODO: Prune cache. + cacheHash := hash.NewFromBytes([]byte(uri)).Hex() + cacheFn, err := xdg.CacheFile(filepath.Join("oasis", artifactCacheDir, cacheHash)) + if err != nil { + cobra.CheckErr(fmt.Errorf("failed to create cache directory for %s artifact: %w", kind, err)) + } + + f, err := os.Create(cacheFn) + if err != nil { + cobra.CheckErr(fmt.Errorf("failed to create file for %s artifact: %w", kind, err)) + } + defer f.Close() + + // Download the remote artifact. + res, err := http.Get(uri) //nolint:gosec,noctx + if err != nil { + cobra.CheckErr(fmt.Errorf("failed to download %s artifact: %w", kind, err)) + } + defer res.Body.Close() + + // Compute the SHA256 hash while downloading the artifact. + h := sha256.New() + rd := io.TeeReader(res.Body, h) + + if _, err = io.Copy(f, rd); err != nil { + cobra.CheckErr(fmt.Errorf("failed to download %s artifact: %w", kind, err)) + } + + // Verify integrity if available. + if knownHash != "" { + artifactHash := fmt.Sprintf("%x", h.Sum(nil)) + if artifactHash != knownHash { + cobra.CheckErr(fmt.Errorf("hash mismatch for %s artifact (expected: %s got: %s)", kind, knownHash, artifactHash)) + } + } + + return cacheFn +} + +// extractArchive extracts the given tar.bz2 archive into the target output directory. +func extractArchive(fn, outputDir string) error { + f, err := os.Open(fn) + if err != nil { + return fmt.Errorf("failed to open archive: %w", err) + } + defer f.Close() + + rd := tar.NewReader(bzip2.NewReader(f)) + + existingPaths := make(map[string]struct{}) + cleanupPath := func(path string) (string, error) { + // Sanitize path to ensure it doesn't escape to any parent directories. + path = filepath.Clean(filepath.Join(outputDir, path)) + if !strings.HasPrefix(path, outputDir) { + return "", fmt.Errorf("malformed path in archive") + } + return path, nil + } + + modTimes := make(map[string]time.Time) + +FILES: + for { + var header *tar.Header + header, err = rd.Next() + switch { + case errors.Is(err, io.EOF): + // We are done. + break FILES + case err != nil: + // Failed to read archive. + return fmt.Errorf("error reading archive: %w", err) + case header == nil: + // Bad archive. + return fmt.Errorf("malformed archive") + } + + var path string + path, err = cleanupPath(header.Name) + if err != nil { + return err + } + if _, ok := existingPaths[path]; ok { + continue // Make sure we never handle a path twice. + } + existingPaths[path] = struct{}{} + modTimes[path] = header.ModTime + + switch header.Typeflag { + case tar.TypeDir: + // Directory. + if err = os.MkdirAll(path, header.FileInfo().Mode()); err != nil { + return fmt.Errorf("failed to create directory: %w", err) + } + case tar.TypeLink: + // Hard link. + var linkPath string + linkPath, err = cleanupPath(header.Linkname) + if err != nil { + return err + } + + if err = os.Link(linkPath, path); err != nil { + return fmt.Errorf("failed to create hard link: %w", err) + } + case tar.TypeSymlink: + // Symbolic link. + if err = os.Symlink(header.Linkname, path); err != nil { + return fmt.Errorf("failed to create soft link: %w", err) + } + case tar.TypeChar, tar.TypeBlock, tar.TypeFifo: + // Device or FIFO node. + if err = extractHandleSpecialNode(path, header); err != nil { + return err + } + case tar.TypeReg: + // Regular file. + if err = os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return fmt.Errorf("failed to create parent directory: %w", err) + } + + var fh *os.File + fh, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, header.FileInfo().Mode()) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + if _, err = io.Copy(fh, rd); err != nil { //nolint:gosec + fh.Close() + return fmt.Errorf("failed to copy data: %w", err) + } + fh.Close() + default: + // Skip unsupported types. + continue + } + } + + // Update all modification times at the end to ensure they are correct. + for path, mtime := range modTimes { + if err = extractChtimes(path, mtime, mtime); err != nil { + return fmt.Errorf("failed to change file '%s' timestamps: %w", path, err) + } + } + + return nil +} + +// copyFile copies the file at path src to a file at path dst using the given mode. +func copyFile(src, dst string, mode os.FileMode) error { + sf, err := os.Open(src) + if err != nil { + return fmt.Errorf("failed to open '%s': %w", src, err) + } + defer sf.Close() + + df, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) + if err != nil { + return fmt.Errorf("failed to create '%s': %w", dst, err) + } + defer df.Close() + + _, err = io.Copy(df, sf) + return err +} + +// computeDirSize computes the size of the given directory. +func computeDirSize(path string) (int64, error) { + var size int64 + err := filepath.WalkDir(path, func(path string, d fs.DirEntry, derr error) error { + if derr != nil { + return derr + } + fi, err := d.Info() + if err != nil { + return err + } + size += fi.Size() + return nil + }) + if err != nil { + return 0, err + } + return size, nil +} + +// createExt4Fs creates an ext4 filesystem in the given file using directory dir to populate it. +// +// Returns the size of the created filesystem image in bytes. +func createExt4Fs(fn, dir string) (int64, error) { + // Compute filesystem size in bytes. + fsSize, err := computeDirSize(dir) + if err != nil { + return 0, err + } + fsSize /= 1024 // Convert to kilobytes. + fsSize = (fsSize * 150) / 100 // Scale by overhead factor of 1.5. + + // Execute mkfs.ext4. + cmd := exec.Command( //nolint:gosec + "mkfs.ext4", + "-E", "root_owner=0:0", + "-d", dir, + fn, + fmt.Sprintf("%dK", fsSize), + ) + var out strings.Builder + cmd.Stderr = &out + if err = cmd.Run(); err != nil { + return 0, fmt.Errorf("%w\n%s", err, out.String()) + } + + // Measure the size of the resulting image. + fi, err := os.Stat(fn) + if err != nil { + return 0, err + } + return fi.Size(), nil +} + +// createVerityHashTree creates the verity Merkle hash tree and returns the root hash. +func createVerityHashTree(fsFn, hashFn string) (string, error) { + rootHashFn := hashFn + ".roothash" + + cmd := exec.Command( //nolint:gosec + "veritysetup", "format", + "--data-block-size=4096", + "--hash-block-size=4096", + "--root-hash-file="+rootHashFn, + fsFn, + hashFn, + ) + if err := cmd.Run(); err != nil { + return "", err + } + + data, err := os.ReadFile(rootHashFn) + if err != nil { + return "", fmt.Errorf("") + } + return string(data), nil +} + +// concatFiles appends the contents of file b to a. +func concatFiles(a, b string) error { + df, err := os.OpenFile(a, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return err + } + defer df.Close() + + sf, err := os.Open(b) + if err != nil { + return err + } + defer sf.Close() + + _, err = io.Copy(df, sf) + return err +} diff --git a/cmd/rofl/build/artifacts_other.go b/cmd/rofl/build/artifacts_other.go new file mode 100644 index 0000000..bd360fd --- /dev/null +++ b/cmd/rofl/build/artifacts_other.go @@ -0,0 +1,17 @@ +//go:build !unix + +package build + +import ( + "archive/tar" + "os" + "time" +) + +func extractHandleSpecialNode(path string, header *tar.Header) error { + return nil +} + +func extractChtimes(path string, atime, mtime time.Time) error { + return os.Chtimes(path, atime, mtime) +} diff --git a/cmd/rofl/build/artifacts_unix.go b/cmd/rofl/build/artifacts_unix.go new file mode 100644 index 0000000..1c326a0 --- /dev/null +++ b/cmd/rofl/build/artifacts_unix.go @@ -0,0 +1,30 @@ +//go:build unix + +package build + +import ( + "archive/tar" + "time" + + "golang.org/x/sys/unix" +) + +func extractHandleSpecialNode(path string, header *tar.Header) error { + mode := uint32(header.Mode & 0o7777) + switch header.Typeflag { + case tar.TypeBlock: + mode |= unix.S_IFBLK + case tar.TypeChar: + mode |= unix.S_IFCHR + case tar.TypeFifo: + mode |= unix.S_IFIFO + } + + return unix.Mknod(path, mode, int(unix.Mkdev(uint32(header.Devmajor), uint32(header.Devminor)))) +} + +func extractChtimes(path string, atime, mtime time.Time) error { + atv := unix.NsecToTimeval(atime.UnixNano()) + mtv := unix.NsecToTimeval(mtime.UnixNano()) + return unix.Lutimes(path, []unix.Timeval{atv, mtv}) +} diff --git a/cmd/rofl/build/build.go b/cmd/rofl/build/build.go index 8b7e787..466497e 100644 --- a/cmd/rofl/build/build.go +++ b/cmd/rofl/build/build.go @@ -62,4 +62,5 @@ func init() { Cmd.PersistentFlags().AddFlagSet(globalFlags) Cmd.AddCommand(sgxCmd) + Cmd.AddCommand(tdxCmd) } diff --git a/cmd/rofl/build/tdx.go b/cmd/rofl/build/tdx.go new file mode 100644 index 0000000..2ce21a2 --- /dev/null +++ b/cmd/rofl/build/tdx.go @@ -0,0 +1,253 @@ +package build + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/spf13/cobra" + flag "github.com/spf13/pflag" + + "github.com/oasisprotocol/oasis-core/go/common/version" + "github.com/oasisprotocol/oasis-core/go/runtime/bundle" + "github.com/oasisprotocol/oasis-core/go/runtime/bundle/component" + + "github.com/oasisprotocol/cli/build/cargo" + "github.com/oasisprotocol/cli/cmd/common" + cliConfig "github.com/oasisprotocol/cli/config" +) + +// TODO: Replace these URIs with a better mechanism for managing releases. +const ( + artifactFirmware = "firmware" + artifactKernel = "kernel" + artifactStage2 = "stage 2 template" + + defaultFirmwareURI = "https://github.com/oasisprotocol/oasis-boot/releases/download/v0.1.0/ovmf.tdx.fd" + defaultKernelURI = "https://github.com/oasisprotocol/oasis-boot/releases/download/v0.1.0/stage1.bin" + defaultStage2TemplateURI = "https://github.com/oasisprotocol/oasis-boot/releases/download/v0.1.0/stage2-basic.tar.bz2" +) + +var knownHashes = map[string]string{ + defaultFirmwareURI: "db47100a7d6a0c1f6983be224137c3f8d7cb09b63bb1c7a5ee7829d8e994a42f", + defaultKernelURI: "b8180d547a06abbb50529266c1bef88bd99a8bf266c155e36699864162c53ecc", + defaultStage2TemplateURI: "c0b110b5749db6b1e3a281ec197111d309111809cab373f9ab01b5105bd2d239", +} + +var ( + tdxFirmwareURI string + tdxFirmwareHash string + tdxKernelURI string + tdxKernelHash string + tdxStage2TemplateURI string + tdxStage2TemplateHash string + + tdxResourcesMemory uint64 + tdxResourcesCPUCount uint8 + + tdxCmd = &cobra.Command{ + Use: "tdx", + Short: "Build a TDX-based ROFL application", + Args: cobra.NoArgs, + Run: func(cmd *cobra.Command, args []string) { + cfg := cliConfig.Global() + npa := common.GetNPASelection(cfg) + + if npa.ParaTime == nil { + cobra.CheckErr("no ParaTime selected") + } + + // Obtain required artifacts. + artifacts := make(map[string]string) + for _, ar := range []struct { + kind string + uri string + knownHash string + }{ + {artifactFirmware, tdxFirmwareURI, tdxFirmwareHash}, + {artifactKernel, tdxKernelURI, tdxKernelHash}, + {artifactStage2, tdxStage2TemplateURI, tdxStage2TemplateHash}, + } { + // Automatically populate known hashes for known URIs. + if ar.knownHash == "" { + ar.knownHash = knownHashes[ar.uri] + } + + artifacts[ar.kind] = maybeDownloadArtifact(ar.kind, ar.uri, ar.knownHash) + } + + fmt.Println("Building a TDX-based Rust ROFL application...") + + detectBuildMode(npa) + tdxSetupBuildEnv() + + // Obtain package metadata. + pkgMeta, err := cargo.GetMetadata() + if err != nil { + cobra.CheckErr(fmt.Errorf("failed to obtain package metadata: %w", err)) + } + + // Start creating the bundle early so we can fail before building anything. + bnd := &bundle.Bundle{ + Manifest: &bundle.Manifest{ + Name: pkgMeta.Name, + ID: npa.ParaTime.Namespace(), + }, + } + bnd.Manifest.Version, err = version.FromString(pkgMeta.Version) + if err != nil { + cobra.CheckErr(fmt.Errorf("unsupported package version format: %w", err)) + } + + fmt.Printf("Name: %s\n", bnd.Manifest.Name) + fmt.Printf("Version: %s\n", bnd.Manifest.Version) + + fmt.Println("Building runtime binary...") + initPath, err := cargo.Build(true, "", nil) + if err != nil { + cobra.CheckErr(fmt.Errorf("failed to build runtime binary: %w", err)) + } + + // Create temporary directory and unpack stage 2 template into it. + fmt.Println("Preparing stage 2 root filesystem...") + tmpDir, err := os.MkdirTemp("", "oasis-build-stage2") + if err != nil { + cobra.CheckErr(fmt.Errorf("failed to create temporary stage 2 build directory: %w", err)) + } + defer os.RemoveAll(tmpDir) // TODO: This doesn't work because of cobra.CheckErr + + rootfsDir := filepath.Join(tmpDir, "rootfs") + if err = os.Mkdir(rootfsDir, 0o755); err != nil { + cobra.CheckErr(fmt.Errorf("failed to create temporary rootfs directory: %w", err)) + } + + // Unpack template into temporary directory. + fmt.Println("Unpacking template...") + if err = extractArchive(artifacts[artifactStage2], rootfsDir); err != nil { + cobra.CheckErr(fmt.Errorf("failed to extract stage 2 template: %w", err)) + } + + // Add runtime as init. + fmt.Println("Adding runtime as init...") + err = copyFile(initPath, filepath.Join(rootfsDir, "init"), 0o755) + cobra.CheckErr(err) + + // Create an ext4 filesystem. + fmt.Println("Creating ext4 filesystem...") + rootfsImage := filepath.Join(tmpDir, "rootfs.ext4") + rootfsSize, err := createExt4Fs(rootfsImage, rootfsDir) + if err != nil { + cobra.CheckErr(fmt.Errorf("failed to create rootfs image: %w", err)) + } + + // Create dm-verity hash tree. + fmt.Println("Creating dm-verity hash tree...") + hashFile := filepath.Join(tmpDir, "rootfs.hash") + rootHash, err := createVerityHashTree(rootfsImage, hashFile) + if err != nil { + cobra.CheckErr(fmt.Errorf("failed to create verity hash tree: %w", err)) + } + + // Concatenate filesystem and hash tree into one image. + if err = concatFiles(rootfsImage, hashFile); err != nil { + cobra.CheckErr(fmt.Errorf("failed to concatenate rootfs and hash tree files: %w", err)) + } + + fmt.Println("Creating ORC bundle...") + + // Add the ROFL component. + firmwareName := "firmware.fd" + kernelName := "kernel.bin" + stage2Name := "stage2.img" + + comp := bundle.Component{ + Kind: component.ROFL, + Name: pkgMeta.Name, + TDX: &bundle.TDXMetadata{ + Firmware: firmwareName, + Kernel: kernelName, + Stage2Image: stage2Name, + ExtraKernelOptions: []string{ + "console=ttyS0", + fmt.Sprintf("oasis.stage2.roothash=%s", rootHash), + fmt.Sprintf("oasis.stage2.hash_offset=%d", rootfsSize), + }, + Resources: bundle.TDXResources{ + Memory: tdxResourcesMemory, + CPUCount: tdxResourcesCPUCount, + }, + }, + } + bnd.Manifest.Components = append(bnd.Manifest.Components, &comp) + + if err = bnd.Manifest.Validate(); err != nil { + cobra.CheckErr(fmt.Errorf("failed to validate manifest: %w", err)) + } + + // Add all files. + fileMap := map[string]string{ + firmwareName: artifacts[artifactFirmware], + kernelName: artifacts[artifactKernel], + stage2Name: rootfsImage, + } + for dst, src := range fileMap { + _ = bnd.Add(dst, bundle.NewFileData(src)) + } + + // Write the bundle out. + outFn := fmt.Sprintf("%s.orc", bnd.Manifest.Name) + if outputFn != "" { + outFn = outputFn + } + if err = bnd.Write(outFn); err != nil { + cobra.CheckErr(fmt.Errorf("failed to write output bundle: %w", err)) + } + + fmt.Printf("ROFL app built and bundle written to '%s'.\n", outFn) + }, + } +) + +// tdxSetupBuildEnv sets up the TDX build environment. +func tdxSetupBuildEnv() { + switch buildMode { + case buildModeProduction, buildModeAuto: + // Production builds. + fmt.Println("Building in production mode.") + + for _, kv := range os.Environ() { + key, _, _ := strings.Cut(kv, "=") + if strings.HasPrefix(key, "OASIS_UNSAFE_") { + os.Unsetenv(key) + } + } + case buildModeUnsafe: + // Unsafe debug builds. + fmt.Println("WARNING: Building in UNSAFE DEBUG mode with MOCK SGX.") + fmt.Println("WARNING: This build will NOT BE DEPLOYABLE outside local test environments.") + + os.Setenv("OASIS_UNSAFE_SKIP_AVR_VERIFY", "1") + os.Setenv("OASIS_UNSAFE_ALLOW_DEBUG_ENCLAVES", "1") + os.Unsetenv("OASIS_UNSAFE_MOCK_SGX") + os.Unsetenv("OASIS_UNSAFE_SKIP_KM_POLICY") + default: + cobra.CheckErr(fmt.Errorf("unsupported build mode: %s", buildMode)) + } +} + +func init() { + tdxFlags := flag.NewFlagSet("", flag.ContinueOnError) + tdxFlags.StringVar(&tdxFirmwareURI, "firmware", defaultFirmwareURI, "URL or path to firmware image") + tdxFlags.StringVar(&tdxFirmwareHash, "firmware-hash", "", "optional SHA256 hash of firmware image") + tdxFlags.StringVar(&tdxKernelURI, "kernel", defaultKernelURI, "URL or path to kernel image") + tdxFlags.StringVar(&tdxKernelHash, "kernel-hash", "", "optional SHA256 hash of kernel image") + tdxFlags.StringVar(&tdxStage2TemplateURI, "template", defaultStage2TemplateURI, "URL or path to stage 2 template") + tdxFlags.StringVar(&tdxStage2TemplateHash, "template-hash", "", "optional SHA256 hash of stage 2 template") + + tdxFlags.Uint64Var(&tdxResourcesMemory, "memory", 512, "required amount of VM memory in megabytes") + tdxFlags.Uint8Var(&tdxResourcesCPUCount, "cpus", 1, "required number of vCPUs") + + tdxCmd.Flags().AddFlagSet(common.SelectorNPFlags) + tdxCmd.Flags().AddFlagSet(tdxFlags) +} diff --git a/cmd/rofl/identity.go b/cmd/rofl/identity.go index 91b8b19..8cb7bbd 100644 --- a/cmd/rofl/identity.go +++ b/cmd/rofl/identity.go @@ -48,9 +48,14 @@ var ( } } - enclaveID, err = bnd.EnclaveIdentity(comp.ID()) - if err != nil { - cobra.CheckErr(fmt.Errorf("failed to generate enclave identity of '%s': %w", comp.ID(), err)) + switch teeKind := comp.TEEKind(); teeKind { + case component.TEEKindSGX: + enclaveID, err = bnd.EnclaveIdentity(comp.ID()) + if err != nil { + cobra.CheckErr(fmt.Errorf("failed to generate enclave identity of '%s': %w", comp.ID(), err)) + } + default: + cobra.CheckErr(fmt.Errorf("identity computation for TEE kind '%s' not supported", teeKind)) } data, _ := enclaveID.MarshalText() diff --git a/go.mod b/go.mod index 274137c..c909556 100644 --- a/go.mod +++ b/go.mod @@ -30,6 +30,7 @@ require ( github.com/tyler-smith/go-bip39 v1.1.0 github.com/zondax/ledger-go v0.15.0 golang.org/x/crypto v0.28.0 + golang.org/x/sys v0.26.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -139,7 +140,6 @@ require ( golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.26.0 // indirect golang.org/x/term v0.25.0 // indirect golang.org/x/text v0.19.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect