diff --git a/.gitignore b/.gitignore index a221ac1..069af35 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ target/ .vscode/ +.cache/ diff --git a/Cargo.lock b/Cargo.lock index d003c87..460582c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -622,6 +622,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.4.2" @@ -1241,6 +1256,40 @@ dependencies = [ "tonic-build", ] +[[package]] +name = "gevulot-rs" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc328be91b31fbc8dd8c82b0d959f2a9c0349e35297f59d661436540597491f2" +dependencies = [ + "backon", + "bip32", + "bytesize", + "const_format", + "cosmos-sdk-proto", + "cosmrs", + "derivative", + "derive_builder", + "hex", + "http 1.1.0", + "humantime", + "log", + "pretty_env_logger", + "prost 0.13.3", + "prost-build", + "rand 0.8.5", + "rand_core 0.6.4", + "semver", + "serde", + "serde_json", + "tendermint", + "thiserror 1.0.64", + "tokio", + "tonic", + "tonic-buf-build", + "tonic-build", +] + [[package]] name = "gimli" version = "0.31.1" @@ -1289,18 +1338,20 @@ version = "0.1.3" dependencies = [ "anyhow", "backhand", + "base64 0.22.1", "bip32", "bytesize", "cargo_metadata 0.19.1", "clap 4.5.20", "clap_complete", "cosmrs", + "crc", "directories", "env_logger 0.11.5", "fatfs", "fs_extra", "fscommon", - "gevulot-rs", + "gevulot-rs 0.1.3", "libz-sys", "log", "mbrman", @@ -1899,14 +1950,14 @@ checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "mia-installer" -version = "0.2.5" -source = "git+https://github.com/gevulotnetwork/mia.git?tag=mia-installer-0.2.5#2733a55e3290db99daaa5def9f4e77215df6c2fd" +version = "0.3.0" +source = "git+https://github.com/gevulotnetwork/mia.git?tag=mia-installer-0.3.0#537af9fd9823a061b1b9e07ceecb5827a57e1798" dependencies = [ "anyhow", "env_logger 0.11.5", "flate2", "fs_extra", - "gevulot-rs", + "gevulot-rs 0.2.0", "log", "octocrab", "reqwest", diff --git a/Cargo.toml b/Cargo.toml index 099d96a..8e56482 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ toml = "0.8.19" openssl = { version = "*", optional = true } # Linux VM builder dependencies -mia-installer = { git = "https://github.com/gevulotnetwork/mia.git", tag = "mia-installer-0.2.5" } +mia-installer = { git = "https://github.com/gevulotnetwork/mia.git", tag = "mia-installer-0.3.0" } anyhow = "1" log = "0.4.22" @@ -38,7 +38,9 @@ thiserror = "1" # Linux VM builder v2 dependencies backhand = { version = "0.20", optional = true, default-features = false, features = ["xz"] } +base64 = { version = "0.22", optional = true } bytesize = { version = "1", optional = true } +crc = { version = "3", optional = true } directories = { version = "5", optional = true } fatfs = { version = "0.3", optional = true } fscommon = { version = "0.1", optional = true } @@ -60,7 +62,9 @@ openssl-vendored = ["openssl/vendored"] # Use new version of Linux VM builder (unstable) vm-builder-v2 = [ "dep:backhand", + "dep:base64", "dep:bytesize", + "dep:crc", "dep:directories", "dep:fatfs", "dep:fscommon", diff --git a/src/builders_v2/linux_vm/image_file.rs b/src/builders_v2/linux_vm/image_file.rs index c96ecfb..c3c8073 100644 --- a/src/builders_v2/linux_vm/image_file.rs +++ b/src/builders_v2/linux_vm/image_file.rs @@ -181,8 +181,11 @@ pub struct UseImageFile; impl Step for UseImageFile { fn run(&mut self, ctx: &mut LinuxVMBuildContext) -> Result<()> { - let base_image_path = ctx.cache().join("base.img"); - if !base_image_path.exists() { + let crc_instance = crc::Crc::::new(&crc::CRC_64_ECMA_182); + let checksum = format!("{:x}", crc_instance.checksum(BASE_IMAGE)); + debug!("base image checksum: {}", &checksum); + let base_image_path = ctx.cache().join(format!("{}.base.img", checksum)); + if !base_image_path.is_file() { info!("creating base image file: {}", base_image_path.display()); let mut file = fs::File::create_new(&base_image_path) .context("failed to create base image file")?; diff --git a/src/builders_v2/linux_vm/kernel.rs b/src/builders_v2/linux_vm/kernel.rs index de08486..22ff830 100644 --- a/src/builders_v2/linux_vm/kernel.rs +++ b/src/builders_v2/linux_vm/kernel.rs @@ -1,4 +1,5 @@ use anyhow::{Context, Result}; +use base64::Engine; use bytesize::ByteSize; use log::{debug, info, trace}; use std::ffi::OsStr; @@ -43,6 +44,9 @@ pub enum Kernel { /// Size of the kernel file. size: u64, + + /// Kernel release string (e.g. 6.12.0). + kernel_release: String, }, } @@ -66,6 +70,7 @@ impl Kernel { } /// Return path to sources if some. + #[allow(unused)] pub fn source_path(&self) -> Option<&Path> { match self { Self::Precompiled { .. } => None, @@ -73,6 +78,14 @@ impl Kernel { } } + /// Kernel release string if some. + pub fn kernel_release(&self) -> Option<&str> { + match self { + Kernel::Precompiled { .. } => None, + Kernel::Sources { kernel_release, .. } => Some(kernel_release), + } + } + /// Size of kernel binary. pub fn size(&self) -> u64 { match self { @@ -82,16 +95,15 @@ impl Kernel { } /// Whether kernel was precompiled or not. + #[allow(unused)] pub fn is_precompiled(&self) -> bool { matches!(self, Self::Precompiled { .. }) } // TODO: use this function. // TODO: maybe use libgit instead of executable? - /// Clone Linux kernel repository into `path/version` returning path to resulting directory. - #[allow(unused)] - fn clone(git_url: &str, version: &str, path: &Path) -> Result { - let target_path = path.join(version); + /// Clone Linux kernel repository into `path`. + fn clone(git_url: &str, version: &str, path: &Path) -> Result<()> { let mut command = vec![ OsStr::new("git"), OsStr::new("clone"), @@ -103,145 +115,120 @@ impl Kernel { command.push(OsStr::new(version)); } command.push(OsStr::new(git_url)); - command.push(target_path.as_os_str()); + command.push(path.as_os_str()); run_command(&command).context("failed to clone kernel repository")?; - Ok(target_path) + Ok(()) + } + + /// Configure kernel. + /// + /// Assumes that CWD is kernel directory. + fn configure() -> Result<()> { + run_command(&["make", "x86_64_defconfig"]).context("Failed to configure kernel")?; + Self::configure_squashfs()?; + Ok(()) + } + + /// Configure SquashFS support in kernel. + /// + /// Assumes that CWD is kernel directory. + fn configure_squashfs() -> Result<()> { + // TODO: ensure stability of this configuration. + // Kernel options can be removed/renamed. This configuration is used for v6.12, + // but it may not work with other versions. + + const ENABLE: &[&str] = &[ + "CONFIG_SQUASHFS", + "CONFIG_SQUASHFS_FILE_DIRECT", + "CONFIG_SQUASHFS_DECOMP_SINGLE", + "CONFIG_SQUASHFS_DECOMP_MULTI", + "CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU", + "CONFIG_SQUASHFS_CHOICE_DECOMP_BY_MOUNT", + "CONFIG_SQUASHFS_MOUNT_DECOMP_THREADS", + "CONFIG_SQUASHFS_XATTR", + "CONFIG_SQUASHFS_ZLIB", + "CONFIG_SQUASHFS_LZ4", + "CONFIG_SQUASHFS_LZO", + "CONFIG_SQUASHFS_XZ", + "CONFIG_SQUASHFS_ZSTD", + "CONFIG_SQUASHFS_4K_DEVBLK_SIZE", + ]; + + const DISABLE: &[&str] = &["CONFIG_SQUASHFS_FILE_CACHE", "CONFIG_SQUASHFS_EMBEDDED"]; + + const SET_VAL: &[(&str, &str)] = &[("3", "CONFIG_SQUASHFS_FRAGMENT_CACHE_SIZE")]; + + for flag in ENABLE { + run_command(&["scripts/config", "--enable", flag]) + .context(format!("failed to enable {} flag to kernel config", flag))?; + } + + for flag in DISABLE { + run_command(&["scripts/config", "--disable", flag]) + .context(format!("failed to disable {} flag to kernel config", flag))?; + } + + for (val, flag) in SET_VAL { + run_command(&["scripts/config", "--set-val", val, flag]) + .context(format!("failed to set {} value to kernel config", flag))?; + } + + Ok(()) } /// Build kernel from sources. - pub fn build(git_url: &str, version: &str) -> Result { + pub fn build(git_url: &str, version: &str, kernel_dir: &Path) -> Result { // TODO: check required tools are available: git, make, gcc - let home_dir = std::env::var("HOME").context("Failed to get HOME environment variable")?; - let kernel_dir = format!("{}/.linux-builds/{}", home_dir, version); - let bzimage_path = format!("{}/arch/x86/boot/bzImage", kernel_dir); - // Check if the bzImage already exists - if Path::new(&bzimage_path).exists() { - debug!("Kernel bzImage already exists, skipping build"); - } else { - // Clone the specific version from the remote repository - - // Check if the kernel directory already exists - if Path::new(&kernel_dir).exists() { - // If it exists, do a git pull - debug!("Kernel directory already exists"); - } else { - debug!("Clonings kernel sources"); - // If it doesn't exist, clone the repository - let clone_args = if version == "latest" { - vec!["git", "clone", "--depth", "1", git_url, &kernel_dir] - } else { - vec![ - "git", - "clone", - "--depth", - "1", - "--branch", - version, - git_url, - &kernel_dir, - ] - }; - run_command(&clone_args).context("Failed to clone kernel repository")?; - } - - debug!("Building sources"); - let current_dir = std::env::current_dir().context("Failed to get current directory")?; - std::env::set_current_dir(&kernel_dir) - .context("Failed to change to kernel directory")?; - - // Configure the kernel - run_command(&["make", "x86_64_defconfig"]).context("Failed to configure kernel")?; - - // SQUASHFS support - run_command(&["scripts/config", "--enable", "CONFIG_SQUASHFS"]) - .context("Failed to enable CONFIG_SQUASHFS flag to kernel config ")?; - - run_command(&["scripts/config", "--disable", "CONFIG_SQUASHFS_FILE_CACHE"]) - .context("Failed to disable CONFIG_SQUASHFS_FILE_CACHE flag to kernel config ")?; - - run_command(&["scripts/config", "--enable", "CONFIG_SQUASHFS_FILE_DIRECT"]) - .context("Failed to enable CONFIG_SQUASHFS_FILE_DIRECT flag to kernel config")?; - - run_command(&[ - "scripts/config", - "--enable", - "CONFIG_SQUASHFS_DECOMP_SINGLE", - ]) - .context("Failed to enable CONFIG_SQUASHFS_DECOMP_SINGLE flag to kernel config")?; - - run_command(&["scripts/config", "--enable", "CONFIG_SQUASHFS_DECOMP_MULTI"]) - .context("Failed to enable CONFIG_SQUASHFS_DECOMP_MULTI flag to kernel config")?; - - run_command(&[ - "scripts/config", - "--enable", - "CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU", - ]) - .context( - "Failed to enable CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU flag to kernel config", - )?; - - run_command(&[ - "scripts/config", - "--enable", - "CONFIG_SQUASHFS_CHOICE_DECOMP_BY_MOUNT", - ]) - .context( - "Failed to enable CONFIG_SQUASHFS_CHOICE_DECOMP_BY_MOUNT flag to kernel config", - )?; - run_command(&[ - "scripts/config", - "--enable", - "CONFIG_SQUASHFS_MOUNT_DECOMP_THREADS", - ]) - .context( - "Failed to enable CONFIG_SQUASHFS_MOUNT_DECOMP_THREADS flag to kernel config", - )?; - run_command(&["scripts/config", "--enable", "CONFIG_SQUASHFS_XATTR"]) - .context("Failed to enable CONFIG_SQUASHFS_XATTR flag to kernel config")?; - run_command(&["scripts/config", "--enable", "CONFIG_SQUASHFS_ZLIB"]) - .context("Failed to enable CONFIG_SQUASHFS_ZLIB flag to kernel config")?; - run_command(&["scripts/config", "--enable", "CONFIG_SQUASHFS_LZ4"]) - .context("Failed to enable CONFIG_SQUASHFS_LZ4 flag to kernel config")?; - run_command(&["scripts/config", "--enable", "CONFIG_SQUASHFS_LZO"]) - .context("Failed to enable CONFIG_SQUASHFS_LZO flag to kernel config")?; - run_command(&["scripts/config", "--enable", "CONFIG_SQUASHFS_XZ"]) - .context("Failed to enable CONFIG_SQUASHFS_XZ flag to kernel config")?; - run_command(&["scripts/config", "--enable", "CONFIG_SQUASHFS_ZSTD"]) - .context("Failed to enable CONFIG_SQUASHFS_ZSTD flag to kernel config")?; - run_command(&[ - "scripts/config", - "--enable", - "CONFIG_SQUASHFS_4K_DEVBLK_SIZE", - ]) - .context("Failed to enable CONFIG_SQUASHFS_4K_DEVBLK_SIZE flag to kernel config")?; - run_command(&[ - "scripts/config", - "--set-val", - "3", - "CONFIG_SQUASHFS_FRAGMENT_CACHE_SIZE", - ]) - .context("Failed to set CONFIG_SQUASHFS_FRAGMENT_CACHE_SIZE value to kernel config")?; - run_command(&["scripts/config", "--disable", "CONFIG_SQUASHFS_EMBEDDED"]) - .context("Failed to disable CONFIG_SQUASHFS_EMBEDDED flag to kernel config ")?; - - // Build the kernel - run_command(&["make", &format!("-j{}", num_cpus::get())]) - .context("Failed to build kernel")?; - - std::env::set_current_dir(current_dir) - .context("Failed to change back to original directory")?; - } + Self::clone(git_url, version, kernel_dir)?; + + debug!("Building sources"); + let current_dir = std::env::current_dir().context("Failed to get current directory")?; + std::env::set_current_dir(&kernel_dir).context("Failed to change to kernel directory")?; + + Self::configure()?; + + // Build the kernel + run_command(["make", &format!("-j{}", num_cpus::get())]) + .context("Failed to build kernel")?; + + std::env::set_current_dir(current_dir) + .context("Failed to change back to original directory")?; + + Self::use_existing(git_url, version, kernel_dir) + } + + /// Use existing kernel compiled from sources. + pub fn use_existing(git_url: &str, version: &str, kernel_dir: &Path) -> Result { + let current_dir = std::env::current_dir().context("Failed to get current directory")?; + std::env::set_current_dir(&kernel_dir).context("Failed to change to kernel directory")?; + + let kernel_release = run_command(["make", "-s", "kernelrelease"]) + .context("failed to get kernel release string")? + .0 + .trim() + .to_string(); + + let bzimage_path = kernel_dir.join( + run_command(["make", "-s", "image_name"]) + .context("failed to get kernel image name")? + .0 + .trim(), + ); + debug!("kernel file: {}", bzimage_path.display()); + + std::env::set_current_dir(current_dir) + .context("Failed to change back to original directory")?; let metadata = fs::metadata(&bzimage_path).context("get kernel file metadata")?; Ok(Self::Sources { git_url: git_url.to_string(), version: version.to_string(), - source_path: PathBuf::from(&kernel_dir), + source_path: kernel_dir.to_path_buf(), path: PathBuf::from(&bzimage_path), size: metadata.len(), + kernel_release, }) } @@ -274,11 +261,30 @@ impl Build { impl Step for Build { fn run(&mut self, ctx: &mut LinuxVMBuildContext) -> Result<()> { info!("building Linux kernel"); - let kernel = Kernel::build(&self.repository_url, &self.version) - .context("failed to build Linux kernel")?; + let build_dir = ctx + .cache() + .join("linux-build") + .join(base64::prelude::BASE64_URL_SAFE_NO_PAD.encode(&self.repository_url)) + .join(&self.version); + debug!("kernel build directory: {}", build_dir.display()); + + // Check if the cache entry (kernel directory) already exists + let kernel = if build_dir.is_dir() { + debug!("using cached kernel"); + Kernel::use_existing(&self.repository_url, &self.version, &build_dir) + .context("failed to use kernel from cache")? + } else { + debug!("Clonings kernel sources"); + fs::create_dir_all(&build_dir).context("failed to create kernel directory")?; + Kernel::build(&self.repository_url, &self.version, &build_dir) + .context("failed to build Linux kernel")? + }; + info!( "kernel ready: {} ({})", - &kernel, + kernel + .kernel_release() + .expect("kernel is expected to be compiled from sources"), ByteSize::b(kernel.size()).to_string_as(true) ); ctx.set("kernel", Box::new(kernel)); diff --git a/src/builders_v2/linux_vm/mia.rs b/src/builders_v2/linux_vm/mia.rs index 88d77aa..36fa73a 100644 --- a/src/builders_v2/linux_vm/mia.rs +++ b/src/builders_v2/linux_vm/mia.rs @@ -1,12 +1,15 @@ -use anyhow::{Context, Result}; +use anyhow::{anyhow, Context, Result}; use log::info; use mia_installer::{runtime_config, RuntimeConfig}; +use std::fs; use std::path::PathBuf; use crate::builders::Step; use super::LinuxVMBuildContext; +const MIA_PLATFORM: &str = "x86_64-unknown-linux-gnu"; + /// Install MIA into root filesystem. /// /// # Context variables required @@ -109,8 +112,49 @@ impl Step for InstallMia { }; let mut install_config = mia_installer::InstallConfig::default(); - install_config.mia_version = self.version.to_string(); - install_config.mia_platform = "x86_64-unknown-linux-gnu".to_string(); + + let version = if self.version.starts_with("file:") { + self.version.clone() + } else { + // Resolve 'latest' to a concrete version + let version = if &self.version == "latest" { + mia_installer::sync::latest_version() + .context("failed to detect latest MIA version")? + } else { + self.version.clone() + }; + + // MIA executable is cached in CACHE/mia//mia- + let cache_dir = ctx.cache().join("mia").join(MIA_PLATFORM); + if !cache_dir.is_dir() { + fs::create_dir_all(&cache_dir).context("failed to create MIA cache dir")?; + } + let cache_entry = cache_dir.join(format!("mia-{}", &version)); + + // Ensure MIA is cached + if !cache_entry.exists() { + info!("downloading MIA ({})", &version); + mia_installer::sync::download(&version, MIA_PLATFORM, &cache_entry) + .context("failed to download MIA (latest)")?; + } else { + info!("using cached MIA ({})", &version); + } + + // Install MIA from local file in cache + format!( + "file:{}", + cache_entry + .as_os_str() + .to_str() + .ok_or(anyhow!("failed to handle MIA cache path: not UTF-8"))? + ) + }; + + // At this point `version` will always point to a local file: user-provided or cached + debug_assert!(version.starts_with("file:")); + + install_config.mia_version = version; + install_config.mia_platform = MIA_PLATFORM.to_string(); let rootfs = ctx.get::("root-fs").expect("root-fs"); @@ -121,7 +165,6 @@ impl Step for InstallMia { install_config.rt_config = Some(rt_config); - // TODO: add 'fetch' option to utilize cache and avoid re-downloading mia_installer::install(&install_config).context("failed to install MIA")?; Ok(()) diff --git a/src/builders_v2/linux_vm/mod.rs b/src/builders_v2/linux_vm/mod.rs index 3a6d759..6fcf5ad 100644 --- a/src/builders_v2/linux_vm/mod.rs +++ b/src/builders_v2/linux_vm/mod.rs @@ -193,6 +193,9 @@ pub struct BuildOpts { /// Mount root filesystem as read-write. pub rw_root: bool, + /// Cache directory. + pub cache_dir: Option, + /// Generate only base (template) of VM image. /// /// This image will include bootloader, partition table and a single partition with filesystem. @@ -216,26 +219,32 @@ impl LinuxVMBuildContext { /// - `cache` - general cache directory (for Linux builds and etc.) pub fn from_opts(opts: BuildOpts) -> Result { let mut ctx = Context::new(); - ctx.set("opts", Box::new(opts)); let tmp = TempDir::new("linux-vm-build").context("failed to create temporary directory")?; log::debug!("temp directory: {} (removed on exit)", tmp.path().display()); ctx.set("tmp", Box::new(tmp)); - let project_dirs = ProjectDirs::from("", "gevulot", "gvltctl"); - // Normally it will be `$HOME/.cache/gvltctl` on Linux - // or `$HOME/Library/Caches/gevulot.gvltctl` on MacOS - let cache_path = project_dirs - .map(|dirs| dirs.cache_dir().to_path_buf()) - .unwrap_or(PathBuf::from(".cache")); - if !cache_path.is_dir() { - fs::create_dir_all(&cache_path).context(format!( - "failed to create cache directory: {}", - cache_path.display() - ))?; - } - log::debug!("cache directory: {}", cache_path.display()); - ctx.set("cache", Box::new(cache_path)); + let cache_dir = if let Some(cache_dir) = &opts.cache_dir { + cache_dir.clone() + } else { + let project_dirs = ProjectDirs::from("", "gevulot", "gvltctl"); + // Normally it will be `$HOME/.cache/gvltctl` on Linux + // or `$HOME/Library/Caches/gevulot.gvltctl` on MacOS + let cache_dir = project_dirs + .map(|dirs| dirs.cache_dir().to_path_buf()) + .unwrap_or(PathBuf::from(".cache")); + if !cache_dir.is_dir() { + fs::create_dir_all(&cache_dir).context(format!( + "failed to create cache directory: {}", + cache_dir.display() + ))?; + } + cache_dir + }; + + log::debug!("cache directory: {}", cache_dir.display()); + ctx.set("cache", Box::new(cache_dir)); + ctx.set("opts", Box::new(opts)); Ok(Self(ctx)) } @@ -315,7 +324,6 @@ fn setup_pipeline(ctx: &mut LinuxVMBuildContext) -> Pipeline = Vec::new(); - // TODO: build artifacts should go to cache match &ctx.opts().kernel_opts { KernelOpts::Precompiled { file } => { steps.push(Box::new(kernel::Precompiled::new(file.clone()))); @@ -357,7 +365,6 @@ fn setup_pipeline(ctx: &mut LinuxVMBuildContext) -> Pipeline Self { + Self { + target_dir, + kernel_release, + } + } + /// Path to drivers directory. pub fn path(&self) -> &Path { - self.target_dir.path() + self.target_dir.as_path() + } + + /// Kernel release string. + pub fn kernel_release(&self) -> &str { + self.kernel_release.as_str() } /// Build and dump drivers. - pub fn build>(kernel_source_dir: P) -> Result { + pub fn build>( + kernel_source_dir: P, + kernel_release: String, + target_dir: PathBuf, + ) -> Result { // TIP: We could do extra initial checks here: // - Checking for kernel source access permissions. // - Checking for dangling symlinks that will not be accessible insider container volume mount. // - Checking for host requierements. - // We need to know kernel name in order to identify location of driver files. - debug!("Extracting kernel version."); - let kernel_release = read_kernel_release(&kernel_source_dir)?; - info!("Detected kernel: '{}'.", kernel_release); - // Copy kernel sources, as we do NOT want them to be directly modified by NVIDIA installer. debug!("Copying kernel source."); let kernel_source_copy = copy_kernel_source(&kernel_source_dir)?; - // Drivers tree will be dumped to temporary directory. - debug!("Preparing target directory."); - let target_dir = TempDir::new("linux-vm-nvidia-drivers").map_err(NvidiaError::Io)?; - // Run container for compiling and dumping NVIDIA drivers. // NOTE: We use it to reduce number of host dependencies **only**, not for isolation. debug!("Running container that prepares custom drivers."); @@ -76,13 +82,13 @@ impl NvidiaDriversFs { return Ok(Self { target_dir, - kernel_release: kernel_release.to_string(), + kernel_release, }); } /// Get size of all files to install. pub fn size(&self) -> Result { - fs_extra::dir::get_size(self.target_dir.path()).map_err(Into::into) + fs_extra::dir::get_size(self.path()).map_err(Into::into) } /// Install drivers returning list of names of installed drivers. @@ -127,23 +133,6 @@ impl NvidiaDriversFs { } } -fn read_kernel_release>(kernel_source_dir: P) -> Result { - let kernel_release_path = kernel_source_dir - .as_ref() - .join("include/config/kernel.release"); - let kernel_release_content = fs::read_to_string(kernel_release_path.clone()).map_err(|e| { - NvidiaError::KernelReleaseRead { - path: kernel_release_path, - source: e, - } - })?; - let kernel_release = kernel_release_content.trim().to_string(); - - info!("Detected kernel: '{}'.", kernel_release); - - Ok(kernel_release) -} - fn copy_kernel_source>(kernel_source_dir: P) -> Result { let kernel_source_copy = TempDir::new("kernel-source-copy").map_err(NvidiaError::Io)?; copy_dir_all(kernel_source_dir, kernel_source_copy.as_ref()) @@ -154,11 +143,11 @@ fn copy_kernel_source>(kernel_source_dir: P) -> Result Result<(), NvidiaError> { let container_image = "docker.io/koxu1996/dump-custom-nvidia-driver:0.3.0"; // TODO: Replace with an official Gmulot image. let kernel_source_copy_str = path_to_str(kernel_source_copy.path())?; - let target_dir_str = path_to_str(target_dir.path())?; + let target_dir_str = path_to_str(target_dir)?; run_command(&[ "podman", @@ -190,14 +179,13 @@ fn prepare_vm_modules_dir>( } fn copy_nvidia_driver>( - target_dir: &TempDir, + target_dir: &Path, vm_root_path: P, kernel_release: &str, driver_name: &str, ) -> Result<(), NvidiaError> { let driver_name = format!("{}.ko", driver_name); let source_path = target_dir - .path() .join("usr/lib/modules") .join(kernel_release) .join("video") @@ -214,11 +202,10 @@ fn copy_nvidia_driver>( } fn copy_driver_libraries>( - target_dir: &TempDir, + target_dir: &Path, vm_root_path: P, ) -> Result<(), NvidiaError> { let source_path = target_dir - .path() .join("usr/lib/x86_64-linux-gnu") // CAUTION: This is okay; it should NOT be the `kernel_release`. .join("libcuda.so.550.120"); // TODO: Dynamically find the filename. let target_path = vm_root_path.as_ref().join("lib").join("libcuda.so.1"); @@ -292,29 +279,59 @@ impl Step for BuildDrivers { info!("building NVIDIA drivers"); let kernel = ctx.get::("kernel").expect("kernel"); - // TODO: should we return error in this case or not? - if kernel.is_precompiled() { - warn!("Installing NVIDIA drivers for precompiled kernel is not supported yet!"); - warn!("Skipping installation."); - return Ok(()); + match kernel { + Kernel::Precompiled { .. } => { + // TODO: should we return error in this case or not? + + warn!("Installing NVIDIA drivers for precompiled kernel is not supported yet!"); + warn!("Skipping installation."); + } + Kernel::Sources { + source_path, + kernel_release, + .. + } => { + let target_dir = ctx + .cache() + .join("nvidia") + .join(DRIVER_VERSION) + .join(&kernel_release); + + let nvidia_drivers = if target_dir.is_dir() { + // If cache entry (directory) already exists, use cached drivers + let nvidia_drivers = + NvidiaDriversFs::use_existing(target_dir, kernel_release.clone()); + info!( + "using cached NVIDIA drivers (kernel release: {}, driver version: {}) - {}", + nvidia_drivers.kernel_release(), + DRIVER_VERSION, + ByteSize::b(nvidia_drivers.size()?) + ); + debug!("drivers cache path: {}", nvidia_drivers.path().display()); + nvidia_drivers + } else { + fs::create_dir_all(&target_dir).map_err(NvidiaError::Io)?; + + let nvidia_drivers = + NvidiaDriversFs::build(source_path, kernel_release.clone(), target_dir) + .context("failed to build NVIDIA drivers")?; + trace!( + "NVIDIA drivers dumped to {}", + nvidia_drivers.path().display() + ); + + info!( + "NVIDIA drivers built (kernel release: {}, driver version: {}) - {}", + nvidia_drivers.kernel_release(), + DRIVER_VERSION, + ByteSize::b(nvidia_drivers.size()?) + ); + nvidia_drivers + }; + + ctx.set("nvidia-drivers", Box::new(nvidia_drivers)); + } } - let kernel_sources = kernel - .source_path() - .ok_or(anyhow!("failed to get path to kernel sources"))?; - - let nvidia_drivers = - NvidiaDriversFs::build(kernel_sources).context("failed to build NVIDIA drivers")?; - trace!( - "NVIDIA drivers dumped to {}", - nvidia_drivers.path().display() - ); - - debug!( - "NVIDIA drivers built: {} ({})", - nvidia_drivers.path().display(), - ByteSize::b(nvidia_drivers.size()?) - ); - ctx.set("nvidia-drivers", Box::new(nvidia_drivers)); Ok(()) } diff --git a/src/commands/build.rs b/src/commands/build.rs index c12156a..13b4576 100644 --- a/src/commands/build.rs +++ b/src/commands/build.rs @@ -182,6 +182,15 @@ pub struct BuildArgs { #[arg(long)] pub from_scratch: bool, + /// Cache directory. + /// + /// Usually defaults to: + /// - '$HOME/.cache/gvltctl' on Linux + /// - '$HOME/Library/Caches/gevulot.gvltctl' on MacOS + #[cfg(feature = "vm-builder-v2")] + #[arg(long, value_name = "DIR", value_hint = ValueHint::FilePath, verbatim_doc_comment)] + pub cache_dir: Option, + /// Generate only base VM image. /// /// If this option is enabled, only base VM image will be generated. @@ -398,6 +407,8 @@ impl TryFrom<&BuildArgs> for linux_vm::LinuxVMBuildContext { } }; + let cache_dir = opts.cache_dir.clone(); + let gen_base_img = opts.generate_base_image; let from_scratch = gen_base_img || opts.from_scratch; let mbr_file = opts.mbr_file.clone(); @@ -412,6 +423,7 @@ impl TryFrom<&BuildArgs> for linux_vm::LinuxVMBuildContext { from_scratch, root_fs_opts, mbr_file, + cache_dir, rw_root, gen_base_img, };