diff --git a/Makefile b/Makefile index 3dcb2ad8..f0fec397 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,9 @@ ifeq ($(SEV),1) INIT_SRC += $(SNP_INIT_SRC) BUILD_INIT = 0 endif +ifeq ($(CCA), 1) + FEATURE_FLAGS := --features cca +endif ifeq ($(GPU),1) FEATURE_FLAGS += --features gpu endif diff --git a/src/arch/src/aarch64/fdt.rs b/src/arch/src/aarch64/fdt.rs index 02b45112..4e051506 100644 --- a/src/arch/src/aarch64/fdt.rs +++ b/src/arch/src/aarch64/fdt.rs @@ -285,7 +285,10 @@ fn create_psci_node(fdt: &mut FdtWriter) -> Result<()> { // Two methods available: hvc and smc. // As per documentation, PSCI calls between a guest and hypervisor may use the HVC conduit instead of SMC. // So, since we are using kvm, we need to use hvc. + #[cfg(not(feature = "cca"))] fdt.property_string("method", "hvc")?; + #[cfg(feature = "cca")] + fdt.property_string("method", "smc")?; fdt.end_node(node)?; Ok(()) diff --git a/src/arch/src/aarch64/linux/regs.rs b/src/arch/src/aarch64/linux/regs.rs index 81146b8a..71dffe5b 100644 --- a/src/arch/src/aarch64/linux/regs.rs +++ b/src/arch/src/aarch64/linux/regs.rs @@ -125,8 +125,10 @@ arm64_sys_reg!(MPIDR_EL1, 3, 0, 0, 0, 5); /// * `boot_ip` - Starting instruction pointer. /// * `mem` - Reserved DRAM for current VM. pub fn setup_regs(vcpu: &VcpuFd, cpu_id: u8, boot_ip: u64, mem: &GuestMemoryMmap) -> Result<()> { - // Get the register index of the PSTATE (Processor State) register. + // PSTATE cannot be accesed from the host in CCA + #[cfg(not(feature = "cca"))] #[allow(deref_nullptr)] + // Get the register index of the PSTATE (Processor State) register. vcpu.set_one_reg(arm64_core_reg!(pstate), &PSTATE_FAULT_BITS_64.to_le_bytes()) .map_err(Error::SetCoreRegister)?; diff --git a/src/arch/src/aarch64/mod.rs b/src/arch/src/aarch64/mod.rs index 9450b94d..23116ea5 100644 --- a/src/arch/src/aarch64/mod.rs +++ b/src/arch/src/aarch64/mod.rs @@ -69,7 +69,7 @@ pub fn arch_memory_regions(size: usize) -> (ArchMemoryInfo, Vec<(GuestAddress, u } else { vec![ (GuestAddress(layout::DRAM_MEM_START), dram_size), - (GuestAddress(shm_start_addr), MMIO_SHM_SIZE as usize), + //(GuestAddress(shm_start_addr), MMIO_SHM_SIZE as usize), ] }; diff --git a/src/vmm/src/builder.rs b/src/vmm/src/builder.rs index 627d7134..234a8a8d 100644 --- a/src/vmm/src/builder.rs +++ b/src/vmm/src/builder.rs @@ -3,8 +3,10 @@ //! Enables pre-boot setup, instantiation and booting of a Firecracker VMM. +use cca::Algo; #[cfg(target_os = "macos")] use crossbeam_channel::{unbounded, Sender}; +use std::cmp::max; use std::fmt::{Display, Formatter}; use std::fs::File; use std::io; @@ -68,7 +70,9 @@ use vm_memory::mmap::MmapRegion; #[cfg(any(target_arch = "aarch64", feature = "tee"))] use vm_memory::Bytes; use vm_memory::GuestMemory; -use vm_memory::{GuestAddress, GuestMemoryMmap}; +use vm_memory::{Address, GuestAddress, GuestMemoryMmap, GuestMemoryRegion}; + +use kvm_bindings::KVM_ARM_VCPU_REC; #[cfg(feature = "efi")] static EDK2_BINARY: &[u8] = include_bytes!("../../../edk2/KRUN_EFI.silent.fd"); @@ -435,17 +439,18 @@ pub fn build_microvm( // On x86_64 always create a serial device, // while on aarch64 only create it if 'console=' is specified in the boot args. - let serial_device = if cfg!(feature = "efi") { + // TODO: to comment this + let serial_device = //if cfg!(feature = "efi") { Some(setup_serial_device( event_manager, None, - None, + //None, // Uncomment this to get EFI output when debugging EDK2. - // Some(Box::new(io::stdout())), - )?) - } else { - None - }; + Some(Box::new(io::stdout())), + )?); + //} else { + // None + //}; let exit_evt = EventFd::new(utils::eventfd::EFD_NONBLOCK) .map_err(Error::EventFd) @@ -559,7 +564,7 @@ pub fn build_microvm( )?; } - #[cfg(not(feature = "tee"))] + #[cfg(all(not(feature = "tee"), not(feature = "cca")))] let _shm_region = Some(VirtioShmRegion { host_addr: guest_memory .get_host_address(GuestAddress(arch_memory_info.shm_start_addr)) @@ -647,6 +652,7 @@ pub fn build_microvm( #[cfg(not(feature = "tee"))] let initrd_config = None; + // after this point guest memory and regs are not accesible anymore vmm.configure_system( vcpus.as_slice(), &initrd_config, @@ -809,7 +815,7 @@ fn load_cmdline(vmm: &Vmm) -> std::result::Result<(), StartMicrovmError> { .map_err(StartMicrovmError::LoadCommandline) } -#[cfg(all(target_os = "linux", not(feature = "tee")))] +#[cfg(all(target_os = "linux", not(feature = "tee"), not(feature = "cca")))] pub(crate) fn setup_vm( guest_memory: &GuestMemoryMmap, ) -> std::result::Result { @@ -824,6 +830,29 @@ pub(crate) fn setup_vm( .map_err(StartMicrovmError::Internal)?; Ok(vm) } +#[cfg(all(target_os = "linux", feature = "cca"))] +pub(crate) fn setup_vm( + guest_memory: &GuestMemoryMmap, +) -> std::result::Result { + let kvm = KvmContext::new() + .map_err(Error::KvmContext) + .map_err(StartMicrovmError::Internal)?; + + // calculate max_addr for max_ipa + let mut max_addr = 0; + for (_index, region) in guest_memory.iter().enumerate() { + max_addr = max(max_addr, region.start_addr().raw_value() + region.len() - 1); + } + + let mut vm = Vm::new(kvm.fd(), max_addr as usize) + .map_err(Error::Vm) + .map_err(StartMicrovmError::Internal)?; + + vm.memory_init(guest_memory, kvm.max_memslots(), true) + .map_err(Error::Vm) + .map_err(StartMicrovmError::Internal)?; + Ok(vm) +} #[cfg(all(target_os = "linux", feature = "tee"))] pub(crate) fn setup_vm( kvm: &KvmContext, @@ -1021,7 +1050,7 @@ fn create_vcpus_aarch64( ) -> super::Result> { let mut vcpus = Vec::with_capacity(vcpu_config.vcpu_count as usize); for cpu_index in 0..vcpu_config.vcpu_count { - let mut vcpu = Vcpu::new_aarch64( + let mut vcpu: Vcpu = Vcpu::new_aarch64( cpu_index, vm.fd(), exit_evt.try_clone().map_err(Error::EventFd)?, diff --git a/src/vmm/src/lib.rs b/src/vmm/src/lib.rs index ea3fdb38..b8874bcc 100644 --- a/src/vmm/src/lib.rs +++ b/src/vmm/src/lib.rs @@ -55,15 +55,20 @@ use crate::vstate::{Vcpu, VcpuHandle, VcpuResponse, Vm}; use arch::ArchMemoryInfo; use arch::DeviceType; use arch::InitrdConfig; +use cca::Algo; #[cfg(target_os = "macos")] use crossbeam_channel::Sender; use devices::virtio::VmmExitObserver; use devices::BusDevice; use kernel::cmdline::Cmdline as KernelCmdline; +use kvm_bindings::KVM_ARM_VCPU_REC; use polly::event_manager::{self, EventManager, Subscriber}; use utils::epoll::{EpollEvent, EventSet}; use utils::eventfd::EventFd; +use vm_memory::Address; +use vm_memory::GuestMemory; use vm_memory::GuestMemoryMmap; +use vm_memory::GuestMemoryRegion; /// Success exit code. pub const FC_EXIT_CODE_OK: u8 = 0; @@ -299,6 +304,31 @@ impl Vmm { _smbios_oem_strings, ) .map_err(Error::ConfigureSystem)?; + + // after activation guest is not accesible anymore + #[cfg(feature = "cca")] + { + let _ = self + .vm + .realm + .configure_measurement(self.vm.fd(), Algo::AlgoSha256); + self.vm.realm.create_realm_descriptor(self.vm.fd()).unwrap(); + + for (_index, region) in self.guest_memory.iter().enumerate() { + self.vm + .realm + .populate(self.vm.fd(), region.start_addr().raw_value(), region.len()) + .unwrap(); + } + let feature = KVM_ARM_VCPU_REC as i32; + + // not really sure if the finalize and the activate should go here + for vcpu in vcpus.iter() { + vcpu.fd.vcpu_finalize(&feature).unwrap(); + } + + self.vm.realm.activate(self.vm.fd()).unwrap(); + } } #[cfg(all(target_arch = "aarch64", target_os = "macos"))] diff --git a/src/vmm/src/linux/vstate.rs b/src/vmm/src/linux/vstate.rs index 439dd203..6dfda147 100644 --- a/src/vmm/src/linux/vstate.rs +++ b/src/vmm/src/linux/vstate.rs @@ -8,6 +8,7 @@ use crossbeam_channel::{unbounded, Receiver, Sender, TryRecvError}; use libc::{c_int, c_void, siginfo_t}; use std::cell::Cell; +use std::cmp::max; use std::fmt::{Display, Formatter}; use std::io; use std::os::fd::RawFd; @@ -15,6 +16,7 @@ use std::os::fd::RawFd; #[cfg(feature = "tee")] use std::os::unix::io::RawFd; +use kvm_ioctls::VcpuExit::Unsupported; use std::result; use std::sync::atomic::{fence, Ordering}; #[cfg(not(test))] @@ -48,8 +50,10 @@ use kvm_bindings::{ KVM_MAX_CPUID_ENTRIES, KVM_PIT_SPEAKER_DUMMY, }; use kvm_bindings::{ - kvm_create_guest_memfd, kvm_userspace_memory_region, kvm_userspace_memory_region2, - KVM_API_VERSION, KVM_MEM_GUEST_MEMFD, + kvm_create_guest_memfd, kvm_memory_attributes, kvm_userspace_memory_region, + kvm_userspace_memory_region2, KVM_API_VERSION, KVM_MEMORY_ATTRIBUTE_PRIVATE, + KVM_MEMORY_EXIT_FLAG_PRIVATE, KVM_MEM_GUEST_MEMFD, KVM_VM_TYPE_ARM_IPA_SIZE_MASK, + KVM_VM_TYPE_ARM_REALM, }; use kvm_ioctls::*; use utils::eventfd::EventFd; @@ -65,6 +69,9 @@ use sev::launch::sev as sev_launch; #[cfg(feature = "amd-sev")] use sev::launch::snp; +#[cfg(feature = "cca")] +use cca::Realm; + /// Signal number (SIGRTMIN) used to kick Vcpus. pub(crate) const VCPU_RTSIG_OFFSET: i32 = 0; @@ -483,11 +490,14 @@ pub struct Vm { #[cfg(feature = "amd-sev")] pub tee: Tee, + + #[cfg(feature = "cca")] + pub realm: Realm, } impl Vm { /// Constructs a new `Vm` using the given `Kvm` instance. - #[cfg(not(feature = "tee"))] + #[cfg(all(not(feature = "tee"), not(feature = "cca")))] pub fn new(kvm: &Kvm) -> Result { //create fd for interacting with kvm-vm specific functions let vm_fd = kvm.create_vm().map_err(Error::VmFd)?; @@ -511,6 +521,26 @@ impl Vm { }) } + #[cfg(feature = "cca")] + pub fn new(kvm: &Kvm, max_ipa: usize) -> Result { + //create fd for interacting with kvm-vm specific functions + let ipa_bits = max(64u32 - max_ipa.leading_zeros() - 1, 32) + 1; + let vm_fd = kvm + .create_vm_with_type( + (KVM_VM_TYPE_ARM_REALM | (ipa_bits & KVM_VM_TYPE_ARM_IPA_SIZE_MASK)).into(), + ) + .map_err(Error::VmFd)?; + + let realm = Realm::new().unwrap(); + + Ok(Vm { + fd: vm_fd, + #[cfg(target_arch = "aarch64")] + irqchip_handle: None, + realm, + }) + } + #[cfg(feature = "amd-sev")] pub fn new(kvm: &Kvm, tee_config: &TeeConfig) -> Result { //create fd for interacting with kvm-vm specific functions @@ -581,7 +611,7 @@ impl Vm { .create_guest_memfd(gmem) .map_err(Error::CreateGuestMemfd)?; - let memory_region = kvm_userspace_memory_region2 { + let memory_region: kvm_userspace_memory_region2 = kvm_userspace_memory_region2 { slot: index as u32, flags: KVM_MEM_GUEST_MEMFD, guest_phys_addr: region.start_addr().raw_value(), @@ -600,6 +630,17 @@ impl Vm { .set_user_memory_region2(memory_region) .map_err(Error::SetUserMemoryRegion2)?; }; + + // set private by default when using guestmemfd + // this imitates QEMU behavior + let attr = kvm_memory_attributes { + address: region.start_addr().raw_value(), + size: region.len(), + attributes: KVM_MEMORY_ATTRIBUTE_PRIVATE as u64, + flags: 0, + }; + + self.fd.set_memory_attributes(attr).unwrap(); } else { let memory_region = kvm_userspace_memory_region { slot: index as u32, @@ -808,7 +849,7 @@ type VcpuCell = Cell>; /// A wrapper around creating and using a kvm-based VCPU. pub struct Vcpu { - fd: VcpuFd, + pub fd: VcpuFd, id: u8, mmio_bus: Option, #[allow(dead_code)] @@ -1267,12 +1308,31 @@ impl Vcpu { info!("Received KVM_EXIT_SHUTDOWN signal"); Ok(VcpuEmulation::Stopped) } + VcpuExit::MemoryFault { + flags, + gpa: _, + size: _, + } => { + // TODO: flags can be private or shared + if flags & !KVM_MEMORY_EXIT_FLAG_PRIVATE as u64 != 0 { + error!("KVM_EXIT_MEMORY_FAULT: Unknown flag {}", flags); + Err(Error::VcpuUnhandledKvmExit) + } else { + // TODO: to transition from shared to private + Ok(VcpuEmulation::Handled) + } + } // Documentation specifies that below kvm exits are considered // errors. VcpuExit::FailEntry(reason, vcpu) => { error!("Received KVM_EXIT_FAIL_ENTRY signal: reason={reason}, vcpu={vcpu}"); Err(Error::VcpuUnhandledKvmExit) } + // TODO: to remove this + Unsupported(39) => { + println!("memory fault!"); + Ok(VcpuEmulation::Handled) + } VcpuExit::InternalError => { error!("Received KVM_EXIT_INTERNAL_ERROR signal"); Err(Error::VcpuUnhandledKvmExit) @@ -1280,6 +1340,7 @@ impl Vcpu { r => { // TODO: Are we sure we want to finish running a vcpu upon // receiving a vm exit that is not necessarily an error? + println!("error! {:?}", r); error!("Unexpected exit reason on vcpu run: {:?}", r); Err(Error::VcpuUnhandledKvmExit) } @@ -1605,7 +1666,9 @@ mod tests { // Create valid memory region and test that the initialization is successful. let gm = GuestMemoryMmap::from_ranges(&[(GuestAddress(0), 0x1000)]).unwrap(); - assert!(vm.memory_init(&gm, kvm_context.max_memslots(), false).is_ok()); + assert!(vm + .memory_init(&gm, kvm_context.max_memslots(), false) + .is_ok()); // Set the maximum number of memory slots to 1 in KvmContext to check the error // path of memory_init. Create 2 non-overlapping memory slots. @@ -1615,7 +1678,9 @@ mod tests { (GuestAddress(0x1001), 0x2000), ]) .unwrap(); - assert!(vm.memory_init(&gm, kvm_context.max_memslots(), false).is_err()); + assert!(vm + .memory_init(&gm, kvm_context.max_memslots(), false) + .is_err()); } #[cfg(target_arch = "x86_64")] diff --git a/src/vmm/src/vmm_config/boot_source.rs b/src/vmm/src/vmm_config/boot_source.rs index 9c70d5b1..34227582 100644 --- a/src/vmm/src/vmm_config/boot_source.rs +++ b/src/vmm/src/vmm_config/boot_source.rs @@ -16,8 +16,10 @@ use std::fmt::{Display, Formatter, Result}; //pub const DEFAULT_KERNEL_CMDLINE: &str = "reboot=k panic=1 pci=off nomodule 8250.nr_uarts=0 \ // i8042.noaux i8042.nomux i8042.nopnp i8042.dumbkbd"; +// TODO: to unchange this #[cfg(all(target_os = "linux", not(feature = "tee")))] -pub const DEFAULT_KERNEL_CMDLINE: &str = "reboot=k panic=-1 panic_print=0 nomodule console=hvc0 \ +pub const DEFAULT_KERNEL_CMDLINE: &str = + "reboot=k panic=-1 panic_print=0 nomodule console=pl011,mmio,0x40001000 \ rootfstype=virtiofs rw quiet no-kvmapf"; #[cfg(feature = "amd-sev")]