From ea414a3a23c23b59ffc37811b7f0484fee2b834a Mon Sep 17 00:00:00 2001 From: Daniel Bittman Date: Tue, 10 Sep 2024 21:53:50 -0700 Subject: [PATCH] Add support for x2apic mode on amd64. --- src/kernel/src/arch/amd64/apic/ipi.rs | 6 +- src/kernel/src/arch/amd64/apic/local.rs | 107 +++++++++++++++++++----- src/kernel/src/arch/amd64/apic/mod.rs | 2 +- src/kernel/src/arch/amd64/interrupt.rs | 10 ++- src/kernel/src/arch/amd64/ioapic.rs | 5 +- src/kernel/src/machine/pc/serial.rs | 3 + src/kernel/src/memory/context.rs | 2 + src/kernel/src/processor.rs | 3 +- 8 files changed, 110 insertions(+), 28 deletions(-) diff --git a/src/kernel/src/arch/amd64/apic/ipi.rs b/src/kernel/src/arch/amd64/apic/ipi.rs index 55e9dade..465e433f 100644 --- a/src/kernel/src/arch/amd64/apic/ipi.rs +++ b/src/kernel/src/arch/amd64/apic/ipi.rs @@ -21,12 +21,10 @@ pub fn send_ipi(dest: Destination, vector: u32) { }; unsafe { let apic = get_lapic(); - apic.write(LAPIC_ICRHI, dest_val); - apic.write( - LAPIC_ICRLO, + apic.write_icr( + dest_val, vector | dest_short << LAPIC_ICRLO_DEST_SHORT_OFFSET, ); - while apic.read(LAPIC_ICRLO) & LAPIC_ICRLO_STATUS_PEND != 0 { core::arch::asm!("pause") } diff --git a/src/kernel/src/arch/amd64/apic/local.rs b/src/kernel/src/arch/amd64/apic/local.rs index 6b1047a1..3a59b515 100644 --- a/src/kernel/src/arch/amd64/apic/local.rs +++ b/src/kernel/src/arch/amd64/apic/local.rs @@ -57,26 +57,31 @@ pub const LAPIC_INT_MASKED: u32 = 1 << 16; // Flags in the APIC base MSR const APIC_BASE_BSP_FLAG: u64 = 1 << 8; const APIC_GLOBAL_ENABLE: u64 = 1 << 11; +const APIC_ENABLE_X2MODE: u64 = 1 << 10; // The APIC can either be a standard APIC or x2APIC. We'll support // x2 eventually. -#[derive(PartialEq, Eq)] +#[derive(PartialEq, Eq, Debug)] enum ApicVersion { XApic, X2Apic, } +#[derive(Debug)] pub struct Lapic { version: ApicVersion, base: VirtAddr, } +fn get_x2_msr_addr(reg: u32) -> u32 { + // See Intel manual chapter of APIC -- all x2 registers are the original register values + // shifted right 4 and offset by 0x800 into the MSR space. + 0x800 | (reg >> 4) +} + impl Lapic { - fn new_apic(base: VirtAddr) -> Self { - Self { - version: ApicVersion::XApic, - base, - } + fn new_apic(base: VirtAddr, version: ApicVersion) -> Self { + Self { version, base } } /// Read a register from the local APIC. @@ -84,7 +89,13 @@ impl Lapic { /// # Safety /// Caller must ensure that reg is a valid register in the APIC register space. pub unsafe fn read(&self, reg: u32) -> u32 { - core::ptr::read_volatile(self.base.offset(reg as usize).unwrap().as_ptr()) + asm!("mfence;"); + match self.version { + ApicVersion::XApic => { + core::ptr::read_volatile(self.base.offset(reg as usize).unwrap().as_ptr()) + } + ApicVersion::X2Apic => rdmsr(get_x2_msr_addr(reg)) as u32, + } } /// Write a value to a register in the local APIC. @@ -93,8 +104,17 @@ impl Lapic { /// Caller must ensure that reg is a valid register in the APIC register space. // Note: this does not need to take &mut self because the APIC is per-CPU. pub unsafe fn write(&self, reg: u32, val: u32) { - core::ptr::write_volatile(self.base.offset(reg as usize).unwrap().as_mut_ptr(), val); - self.read(LAPIC_ID); + asm!("mfence;"); + match self.version { + ApicVersion::XApic => { + core::ptr::write_volatile( + self.base.offset(reg as usize).unwrap().as_mut_ptr(), + val, + ); + self.read(LAPIC_ID); + } + ApicVersion::X2Apic => wrmsr(get_x2_msr_addr(reg), val.into()), + } } unsafe fn local_enable_set(&self, enable: bool) { @@ -115,6 +135,20 @@ impl Lapic { } } + /// Write the interrupt control register. + pub fn write_icr(&self, hi: u32, lo: u32) { + match self.version { + ApicVersion::XApic => unsafe { + self.write(LAPIC_ICRHI, hi); + self.write(LAPIC_ICRLO, lo); + }, + ApicVersion::X2Apic => { + let val = ((hi as u64) << 32) | lo as u64; + unsafe { wrmsr(get_x2_msr_addr(LAPIC_ICRLO), val) } + } + } + } + /// Clear the error status register. pub fn clear_err(&self) { unsafe { @@ -133,11 +167,15 @@ impl Lapic { self.write(LAPIC_LINT1, LAPIC_INT_MASKED); self.write(LAPIC_ERROR, LAPIC_ERR_VECTOR as u32); self.write(LAPIC_ESR, 0); - self.write(LAPIC_DFR, !0); + if matches!(self.version, ApicVersion::XApic) { + self.write(LAPIC_DFR, !0); + } self.write(LAPIC_TPR, 0); - // Assign all processors to group 1 in the logical addressing mode. - self.write(LAPIC_LDR, 1 << 24); + if matches!(self.version, ApicVersion::XApic) { + // Assign all processors to group 1 in the logical addressing mode. + self.write(LAPIC_LDR, 1 << 24); + } // Signal EOI self.write(LAPIC_EOI, 0); @@ -187,13 +225,28 @@ fn supports_deadline() -> bool { }) } -fn global_enable() -> PhysAddr { +fn supports_x2_mode() -> bool { + let cpuid = x86::cpuid::CpuId::new(); + let features = cpuid.get_feature_info().unwrap(); + features.has_x2apic() +} + +fn global_enable() -> (PhysAddr, ApicVersion) { let mut base = unsafe { rdmsr(APIC_BASE) }; - if base & APIC_GLOBAL_ENABLE == 0 { - base |= APIC_GLOBAL_ENABLE; - unsafe { wrmsr(APIC_BASE, base) }; + base |= APIC_GLOBAL_ENABLE; + if supports_x2_mode() { + base |= APIC_ENABLE_X2MODE; } - PhysAddr::new(base & !0xfff).expect("invalid APIC base address") + unsafe { wrmsr(APIC_BASE, base) }; + let vers = if supports_x2_mode() { + ApicVersion::X2Apic + } else { + ApicVersion::XApic + }; + ( + PhysAddr::new(base & !0xfff).expect("invalid APIC base address"), + vers, + ) } static LAPIC: Once = Once::new(); @@ -204,14 +257,30 @@ pub fn get_lapic() -> &'static Lapic { LAPIC.poll().expect("must initialize APIC before use") } +/// Get a handle to the local APIC for this core. Note that the returned pointer +/// is actually shared by all cores, since there is no CPU-local state we need to keep +/// for now. If the LAPIC is not initialized, return None. +pub fn try_get_lapic() -> Option<&'static Lapic> { + LAPIC.poll() +} + pub fn init(bsp: bool) { let apic = if bsp { - let base = global_enable(); - let apic = Lapic::new_apic(phys_to_virt(base)); + let (base, version) = global_enable(); + logln!("[x86::apic] initializing APIC version {:?}", version); + // TODO: make uncachable + let apic = Lapic::new_apic(phys_to_virt(base), version); LAPIC.call_once(|| apic) } else { get_lapic() }; + if matches!(apic.version, ApicVersion::X2Apic) && !bsp { + let mut base = unsafe { rdmsr(APIC_BASE) } | APIC_GLOBAL_ENABLE; + if supports_x2_mode() { + base |= APIC_ENABLE_X2MODE; + } + unsafe { wrmsr(APIC_BASE, base) }; + } apic.reset(); } diff --git a/src/kernel/src/arch/amd64/apic/mod.rs b/src/kernel/src/arch/amd64/apic/mod.rs index 5adfddb9..bd928e51 100644 --- a/src/kernel/src/arch/amd64/apic/mod.rs +++ b/src/kernel/src/arch/amd64/apic/mod.rs @@ -3,5 +3,5 @@ mod local; mod trampolines; pub use ipi::send_ipi; -pub(super) use local::{get_lapic, init, lapic_interrupt}; +pub(super) use local::{get_lapic, init, lapic_interrupt, try_get_lapic}; pub use trampolines::poke_cpu; diff --git a/src/kernel/src/arch/amd64/interrupt.rs b/src/kernel/src/arch/amd64/interrupt.rs index 3cc45bbb..f1d70a1e 100644 --- a/src/kernel/src/arch/amd64/interrupt.rs +++ b/src/kernel/src/arch/amd64/interrupt.rs @@ -13,7 +13,7 @@ use super::{ thread::{Registers, UpcallAble}, }; use crate::{ - arch::amd64::apic::get_lapic, + arch::amd64::apic::try_get_lapic, interrupt::{Destination, DynamicInterrupt}, memory::{context::virtmem::PageFaultFlags, VirtAddr}, processor::current_processor, @@ -467,7 +467,13 @@ fn generic_isr_handler(ctx: *mut IsrContext, number: u64, user: bool) { ); } - get_lapic().eoi(); + let Some(lapic) = try_get_lapic() else { + panic!( + "got interrupt before initializing APIC: {}, {:#?}", + number, ctx + ); + }; + lapic.eoi(); match number as u32 { 14 => { let cr2 = unsafe { x86::controlregs::cr2() }; diff --git a/src/kernel/src/arch/amd64/ioapic.rs b/src/kernel/src/arch/amd64/ioapic.rs index 39184df8..75b18c0e 100644 --- a/src/kernel/src/arch/amd64/ioapic.rs +++ b/src/kernel/src/arch/amd64/ioapic.rs @@ -219,8 +219,11 @@ fn disable_pic() { x86::io::outb(PIC1_DATA, mask1); iowait(); x86::io::outb(PIC2_DATA, mask2); + iowait(); - x86::io::outb(PIC2_DATA, 0xff); x86::io::outb(PIC1_DATA, 0xff); + iowait(); + x86::io::outb(PIC2_DATA, 0xff); + iowait(); } } diff --git a/src/kernel/src/machine/pc/serial.rs b/src/kernel/src/machine/pc/serial.rs index e8df1dc5..c07af4d6 100755 --- a/src/kernel/src/machine/pc/serial.rs +++ b/src/kernel/src/machine/pc/serial.rs @@ -124,6 +124,9 @@ impl core::fmt::Write for SerialPort { fn write_str(&mut self, s: &str) -> core::fmt::Result { for byte in s.bytes() { self.send(byte); + if byte == b'\n' { + self.send(b'\r'); + } } Ok(()) } diff --git a/src/kernel/src/memory/context.rs b/src/kernel/src/memory/context.rs index 66f6783f..862aad7f 100644 --- a/src/kernel/src/memory/context.rs +++ b/src/kernel/src/memory/context.rs @@ -14,7 +14,9 @@ use twizzler_abi::{ }; use self::virtmem::KernelObjectVirtHandle; +use super::{PhysAddr, VirtAddr}; use crate::{ + memory::pagetables::{ContiguousProvider, MappingFlags, MappingSettings}, obj::{InvalidateMode, ObjectRef, PageNumber}, syscall::object::ObjectHandle, }; diff --git a/src/kernel/src/processor.rs b/src/kernel/src/processor.rs index 98abe0e0..66f0a006 100644 --- a/src/kernel/src/processor.rs +++ b/src/kernel/src/processor.rs @@ -394,7 +394,8 @@ pub fn boot_all_secondaries(tls_template: TlsInfo) { pub fn register(id: u32, bsp_id: u32) { if id as usize >= unsafe { &ALL_PROCESSORS }.len() { - unimplemented!("processor ID too large"); + logln!("processor ID {} not supported (too large)", id); + return; } unsafe {