Add support for x2apic mode on amd64

src/kernel/src/arch/amd64/apic/ipi.rs

@@ -21,12 +21,10 @@ pub fn send_ipi(dest: Destination, vector: u32) {
     };
     unsafe {
         let apic = get_lapic();
-        apic.write(LAPIC_ICRHI, dest_val);
-        apic.write(
-            LAPIC_ICRLO,
+        apic.write_icr(
+            dest_val,
             vector | dest_short << LAPIC_ICRLO_DEST_SHORT_OFFSET,
         );
-
         while apic.read(LAPIC_ICRLO) & LAPIC_ICRLO_STATUS_PEND != 0 {
             core::arch::asm!("pause")
         }

src/kernel/src/arch/amd64/apic/local.rs

@@ -57,34 +57,45 @@ pub const LAPIC_INT_MASKED: u32 = 1 << 16;
 // Flags in the APIC base MSR
 const APIC_BASE_BSP_FLAG: u64 = 1 << 8;
 const APIC_GLOBAL_ENABLE: u64 = 1 << 11;
+const APIC_ENABLE_X2MODE: u64 = 1 << 10;
 
 // The APIC can either be a standard APIC or x2APIC. We'll support
 // x2 eventually.
-#[derive(PartialEq, Eq)]
+#[derive(PartialEq, Eq, Debug)]
 enum ApicVersion {
     XApic,
     X2Apic,
 }
 
+#[derive(Debug)]
 pub struct Lapic {
     version: ApicVersion,
     base: VirtAddr,
 }
 
+fn get_x2_msr_addr(reg: u32) -> u32 {
+    // See Intel manual chapter of APIC -- all x2 registers are the original register values
+    // shifted right 4 and offset by 0x800 into the MSR space.
+    0x800 | (reg >> 4)
+}
+
 impl Lapic {
-    fn new_apic(base: VirtAddr) -> Self {
-        Self {
-            version: ApicVersion::XApic,
-            base,
-        }
+    fn new_apic(base: VirtAddr, version: ApicVersion) -> Self {
+        Self { version, base }
     }
 
     /// Read a register from the local APIC.
     ///
     /// # Safety
     /// Caller must ensure that reg is a valid register in the APIC register space.
     pub unsafe fn read(&self, reg: u32) -> u32 {
-        core::ptr::read_volatile(self.base.offset(reg as usize).unwrap().as_ptr())
+        asm!("mfence;");
+        match self.version {
+            ApicVersion::XApic => {
+                core::ptr::read_volatile(self.base.offset(reg as usize).unwrap().as_ptr())
+            }
+            ApicVersion::X2Apic => rdmsr(get_x2_msr_addr(reg)) as u32,
+        }
     }
 
     /// Write a value to a register in the local APIC.
@@ -93,8 +104,17 @@ impl Lapic {
     /// Caller must ensure that reg is a valid register in the APIC register space.
     // Note: this does not need to take &mut self because the APIC is per-CPU.
     pub unsafe fn write(&self, reg: u32, val: u32) {
-        core::ptr::write_volatile(self.base.offset(reg as usize).unwrap().as_mut_ptr(), val);
-        self.read(LAPIC_ID);
+        asm!("mfence;");
+        match self.version {
+            ApicVersion::XApic => {
+                core::ptr::write_volatile(
+                    self.base.offset(reg as usize).unwrap().as_mut_ptr(),
+                    val,
+                );
+                self.read(LAPIC_ID);
+            }
+            ApicVersion::X2Apic => wrmsr(get_x2_msr_addr(reg), val.into()),
+        }
     }
 
     unsafe fn local_enable_set(&self, enable: bool) {
@@ -115,6 +135,20 @@ impl Lapic {
         }
     }
 
+    /// Write the interrupt control register.
+    pub fn write_icr(&self, hi: u32, lo: u32) {
+        match self.version {
+            ApicVersion::XApic => unsafe {
+                self.write(LAPIC_ICRHI, hi);
+                self.write(LAPIC_ICRLO, lo);
+            },
+            ApicVersion::X2Apic => {
+                let val = ((hi as u64) << 32) | lo as u64;
+                unsafe { wrmsr(get_x2_msr_addr(LAPIC_ICRLO), val) }
+            }
+        }
+    }
+
     /// Clear the error status register.
     pub fn clear_err(&self) {
         unsafe {
@@ -133,11 +167,15 @@ impl Lapic {
             self.write(LAPIC_LINT1, LAPIC_INT_MASKED);
             self.write(LAPIC_ERROR, LAPIC_ERR_VECTOR as u32);
             self.write(LAPIC_ESR, 0);
-            self.write(LAPIC_DFR, !0);
+            if matches!(self.version, ApicVersion::XApic) {
+                self.write(LAPIC_DFR, !0);
+            }
             self.write(LAPIC_TPR, 0);
 
-            // Assign all processors to group 1 in the logical addressing mode.
-            self.write(LAPIC_LDR, 1 << 24);
+            if matches!(self.version, ApicVersion::XApic) {
+                // Assign all processors to group 1 in the logical addressing mode.
+                self.write(LAPIC_LDR, 1 << 24);
+            }
 
             // Signal EOI
             self.write(LAPIC_EOI, 0);
@@ -187,13 +225,28 @@ fn supports_deadline() -> bool {
     })
 }
 
-fn global_enable() -> PhysAddr {
+fn supports_x2_mode() -> bool {
+    let cpuid = x86::cpuid::CpuId::new();
+    let features = cpuid.get_feature_info().unwrap();
+    features.has_x2apic()
+}
+
+fn global_enable() -> (PhysAddr, ApicVersion) {
     let mut base = unsafe { rdmsr(APIC_BASE) };
-    if base & APIC_GLOBAL_ENABLE == 0 {
-        base |= APIC_GLOBAL_ENABLE;
-        unsafe { wrmsr(APIC_BASE, base) };
+    base |= APIC_GLOBAL_ENABLE;
+    if supports_x2_mode() {
+        base |= APIC_ENABLE_X2MODE;
     }
-    PhysAddr::new(base & !0xfff).expect("invalid APIC base address")
+    unsafe { wrmsr(APIC_BASE, base) };
+    let vers = if supports_x2_mode() {
+        ApicVersion::X2Apic
+    } else {
+        ApicVersion::XApic
+    };
+    (
+        PhysAddr::new(base & !0xfff).expect("invalid APIC base address"),
+        vers,
+    )
 }
 
 static LAPIC: Once<Lapic> = Once::new();
@@ -204,14 +257,30 @@ pub fn get_lapic() -> &'static Lapic {
     LAPIC.poll().expect("must initialize APIC before use")
 }
 
+/// Get a handle to the local APIC for this core. Note that the returned pointer
+/// is actually shared by all cores, since there is no CPU-local state we need to keep
+/// for now. If the LAPIC is not initialized, return None.
+pub fn try_get_lapic() -> Option<&'static Lapic> {
+    LAPIC.poll()
+}
+
 pub fn init(bsp: bool) {
     let apic = if bsp {
-        let base = global_enable();
-        let apic = Lapic::new_apic(phys_to_virt(base));
+        let (base, version) = global_enable();
+        logln!("[x86::apic] initializing APIC version {:?}", version);
+        // TODO: make uncachable
+        let apic = Lapic::new_apic(phys_to_virt(base), version);
         LAPIC.call_once(|| apic)
     } else {
         get_lapic()
     };
+    if matches!(apic.version, ApicVersion::X2Apic) && !bsp {
+        let mut base = unsafe { rdmsr(APIC_BASE) } | APIC_GLOBAL_ENABLE;
+        if supports_x2_mode() {
+            base |= APIC_ENABLE_X2MODE;
+        }
+        unsafe { wrmsr(APIC_BASE, base) };
+    }
     apic.reset();
 }
 

src/kernel/src/arch/amd64/apic/mod.rs

@@ -3,5 +3,5 @@ mod local;
 mod trampolines;
 
 pub use ipi::send_ipi;
-pub(super) use local::{get_lapic, init, lapic_interrupt};
+pub(super) use local::{get_lapic, init, lapic_interrupt, try_get_lapic};
 pub use trampolines::poke_cpu;

src/kernel/src/arch/amd64/interrupt.rs

@@ -13,7 +13,7 @@ use super::{
     thread::{Registers, UpcallAble},
 };
 use crate::{
-    arch::amd64::apic::get_lapic,
+    arch::amd64::apic::try_get_lapic,
     interrupt::{Destination, DynamicInterrupt},
     memory::{context::virtmem::PageFaultFlags, VirtAddr},
     processor::current_processor,
@@ -467,7 +467,13 @@ fn generic_isr_handler(ctx: *mut IsrContext, number: u64, user: bool) {
         );
     }
 
-    get_lapic().eoi();
+    let Some(lapic) = try_get_lapic() else {
+        panic!(
+            "got interrupt before initializing APIC: {}, {:#?}",
+            number, ctx
+        );
+    };
+    lapic.eoi();
     match number as u32 {
         14 => {
             let cr2 = unsafe { x86::controlregs::cr2() };

src/kernel/src/arch/amd64/ioapic.rs

@@ -219,8 +219,11 @@ fn disable_pic() {
         x86::io::outb(PIC1_DATA, mask1);
         iowait();
         x86::io::outb(PIC2_DATA, mask2);
+        iowait();
 
-        x86::io::outb(PIC2_DATA, 0xff);
         x86::io::outb(PIC1_DATA, 0xff);
+        iowait();
+        x86::io::outb(PIC2_DATA, 0xff);
+        iowait();
     }
 }

src/kernel/src/machine/pc/serial.rs

@@ -124,6 +124,9 @@ impl core::fmt::Write for SerialPort {
     fn write_str(&mut self, s: &str) -> core::fmt::Result {
         for byte in s.bytes() {
             self.send(byte);
+            if byte == b'\n' {
+                self.send(b'\r');
+            }
         }
         Ok(())
     }

src/kernel/src/memory/context.rs

@@ -14,7 +14,9 @@ use twizzler_abi::{
 };
 
 use self::virtmem::KernelObjectVirtHandle;
+use super::{PhysAddr, VirtAddr};
 use crate::{
+    memory::pagetables::{ContiguousProvider, MappingFlags, MappingSettings},
     obj::{InvalidateMode, ObjectRef, PageNumber},
     syscall::object::ObjectHandle,
 };

src/kernel/src/processor.rs

@@ -394,7 +394,8 @@ pub fn boot_all_secondaries(tls_template: TlsInfo) {
 
 pub fn register(id: u32, bsp_id: u32) {
     if id as usize >= unsafe { &ALL_PROCESSORS }.len() {
-        unimplemented!("processor ID too large");
+        logln!("processor ID {} not supported (too large)", id);
+        return;
     }
 
     unsafe {