aarch64: support FIQs and use them for TLB shootdown IPIs (#1039)

* Add support for fast interrupts on aarch64, aka FIQs.
  FIQs are designed to be fast and can thus interrupt regular interrupts (IRQs)
  that are in the process of being handled. They are similar to NMIs
  on x86_64 in this regard, but can also be explicitly enabled/disabled.
  * Updated the GIC driver to support both Group 0 (FIQs) and Group 1 (IRQs).
  * `nano_core`, `captain`, and `ap_start` now enable/disable FIQs.

* Broadcasting TLB shootdown IPIs now uses FIQs to ensure that
  TLB shootdowns occur instantly even if regular interrupts are disabled
  on one or more other CPUs.

* Add a separate trait `Aarch64LocalInterruptController` for arch-specific
  features, which keeps the `LocalInterruptController` arch-agnostic.
  * This trait is primarily for configuring/handling fast interrupts (FIQs),
    but also for acknowledging interrupts, which x86_64 does not require.
  * The interrupt controller now allows enabling/disabling SPIs too.

---------

Co-authored-by: Kevin Boos <[email protected]>

kernel/ap_start/src/lib.rs

@@ -52,6 +52,8 @@ pub fn kstart_ap(
     nmi_flags: u16,
 ) -> ! {
     irq_safety::disable_interrupts();
+    #[cfg(target_arch = "aarch64")]
+    irq_safety::disable_fast_interrupts();
 
     info!("Booted CPU {}, proc: {}, stack: {:#X} to {:#X}, nmi_lint: {}, nmi_flags: {:#X}",
         cpu_id, processor_id, _stack_start, _stack_end, nmi_lint, nmi_flags
@@ -100,6 +102,7 @@ pub fn kstart_ap(
 
     #[cfg(target_arch = "aarch64")] {
         interrupts::init_ap();
+        irq_safety::enable_fast_interrupts();
 
         // Register this CPU as online in the system
         // This is the equivalent of `LocalApic::init` on aarch64

kernel/captain/src/lib.rs

@@ -123,6 +123,7 @@ pub fn init(
         interrupt_controller::init()?;
 
         interrupts::init()?;
+        irq_safety::enable_fast_interrupts();
 
         // register BSP CpuId
         cpu::register_cpu(true)?;

kernel/context_switch_regular/src/aarch64.rs

@@ -85,21 +85,21 @@ macro_rules! save_registers_regular {
         // Save all general purpose registers into the previous task.
         r#"
             // Make room on the stack for the registers.
-            sub sp,  sp,  #8 * 2 * 6
+            sub sp,  sp,  #8 * 13
 
             // Push registers on the stack, two at a time.
-            stp x19, x20, [sp, #8 * 2 * 0]
-            stp x21, x22, [sp, #8 * 2 * 1]
-            stp x23, x24, [sp, #8 * 2 * 2]
-            stp x25, x26, [sp, #8 * 2 * 3]
-            stp x27, x28, [sp, #8 * 2 * 4]
-            stp x29, x30, [sp, #8 * 2 * 5]
+            stp x19, x20, [sp, #8 * 0]
+            stp x21, x22, [sp, #8 * 2]
+            stp x23, x24, [sp, #8 * 4]
+            stp x25, x26, [sp, #8 * 6]
+            stp x27, x28, [sp, #8 * 8]
+            stp x29, x30, [sp, #8 * 10]
 
             // Push an OR of DAIF and NZCV flags of PSTATE
             mrs x29, DAIF
             mrs x30, NZCV
             orr x29, x29, x30
-            str x29, [sp, #8 * 2 * 6]
+            str x29, [sp, #8 * 12]
         "#
     );
 }
@@ -133,20 +133,20 @@ macro_rules! restore_registers_regular {
         r#"
             // Pop DAIF and NZCV flags of PSTATE
             // These MSRs discard irrelevant bits; no AND is required.
-            ldr x29, [sp, #8 * 2 * 6]
+            ldr x29, [sp, #8 * 12]
             msr DAIF, x29
             msr NZCV, x29
 
             // Pop registers from the stack, two at a time.
-            ldp x29, x30, [sp, #8 * 2 * 5]
-            ldp x27, x28, [sp, #8 * 2 * 4]
-            ldp x25, x26, [sp, #8 * 2 * 3]
-            ldp x23, x24, [sp, #8 * 2 * 2]
-            ldp x21, x22, [sp, #8 * 2 * 1]
-            ldp x19, x20, [sp, #8 * 2 * 0]
+            ldp x29, x30, [sp, #8 * 10]
+            ldp x27, x28, [sp, #8 * 8]
+            ldp x25, x26, [sp, #8 * 6]
+            ldp x23, x24, [sp, #8 * 4]
+            ldp x21, x22, [sp, #8 * 2]
+            ldp x19, x20, [sp, #8 * 0]
 
             // Move the stack pointer back up.
-            add sp,  sp,  #8 * 2 * 6
+            add sp,  sp,  #8 * 13
         "#
     );
 }

kernel/gic/src/gic/cpu_interface_gicv2.rs

@@ -8,6 +8,7 @@
 
 use super::Priority;
 use super::InterruptNumber;
+use super::SPURIOUS_INTERRUPT_NUM;
 
 use volatile::{Volatile, ReadOnly, WriteOnly};
 use zerocopy::FromBytes;
@@ -42,7 +43,7 @@ pub struct CpuRegsP1 {            // base offset
 }
 
 // enable group 0
-// const CTLR_ENGRP0: u32 = 0b01;
+const CTLR_ENGRP0: u32 = 0b01;
 
 // enable group 1
 const CTLR_ENGRP1: u32 = 0b10;
@@ -51,6 +52,7 @@ impl CpuRegsP1 {
     /// Enables routing of group 1 interrupts for the current CPU.
     pub fn init(&mut self) {
         let mut reg = self.ctlr.read();
+        reg |= CTLR_ENGRP0;
         reg |= CTLR_ENGRP1;
         self.ctlr.write(reg);
     }
@@ -87,12 +89,17 @@ impl CpuRegsP1 {
     ///
     /// This tells the GIC that the requested interrupt is being
     /// handled by this CPU.
-    pub fn acknowledge_interrupt(&mut self) -> (InterruptNumber, Priority) {
+    ///
+    /// Returns None if a spurious interrupt is detected.
+    pub fn acknowledge_interrupt(&mut self) -> Option<(InterruptNumber, Priority)> {
         // Reading the interrupt number has the side effect
         // of acknowledging the interrupt.
         let int_num = self.acknowledge.read() as InterruptNumber;
         let priority = self.running_prio.read() as u8;
 
-        (int_num, priority)
+        match int_num {
+            SPURIOUS_INTERRUPT_NUM => None,
+            _ => Some((int_num, priority))
+        }
     }
 }

kernel/gic/src/gic/cpu_interface_gicv3.rs

@@ -11,6 +11,8 @@ use core::arch::asm;
 use super::IpiTargetCpu;
 use super::Priority;
 use super::InterruptNumber;
+use super::InterruptGroup;
+use super::SPURIOUS_INTERRUPT_NUM;
 
 const SGIR_TARGET_ALL_OTHER_PE: u64 = 1 << 40;
 const IGRPEN_ENABLED: u64 = 1;
@@ -28,7 +30,7 @@ pub fn init() {
 
     // Enable Group 0
     // bit 0 = group 0 enable
-    // unsafe { asm!("msr ICC_IGRPEN0_EL1, {}", in(reg) IGRPEN_ENABLED) };
+    unsafe { asm!("msr ICC_IGRPEN0_EL1, {}", in(reg) IGRPEN_ENABLED) };
 
     // Enable Groupe 1 (non-secure)
     // bit 0 = group 1 (non-secure) enable
@@ -61,34 +63,47 @@ pub fn set_minimum_priority(priority: Priority) {
 /// the current CPU.
 ///
 /// This implies that the CPU is ready to process interrupts again.
-pub fn end_of_interrupt(int: InterruptNumber) {
+pub fn end_of_interrupt(int: InterruptNumber, group: InterruptGroup) {
     let reg_value = int as u64;
-    unsafe { asm!("msr ICC_EOIR1_EL1, {}", in(reg) reg_value) };
+
+    match group {
+        InterruptGroup::Group0 => unsafe { asm!("msr ICC_EOIR0_EL1, {}", in(reg) reg_value) },
+        InterruptGroup::Group1 => unsafe { asm!("msr ICC_EOIR1_EL1, {}", in(reg) reg_value) },
+    }
 }
 
 /// Acknowledge the currently serviced interrupt and fetches its
 /// number.
 ///
 /// This tells the GIC that the requested interrupt is being
 /// handled by this CPU.
-pub fn acknowledge_interrupt() -> (InterruptNumber, Priority) {
+///
+/// Returns None if a spurious interrupt is detected.
+pub fn acknowledge_interrupt(group: InterruptGroup) -> Option<(InterruptNumber, Priority)> {
     let int_num: u64;
     let priority: u64;
 
     // Reading the interrupt number has the side effect
     // of acknowledging the interrupt.
+    match group {
+        InterruptGroup::Group0 => unsafe { asm!("mrs {}, ICC_IAR0_EL1", out(reg) int_num) },
+        InterruptGroup::Group1 => unsafe { asm!("mrs {}, ICC_IAR1_EL1", out(reg) int_num) },
+    }
+
     unsafe {
-        asm!("mrs {}, ICC_IAR1_EL1", out(reg) int_num);
         asm!("mrs {}, ICC_RPR_EL1", out(reg) priority);
     }
 
     let int_num = int_num & 0xffffff;
     let priority = priority & 0xff;
-    (int_num as InterruptNumber, priority as u8)
+    match int_num as InterruptNumber {
+        SPURIOUS_INTERRUPT_NUM => None,
+        n => Some((n, priority as u8)),
+    }
 }
 
 /// Generates an interrupt in CPU interfaces of the system
-pub fn send_ipi(int_num: InterruptNumber, target: IpiTargetCpu) {
+pub fn send_ipi(int_num: InterruptNumber, target: IpiTargetCpu, group: InterruptGroup) {
     let mut value = match target {
         IpiTargetCpu::Specific(cpu) => {
             let mpidr: cpu::MpidrValue = cpu.into();
@@ -118,5 +133,8 @@ pub fn send_ipi(int_num: InterruptNumber, target: IpiTargetCpu) {
     };
 
     value |= (int_num as u64) << 24;
-    unsafe { asm!("msr ICC_SGI1R_EL1, {}", in(reg) value) };
+    match group {
+        InterruptGroup::Group0 => unsafe { asm!("msr ICC_SGI0R_EL1, {}", in(reg) value) },
+        InterruptGroup::Group1 => unsafe { asm!("msr ICC_SGI1R_EL1, {}", in(reg) value) },
+    }
 }

kernel/gic/src/gic/dist_interface.rs

@@ -16,6 +16,7 @@
 use super::IpiTargetCpu;
 use super::SpiDestination;
 use super::InterruptNumber;
+use super::InterruptGroup;
 use super::Enabled;
 use super::Priority;
 use super::TargetList;
@@ -75,7 +76,7 @@ pub struct DistRegsP6 {     // base offset
 }
 
 // enable group 0
-// const CTLR_ENGRP0: u32 = 0b01;
+const CTLR_ENGRP0: u32 = 0b01;
 
 // enable group 1
 const CTLR_ENGRP1: u32 = 0b10;
@@ -96,9 +97,6 @@ const SGIR_TARGET_ALL_OTHER_PE: u32 = 1 << 24;
 //   0 = route to specific PE
 const P6IROUTER_ANY_AVAILABLE_PE: u64 = 1 << 31;
 
-// const GROUP_0: u32 = 0;
-const GROUP_1: u32 = 1;
-
 // bit 15: which interrupt group to target
 const SGIR_NSATT_GRP1: u32 = 1 << 15;
 
@@ -112,6 +110,7 @@ impl DistRegsP1 {
     /// states.
     pub fn init(&mut self) -> Enabled {
         let mut reg = self.ctlr.read();
+        reg |= CTLR_ENGRP0;
         reg |= CTLR_ENGRP1;
         reg |= CTLR_E1NWF;
         self.ctlr.write(reg);
@@ -123,26 +122,26 @@ impl DistRegsP1 {
     }
 
     /// Returns whether the given SPI (shared peripheral interrupt) will be
-    /// forwarded by the distributor
-    pub fn is_spi_enabled(&self, int: InterruptNumber) -> Enabled {
-        // enabled?
-        read_array_volatile::<32>(&self.set_enable, int) > 0
-        &&
-        // part of group 1?
-        read_array_volatile::<32>(&self.group, int) == GROUP_1
+    /// forwarded by the distributor.
+    pub fn get_spi_state(&self, int: InterruptNumber) -> Option<InterruptGroup> {
+        if read_array_volatile::<32>(&self.set_enable, int) == 1 {
+            match read_array_volatile::<32>(&self.group, int) {
+                0 => return Some(InterruptGroup::Group0),
+                1 => return Some(InterruptGroup::Group1),
+                _ => { }
+            }
+        }
+        None
     }
 
-    /// Enables or disables the forwarding of a particular SPI (shared peripheral interrupt)
-    pub fn enable_spi(&mut self, int: InterruptNumber, enabled: Enabled) {
-        let reg_base = match enabled {
-            true  => &mut self.set_enable,
-            false => &mut self.clear_enable,
-        };
-        write_array_volatile::<32>(reg_base, int, 1);
-
-        // whether we're enabling or disabling,
-        // set as part of group 1
-        write_array_volatile::<32>(&mut self.group, int, GROUP_1);
+    /// Enables or disables the forwarding of a particular SPI (shared peripheral interrupt).
+    pub fn set_spi_state(&mut self, int: InterruptNumber, state: Option<InterruptGroup>) {
+        if let Some(group) = state {
+            write_array_volatile::<32>(&mut self.group, int, group as u32);
+            write_array_volatile::<32>(&mut self.set_enable, int, 1);
+        } else {
+            write_array_volatile::<32>(&mut self.clear_enable, int, 1);
+        }
     }
 
     /// Returns the priority of an SPI.