Merge branch 'fix/freertos_optimize_critical_sections' into 'master'

fix(riscv): Updated RISC-V functions to set interrupt threshold for CLIC targets

Closes IDFCI-2033, IDFCI-2034, IDF-8090, and IDF-8117

See merge request espressif/esp-idf!29055

components/esp_hw_support/include/spinlock.h

@@ -1,5 +1,5 @@
 /*
- * SPDX-FileCopyrightText: 2015-2023 Espressif Systems (Shanghai) CO LTD
+ * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD
  *
  * SPDX-License-Identifier: Apache-2.0
  */
@@ -87,7 +87,7 @@ static inline bool __attribute__((always_inline)) spinlock_acquire(spinlock_t *l
     core_owner_id = xt_utils_get_raw_core_id();
 #else  //__riscv
 
-    irq_status = rv_utils_mask_int_level_lower_than(RVHAL_EXCM_LEVEL);
+    irq_status = rv_utils_set_intlevel_regval(RVHAL_EXCM_LEVEL_CLIC);
     core_owner_id = rv_utils_get_core_id() == 0 ? SPINLOCK_OWNER_ID_0 : SPINLOCK_OWNER_ID_1;
 #endif
     other_core_owner_id = CORE_ID_REGVAL_XOR_SWAP ^ core_owner_id;
@@ -106,7 +106,7 @@ static inline bool __attribute__((always_inline)) spinlock_acquire(spinlock_t *l
 #if __XTENSA__
         XTOS_RESTORE_INTLEVEL(irq_status);
 #else
-        rv_utils_restore_intlevel(irq_status);
+        rv_utils_restore_intlevel_regval(irq_status);
 #endif
         return true;
     }
@@ -147,7 +147,7 @@ static inline bool __attribute__((always_inline)) spinlock_acquire(spinlock_t *l
 #if __XTENSA__
     XTOS_RESTORE_INTLEVEL(irq_status);
 #else
-    rv_utils_restore_intlevel(irq_status);
+    rv_utils_restore_intlevel_regval(irq_status);
 #endif
     return lock_set;
 
@@ -181,7 +181,7 @@ static inline void __attribute__((always_inline)) spinlock_release(spinlock_t *l
 
     core_owner_id = xt_utils_get_raw_core_id();
 #else
-    irq_status = rv_utils_mask_int_level_lower_than(RVHAL_EXCM_LEVEL);
+    irq_status = rv_utils_set_intlevel_regval(RVHAL_EXCM_LEVEL_CLIC);
     core_owner_id = rv_utils_get_core_id() == 0 ? SPINLOCK_OWNER_ID_0 : SPINLOCK_OWNER_ID_1;
 #endif
     assert(core_owner_id == lock->owner); // This is a lock that we didn't acquire, or the lock is corrupt
@@ -196,7 +196,7 @@ static inline void __attribute__((always_inline)) spinlock_release(spinlock_t *l
 #if __XTENSA__
     XTOS_RESTORE_INTLEVEL(irq_status);
 #else
-    rv_utils_restore_intlevel(irq_status);
+    rv_utils_restore_intlevel_regval(irq_status);
 #endif  //#if __XTENSA__
 #endif  //#if !CONFIG_ESP_SYSTEM_SINGLE_CORE_MODE && !BOOTLOADER_BUILD
 }

components/freertos/FreeRTOS-Kernel/portable/riscv/include/freertos/portmacro.h

@@ -466,7 +466,11 @@ void vPortTCBPreDeleteHook( void *pxTCB );
 // --------------------- Interrupts ------------------------
 
 #define portDISABLE_INTERRUPTS()            portSET_INTERRUPT_MASK_FROM_ISR()
+#if !SOC_INT_CLIC_SUPPORTED
 #define portENABLE_INTERRUPTS()             portCLEAR_INTERRUPT_MASK_FROM_ISR(RVHAL_INTR_ENABLE_THRESH)
+#else
+#define portENABLE_INTERRUPTS()             portCLEAR_INTERRUPT_MASK_FROM_ISR(RVHAL_INTR_ENABLE_THRESH_CLIC)
+#endif /* !SOC_INT_CLIC_SUPPORTED */
 
 /**
  * ISR versions to enable/disable interrupts

components/freertos/FreeRTOS-Kernel/portable/riscv/port.c

@@ -453,7 +453,7 @@ UBaseType_t xPortSetInterruptMaskFromISR(void)
     RV_SET_CSR(mstatus, old_mstatus & MSTATUS_MIE);
 #else
     /* When CLIC is supported, all interrupt priority levels less than or equal to the threshold level are masked. */
-    prev_int_level = rv_utils_mask_int_level_lower_than(RVHAL_EXCM_LEVEL);
+    prev_int_level = rv_utils_set_intlevel_regval(RVHAL_EXCM_LEVEL_CLIC);
 #endif /* !SOC_INIT_CLIC_SUPPORTED */
     /**
      * In theory, this function should not return immediately as there is a
@@ -474,7 +474,7 @@ void vPortClearInterruptMaskFromISR(UBaseType_t prev_int_level)
 #if !SOC_INT_CLIC_SUPPORTED
     REG_WRITE(INTERRUPT_CURRENT_CORE_INT_THRESH_REG, prev_int_level);
 #else
-    rv_utils_restore_intlevel(prev_int_level);
+    rv_utils_restore_intlevel_regval(prev_int_level);
 #endif /* SOC_INIT_CLIC_SUPPORTED */
     /**
      * The delay between the moment we unmask the interrupt threshold register

components/idf_test/include/esp32p4/idf_performance_target.h

@@ -10,3 +10,7 @@
 #define IDF_PERFORMANCE_MAX_SPI_PER_TRANS_POLLING                               1000
 #define IDF_PERFORMANCE_MAX_SPI_PER_TRANS_NO_POLLING_NO_DMA                     1000
 #define IDF_PERFORMANCE_MAX_SPI_PER_TRANS_POLLING_NO_DMA                        1000
+
+/* Spinlock performance on esp32p4 is slower. May need to adjust these values once IDF-7898 is fixed */
+#define IDF_PERFORMANCE_MAX_FREERTOS_SPINLOCK_CYCLES_PER_OP                     380
+#define IDF_PERFORMANCE_MAX_FREERTOS_SPINLOCK_CYCLES_PER_OP_UNICORE             135

components/riscv/include/riscv/rv_utils.h

@@ -51,6 +51,19 @@ extern "C" {
 #define RVHAL_INTR_ENABLE_THRESH    1
 #endif /* SOC_INT_CLIC_SUPPORTED */
 
+/* On CLIC, the interrupt threshold is stored in the upper (NLBITS) of the mintthresh register, with the other (8 - NLBITS)
+ * defaulted to 1. We form the interrupt level bits here to avoid doing this at run time */
+#if SOC_INT_CLIC_SUPPORTED
+/* Helper macro to translate absolute interrupt level to CLIC interrupt threshold bits in the mintthresh reg */
+#define CLIC_INT_THRESH(intlevel)       (((((intlevel) << (8 - NLBITS))) | 0x1f) << CLIC_CPU_INT_THRESH_S)
+
+/* Helper macro to set interrupt level RVHAL_EXCM_LEVEL. Used during critical sections */
+#define RVHAL_EXCM_LEVEL_CLIC           (CLIC_INT_THRESH(RVHAL_EXCM_LEVEL - 1))
+
+/* Helper macro to enable interrupts. */
+#define RVHAL_INTR_ENABLE_THRESH_CLIC   (CLIC_INT_THRESH(RVHAL_INTR_ENABLE_THRESH))
+#endif /* SOC_INT_CLIC_SUPPORTED */
+
 /* --------------------------------------------------- CPU Control -----------------------------------------------------
  *
  * ------------------------------------------------------------------------------------------------------------------ */
@@ -192,6 +205,38 @@ FORCE_INLINE_ATTR uint32_t __attribute__((always_inline)) rv_utils_set_intlevel(
     return old_thresh;
 }
 
+/* Direct register write version of rv_utils_restore_intlevel(). Used to speed up critical sections. */
+FORCE_INLINE_ATTR void __attribute__((always_inline)) rv_utils_restore_intlevel_regval(uint32_t restoreval)
+{
+    /* This function expects restoreval to be in the format needed to restore the interrupt level with
+     * a single write to the mintthresh register without further manipulations needed.
+     * This is done to quicken up exit for critical sections */
+    REG_WRITE(CLIC_INT_THRESH_REG, restoreval);
+}
+
+/* Direct register write version of rv_utils_set_intlevel(). Used to speed up critical sections. */
+FORCE_INLINE_ATTR uint32_t __attribute__((always_inline)) rv_utils_set_intlevel_regval(uint32_t intlevel)
+{
+    uint32_t old_mstatus = RV_CLEAR_CSR(mstatus, MSTATUS_MIE);
+    uint32_t old_thresh = REG_READ(CLIC_INT_THRESH_REG);
+
+    /* This function expects the interrupt level to be available in the format needed for mintthresh reg.
+     * Providing an absolute interrupt level will result in incorrect behavior.
+     * See CLIC_INT_THRESH() macro for details of how the interrupt level must be provided. */
+    REG_WRITE(CLIC_INT_THRESH_REG, intlevel);
+    /**
+     * After writing the threshold register, the new threshold is not directly taken into account by the CPU.
+     * By executing ~8 nop instructions, or by performing a memory load right now, the previous memory write
+     * operations is forced, making the new threshold active. It is then safe to re-enable MIE bit in mstatus.
+     */
+    REG_READ(CLIC_INT_THRESH_REG);
+    RV_SET_CSR(mstatus, old_mstatus & MSTATUS_MIE);
+
+    /* We return the mintthresh register value and NOT the absolute interrupt threshold level.
+     * This is done to avoid extra bit manipulations during critical sections. */
+    return old_thresh;
+}
+
 FORCE_INLINE_ATTR uint32_t __attribute__((always_inline)) rv_utils_mask_int_level_lower_than(uint32_t intlevel)
 {
     /* CLIC's set interrupt level is inclusive, i.e. it does mask the set level  */