diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index e17cf3f9b31..ef9a20d0cc9 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -53,6 +53,16 @@ config ARM_ON_ENTER_CPU_IDLE_HOOK If needed, this hook can be used to prevent the CPU from actually entering sleep by skipping the WFE/WFI instruction. +config ARM_ON_ENTER_CPU_IDLE_PREPARE_HOOK + bool + help + Enables a hook (z_arm_on_enter_cpu_idle_prepare()) that is called when + the CPU is made idle (by k_cpu_idle() or k_cpu_atomic_idle()). + If needed, this hook can prepare data to upcoming call to + z_arm_on_enter_cpu_idle(). The z_arm_on_enter_cpu_idle_prepare differs + from z_arm_on_enter_cpu_idle because it is called before interrupts are + disabled. + config ARM_ON_EXIT_CPU_IDLE bool help diff --git a/arch/arm/core/aarch32/cpu_idle.S b/arch/arm/core/aarch32/cpu_idle.S index 8164959ab29..95e37917180 100644 --- a/arch/arm/core/aarch32/cpu_idle.S +++ b/arch/arm/core/aarch32/cpu_idle.S @@ -85,16 +85,24 @@ _skip_\@: .endm SECTION_FUNC(TEXT, arch_cpu_idle) -#ifdef CONFIG_TRACING +#if defined(CONFIG_TRACING) || \ + defined(CONFIG_ARM_ON_ENTER_CPU_IDLE_PREPARE_HOOK) push {r0, lr} + +#ifdef CONFIG_TRACING bl sys_trace_idle +#endif +#ifdef CONFIG_ARM_ON_ENTER_CPU_IDLE_PREPARE_HOOK + bl z_arm_on_enter_cpu_idle_prepare +#endif + #if defined(CONFIG_ARMV6_M_ARMV8_M_BASELINE) pop {r0, r1} mov lr, r1 #else pop {r0, lr} #endif /* CONFIG_ARMV6_M_ARMV8_M_BASELINE */ -#endif /* CONFIG_TRACING */ +#endif #if defined(CONFIG_ARMV7_M_ARMV8_M_MAINLINE) /* @@ -138,17 +146,24 @@ SECTION_FUNC(TEXT, arch_cpu_idle) bx lr SECTION_FUNC(TEXT, arch_cpu_atomic_idle) -#ifdef CONFIG_TRACING +#if defined(CONFIG_TRACING) || \ + defined(CONFIG_ARM_ON_ENTER_CPU_IDLE_PREPARE_HOOK) push {r0, lr} + +#ifdef CONFIG_TRACING bl sys_trace_idle +#endif +#ifdef CONFIG_ARM_ON_ENTER_CPU_IDLE_PREPARE_HOOK + bl z_arm_on_enter_cpu_idle_prepare +#endif + #if defined(CONFIG_ARMV6_M_ARMV8_M_BASELINE) pop {r0, r1} mov lr, r1 #else pop {r0, lr} #endif /* CONFIG_ARMV6_M_ARMV8_M_BASELINE */ -#endif /* CONFIG_TRACING */ - +#endif /* * Lock PRIMASK while sleeping: wfe will still get interrupted by * incoming interrupts but the CPU will not service them right away. diff --git a/drivers/timer/Kconfig.nrf_rtc b/drivers/timer/Kconfig.nrf_rtc index cda05d4dabe..acb6f123afe 100644 --- a/drivers/timer/Kconfig.nrf_rtc +++ b/drivers/timer/Kconfig.nrf_rtc @@ -19,6 +19,7 @@ if NRF_RTC_TIMER config NRF_RTC_TIMER_USER_CHAN_COUNT int "Additional channels that can be used" + default 2 if NRF_802154_RADIO_DRIVER && SOC_NRF5340_CPUNET default 3 if NRF_802154_RADIO_DRIVER default 0 help diff --git a/drivers/timer/nrf_rtc_timer.c b/drivers/timer/nrf_rtc_timer.c index ef7b2aab574..ff5b55585fe 100644 --- a/drivers/timer/nrf_rtc_timer.c +++ b/drivers/timer/nrf_rtc_timer.c @@ -16,15 +16,18 @@ #include #include +#define RTC_PRETICK (IS_ENABLED(CONFIG_SOC_NRF53_RTC_PRETICK) && \ + IS_ENABLED(CONFIG_SOC_NRF5340_CPUNET)) + #define EXT_CHAN_COUNT CONFIG_NRF_RTC_TIMER_USER_CHAN_COUNT #define CHAN_COUNT (EXT_CHAN_COUNT + 1) #define RTC NRF_RTC1 #define RTC_IRQn NRFX_IRQ_NUMBER_GET(RTC) #define RTC_LABEL rtc1 -#define RTC_CH_COUNT RTC1_CC_NUM +#define CHAN_COUNT_MAX (RTC1_CC_NUM - (RTC_PRETICK ? 1 : 0)) -BUILD_ASSERT(CHAN_COUNT <= RTC_CH_COUNT, "Not enough compare channels"); +BUILD_ASSERT(CHAN_COUNT <= CHAN_COUNT_MAX, "Not enough compare channels"); /* Ensure that counter driver for RTC1 is not enabled. */ BUILD_ASSERT(DT_NODE_HAS_STATUS(DT_NODELABEL(RTC_LABEL), disabled), "Counter for RTC1 must be disabled"); @@ -43,6 +46,8 @@ BUILD_ASSERT(DT_NODE_HAS_STATUS(DT_NODELABEL(RTC_LABEL), disabled), #define ANCHOR_RANGE_END (7 * COUNTER_SPAN / 8) #define TARGET_TIME_INVALID (UINT64_MAX) +extern void rtc_pretick_rtc1_isr_hook(void); + static volatile uint32_t overflow_cnt; static volatile uint64_t anchor; static uint64_t last_count; @@ -231,9 +236,20 @@ uint64_t z_nrf_rtc_timer_get_ticks(k_timeout_t t) * @param[in] chan A channel for which a new CC value is to be set. * * @param[in] req_cc Requested CC register value to be set. + * + * @param[in] exact Use @c false to allow CC adjustment if @c req_cc value is + * close to the current value of the timer. + * Use @c true to disallow CC adjustment. The function can + * fail with -EINVAL result if @p req_cc is too close to the + * current value. + * + * @retval 0 The requested CC has been set successfully. + * @retval -EINVAL The requested CC value could not be reliably set. */ -static void set_alarm(int32_t chan, uint32_t req_cc) +static int set_alarm(int32_t chan, uint32_t req_cc, bool exact) { + int ret = 0; + /* Ensure that the value exposed in this driver API is consistent with * assumptions of this function. */ @@ -300,9 +316,16 @@ static void set_alarm(int32_t chan, uint32_t req_cc) now = counter(); if (counter_sub(now, req_cc) > COUNTER_HALF_SPAN) { event_clear(chan); + if (exact) { + ret = -EINVAL; + break; + } } else { break; } + } else if (exact) { + ret = -EINVAL; + break; } cc_val = now + cc_inc; @@ -311,11 +334,13 @@ static void set_alarm(int32_t chan, uint32_t req_cc) break; } } + + return ret; } static int compare_set_nolocks(int32_t chan, uint64_t target_time, z_nrf_rtc_timer_compare_handler_t handler, - void *user_data) + void *user_data, bool exact) { int ret = 0; uint32_t cc_value = absolute_time_to_cc(target_time); @@ -331,29 +356,33 @@ static int compare_set_nolocks(int32_t chan, uint64_t target_time, /* Target time is valid and is different than currently set. * Set CC value. */ - set_alarm(chan, cc_value); + ret = set_alarm(chan, cc_value, exact); } - } else { + } else if (!exact) { /* Force ISR handling when exiting from critical section. */ atomic_or(&force_isr_mask, BIT(chan)); + } else { + ret = -EINVAL; } - cc_data[chan].target_time = target_time; - cc_data[chan].callback = handler; - cc_data[chan].user_context = user_data; + if (ret == 0) { + cc_data[chan].target_time = target_time; + cc_data[chan].callback = handler; + cc_data[chan].user_context = user_data; + } return ret; } static int compare_set(int32_t chan, uint64_t target_time, z_nrf_rtc_timer_compare_handler_t handler, - void *user_data) + void *user_data, bool exact) { bool key; key = compare_int_lock(chan); - int ret = compare_set_nolocks(chan, target_time, handler, user_data); + int ret = compare_set_nolocks(chan, target_time, handler, user_data, exact); compare_int_unlock(chan, key); @@ -366,7 +395,16 @@ int z_nrf_rtc_timer_set(int32_t chan, uint64_t target_time, { __ASSERT_NO_MSG(chan > 0 && chan < CHAN_COUNT); - return compare_set(chan, target_time, handler, user_data); + return compare_set(chan, target_time, handler, user_data, false); +} + +int z_nrf_rtc_timer_exact_set(int32_t chan, uint64_t target_time, + z_nrf_rtc_timer_compare_handler_t handler, + void *user_data) +{ + __ASSERT_NO_MSG(chan > 0 && chan < CHAN_COUNT); + + return compare_set(chan, target_time, handler, user_data, true); } void z_nrf_rtc_timer_abort(int32_t chan) @@ -447,7 +485,7 @@ static void sys_clock_timeout_handler(int32_t chan, * so it won't get preempted by the interrupt. */ compare_set(chan, last_count + CYC_PER_TICK, - sys_clock_timeout_handler, NULL); + sys_clock_timeout_handler, NULL, false); } sys_clock_announce(dticks); @@ -525,6 +563,10 @@ void rtc_nrf_isr(const void *arg) { ARG_UNUSED(arg); + if (RTC_PRETICK) { + rtc_pretick_rtc1_isr_hook(); + } + if (nrf_rtc_int_enable_check(RTC, NRF_RTC_INT_OVERFLOW_MASK) && nrf_rtc_event_check(RTC, NRF_RTC_EVENT_OVERFLOW)) { nrf_rtc_event_clear(RTC, NRF_RTC_EVENT_OVERFLOW); @@ -643,7 +685,7 @@ void sys_clock_set_timeout(int32_t ticks, bool idle) uint64_t target_time = cyc + last_count; - compare_set(0, target_time, sys_clock_timeout_handler, NULL); + compare_set(0, target_time, sys_clock_timeout_handler, NULL, false); } uint32_t sys_clock_elapsed(void) @@ -721,7 +763,7 @@ static int sys_clock_driver_init(void) uint32_t initial_timeout = IS_ENABLED(CONFIG_TICKLESS_KERNEL) ? MAX_CYCLES : CYC_PER_TICK; - compare_set(0, initial_timeout, sys_clock_timeout_handler, NULL); + compare_set(0, initial_timeout, sys_clock_timeout_handler, NULL, false); z_nrf_clock_control_lf_on(mode); diff --git a/include/zephyr/arch/arm/aarch32/misc.h b/include/zephyr/arch/arm/aarch32/misc.h index 24ed69f663c..ab67a35e94c 100644 --- a/include/zephyr/arch/arm/aarch32/misc.h +++ b/include/zephyr/arch/arm/aarch32/misc.h @@ -51,6 +51,15 @@ extern bool z_arm_thread_is_in_user_mode(void); bool z_arm_on_enter_cpu_idle(void); #endif +#if defined(CONFIG_ARM_ON_ENTER_CPU_IDLE_PREPARE_HOOK) +/* Prototype of a hook that can be enabled to be called every time the CPU is + * made idle (the calls will be done from k_cpu_idle() and k_cpu_atomic_idle()). + * The function is called before interrupts are disabled and can prepare to + * upcoming call to z_arm_on_enter_cpu_idle. + */ +void z_arm_on_enter_cpu_idle_prepare(void); +#endif + #endif #ifdef __cplusplus diff --git a/include/zephyr/drivers/timer/nrf_rtc_timer.h b/include/zephyr/drivers/timer/nrf_rtc_timer.h index b11f201386e..c5992bee4bd 100644 --- a/include/zephyr/drivers/timer/nrf_rtc_timer.h +++ b/include/zephyr/drivers/timer/nrf_rtc_timer.h @@ -127,11 +127,42 @@ uint32_t z_nrf_rtc_timer_compare_read(int32_t chan); * @retval 0 if the compare channel was set successfully. * @retval -EINVAL if provided target time was further than * @c NRF_RTC_TIMER_MAX_SCHEDULE_SPAN ticks in the future. + * + * @sa @ref z_nrf_rtc_timer_exact_set */ int z_nrf_rtc_timer_set(int32_t chan, uint64_t target_time, z_nrf_rtc_timer_compare_handler_t handler, void *user_data); +/** @brief Try to set compare channel exactly to given value. + * + * @note This function is similar to @ref z_nrf_rtc_timer_set, but the compare + * channel will be set to expected value only when it can be guaranteed that + * the hardware event will be generated exactly at expected @c target_time in + * the future. If the @c target_time is in the past or so close in the future + * that the reliable generation of event would require adjustment of compare + * value (as would @ref z_nrf_rtc_timer_set function do), neither the hardware + * event nor interrupt will be generated and the function fails. + * + * @param chan Channel ID between 1 and CONFIG_NRF_RTC_TIMER_USER_CHAN_COUNT. + * + * @param target_time Absolute target time in ticks. + * + * @param handler User function called in the context of the RTC interrupt. + * + * @param user_data Data passed to the handler. + * + * @retval 0 if the compare channel was set successfully. + * @retval -EINVAL if provided target time was further than + * @c NRF_RTC_TIMER_MAX_SCHEDULE_SPAN ticks in the future + * or the target time is in the past or is so close in the future that + * event generation could not be guaranteed without adjusting + * compare value of that channel. + */ +int z_nrf_rtc_timer_exact_set(int32_t chan, uint64_t target_time, + z_nrf_rtc_timer_compare_handler_t handler, + void *user_data); + /** @brief Abort a timer requested with @ref z_nrf_rtc_timer_set. * * If an abort operation is performed too late it is still possible for an event diff --git a/include/zephyr/sys/barrier.h b/include/zephyr/sys/barrier.h new file mode 100644 index 00000000000..6acc0476ec1 --- /dev/null +++ b/include/zephyr/sys/barrier.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2023 Carlo Caione + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ZEPHYR_INCLUDE_SYS_BARRIER_H_ +#define ZEPHYR_INCLUDE_SYS_BARRIER_H_ + +#include + +#if defined(CONFIG_BARRIER_OPERATIONS_ARCH) +/* Empty */ +#elif defined(CONFIG_BARRIER_OPERATIONS_BUILTIN) +#include +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @addtogroup barrier_apis Barrier Services APIs + * @ingroup kernel_apis + * @{ + */ + +/** + * @brief Full/sequentially-consistent data memory barrier. + * + * This routine acts as a synchronization fence between threads and prevents + * re-ordering of data accesses instructions across the barrier instruction. + */ +static ALWAYS_INLINE void barrier_dmem_fence_full(void) +{ +#if defined(CONFIG_BARRIER_OPERATIONS_ARCH) || defined(CONFIG_BARRIER_OPERATIONS_BUILTIN) + z_barrier_dmem_fence_full(); +#endif +} + +/** @} */ + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* ZEPHYR_INCLUDE_SYS_ATOMIC_H_ */ diff --git a/include/zephyr/sys/barrier_builtin.h b/include/zephyr/sys/barrier_builtin.h new file mode 100644 index 00000000000..98bec6eaa36 --- /dev/null +++ b/include/zephyr/sys/barrier_builtin.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2023 Carlo Caione + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef ZEPHYR_INCLUDE_SYS_BARRIER_BUILTIN_H_ +#define ZEPHYR_INCLUDE_SYS_BARRIER_BUILTIN_H_ + +#ifndef ZEPHYR_INCLUDE_SYS_BARRIER_H_ +#error Please include +#endif + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +static ALWAYS_INLINE void z_barrier_dmem_fence_full(void) +{ + __atomic_thread_fence(__ATOMIC_SEQ_CST); +} + +#ifdef __cplusplus +} +#endif + +#endif /* ZEPHYR_INCLUDE_SYS_BARRIER_BUILTIN_H_ */ diff --git a/kernel/Kconfig b/kernel/Kconfig index 797a98beb2a..e69b43f201e 100644 --- a/kernel/Kconfig +++ b/kernel/Kconfig @@ -452,6 +452,21 @@ config SYSTEM_WORKQUEUE_NO_YIELD endmenu +menu "Barrier Operations" +config BARRIER_OPERATIONS_BUILTIN + bool + help + Use the compiler builtin functions for barrier operations. This is + the preferred method. However, support for all arches in GCC is + incomplete. + +config BARRIER_OPERATIONS_ARCH + bool + help + Use when there isn't support for compiler built-ins, but you have + written optimized assembly code under arch/ which implements these. +endmenu + menu "Atomic Operations" config ATOMIC_OPERATIONS_BUILTIN bool diff --git a/soc/arm/nordic_nrf/nrf53/Kconfig.soc b/soc/arm/nordic_nrf/nrf53/Kconfig.soc index 67cb8a08a6f..3d1e3bee7fa 100644 --- a/soc/arm/nordic_nrf/nrf53/Kconfig.soc +++ b/soc/arm/nordic_nrf/nrf53/Kconfig.soc @@ -9,12 +9,16 @@ config SOC_NRF5340_CPUAPP select CPU_HAS_NRF_IDAU select CPU_HAS_FPU select ARMV8_M_DSP + imply SOC_NRF53_RTC_PRETICK + imply SOC_NRF53_ANOMALY_168_WORKAROUND config SOC_NRF5340_CPUNET bool select HAS_NO_PM select ARM_ON_EXIT_CPU_IDLE imply SOC_NRF53_ANOMALY_160_WORKAROUND_NEEDED + imply SOC_NRF53_RTC_PRETICK if !WDT_NRFX + imply SOC_NRF53_ANOMALY_168_WORKAROUND choice prompt "nRF53x MCU Selection" @@ -47,6 +51,53 @@ config SOC_NRF53_ANOMALY_160_WORKAROUND depends on SYS_CLOCK_EXISTS select ARM_ON_ENTER_CPU_IDLE_HOOK +config SOC_NRF53_RTC_PRETICK + bool "Pre-tick workaround for nRF5340 anomaly 165" + depends on (SYS_CLOCK_EXISTS && SOC_NRF5340_CPUNET) || SOC_NRF5340_CPUAPP + select NRFX_DPPI + select ARM_ON_ENTER_CPU_IDLE_HOOK if SOC_NRF5340_CPUNET + select ARM_ON_ENTER_CPU_IDLE_PREPARE_HOOK if SOC_NRF5340_CPUNET + help + Indicates that the pre-tick workaround for the anomaly 165 that affects + the nRF5340 SoC should be applied. The workaround applies to wake ups caused + by EVENTS_COMPARE and EVENTS_OVRFLW on RTC0 and RTC1 for which interrupts are + enabled through INTENSET register. The case when these events are generated + by EVTEN but without interrupts enabled through INTENSET is not handled. + The EVENTS_TICK event is not handled. + +if SOC_NRF53_RTC_PRETICK + +config SOC_NRF53_RTC_PRETICK_IPC_CH_FROM_NET + int "IPC 0 channel for RTC pretick" + range 0 15 + default 10 + +config SOC_NRF53_RTC_PRETICK_IPC_CH_TO_NET + int "IPC 1 channel for RTC pretick" + range 0 15 + default 11 + +endif + +config SOC_NRF53_ANOMALY_168_WORKAROUND + bool "Workaround for nRF5340 anomaly 168" + select ARM_ON_EXIT_CPU_IDLE + help + Indicates that the workaround for the anomaly 168 that affects + the nRF5340 SoC should be applied. + The workaround involves execution of 8 NOP instructions when the CPU + exist its idle state (when the WFI/WFE instruction returns) and it is + enabled by default for both the application and network core. + +config SOC_NRF53_ANOMALY_168_WORKAROUND_FOR_EXECUTION_FROM_RAM + bool "Extend the workaround to execution at 128 MHz from RAM" + depends on SOC_NRF53_ANOMALY_168_WORKAROUND && SOC_NRF5340_CPUAPP + help + Indicates that the anomaly 168 workaround is to be extended to cover + also a specific case when the WFI/WFE instruction is executed at 128 + MHz from RAM. Then, 26 instead of 8 NOP instructions needs to be + executed after WFI/WFE. This extension is not enabled by default. + if SOC_NRF5340_CPUAPP config SOC_DCDC_NRF53X_APP @@ -176,6 +227,7 @@ config NRF53_SYNC_RTC_INIT_PRIORITY nRF53 Synchronized RTC initialization priority. config NRF_RTC_TIMER_USER_CHAN_COUNT + default 2 if NRF_802154_RADIO_DRIVER && SOC_NRF5340_CPUNET default 3 if NRF_802154_RADIO_DRIVER default 1 diff --git a/soc/arm/nordic_nrf/nrf53/soc.c b/soc/arm/nordic_nrf/nrf53/soc.c index 7f8e1b5ba81..5193d79162e 100644 --- a/soc/arm/nordic_nrf/nrf53/soc.c +++ b/soc/arm/nordic_nrf/nrf53/soc.c @@ -16,10 +16,13 @@ #include #include #include +#include #include #include #include #include +#include +#include #if defined(CONFIG_SOC_NRF5340_CPUAPP) #include #include @@ -33,11 +36,30 @@ #if defined(CONFIG_PM_S2RAM) #include #endif +#include +#include #include #define PIN_XL1 0 #define PIN_XL2 1 + +/** @brief Symbol specifying maximum number of available compare channels. */ +#define NRF_RTC_CC_COUNT_MAX NRFX_ARRAY_SIZE(((NRF_RTC_Type *)0)->EVENTS_COMPARE) + +/** @brief Macro for creating the interrupt bitmask for the specified compare channel. */ +#define NRF_RTC_CHANNEL_INT_MASK(ch) ((uint32_t)(NRF_RTC_INT_COMPARE0_MASK) << (ch)) + +/** @brief Macro for obtaining the compare event for the specified channel. */ +#define NRF_RTC_CHANNEL_EVENT_ADDR(ch) \ + (nrf_rtc_event_t)((NRF_RTC_EVENT_COMPARE_0) + (ch) * sizeof(uint32_t)) + +#define RTC1_PRETICK_CC_CHAN (RTC1_CC_NUM - 1) + +/* Mask of CC channels capable of generating interrupts, see nrf_rtc_timer.c */ +#define RTC1_PRETICK_SELECTED_CC_MASK BIT_MASK(CONFIG_NRF_RTC_TIMER_USER_CHAN_COUNT + 1U) +#define RTC0_PRETICK_SELECTED_CC_MASK BIT_MASK(NRF_RTC_CC_COUNT_MAX) + #if defined(CONFIG_SOC_NRF_GPIO_FORWARDER_FOR_NRF5340) #define GPIOS_PSEL_BY_IDX(node_id, prop, idx) \ NRF_DT_GPIOS_TO_PSEL_BY_IDX(node_id, prop, idx), @@ -122,25 +144,359 @@ static bool nrf53_anomaly_160_check(void) return true; } +#endif /* CONFIG_SOC_NRF53_ANOMALY_160_WORKAROUND */ + +#if defined(CONFIG_SOC_NRF53_RTC_PRETICK) && defined(CONFIG_SOC_NRF5340_CPUNET) + +BUILD_ASSERT(!IS_ENABLED(CONFIG_WDT_NRFX), + "For CONFIG_SOC_NRF53_RTC_PRETICK watchdog is used internally for the pre-tick workaround on nRF5340 cpunet. Application cannot use the watchdog."); + +static inline uint32_t rtc_counter_sub(uint32_t a, uint32_t b) +{ + return (a - b) & NRF_RTC_COUNTER_MAX; +} + +static bool rtc_ticks_to_next_event_get(NRF_RTC_Type *rtc, uint32_t selected_cc_mask, uint32_t cntr, + uint32_t *ticks_to_next_event) +{ + bool result = false; + + /* Let's preload register to speed-up. */ + uint32_t reg_intenset = rtc->INTENSET; + + /* Note: TICK event not handled. */ + + if (reg_intenset & NRF_RTC_INT_OVERFLOW_MASK) { + /* Overflow can generate an interrupt. */ + *ticks_to_next_event = NRF_RTC_COUNTER_MAX + 1U - cntr; + result = true; + } + + for (uint32_t chan = 0; chan < NRF_RTC_CC_COUNT_MAX; chan++) { + if ((selected_cc_mask & (1U << chan)) && + (reg_intenset & NRF_RTC_CHANNEL_INT_MASK(chan))) { + /* The CC is in selected mask and is can generate an interrupt. */ + uint32_t cc = nrf_rtc_cc_get(rtc, chan); + uint32_t ticks_to_fire = rtc_counter_sub(cc, cntr); + + if (ticks_to_fire == 0U) { + /* When ticks_to_fire == 0, the event should have been just + * generated the interrupt can be already handled or be pending. + * However the next event is expected to be after counter wraps. + */ + ticks_to_fire = NRF_RTC_COUNTER_MAX + 1U; + } + + if (!result) { + *ticks_to_next_event = ticks_to_fire; + result = true; + } else if (ticks_to_fire < *ticks_to_next_event) { + *ticks_to_next_event = ticks_to_fire; + result = true; + } else { + /* CC that fires no earlier than already found. */ + } + } + } + + return result; +} + +static void rtc_counter_synchronized_get(NRF_RTC_Type *rtc_a, NRF_RTC_Type *rtc_b, + uint32_t *counter_a, uint32_t *counter_b) +{ + do { + *counter_a = nrf_rtc_counter_get(rtc_a); + barrier_dmem_fence_full(); + *counter_b = nrf_rtc_counter_get(rtc_b); + barrier_dmem_fence_full(); + } while (*counter_a != nrf_rtc_counter_get(rtc_a)); +} + +static uint8_t cpu_idle_prepare_monitor_dummy; +static bool cpu_idle_prepare_allows_sleep; + +static void cpu_idle_prepare_monitor_begin(void) +{ + __LDREXB(&cpu_idle_prepare_monitor_dummy); +} + +/* Returns 0 if no exception preempted since the last call to cpu_idle_prepare_monitor_begin. */ +static bool cpu_idle_prepare_monitor_end(void) +{ + /* The value stored is irrelevant. If any exception took place after + * cpu_idle_prepare_monitor_begin, the the local monitor is cleared and + * the store fails returning 1. + * See Arm v8-M Architecture Reference Manual: + * Chapter B9.2 The local monitors + * Chapter B9.4 Exclusive access instructions and the monitors + * See Arm Cortex-M33 Processor Technical Reference Manual + * Chapter 3.5 Exclusive monitor + */ + return __STREXB(0U, &cpu_idle_prepare_monitor_dummy); +} + +static void rtc_pretick_finish_previous(void) +{ + NRF_IPC->PUBLISH_RECEIVE[CONFIG_SOC_NRF53_RTC_PRETICK_IPC_CH_TO_NET] &= + ~IPC_PUBLISH_RECEIVE_EN_Msk; + + nrf_rtc_event_clear(NRF_RTC1, NRF_RTC_CHANNEL_EVENT_ADDR(RTC1_PRETICK_CC_CHAN)); +} + + +void z_arm_on_enter_cpu_idle_prepare(void) +{ + bool ok_to_sleep = true; + + cpu_idle_prepare_monitor_begin(); + + uint32_t rtc_counter = 0U; + uint32_t rtc_ticks_to_next_event = 0U; + uint32_t rtc0_counter = 0U; + uint32_t rtc0_ticks_to_next_event = 0U; + + rtc_counter_synchronized_get(NRF_RTC1, NRF_RTC0, &rtc_counter, &rtc0_counter); + + bool rtc_scheduled = rtc_ticks_to_next_event_get(NRF_RTC1, RTC1_PRETICK_SELECTED_CC_MASK, + rtc_counter, &rtc_ticks_to_next_event); + + if (rtc_ticks_to_next_event_get(NRF_RTC0, RTC0_PRETICK_SELECTED_CC_MASK, rtc0_counter, + &rtc0_ticks_to_next_event)) { + /* An event is scheduled on RTC0. */ + if (!rtc_scheduled) { + rtc_ticks_to_next_event = rtc0_ticks_to_next_event; + rtc_scheduled = true; + } else if (rtc0_ticks_to_next_event < rtc_ticks_to_next_event) { + rtc_ticks_to_next_event = rtc0_ticks_to_next_event; + } else { + /* Event on RTC0 will not happen earlier than already found. */ + } + } + + if (rtc_scheduled) { + static bool rtc_pretick_cc_set_on_time; + /* The pretick should happen 1 tick before the earliest scheduled event + * that can trigger an interrupt. + */ + uint32_t rtc_pretick_cc_val = (rtc_counter + rtc_ticks_to_next_event - 1U) + & NRF_RTC_COUNTER_MAX; + + if (rtc_pretick_cc_val != nrf_rtc_cc_get(NRF_RTC1, RTC1_PRETICK_CC_CHAN)) { + /* The CC for pretick needs to be updated. */ + rtc_pretick_finish_previous(); + nrf_rtc_cc_set(NRF_RTC1, RTC1_PRETICK_CC_CHAN, rtc_pretick_cc_val); + + if (rtc_ticks_to_next_event >= NRF_RTC_COUNTER_MAX/2) { + /* Pretick is scheduled so far in the future, assumed on time. */ + rtc_pretick_cc_set_on_time = true; + } else { + /* Let's check if we updated CC on time, so that the CC can + * take effect. + */ + barrier_dmem_fence_full(); + rtc_counter = nrf_rtc_counter_get(NRF_RTC1); + uint32_t pretick_cc_to_counter = + rtc_counter_sub(rtc_pretick_cc_val, rtc_counter); + + if ((pretick_cc_to_counter < 3) || + (pretick_cc_to_counter >= NRF_RTC_COUNTER_MAX/2)) { + /* The COUNTER value is close enough to the expected + * pretick CC or has just expired, so the pretick event + * generation is not guaranteed. + */ + rtc_pretick_cc_set_on_time = false; + } else { + /* The written rtc_pretick_cc is guaranteed to to trigger + * compare event. + */ + rtc_pretick_cc_set_on_time = true; + } + } + } else { + /* The CC for pretick doesn't need to be updated, however + * rtc_pretick_cc_set_on_time still holds if we managed to set it on time. + */ + } + + /* If the CC for pretick is set on time, so the pretick CC event can be reliably + * generated then allow to sleep. Otherwise (the CC for pretick cannot be reliably + * generated, because CC was set very short to it's fire time) sleep not at all. + */ + ok_to_sleep = rtc_pretick_cc_set_on_time; + } else { + /* No events on any RTC timers are scheduled. */ + } + + if (ok_to_sleep) { + NRF_IPC->PUBLISH_RECEIVE[CONFIG_SOC_NRF53_RTC_PRETICK_IPC_CH_TO_NET] |= + IPC_PUBLISH_RECEIVE_EN_Msk; + if (!nrf_rtc_event_check(NRF_RTC1, + NRF_RTC_CHANNEL_EVENT_ADDR(RTC1_PRETICK_CC_CHAN))) { + NRF_WDT->TASKS_STOP = 1; + /* Check if any event did not occur after we checked for + * stopping condition. If yes, we might have stopped WDT + * when it should be running. Restart it. + */ + if (nrf_rtc_event_check(NRF_RTC1, + NRF_RTC_CHANNEL_EVENT_ADDR(RTC1_PRETICK_CC_CHAN))) { + NRF_WDT->TASKS_START = 1; + } + } + } + + cpu_idle_prepare_allows_sleep = ok_to_sleep; +} +#endif /* CONFIG_SOC_NRF53_RTC_PRETICK && CONFIG_SOC_NRF5340_CPUNET */ +#if defined(CONFIG_SOC_NRF53_ANOMALY_160_WORKAROUND) || \ + (defined(CONFIG_SOC_NRF53_RTC_PRETICK) && defined(CONFIG_SOC_NRF5340_CPUNET)) bool z_arm_on_enter_cpu_idle(void) { - bool ok_to_sleep = nrf53_anomaly_160_check(); + bool ok_to_sleep = true; -#if (LOG_LEVEL >= LOG_LEVEL_DBG) - static bool suppress_message; +#if defined(CONFIG_SOC_NRF53_RTC_PRETICK) && defined(CONFIG_SOC_NRF5340_CPUNET) + if (cpu_idle_prepare_monitor_end() == 0) { + /* No exception happened since cpu_idle_prepare_monitor_begin. + * We can trust the outcome of. z_arm_on_enter_cpu_idle_prepare + */ + ok_to_sleep = cpu_idle_prepare_allows_sleep; + } else { + /* Exception happened since cpu_idle_prepare_monitor_begin. + * The values which z_arm_on_enter_cpu_idle_prepare could be changed + * by the exception, so we can not trust to it's outcome. + * Do not sleep at all, let's try in the next iteration of idle loop. + */ + ok_to_sleep = false; + } +#endif +#if defined(CONFIG_SOC_NRF53_ANOMALY_160_WORKAROUND) if (ok_to_sleep) { - suppress_message = false; - } else if (!suppress_message) { - LOG_DBG("Anomaly 160 trigger conditions detected."); - suppress_message = true; + ok_to_sleep = nrf53_anomaly_160_check(); + +#if (LOG_LEVEL >= LOG_LEVEL_DBG) + static bool suppress_message; + + if (ok_to_sleep) { + suppress_message = false; + } else if (!suppress_message) { + LOG_DBG("Anomaly 160 trigger conditions detected."); + suppress_message = true; + } +#endif + } +#endif /* CONFIG_SOC_NRF53_ANOMALY_160_WORKAROUND */ + +#if defined(CONFIG_SOC_NRF53_RTC_PRETICK) && defined(CONFIG_SOC_NRF5340_CPUNET) + if (!ok_to_sleep) { + NRF_IPC->PUBLISH_RECEIVE[CONFIG_SOC_NRF53_RTC_PRETICK_IPC_CH_TO_NET] &= + ~IPC_PUBLISH_RECEIVE_EN_Msk; + NRF_WDT->TASKS_STOP = 1; } #endif return ok_to_sleep; } -#endif /* CONFIG_SOC_NRF53_ANOMALY_160_WORKAROUND */ +#endif /* CONFIG_SOC_NRF53_ANOMALY_160_WORKAROUND || + * (CONFIG_SOC_NRF53_RTC_PRETICK && CONFIG_SOC_NRF5340_CPUNET) + */ + +#if CONFIG_SOC_NRF53_RTC_PRETICK +#ifdef CONFIG_SOC_NRF5340_CPUAPP +/* RTC pretick - application core part. */ +static int rtc_pretick_cpuapp_init(void) +{ + uint8_t ch; + nrfx_err_t err; + nrf_ipc_event_t ipc_event = + nrf_ipc_receive_event_get(CONFIG_SOC_NRF53_RTC_PRETICK_IPC_CH_FROM_NET); + nrf_ipc_task_t ipc_task = + nrf_ipc_send_task_get(CONFIG_SOC_NRF53_RTC_PRETICK_IPC_CH_TO_NET); + uint32_t task_ipc = nrf_ipc_task_address_get(NRF_IPC, ipc_task); + uint32_t evt_ipc = nrf_ipc_event_address_get(NRF_IPC, ipc_event); + + err = nrfx_gppi_channel_alloc(&ch); + if (err != NRFX_SUCCESS) { + return -ENOMEM; + } + + nrf_ipc_receive_config_set(NRF_IPC, CONFIG_SOC_NRF53_RTC_PRETICK_IPC_CH_FROM_NET, + BIT(CONFIG_SOC_NRF53_RTC_PRETICK_IPC_CH_FROM_NET)); + nrf_ipc_send_config_set(NRF_IPC, CONFIG_SOC_NRF53_RTC_PRETICK_IPC_CH_TO_NET, + BIT(CONFIG_SOC_NRF53_RTC_PRETICK_IPC_CH_TO_NET)); + + nrfx_gppi_task_endpoint_setup(ch, task_ipc); + nrfx_gppi_event_endpoint_setup(ch, evt_ipc); + nrfx_gppi_channels_enable(BIT(ch)); + + return 0; +} +#else /* CONFIG_SOC_NRF5340_CPUNET */ + +void rtc_pretick_rtc0_isr_hook(void) +{ + rtc_pretick_finish_previous(); +} + +void rtc_pretick_rtc1_isr_hook(void) +{ + rtc_pretick_finish_previous(); +} + +static int rtc_pretick_cpunet_init(void) +{ + uint8_t ppi_ch; + nrf_ipc_task_t ipc_task = + nrf_ipc_send_task_get(CONFIG_SOC_NRF53_RTC_PRETICK_IPC_CH_FROM_NET); + nrf_ipc_event_t ipc_event = + nrf_ipc_receive_event_get(CONFIG_SOC_NRF53_RTC_PRETICK_IPC_CH_TO_NET); + uint32_t task_ipc = nrf_ipc_task_address_get(NRF_IPC, ipc_task); + uint32_t evt_ipc = nrf_ipc_event_address_get(NRF_IPC, ipc_event); + uint32_t task_wdt = nrf_wdt_task_address_get(NRF_WDT, NRF_WDT_TASK_START); + uint32_t evt_cc = nrf_rtc_event_address_get(NRF_RTC1, + NRF_RTC_CHANNEL_EVENT_ADDR(RTC1_PRETICK_CC_CHAN)); + + /* Configure Watchdog to allow stopping. */ + nrf_wdt_behaviour_set(NRF_WDT, WDT_CONFIG_STOPEN_Msk | BIT(4)); + *((volatile uint32_t *)0x41203120) = 0x14; + + /* Configure IPC */ + nrf_ipc_receive_config_set(NRF_IPC, CONFIG_SOC_NRF53_RTC_PRETICK_IPC_CH_TO_NET, + BIT(CONFIG_SOC_NRF53_RTC_PRETICK_IPC_CH_TO_NET)); + nrf_ipc_send_config_set(NRF_IPC, CONFIG_SOC_NRF53_RTC_PRETICK_IPC_CH_FROM_NET, + BIT(CONFIG_SOC_NRF53_RTC_PRETICK_IPC_CH_FROM_NET)); + + /* Allocate PPI channel for RTC Compare event publishers that starts WDT. */ + nrfx_err_t err = nrfx_gppi_channel_alloc(&ppi_ch); + + if (err != NRFX_SUCCESS) { + return -ENOMEM; + } + + nrfx_gppi_event_endpoint_setup(ppi_ch, evt_cc); + nrfx_gppi_task_endpoint_setup(ppi_ch, task_ipc); + nrfx_gppi_event_endpoint_setup(ppi_ch, evt_ipc); + nrfx_gppi_task_endpoint_setup(ppi_ch, task_wdt); + nrfx_gppi_channels_enable(BIT(ppi_ch)); + + nrf_rtc_event_enable(NRF_RTC1, NRF_RTC_CHANNEL_INT_MASK(RTC1_PRETICK_CC_CHAN)); + nrf_rtc_event_clear(NRF_RTC1, NRF_RTC_CHANNEL_EVENT_ADDR(RTC1_PRETICK_CC_CHAN)); + + return 0; +} +#endif /* CONFIG_SOC_NRF5340_CPUNET */ + +static int rtc_pretick_init(void) +{ +#ifdef CONFIG_SOC_NRF5340_CPUAPP + return rtc_pretick_cpuapp_init(); +#else + return rtc_pretick_cpunet_init(); +#endif +} +#endif /* CONFIG_SOC_NRF53_RTC_PRETICK */ + static int nordicsemi_nrf53_init(void) { @@ -253,3 +609,7 @@ void arch_busy_wait(uint32_t time_us) } SYS_INIT(nordicsemi_nrf53_init, PRE_KERNEL_1, 0); + +#ifdef CONFIG_SOC_NRF53_RTC_PRETICK +SYS_INIT(rtc_pretick_init, POST_KERNEL, 0); +#endif diff --git a/soc/arm/nordic_nrf/nrf53/soc_cpu_idle.h b/soc/arm/nordic_nrf/nrf53/soc_cpu_idle.h index b6cd92ca092..c02c9451419 100644 --- a/soc/arm/nordic_nrf/nrf53/soc_cpu_idle.h +++ b/soc/arm/nordic_nrf/nrf53/soc_cpu_idle.h @@ -11,10 +11,16 @@ #if defined(_ASMLANGUAGE) +#if defined(CONFIG_SOC_NRF53_ANOMALY_168_WORKAROUND_FOR_EXECUTION_FROM_RAM) #define SOC_ON_EXIT_CPU_IDLE \ + .rept 26; \ nop; \ + .endr +#elif defined(CONFIG_SOC_NRF53_ANOMALY_168_WORKAROUND) +#define SOC_ON_EXIT_CPU_IDLE \ + .rept 8; \ nop; \ - nop; \ - nop; + .endr +#endif #endif /* _ASMLANGUAGE */