diff --git a/include/nuttx/sched.h b/include/nuttx/sched.h index 8ac19ec584097..960847b5b46bb 100644 --- a/include/nuttx/sched.h +++ b/include/nuttx/sched.h @@ -232,6 +232,8 @@ # define get_task_name(tcb) "" #endif +#define SMP_CALL_INITIALIZER(func, arg) {(func), (arg)} + /* These are macros to access the current CPU and the current task on a CPU. * These macros are intended to support a future SMP implementation. */ @@ -833,6 +835,15 @@ typedef CODE void (*nxsched_foreach_t)(FAR struct tcb_s *tcb, FAR void *arg); #ifdef CONFIG_SMP typedef CODE int (*nxsched_smp_call_t)(FAR void *arg); + +struct smp_call_cookie_s; +struct smp_call_data_s +{ + nxsched_smp_call_t func; + FAR void *arg; + FAR struct smp_call_cookie_s *cookie; + sq_entry_t node[CONFIG_SMP_NCPUS]; +}; #endif #endif /* __ASSEMBLY__ */ @@ -1698,17 +1709,35 @@ void nxsched_dumponexit(void); int nxsched_smp_call_handler(int irq, FAR void *context, FAR void *arg); +/**************************************************************************** + * Name: nxsched_smp_call_init + * + * Description: + * Init call_data + * + * Input Parameters: + * data - Call data + * func - Function + * arg - Function args + * + * Returned Value: + * Result + * + ****************************************************************************/ + +void nxsched_smp_call_init(FAR struct smp_call_data_s *data, + nxsched_smp_call_t func, FAR void *arg); + /**************************************************************************** * Name: nxsched_smp_call_single * * Description: - * Call function on single processor + * Call function on single processor, wait function callback * * Input Parameters: * cpuid - Target cpu id * func - Function * arg - Function args - * wait - Wait function callback or not * * Returned Value: * Result @@ -1716,19 +1745,18 @@ int nxsched_smp_call_handler(int irq, FAR void *context, ****************************************************************************/ int nxsched_smp_call_single(int cpuid, nxsched_smp_call_t func, - FAR void *arg, bool wait); + FAR void *arg); /**************************************************************************** * Name: nxsched_smp_call * * Description: - * Call function on multi processors + * Call function on multi processors, wait function callback * * Input Parameters: * cpuset - Target cpuset * func - Function * arg - Function args - * wait - Wait function callback or not * * Returned Value: * Result @@ -1736,7 +1764,43 @@ int nxsched_smp_call_single(int cpuid, nxsched_smp_call_t func, ****************************************************************************/ int nxsched_smp_call(cpu_set_t cpuset, nxsched_smp_call_t func, - FAR void *arg, bool wait); + FAR void *arg); + +/**************************************************************************** + * Name: nxsched_smp_call_single_async + * + * Description: + * Call function on single processor async + * + * Input Parameters: + * cpuset - Target cpuset + * data - Call data + * + * Returned Value: + * Result + * + ****************************************************************************/ + +int nxsched_smp_call_single_async(int cpuid, + FAR struct smp_call_data_s *data); + +/**************************************************************************** + * Name: nxsched_smp_call_async + * + * Description: + * Call function on multi processors async + * + * Input Parameters: + * cpuset - Target cpuset + * data - Call data + * + * Returned Value: + * Result + * + ****************************************************************************/ + +int nxsched_smp_call_async(cpu_set_t cpuset, + FAR struct smp_call_data_s *data); #endif #undef EXTERN diff --git a/libs/libc/gdbstub/lib_gdbstub.c b/libs/libc/gdbstub/lib_gdbstub.c index a8e4f5689077b..45660591f5cda 100644 --- a/libs/libc/gdbstub/lib_gdbstub.c +++ b/libs/libc/gdbstub/lib_gdbstub.c @@ -1879,7 +1879,7 @@ int gdb_debugpoint_add(int type, FAR void *addr, size_t size, point.callback = callback; point.arg = arg; return nxsched_smp_call((1 << CONFIG_SMP_NCPUS) - 1, - gdb_smp_debugpoint_add, &point, true); + gdb_smp_debugpoint_add, &point); #else return up_debugpoint_add(type, addr, size, callback, arg); #endif @@ -1897,8 +1897,8 @@ int gdb_debugpoint_remove(int type, FAR void *addr, size_t size) point.addr = addr; point.size = size; - return nxsched_smp_call((1 << CONFIG_SMP_NCPUS) - 1, - gdb_smp_debugpoint_remove, &point, true); + retrun nxsched_smp_call((1 << CONFIG_SMP_NCPUS) - 1, + gdb_smp_debugpoint_remove, &point); #else return up_debugpoint_remove(type, addr, size); #endif diff --git a/sched/misc/assert.c b/sched/misc/assert.c index c77d1d617c6ba..0864e60205c32 100644 --- a/sched/misc/assert.c +++ b/sched/misc/assert.c @@ -98,6 +98,14 @@ # define XCPTCONTEXT_ALIGN 16 #endif +/**************************************************************************** + * Private Types + ****************************************************************************/ + +#ifdef CONFIG_SMP +static noreturn_function int pause_cpu_handler(FAR void *arg); +#endif + /**************************************************************************** * Private Data ****************************************************************************/ @@ -124,6 +132,11 @@ static FAR const char * const g_ttypenames[4] = }; #endif +#ifdef CONFIG_SMP +static struct smp_call_data_s g_call_data = +SMP_CALL_INITIALIZER(pause_cpu_handler, NULL); +#endif + /**************************************************************************** * Private Functions ****************************************************************************/ @@ -610,7 +623,7 @@ static void pause_all_cpu(void) int delay = CONFIG_ASSERT_PAUSE_CPU_TIMEOUT; CPU_CLR(this_cpu(), &cpus); - nxsched_smp_call(cpus, pause_cpu_handler, NULL, false); + nxsched_smp_call_async(cpus, &g_call_data); g_cpu_paused[this_cpu()] = true; /* Check if all CPUs paused with timeout */ diff --git a/sched/sched/sched_backtrace.c b/sched/sched/sched_backtrace.c index dec02e4854edf..91ebff8b2571b 100644 --- a/sched/sched/sched_backtrace.c +++ b/sched/sched/sched_backtrace.c @@ -140,7 +140,7 @@ int sched_backtrace(pid_t tid, FAR void **buffer, int size, int skip) arg.skip = skip; ret = nxsched_smp_call_single(tcb->cpu, sched_backtrace_handler, - &arg, true); + &arg); } else #endif diff --git a/sched/sched/sched_profil.c b/sched/sched/sched_profil.c index 21294f6205ab7..40c276a0e79a3 100644 --- a/sched/sched/sched_profil.c +++ b/sched/sched/sched_profil.c @@ -50,12 +50,21 @@ struct profinfo_s spinlock_t lock; /* Lock for this structure */ }; +#ifdef CONFIG_SMP +static int profil_timer_handler_cpu(FAR void *arg); +#endif + /**************************************************************************** * Private Data ****************************************************************************/ static struct profinfo_s g_prof; +#ifdef CONFIG_SMP +static struct smp_call_data_s g_call_data = +SMP_CALL_INITIALIZER(profil_timer_handler_cpu, &g_prof); +#endif + /**************************************************************************** * Private Functions ****************************************************************************/ @@ -89,8 +98,7 @@ static void profil_timer_handler(wdparm_t arg) #ifdef CONFIG_SMP cpu_set_t cpus = (1 << CONFIG_SMP_NCPUS) - 1; CPU_CLR(this_cpu(), &cpus); - nxsched_smp_call(cpus, profil_timer_handler_cpu, - (FAR void *)arg, false); + nxsched_smp_call_async(cpus, &g_call_data); #endif profil_timer_handler_cpu(prof); diff --git a/sched/sched/sched_roundrobin.c b/sched/sched/sched_roundrobin.c index 5ebae49859b5f..13b0bc7816a7d 100644 --- a/sched/sched/sched_roundrobin.c +++ b/sched/sched/sched_roundrobin.c @@ -40,6 +40,15 @@ #if CONFIG_RR_INTERVAL > 0 #ifdef CONFIG_SMP + +/**************************************************************************** + * Private Data + ****************************************************************************/ + +#ifdef CONFIG_SMP +static struct smp_call_data_s g_call_data; +#endif + /**************************************************************************** * Private Type Declarations ****************************************************************************/ @@ -55,12 +64,12 @@ struct roundrobin_arg_s static int nxsched_roundrobin_handler(FAR void *cookie) { - FAR struct roundrobin_arg_s *arg = cookie; + pid_t pid = (uintptr_t)cookie; FAR struct tcb_s *tcb; irqstate_t flags; flags = enter_critical_section(); - tcb = nxsched_get_tcb(arg->pid); + tcb = nxsched_get_tcb(pid); if (!tcb || tcb->task_state == TSTATE_TASK_INVALID || (tcb->flags & TCB_FLAG_EXIT_PROCESSING) != 0) @@ -185,12 +194,10 @@ uint32_t nxsched_process_roundrobin(FAR struct tcb_s *tcb, uint32_t ticks, if (tcb->task_state == TSTATE_TASK_RUNNING && tcb->cpu != this_cpu()) { - struct roundrobin_arg_s arg; - - arg.pid = tcb->pid; - nxsched_smp_call_single(tcb->cpu, - nxsched_roundrobin_handler, - &arg, false); + nxsched_smp_call_init(&g_call_data, + nxsched_roundrobin_handler, + (FAR void *)(uintptr_t)tcb->pid); + nxsched_smp_call_single_async(tcb->cpu, &g_call_data); } else #endif diff --git a/sched/sched/sched_setpriority.c b/sched/sched/sched_setpriority.c index 2eabdd1be2001..caa10cfbc37fc 100644 --- a/sched/sched/sched_setpriority.c +++ b/sched/sched/sched_setpriority.c @@ -246,8 +246,7 @@ static inline void nxsched_running_setpriority(FAR struct tcb_s *tcb, } arg.sched_priority = sched_priority; - nxsched_smp_call_single(tcb->cpu, reprioritize_handler, - &arg, true); + nxsched_smp_call_single(tcb->cpu, reprioritize_handler, &arg); } else #endif diff --git a/sched/sched/sched_smp.c b/sched/sched/sched_smp.c index ff2607964e473..0a035d5e52609 100644 --- a/sched/sched/sched_smp.c +++ b/sched/sched/sched_smp.c @@ -46,22 +46,11 @@ struct smp_call_cookie_s int error; }; -struct smp_call_data_s -{ - sq_entry_t node[CONFIG_SMP_NCPUS]; - nxsched_smp_call_t func; - FAR void *arg; - FAR struct smp_call_cookie_s *cookie; - spinlock_t lock; - volatile int refcount; -}; - /**************************************************************************** * Private Data ****************************************************************************/ static sq_queue_t g_smp_call_queue[CONFIG_SMP_NCPUS]; -static struct smp_call_data_s g_smp_call_data; static spinlock_t g_smp_call_lock; /**************************************************************************** @@ -76,7 +65,7 @@ static spinlock_t g_smp_call_lock; * * Input Parameters: * cpu - Target cpu id - * call_data - Call data + * data - Call data * * Returned Value: * None @@ -84,12 +73,16 @@ static spinlock_t g_smp_call_lock; ****************************************************************************/ static void nxsched_smp_call_add(int cpu, - FAR struct smp_call_data_s *call_data) + FAR struct smp_call_data_s *data) { irqstate_t flags; flags = spin_lock_irqsave(&g_smp_call_lock); - sq_addlast(&call_data->node[cpu], &g_smp_call_queue[cpu]); + if (!sq_inqueue(&data->node[cpu], &g_smp_call_queue[cpu])) + { + sq_addlast(&data->node[cpu], &g_smp_call_queue[cpu]); + } + spin_unlock_irqrestore(&g_smp_call_lock, flags); } @@ -127,33 +120,26 @@ int nxsched_smp_call_handler(int irq, FAR void *context, sq_for_every_safe(call_queue, curr, next) { - FAR struct smp_call_data_s *call_data = + FAR struct smp_call_data_s *data = container_of(curr, struct smp_call_data_s, node[cpu]); int ret; - sq_rem(&call_data->node[cpu], call_queue); + sq_rem(&data->node[cpu], call_queue); spin_unlock_irqrestore(&g_smp_call_lock, flags); - ret = call_data->func(call_data->arg); + ret = data->func(data->arg); flags = spin_lock_irqsave(&g_smp_call_lock); - if (spin_is_locked(&call_data->lock)) - { - if (--call_data->refcount == 0) - { - spin_unlock(&call_data->lock); - } - } - if (call_data->cookie != NULL) + if (data->cookie != NULL) { if (ret < 0) { - call_data->cookie->error = ret; + data->cookie->error = ret; } - nxsem_post(&call_data->cookie->sem); + nxsem_post(&data->cookie->sem); } } @@ -161,17 +147,42 @@ int nxsched_smp_call_handler(int irq, FAR void *context, return OK; } +/**************************************************************************** + * Name: nxsched_smp_call_init + * + * Description: + * Init call_data + * + * Input Parameters: + * data - Call data + * func - Function + * arg - Function args + * + * Returned Value: + * Result + * + ****************************************************************************/ + +void nxsched_smp_call_init(FAR struct smp_call_data_s *data, + nxsched_smp_call_t func, FAR void *arg) +{ + DEBUGASSERT(data != NULL && func != NULL); + + memset(data, 0, sizeof(struct smp_call_data_s)); + data->func = func; + data->arg = arg; +} + /**************************************************************************** * Name: nxsched_smp_call_single * * Description: - * Call function on single processor + * Call function on single processor, wait function callback * * Input Parameters: * cpuid - Target cpu id * func - Function * arg - Function args - * wait - Wait function callback or not * * Returned Value: * Result @@ -179,26 +190,25 @@ int nxsched_smp_call_handler(int irq, FAR void *context, ****************************************************************************/ int nxsched_smp_call_single(int cpuid, nxsched_smp_call_t func, - FAR void *arg, bool wait) + FAR void *arg) { cpu_set_t cpuset; CPU_ZERO(&cpuset); CPU_SET(cpuid, &cpuset); - return nxsched_smp_call(cpuset, func, arg, wait); + return nxsched_smp_call(cpuset, func, arg); } /**************************************************************************** * Name: nxsched_smp_call * * Description: - * Call function on multi processors + * Call function on multi processors, wait function callback * * Input Parameters: * cpuset - Target cpuset * func - Function * arg - Function args - * wait - Wait function callback or not * * Returned Value: * Result @@ -206,99 +216,124 @@ int nxsched_smp_call_single(int cpuid, nxsched_smp_call_t func, ****************************************************************************/ int nxsched_smp_call(cpu_set_t cpuset, nxsched_smp_call_t func, - FAR void *arg, bool wait) + FAR void *arg) { - struct smp_call_data_s call_data_stack = - { - 0 - }; - - struct smp_call_cookie_s cookie = - { - 0 - }; - - FAR struct smp_call_data_s *call_data; - int remote_cpus; + struct smp_call_data_s data; + struct smp_call_cookie_s cookie; + int cpucnt; int ret = OK; int i; /* Cannot wait in interrupt context. */ - DEBUGASSERT(!(wait && up_interrupt_context())); + DEBUGASSERT(!up_interrupt_context()); + nxsched_smp_call_init(&data, func, arg); + cookie.error = 0; + nxsem_init(&cookie.sem, 0, 0); - /* Prevent reschedule on another processor */ + data.cookie = &cookie; + ret = nxsched_smp_call_async(cpuset, &data); - if (!up_interrupt_context()) + if (ret < 0) { - sched_lock(); + nxsem_destroy(&cookie.sem); + return ret; } - if (CPU_ISSET(this_cpu(), &cpuset)) + cpucnt = CPU_COUNT(&cpuset); + for (i = 0; i < cpucnt; i++) { - ret = func(arg); - if (ret < 0) + int rc = nxsem_wait_uninterruptible(&cookie.sem); + if (rc < 0) { - goto out; + ret = rc; } - - CPU_CLR(this_cpu(), &cpuset); } - remote_cpus = CPU_COUNT(&cpuset); - if (remote_cpus == 0) + if (cookie.error < 0) { - goto out; + ret = cookie.error; } - /* If waiting is necessary, initialize and wait for the cookie. */ + nxsem_destroy(&cookie.sem); - if (wait) - { - nxsem_init(&cookie.sem, 0, 0); + return ret; +} - call_data = &call_data_stack; - call_data->cookie = &cookie; - } - else - { - call_data = &g_smp_call_data; - spin_lock(&call_data->lock); - } +/**************************************************************************** + * Name: nxsched_smp_call_single_async + * + * Description: + * Call function on single processor async + * + * Input Parameters: + * cpuset - Target cpuset + * data - Call data + * + * Returned Value: + * Result + * + ****************************************************************************/ + +int nxsched_smp_call_single_async(int cpuid, + FAR struct smp_call_data_s *data) +{ + cpu_set_t cpuset; - call_data->func = func; - call_data->arg = arg; - call_data->refcount = remote_cpus; + CPU_ZERO(&cpuset); + CPU_SET(cpuid, &cpuset); + return nxsched_smp_call_async(cpuset, data); +} - for (i = 0; i < CONFIG_SMP_NCPUS; i++) +/**************************************************************************** + * Name: nxsched_smp_call_async + * + * Description: + * Call function on multi processors async + * + * Input Parameters: + * cpuset - Target cpuset + * data - Call data + * + * Returned Value: + * Result + * + ****************************************************************************/ + +int nxsched_smp_call_async(cpu_set_t cpuset, + FAR struct smp_call_data_s *data) +{ + int cpucnt; + int ret = OK; + int i; + + /* Prevent reschedule on another processor */ + + if (!up_interrupt_context()) { - if (CPU_ISSET(i, &cpuset)) - { - nxsched_smp_call_add(i, call_data); - } + sched_lock(); } - up_send_smp_call(cpuset); + cpucnt = CPU_COUNT(&cpuset); + if (cpucnt == 0) + { + goto out; + } - if (wait) + for (i = 0; i < CONFIG_SMP_NCPUS; i++) { - for (i = 0; i < remote_cpus; i++) + if (CPU_ISSET(i, &cpuset)) { - int wait_ret = nxsem_wait_uninterruptible(&cookie.sem); - if (wait_ret < 0) + nxsched_smp_call_add(i, data); + if (--cpucnt == 0) { - ret = wait_ret; + break; } } - - if (cookie.error < 0) - { - ret = cookie.error; - } - - nxsem_destroy(&cookie.sem); } + up_send_smp_call(cpuset); + out: if (!up_interrupt_context()) { diff --git a/sched/sched/sched_suspend.c b/sched/sched/sched_suspend.c index fbd0235f3d38b..c0a3e31a0f073 100644 --- a/sched/sched/sched_suspend.c +++ b/sched/sched/sched_suspend.c @@ -166,8 +166,7 @@ void nxsched_suspend(FAR struct tcb_s *tcb) CPU_SET(tcb->cpu, &tcb->affinity); } - nxsched_smp_call_single(tcb->cpu, nxsched_suspend_handler, - &arg, true); + nxsched_smp_call_single(tcb->cpu, nxsched_suspend_handler, &arg); } else #endif diff --git a/sched/signal/sig_dispatch.c b/sched/signal/sig_dispatch.c index 35cbc7409dd10..9116e711caacb 100644 --- a/sched/signal/sig_dispatch.c +++ b/sched/signal/sig_dispatch.c @@ -187,8 +187,7 @@ static int nxsig_queue_action(FAR struct tcb_s *stcb, siginfo_t *info) } arg.pid = stcb->pid; - nxsched_smp_call_single(stcb->cpu, sig_handler, &arg, - true); + nxsched_smp_call_single(stcb->cpu, sig_handler, &arg); } else #endif diff --git a/sched/task/task_restart.c b/sched/task/task_restart.c index f95c9b0d2935e..b4a6f91784811 100644 --- a/sched/task/task_restart.c +++ b/sched/task/task_restart.c @@ -246,7 +246,7 @@ static int nxtask_restart(pid_t pid) CPU_SET(tcb->cpu, &tcb->affinity); } - nxsched_smp_call_single(tcb->cpu, restart_handler, &arg, true); + nxsched_smp_call_single(tcb->cpu, restart_handler, &arg); tcb = nxsched_get_tcb(pid); if (!tcb || tcb->task_state != TSTATE_TASK_INVALID || diff --git a/sched/task/task_terminate.c b/sched/task/task_terminate.c index 8eee41b199922..4e6f334756d9c 100644 --- a/sched/task/task_terminate.c +++ b/sched/task/task_terminate.c @@ -147,8 +147,7 @@ int nxtask_terminate(pid_t pid) CPU_SET(dtcb->cpu, &dtcb->affinity); ret = nxsched_smp_call_single(dtcb->cpu, terminat_handler, - (FAR void *)(uintptr_t)pid, - true); + (FAR void *)(uintptr_t)pid); if (ret < 0) {