From 8aa7972223f84bb26f5f479db73bcc8c4327affe Mon Sep 17 00:00:00 2001 From: Sultan Alsawaf Date: Wed, 20 Apr 2022 01:15:01 -0700 Subject: [PATCH] sched/core: Skip superfluous acquire barrier in ttwu ttwu_remote() unconditionally locks the task's runqueue lock, which implies a full barrier across the lock and unlock, so the acquire barrier after the control dependency is only needed when the task isn't on the runqueue. Signed-off-by: Sultan Alsawaf Change-Id: Ibb988fe341ba3109d381682a8725fbd2e7a648e3 --- kernel/sched/core.c | 55 ++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 141479ce13e5..d4e4799df8ca 100755 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2217,33 +2217,38 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags, * current. */ smp_rmb(); - if (READ_ONCE(p->on_rq) && ttwu_remote(p, wake_flags)) - goto unlock; - + if (READ_ONCE(p->on_rq)) { + if (ttwu_remote(p, wake_flags)) + goto unlock; + } else { #ifdef CONFIG_SMP - /* - * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be - * possible to, falsely, observe p->on_cpu == 0. - * - * One must be running (->on_cpu == 1) in order to remove oneself - * from the runqueue. - * - * [S] ->on_cpu = 1; [L] ->on_rq - * UNLOCK rq->lock - * RMB - * LOCK rq->lock - * [S] ->on_rq = 0; [L] ->on_cpu - * - * Pairs with the full barrier implied in the UNLOCK+LOCK on rq->lock - * from the consecutive calls to schedule(); the first switching to our - * task, the second putting it to sleep. - * - * Form a control-dep-acquire with p->on_rq == 0 above, to ensure - * schedule()'s deactivate_task() has 'happened' and p will no longer - * care about it's own p->state. See the comment in __schedule(). - */ - smp_acquire__after_ctrl_dep(); + /* + * Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would + * be possible to, falsely, observe p->on_cpu == 0. + * + * One must be running (->on_cpu == 1) in order to remove + * oneself from the runqueue. + * + * [S] ->on_cpu = 1; [L] ->on_rq + * UNLOCK rq->lock + * RMB + * LOCK rq->lock + * [S] ->on_rq = 0; [L] ->on_cpu + * + * Pairs with the full barrier implied in the UNLOCK+LOCK on + * rq->lock from the consecutive calls to schedule(); the first + * switching to our task, the second putting it to sleep. + * + * Form a control-dep-acquire with p->on_rq == 0 above, to + * ensure schedule()'s deactivate_task() has 'happened' and p + * will no longer care about it's own p->state. See the comment + * in __schedule(). + */ + smp_acquire__after_ctrl_dep(); +#endif + } +#ifdef CONFIG_SMP /* * We're doing the wakeup (@success == 1), they did a dequeue (p->on_rq * == 0), which means we need to do an enqueue, change p->state to