diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 2e2780c1abe2..d08ac2efa52e 100755 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2099,6 +2099,29 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags, unsigned long flags; int cpu, success = 0; + if (p == current) { + /* + * We're waking current, this means 'p->on_rq' and 'task_cpu(p) + * == smp_processor_id()'. Together this means we can special + * case the whole 'p->on_rq && ttwu_remote()' case below + * without taking any locks. + * + * In particular: + * - we rely on Program-Order guarantees for all the ordering, + * - we're serialized against set_special_state() by virtue of + * it disabling IRQs (this allows not taking ->pi_lock). + */ + if (!(p->state & state)) + return false; + + success = 1; + cpu = task_cpu(p); + trace_sched_waking(p); + p->state = TASK_RUNNING; + trace_sched_wakeup(p); + goto out; + } + /* * If we are going to wake up a thread waiting for CONDITION we * need to ensure that CONDITION=1 done by the caller can not be @@ -2108,7 +2131,7 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags, raw_spin_lock_irqsave(&p->pi_lock, flags); smp_mb__after_spinlock(); if (!(p->state & state)) - goto out; + goto unlock; trace_sched_waking(p); @@ -2139,7 +2162,7 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags, */ smp_rmb(); if (p->on_rq && ttwu_remote(p, wake_flags)) - goto stat; + goto unlock; #ifdef CONFIG_SMP /* @@ -2200,10 +2223,11 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags, #endif /* CONFIG_SMP */ ttwu_queue(p, cpu, wake_flags); -stat: - ttwu_stat(p, cpu, wake_flags); -out: +unlock: raw_spin_unlock_irqrestore(&p->pi_lock, flags); +out: + if (success) + ttwu_stat(p, cpu, wake_flags); #ifdef CONFIG_SCHED_WALT if (success && sched_predl) {