aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/rcutree_plugin.h
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/rcutree_plugin.h')
-rw-r--r--kernel/rcutree_plugin.h20
1 files changed, 10 insertions, 10 deletions
diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index ed70f6bf4c31..eeb38ee8ebba 100644
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -306,6 +306,9 @@ static noinline void rcu_read_unlock_special(struct task_struct *t)
306 int empty_exp; 306 int empty_exp;
307 unsigned long flags; 307 unsigned long flags;
308 struct list_head *np; 308 struct list_head *np;
309#ifdef CONFIG_RCU_BOOST
310 struct rt_mutex *rbmp = NULL;
311#endif /* #ifdef CONFIG_RCU_BOOST */
309 struct rcu_node *rnp; 312 struct rcu_node *rnp;
310 int special; 313 int special;
311 314
@@ -351,6 +354,7 @@ static noinline void rcu_read_unlock_special(struct task_struct *t)
351 smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */ 354 smp_mb(); /* ensure expedited fastpath sees end of RCU c-s. */
352 np = rcu_next_node_entry(t, rnp); 355 np = rcu_next_node_entry(t, rnp);
353 list_del_init(&t->rcu_node_entry); 356 list_del_init(&t->rcu_node_entry);
357 t->rcu_blocked_node = NULL;
354 trace_rcu_unlock_preempted_task("rcu_preempt", 358 trace_rcu_unlock_preempted_task("rcu_preempt",
355 rnp->gpnum, t->pid); 359 rnp->gpnum, t->pid);
356 if (&t->rcu_node_entry == rnp->gp_tasks) 360 if (&t->rcu_node_entry == rnp->gp_tasks)
@@ -360,13 +364,12 @@ static noinline void rcu_read_unlock_special(struct task_struct *t)
360#ifdef CONFIG_RCU_BOOST 364#ifdef CONFIG_RCU_BOOST
361 if (&t->rcu_node_entry == rnp->boost_tasks) 365 if (&t->rcu_node_entry == rnp->boost_tasks)
362 rnp->boost_tasks = np; 366 rnp->boost_tasks = np;
363 /* Snapshot and clear ->rcu_boosted with rcu_node lock held. */ 367 /* Snapshot/clear ->rcu_boost_mutex with rcu_node lock held. */
364 if (t->rcu_boosted) { 368 if (t->rcu_boost_mutex) {
365 special |= RCU_READ_UNLOCK_BOOSTED; 369 rbmp = t->rcu_boost_mutex;
366 t->rcu_boosted = 0; 370 t->rcu_boost_mutex = NULL;
367 } 371 }
368#endif /* #ifdef CONFIG_RCU_BOOST */ 372#endif /* #ifdef CONFIG_RCU_BOOST */
369 t->rcu_blocked_node = NULL;
370 373
371 /* 374 /*
372 * If this was the last task on the current list, and if 375 * If this was the last task on the current list, and if
@@ -387,10 +390,8 @@ static noinline void rcu_read_unlock_special(struct task_struct *t)
387 390
388#ifdef CONFIG_RCU_BOOST 391#ifdef CONFIG_RCU_BOOST
389 /* Unboost if we were boosted. */ 392 /* Unboost if we were boosted. */
390 if (special & RCU_READ_UNLOCK_BOOSTED) { 393 if (rbmp)
391 rt_mutex_unlock(t->rcu_boost_mutex); 394 rt_mutex_unlock(rbmp);
392 t->rcu_boost_mutex = NULL;
393 }
394#endif /* #ifdef CONFIG_RCU_BOOST */ 395#endif /* #ifdef CONFIG_RCU_BOOST */
395 396
396 /* 397 /*
@@ -1206,7 +1207,6 @@ static int rcu_boost(struct rcu_node *rnp)
1206 t = container_of(tb, struct task_struct, rcu_node_entry); 1207 t = container_of(tb, struct task_struct, rcu_node_entry);
1207 rt_mutex_init_proxy_locked(&mtx, t); 1208 rt_mutex_init_proxy_locked(&mtx, t);
1208 t->rcu_boost_mutex = &mtx; 1209 t->rcu_boost_mutex = &mtx;
1209 t->rcu_boosted = 1;
1210 raw_spin_unlock_irqrestore(&rnp->lock, flags); 1210 raw_spin_unlock_irqrestore(&rnp->lock, flags);
1211 rt_mutex_lock(&mtx); /* Side effect: boosts task t's priority. */ 1211 rt_mutex_lock(&mtx); /* Side effect: boosts task t's priority. */
1212 rt_mutex_unlock(&mtx); /* Keep lockdep happy. */ 1212 rt_mutex_unlock(&mtx); /* Keep lockdep happy. */