aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/rcutiny_plugin.h163
1 files changed, 89 insertions, 74 deletions
diff --git a/kernel/rcutiny_plugin.h b/kernel/rcutiny_plugin.h
index 2b8d5293c5f4..f259c676195f 100644
--- a/kernel/rcutiny_plugin.h
+++ b/kernel/rcutiny_plugin.h
@@ -106,15 +106,22 @@ struct rcu_preempt_ctrlblk {
106 unsigned long n_grace_periods; 106 unsigned long n_grace_periods;
107#ifdef CONFIG_RCU_BOOST 107#ifdef CONFIG_RCU_BOOST
108 unsigned long n_tasks_boosted; 108 unsigned long n_tasks_boosted;
109 /* Total number of tasks boosted. */
109 unsigned long n_exp_boosts; 110 unsigned long n_exp_boosts;
111 /* Number of tasks boosted for expedited GP. */
110 unsigned long n_normal_boosts; 112 unsigned long n_normal_boosts;
111 unsigned long n_normal_balk_blkd_tasks; 113 /* Number of tasks boosted for normal GP. */
112 unsigned long n_normal_balk_gp_tasks; 114 unsigned long n_balk_blkd_tasks;
113 unsigned long n_normal_balk_boost_tasks; 115 /* Refused to boost: no blocked tasks. */
114 unsigned long n_normal_balk_notyet; 116 unsigned long n_balk_exp_gp_tasks;
115 unsigned long n_normal_balk_nos; 117 /* Refused to boost: nothing blocking GP. */
116 unsigned long n_exp_balk_blkd_tasks; 118 unsigned long n_balk_boost_tasks;
117 unsigned long n_exp_balk_nos; 119 /* Refused to boost: already boosting. */
120 unsigned long n_balk_notyet;
121 /* Refused to boost: not yet time. */
122 unsigned long n_balk_nos;
123 /* Refused to boost: not sure why, though. */
124 /* This can happen due to race conditions. */
118#endif /* #ifdef CONFIG_RCU_BOOST */ 125#endif /* #ifdef CONFIG_RCU_BOOST */
119#endif /* #ifdef CONFIG_RCU_TRACE */ 126#endif /* #ifdef CONFIG_RCU_TRACE */
120}; 127};
@@ -199,7 +206,6 @@ static struct list_head *rcu_next_node_entry(struct task_struct *t)
199 206
200#ifdef CONFIG_RCU_BOOST 207#ifdef CONFIG_RCU_BOOST
201static void rcu_initiate_boost_trace(void); 208static void rcu_initiate_boost_trace(void);
202static void rcu_initiate_exp_boost_trace(void);
203#endif /* #ifdef CONFIG_RCU_BOOST */ 209#endif /* #ifdef CONFIG_RCU_BOOST */
204 210
205/* 211/*
@@ -225,16 +231,13 @@ static void show_tiny_preempt_stats(struct seq_file *m)
225 rcu_preempt_ctrlblk.n_normal_boosts, 231 rcu_preempt_ctrlblk.n_normal_boosts,
226 (int)(jiffies & 0xffff), 232 (int)(jiffies & 0xffff),
227 (int)(rcu_preempt_ctrlblk.boost_time & 0xffff)); 233 (int)(rcu_preempt_ctrlblk.boost_time & 0xffff));
228 seq_printf(m, " %s: nt=%lu gt=%lu bt=%lu ny=%lu nos=%lu\n", 234 seq_printf(m, "%s: nt=%lu egt=%lu bt=%lu ny=%lu nos=%lu\n",
229 "normal balk", 235 " balk",
230 rcu_preempt_ctrlblk.n_normal_balk_blkd_tasks, 236 rcu_preempt_ctrlblk.n_balk_blkd_tasks,
231 rcu_preempt_ctrlblk.n_normal_balk_gp_tasks, 237 rcu_preempt_ctrlblk.n_balk_exp_gp_tasks,
232 rcu_preempt_ctrlblk.n_normal_balk_boost_tasks, 238 rcu_preempt_ctrlblk.n_balk_boost_tasks,
233 rcu_preempt_ctrlblk.n_normal_balk_notyet, 239 rcu_preempt_ctrlblk.n_balk_notyet,
234 rcu_preempt_ctrlblk.n_normal_balk_nos); 240 rcu_preempt_ctrlblk.n_balk_nos);
235 seq_printf(m, " exp balk: bt=%lu nos=%lu\n",
236 rcu_preempt_ctrlblk.n_exp_balk_blkd_tasks,
237 rcu_preempt_ctrlblk.n_exp_balk_nos);
238#endif /* #ifdef CONFIG_RCU_BOOST */ 241#endif /* #ifdef CONFIG_RCU_BOOST */
239} 242}
240 243
@@ -252,23 +255,59 @@ static int rcu_boost(void)
252{ 255{
253 unsigned long flags; 256 unsigned long flags;
254 struct rt_mutex mtx; 257 struct rt_mutex mtx;
255 struct list_head *np;
256 struct task_struct *t; 258 struct task_struct *t;
259 struct list_head *tb;
257 260
258 if (rcu_preempt_ctrlblk.boost_tasks == NULL) 261 if (rcu_preempt_ctrlblk.boost_tasks == NULL &&
262 rcu_preempt_ctrlblk.exp_tasks == NULL)
259 return 0; /* Nothing to boost. */ 263 return 0; /* Nothing to boost. */
264
260 raw_local_irq_save(flags); 265 raw_local_irq_save(flags);
261 t = container_of(rcu_preempt_ctrlblk.boost_tasks, struct task_struct, 266
262 rcu_node_entry); 267 /*
263 np = rcu_next_node_entry(t); 268 * Recheck with irqs disabled: all tasks in need of boosting
269 * might exit their RCU read-side critical sections on their own
270 * if we are preempted just before disabling irqs.
271 */
272 if (rcu_preempt_ctrlblk.boost_tasks == NULL &&
273 rcu_preempt_ctrlblk.exp_tasks == NULL) {
274 raw_local_irq_restore(flags);
275 return 0;
276 }
277
278 /*
279 * Preferentially boost tasks blocking expedited grace periods.
280 * This cannot starve the normal grace periods because a second
281 * expedited grace period must boost all blocked tasks, including
282 * those blocking the pre-existing normal grace period.
283 */
284 if (rcu_preempt_ctrlblk.exp_tasks != NULL) {
285 tb = rcu_preempt_ctrlblk.exp_tasks;
286 RCU_TRACE(rcu_preempt_ctrlblk.n_exp_boosts++);
287 } else {
288 tb = rcu_preempt_ctrlblk.boost_tasks;
289 RCU_TRACE(rcu_preempt_ctrlblk.n_normal_boosts++);
290 }
291 RCU_TRACE(rcu_preempt_ctrlblk.n_tasks_boosted++);
292
293 /*
294 * We boost task t by manufacturing an rt_mutex that appears to
295 * be held by task t. We leave a pointer to that rt_mutex where
296 * task t can find it, and task t will release the mutex when it
297 * exits its outermost RCU read-side critical section. Then
298 * simply acquiring this artificial rt_mutex will boost task
299 * t's priority. (Thanks to tglx for suggesting this approach!)
300 */
301 t = container_of(tb, struct task_struct, rcu_node_entry);
264 rt_mutex_init_proxy_locked(&mtx, t); 302 rt_mutex_init_proxy_locked(&mtx, t);
265 t->rcu_boost_mutex = &mtx; 303 t->rcu_boost_mutex = &mtx;
266 t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BOOSTED; 304 t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BOOSTED;
267 raw_local_irq_restore(flags); 305 raw_local_irq_restore(flags);
268 rt_mutex_lock(&mtx); 306 rt_mutex_lock(&mtx);
269 RCU_TRACE(rcu_preempt_ctrlblk.n_tasks_boosted++); 307 rt_mutex_unlock(&mtx); /* Keep lockdep happy. */
270 rt_mutex_unlock(&mtx); 308
271 return rcu_preempt_ctrlblk.boost_tasks != NULL; 309 return rcu_preempt_ctrlblk.boost_tasks != NULL ||
310 rcu_preempt_ctrlblk.exp_tasks != NULL;
272} 311}
273 312
274/* 313/*
@@ -283,39 +322,24 @@ static int rcu_boost(void)
283 */ 322 */
284static int rcu_initiate_boost(void) 323static int rcu_initiate_boost(void)
285{ 324{
286 if (!rcu_preempt_blocked_readers_cgp()) { 325 if (!rcu_preempt_blocked_readers_cgp() &&
287 RCU_TRACE(rcu_preempt_ctrlblk.n_normal_balk_blkd_tasks++); 326 rcu_preempt_ctrlblk.exp_tasks == NULL) {
327 RCU_TRACE(rcu_preempt_ctrlblk.n_balk_exp_gp_tasks++);
288 return 0; 328 return 0;
289 } 329 }
290 if (rcu_preempt_ctrlblk.gp_tasks != NULL && 330 if (rcu_preempt_ctrlblk.exp_tasks != NULL ||
291 rcu_preempt_ctrlblk.boost_tasks == NULL && 331 (rcu_preempt_ctrlblk.gp_tasks != NULL &&
292 ULONG_CMP_GE(jiffies, rcu_preempt_ctrlblk.boost_time)) { 332 rcu_preempt_ctrlblk.boost_tasks == NULL &&
293 rcu_preempt_ctrlblk.boost_tasks = rcu_preempt_ctrlblk.gp_tasks; 333 ULONG_CMP_GE(jiffies, rcu_preempt_ctrlblk.boost_time))) {
334 if (rcu_preempt_ctrlblk.exp_tasks == NULL)
335 rcu_preempt_ctrlblk.boost_tasks =
336 rcu_preempt_ctrlblk.gp_tasks;
294 invoke_rcu_kthread(); 337 invoke_rcu_kthread();
295 RCU_TRACE(rcu_preempt_ctrlblk.n_normal_boosts++);
296 } else 338 } else
297 RCU_TRACE(rcu_initiate_boost_trace()); 339 RCU_TRACE(rcu_initiate_boost_trace());
298 return 1; 340 return 1;
299} 341}
300 342
301/*
302 * Initiate boosting for an expedited grace period.
303 */
304static void rcu_initiate_expedited_boost(void)
305{
306 unsigned long flags;
307
308 raw_local_irq_save(flags);
309 if (!list_empty(&rcu_preempt_ctrlblk.blkd_tasks)) {
310 rcu_preempt_ctrlblk.boost_tasks =
311 rcu_preempt_ctrlblk.blkd_tasks.next;
312 invoke_rcu_kthread();
313 RCU_TRACE(rcu_preempt_ctrlblk.n_exp_boosts++);
314 } else
315 RCU_TRACE(rcu_initiate_exp_boost_trace());
316 raw_local_irq_restore(flags);
317}
318
319#define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000) 343#define RCU_BOOST_DELAY_JIFFIES DIV_ROUND_UP(CONFIG_RCU_BOOST_DELAY * HZ, 1000)
320 344
321/* 345/*
@@ -347,13 +371,6 @@ static int rcu_initiate_boost(void)
347} 371}
348 372
349/* 373/*
350 * If there is no RCU priority boosting, we don't initiate expedited boosting.
351 */
352static void rcu_initiate_expedited_boost(void)
353{
354}
355
356/*
357 * If there is no RCU priority boosting, nothing to do at grace-period start. 374 * If there is no RCU priority boosting, nothing to do at grace-period start.
358 */ 375 */
359static void rcu_preempt_boost_start_gp(void) 376static void rcu_preempt_boost_start_gp(void)
@@ -786,13 +803,16 @@ void synchronize_rcu_expedited(void)
786 rpcp->exp_tasks = rpcp->blkd_tasks.next; 803 rpcp->exp_tasks = rpcp->blkd_tasks.next;
787 if (rpcp->exp_tasks == &rpcp->blkd_tasks) 804 if (rpcp->exp_tasks == &rpcp->blkd_tasks)
788 rpcp->exp_tasks = NULL; 805 rpcp->exp_tasks = NULL;
789 local_irq_restore(flags);
790 806
791 /* Wait for tail of ->blkd_tasks list to drain. */ 807 /* Wait for tail of ->blkd_tasks list to drain. */
792 if (rcu_preempted_readers_exp()) 808 if (!rcu_preempted_readers_exp())
793 rcu_initiate_expedited_boost(); 809 local_irq_restore(flags);
810 else {
811 rcu_initiate_boost();
812 local_irq_restore(flags);
794 wait_event(sync_rcu_preempt_exp_wq, 813 wait_event(sync_rcu_preempt_exp_wq,
795 !rcu_preempted_readers_exp()); 814 !rcu_preempted_readers_exp());
815 }
796 816
797 /* Clean up and exit. */ 817 /* Clean up and exit. */
798 barrier(); /* ensure expedited GP seen before counter increment. */ 818 barrier(); /* ensure expedited GP seen before counter increment. */
@@ -905,22 +925,17 @@ void __init rcu_scheduler_starting(void)
905 925
906static void rcu_initiate_boost_trace(void) 926static void rcu_initiate_boost_trace(void)
907{ 927{
908 if (rcu_preempt_ctrlblk.gp_tasks == NULL) 928 if (list_empty(&rcu_preempt_ctrlblk.blkd_tasks))
909 rcu_preempt_ctrlblk.n_normal_balk_gp_tasks++; 929 rcu_preempt_ctrlblk.n_balk_blkd_tasks++;
930 else if (rcu_preempt_ctrlblk.gp_tasks == NULL &&
931 rcu_preempt_ctrlblk.exp_tasks == NULL)
932 rcu_preempt_ctrlblk.n_balk_exp_gp_tasks++;
910 else if (rcu_preempt_ctrlblk.boost_tasks != NULL) 933 else if (rcu_preempt_ctrlblk.boost_tasks != NULL)
911 rcu_preempt_ctrlblk.n_normal_balk_boost_tasks++; 934 rcu_preempt_ctrlblk.n_balk_boost_tasks++;
912 else if (!ULONG_CMP_GE(jiffies, rcu_preempt_ctrlblk.boost_time)) 935 else if (!ULONG_CMP_GE(jiffies, rcu_preempt_ctrlblk.boost_time))
913 rcu_preempt_ctrlblk.n_normal_balk_notyet++; 936 rcu_preempt_ctrlblk.n_balk_notyet++;
914 else
915 rcu_preempt_ctrlblk.n_normal_balk_nos++;
916}
917
918static void rcu_initiate_exp_boost_trace(void)
919{
920 if (list_empty(&rcu_preempt_ctrlblk.blkd_tasks))
921 rcu_preempt_ctrlblk.n_exp_balk_blkd_tasks++;
922 else 937 else
923 rcu_preempt_ctrlblk.n_exp_balk_nos++; 938 rcu_preempt_ctrlblk.n_balk_nos++;
924} 939}
925 940
926#endif /* #ifdef CONFIG_RCU_BOOST */ 941#endif /* #ifdef CONFIG_RCU_BOOST */