aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
authorDipankar Sarma <dipankar@in.ibm.com>2006-03-08 00:55:33 -0500
committerLinus Torvalds <torvalds@g5.osdl.org>2006-03-08 17:14:01 -0500
commit21a1ea9eb40411d4ee29448c53b9e4c0654d6ceb (patch)
tree86e4b681858ac562c953bd3ea3c038defd8ec4f4 /kernel
parente2bab3d92486fb781f4d06f56339264ed1492392 (diff)
[PATCH] rcu batch tuning
This patch adds new tunables for RCU queue and finished batches. There are two types of controls - number of completed RCU updates invoked in a batch (blimit) and monitoring for high rate of incoming RCUs on a cpu (qhimark, qlowmark). By default, the per-cpu batch limit is set to a small value. If the input RCU rate exceeds the high watermark, we do two things - force quiescent state on all cpus and set the batch limit of the CPU to INTMAX. Setting batch limit to INTMAX forces all finished RCUs to be processed in one shot. If we have more than INTMAX RCUs queued up, then we have bigger problems anyway. Once the incoming queued RCUs fall below the low watermark, the batch limit is set to the default. Signed-off-by: Dipankar Sarma <dipankar@in.ibm.com> Cc: "Paul E. McKenney" <paulmck@us.ibm.com> Cc: "David S. Miller" <davem@davemloft.net> Signed-off-by: Andrew Morton <akpm@osdl.org> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Diffstat (limited to 'kernel')
-rw-r--r--kernel/rcupdate.c76
1 files changed, 58 insertions, 18 deletions
diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index 0cf8146bd585..8cf15a569fcd 100644
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -67,7 +67,43 @@ DEFINE_PER_CPU(struct rcu_data, rcu_bh_data) = { 0L };
67 67
68/* Fake initialization required by compiler */ 68/* Fake initialization required by compiler */
69static DEFINE_PER_CPU(struct tasklet_struct, rcu_tasklet) = {NULL}; 69static DEFINE_PER_CPU(struct tasklet_struct, rcu_tasklet) = {NULL};
70static int maxbatch = 10000; 70static int blimit = 10;
71static int qhimark = 10000;
72static int qlowmark = 100;
73#ifdef CONFIG_SMP
74static int rsinterval = 1000;
75#endif
76
77static atomic_t rcu_barrier_cpu_count;
78static struct semaphore rcu_barrier_sema;
79static struct completion rcu_barrier_completion;
80
81#ifdef CONFIG_SMP
82static void force_quiescent_state(struct rcu_data *rdp,
83 struct rcu_ctrlblk *rcp)
84{
85 int cpu;
86 cpumask_t cpumask;
87 set_need_resched();
88 if (unlikely(rdp->qlen - rdp->last_rs_qlen > rsinterval)) {
89 rdp->last_rs_qlen = rdp->qlen;
90 /*
91 * Don't send IPI to itself. With irqs disabled,
92 * rdp->cpu is the current cpu.
93 */
94 cpumask = rcp->cpumask;
95 cpu_clear(rdp->cpu, cpumask);
96 for_each_cpu_mask(cpu, cpumask)
97 smp_send_reschedule(cpu);
98 }
99}
100#else
101static inline void force_quiescent_state(struct rcu_data *rdp,
102 struct rcu_ctrlblk *rcp)
103{
104 set_need_resched();
105}
106#endif
71 107
72/** 108/**
73 * call_rcu - Queue an RCU callback for invocation after a grace period. 109 * call_rcu - Queue an RCU callback for invocation after a grace period.
@@ -92,17 +128,13 @@ void fastcall call_rcu(struct rcu_head *head,
92 rdp = &__get_cpu_var(rcu_data); 128 rdp = &__get_cpu_var(rcu_data);
93 *rdp->nxttail = head; 129 *rdp->nxttail = head;
94 rdp->nxttail = &head->next; 130 rdp->nxttail = &head->next;
95 131 if (unlikely(++rdp->qlen > qhimark)) {
96 if (unlikely(++rdp->count > 10000)) 132 rdp->blimit = INT_MAX;
97 set_need_resched(); 133 force_quiescent_state(rdp, &rcu_ctrlblk);
98 134 }
99 local_irq_restore(flags); 135 local_irq_restore(flags);
100} 136}
101 137
102static atomic_t rcu_barrier_cpu_count;
103static struct semaphore rcu_barrier_sema;
104static struct completion rcu_barrier_completion;
105
106/** 138/**
107 * call_rcu_bh - Queue an RCU for invocation after a quicker grace period. 139 * call_rcu_bh - Queue an RCU for invocation after a quicker grace period.
108 * @head: structure to be used for queueing the RCU updates. 140 * @head: structure to be used for queueing the RCU updates.
@@ -131,12 +163,12 @@ void fastcall call_rcu_bh(struct rcu_head *head,
131 rdp = &__get_cpu_var(rcu_bh_data); 163 rdp = &__get_cpu_var(rcu_bh_data);
132 *rdp->nxttail = head; 164 *rdp->nxttail = head;
133 rdp->nxttail = &head->next; 165 rdp->nxttail = &head->next;
134 rdp->count++; 166
135/* 167 if (unlikely(++rdp->qlen > qhimark)) {
136 * Should we directly call rcu_do_batch() here ? 168 rdp->blimit = INT_MAX;
137 * if (unlikely(rdp->count > 10000)) 169 force_quiescent_state(rdp, &rcu_bh_ctrlblk);
138 * rcu_do_batch(rdp); 170 }
139 */ 171
140 local_irq_restore(flags); 172 local_irq_restore(flags);
141} 173}
142 174
@@ -199,10 +231,12 @@ static void rcu_do_batch(struct rcu_data *rdp)
199 next = rdp->donelist = list->next; 231 next = rdp->donelist = list->next;
200 list->func(list); 232 list->func(list);
201 list = next; 233 list = next;
202 rdp->count--; 234 rdp->qlen--;
203 if (++count >= maxbatch) 235 if (++count >= rdp->blimit)
204 break; 236 break;
205 } 237 }
238 if (rdp->blimit == INT_MAX && rdp->qlen <= qlowmark)
239 rdp->blimit = blimit;
206 if (!rdp->donelist) 240 if (!rdp->donelist)
207 rdp->donetail = &rdp->donelist; 241 rdp->donetail = &rdp->donelist;
208 else 242 else
@@ -473,6 +507,7 @@ static void rcu_init_percpu_data(int cpu, struct rcu_ctrlblk *rcp,
473 rdp->quiescbatch = rcp->completed; 507 rdp->quiescbatch = rcp->completed;
474 rdp->qs_pending = 0; 508 rdp->qs_pending = 0;
475 rdp->cpu = cpu; 509 rdp->cpu = cpu;
510 rdp->blimit = blimit;
476} 511}
477 512
478static void __devinit rcu_online_cpu(int cpu) 513static void __devinit rcu_online_cpu(int cpu)
@@ -567,7 +602,12 @@ void synchronize_kernel(void)
567 synchronize_rcu(); 602 synchronize_rcu();
568} 603}
569 604
570module_param(maxbatch, int, 0); 605module_param(blimit, int, 0);
606module_param(qhimark, int, 0);
607module_param(qlowmark, int, 0);
608#ifdef CONFIG_SMP
609module_param(rsinterval, int, 0);
610#endif
571EXPORT_SYMBOL_GPL(rcu_batches_completed); 611EXPORT_SYMBOL_GPL(rcu_batches_completed);
572EXPORT_SYMBOL(call_rcu); /* WARNING: GPL-only in April 2006. */ 612EXPORT_SYMBOL(call_rcu); /* WARNING: GPL-only in April 2006. */
573EXPORT_SYMBOL(call_rcu_bh); /* WARNING: GPL-only in April 2006. */ 613EXPORT_SYMBOL(call_rcu_bh); /* WARNING: GPL-only in April 2006. */