aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPaul E. McKenney <paulmck@linux.vnet.ibm.com>2009-08-22 16:56:51 -0400
committerIngo Molnar <mingo@elte.hu>2009-08-23 04:32:39 -0400
commita157229cabd6dd8cfa82525fc9bf730c94cc9ac2 (patch)
tree886013efc2f25dab915640acf46151fac7ae57b4
parent22f00b69f6a7e1e18e821979a23e8307c2de9888 (diff)
rcu: Simplify rcu_pending()/rcu_check_callbacks() API
All calls from outside RCU are of the form: if (rcu_pending(cpu)) rcu_check_callbacks(cpu, user); This is silly, instead we put a call to rcu_pending() in rcu_check_callbacks(), and then make the outside calls be to rcu_check_callbacks(). This cuts down on the code a bit and also gives the compiler a better chance of optimizing. Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> Cc: laijs@cn.fujitsu.com Cc: dipankar@in.ibm.com Cc: akpm@linux-foundation.org Cc: mathieu.desnoyers@polymtl.ca Cc: josht@linux.vnet.ibm.com Cc: dvhltc@us.ibm.com Cc: niv@us.ibm.com Cc: peterz@infradead.org Cc: rostedt@goodmis.org LKML-Reference: <125097461311-git-send-email-> Signed-off-by: Ingo Molnar <mingo@elte.hu>
-rw-r--r--arch/ia64/xen/time.c3
-rw-r--r--include/linux/rcupreempt.h1
-rw-r--r--include/linux/rcutree.h1
-rw-r--r--kernel/rcupreempt.c10
-rw-r--r--kernel/rcutree.c5
-rw-r--r--kernel/timer.c3
6 files changed, 14 insertions, 9 deletions
diff --git a/arch/ia64/xen/time.c b/arch/ia64/xen/time.c
index fb8332690179..dbeadb9c8e20 100644
--- a/arch/ia64/xen/time.c
+++ b/arch/ia64/xen/time.c
@@ -133,8 +133,7 @@ consider_steal_time(unsigned long new_itm)
133 account_idle_ticks(blocked); 133 account_idle_ticks(blocked);
134 run_local_timers(); 134 run_local_timers();
135 135
136 if (rcu_pending(cpu)) 136 rcu_check_callbacks(cpu, user_mode(get_irq_regs()));
137 rcu_check_callbacks(cpu, user_mode(get_irq_regs()));
138 137
139 scheduler_tick(); 138 scheduler_tick();
140 run_posix_cpu_timers(p); 139 run_posix_cpu_timers(p);
diff --git a/include/linux/rcupreempt.h b/include/linux/rcupreempt.h
index 6c9dd9cf8b8e..aff4772fb49e 100644
--- a/include/linux/rcupreempt.h
+++ b/include/linux/rcupreempt.h
@@ -66,7 +66,6 @@ extern void call_rcu_sched(struct rcu_head *head,
66 66
67extern void __rcu_read_lock(void); 67extern void __rcu_read_lock(void);
68extern void __rcu_read_unlock(void); 68extern void __rcu_read_unlock(void);
69extern int rcu_pending(int cpu);
70extern int rcu_needs_cpu(int cpu); 69extern int rcu_needs_cpu(int cpu);
71 70
72#define __rcu_read_lock_bh() { rcu_read_lock(); local_bh_disable(); } 71#define __rcu_read_lock_bh() { rcu_read_lock(); local_bh_disable(); }
diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h
index 8a0222ce3b13..c739d90f5e68 100644
--- a/include/linux/rcutree.h
+++ b/include/linux/rcutree.h
@@ -33,7 +33,6 @@
33extern void rcu_sched_qs(int cpu); 33extern void rcu_sched_qs(int cpu);
34extern void rcu_bh_qs(int cpu); 34extern void rcu_bh_qs(int cpu);
35 35
36extern int rcu_pending(int cpu);
37extern int rcu_needs_cpu(int cpu); 36extern int rcu_needs_cpu(int cpu);
38 37
39static inline void __rcu_read_lock(void) 38static inline void __rcu_read_lock(void)
diff --git a/kernel/rcupreempt.c b/kernel/rcupreempt.c
index 7d777c9f394c..0053ce56e326 100644
--- a/kernel/rcupreempt.c
+++ b/kernel/rcupreempt.c
@@ -159,6 +159,8 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_dyntick_sched, rcu_dyntick_sched
159 .dynticks = 1, 159 .dynticks = 1,
160}; 160};
161 161
162static int rcu_pending(int cpu);
163
162void rcu_sched_qs(int cpu) 164void rcu_sched_qs(int cpu)
163{ 165{
164 struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu); 166 struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
@@ -961,7 +963,10 @@ static void rcu_check_mb(int cpu)
961void rcu_check_callbacks(int cpu, int user) 963void rcu_check_callbacks(int cpu, int user)
962{ 964{
963 unsigned long flags; 965 unsigned long flags;
964 struct rcu_data *rdp = RCU_DATA_CPU(cpu); 966 struct rcu_data *rdp;
967
968 if (!rcu_pending(cpu))
969 return; /* if nothing for RCU to do. */
965 970
966 /* 971 /*
967 * If this CPU took its interrupt from user mode or from the 972 * If this CPU took its interrupt from user mode or from the
@@ -976,6 +981,7 @@ void rcu_check_callbacks(int cpu, int user)
976 * CPUs to happen after any such write. 981 * CPUs to happen after any such write.
977 */ 982 */
978 983
984 rdp = RCU_DATA_CPU(cpu);
979 if (user || 985 if (user ||
980 (idle_cpu(cpu) && !in_softirq() && 986 (idle_cpu(cpu) && !in_softirq() &&
981 hardirq_count() <= (1 << HARDIRQ_SHIFT))) { 987 hardirq_count() <= (1 << HARDIRQ_SHIFT))) {
@@ -1382,7 +1388,7 @@ int rcu_needs_cpu(int cpu)
1382 rdp->waitschedlist != NULL); 1388 rdp->waitschedlist != NULL);
1383} 1389}
1384 1390
1385int rcu_pending(int cpu) 1391static int rcu_pending(int cpu)
1386{ 1392{
1387 struct rcu_data *rdp = RCU_DATA_CPU(cpu); 1393 struct rcu_data *rdp = RCU_DATA_CPU(cpu);
1388 1394
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 7c515082ae84..4ce3adcfa94d 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -111,6 +111,7 @@ static int qhimark = 10000; /* If this many pending, ignore blimit. */
111static int qlowmark = 100; /* Once only this many pending, use blimit. */ 111static int qlowmark = 100; /* Once only this many pending, use blimit. */
112 112
113static void force_quiescent_state(struct rcu_state *rsp, int relaxed); 113static void force_quiescent_state(struct rcu_state *rsp, int relaxed);
114static int rcu_pending(int cpu);
114 115
115/* 116/*
116 * Return the number of RCU-sched batches processed thus far for debug & stats. 117 * Return the number of RCU-sched batches processed thus far for debug & stats.
@@ -974,6 +975,8 @@ static void rcu_do_batch(struct rcu_data *rdp)
974 */ 975 */
975void rcu_check_callbacks(int cpu, int user) 976void rcu_check_callbacks(int cpu, int user)
976{ 977{
978 if (!rcu_pending(cpu))
979 return; /* if nothing for RCU to do. */
977 if (user || 980 if (user ||
978 (idle_cpu(cpu) && rcu_scheduler_active && 981 (idle_cpu(cpu) && rcu_scheduler_active &&
979 !in_softirq() && hardirq_count() <= (1 << HARDIRQ_SHIFT))) { 982 !in_softirq() && hardirq_count() <= (1 << HARDIRQ_SHIFT))) {
@@ -1329,7 +1332,7 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
1329 * by the current CPU, returning 1 if so. This function is part of the 1332 * by the current CPU, returning 1 if so. This function is part of the
1330 * RCU implementation; it is -not- an exported member of the RCU API. 1333 * RCU implementation; it is -not- an exported member of the RCU API.
1331 */ 1334 */
1332int rcu_pending(int cpu) 1335static int rcu_pending(int cpu)
1333{ 1336{
1334 return __rcu_pending(&rcu_sched_state, &per_cpu(rcu_sched_data, cpu)) || 1337 return __rcu_pending(&rcu_sched_state, &per_cpu(rcu_sched_data, cpu)) ||
1335 __rcu_pending(&rcu_bh_state, &per_cpu(rcu_bh_data, cpu)); 1338 __rcu_pending(&rcu_bh_state, &per_cpu(rcu_bh_data, cpu));
diff --git a/kernel/timer.c b/kernel/timer.c
index a7f07d5a6241..a3d25f415019 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -1156,8 +1156,7 @@ void update_process_times(int user_tick)
1156 /* Note: this timer irq context must be accounted for as well. */ 1156 /* Note: this timer irq context must be accounted for as well. */
1157 account_process_tick(p, user_tick); 1157 account_process_tick(p, user_tick);
1158 run_local_timers(); 1158 run_local_timers();
1159 if (rcu_pending(cpu)) 1159 rcu_check_callbacks(cpu, user_tick);
1160 rcu_check_callbacks(cpu, user_tick);
1161 printk_tick(); 1160 printk_tick();
1162 scheduler_tick(); 1161 scheduler_tick();
1163 run_posix_cpu_timers(p); 1162 run_posix_cpu_timers(p);