diff options
-rw-r--r-- | arch/ia64/xen/time.c | 3 | ||||
-rw-r--r-- | include/linux/rcupreempt.h | 1 | ||||
-rw-r--r-- | include/linux/rcutree.h | 1 | ||||
-rw-r--r-- | kernel/rcupreempt.c | 10 | ||||
-rw-r--r-- | kernel/rcutree.c | 5 | ||||
-rw-r--r-- | kernel/timer.c | 3 |
6 files changed, 14 insertions, 9 deletions
diff --git a/arch/ia64/xen/time.c b/arch/ia64/xen/time.c index fb833269017..dbeadb9c8e2 100644 --- a/arch/ia64/xen/time.c +++ b/arch/ia64/xen/time.c | |||
@@ -133,8 +133,7 @@ consider_steal_time(unsigned long new_itm) | |||
133 | account_idle_ticks(blocked); | 133 | account_idle_ticks(blocked); |
134 | run_local_timers(); | 134 | run_local_timers(); |
135 | 135 | ||
136 | if (rcu_pending(cpu)) | 136 | rcu_check_callbacks(cpu, user_mode(get_irq_regs())); |
137 | rcu_check_callbacks(cpu, user_mode(get_irq_regs())); | ||
138 | 137 | ||
139 | scheduler_tick(); | 138 | scheduler_tick(); |
140 | run_posix_cpu_timers(p); | 139 | run_posix_cpu_timers(p); |
diff --git a/include/linux/rcupreempt.h b/include/linux/rcupreempt.h index 6c9dd9cf8b8..aff4772fb49 100644 --- a/include/linux/rcupreempt.h +++ b/include/linux/rcupreempt.h | |||
@@ -66,7 +66,6 @@ extern void call_rcu_sched(struct rcu_head *head, | |||
66 | 66 | ||
67 | extern void __rcu_read_lock(void); | 67 | extern void __rcu_read_lock(void); |
68 | extern void __rcu_read_unlock(void); | 68 | extern void __rcu_read_unlock(void); |
69 | extern int rcu_pending(int cpu); | ||
70 | extern int rcu_needs_cpu(int cpu); | 69 | extern int rcu_needs_cpu(int cpu); |
71 | 70 | ||
72 | #define __rcu_read_lock_bh() { rcu_read_lock(); local_bh_disable(); } | 71 | #define __rcu_read_lock_bh() { rcu_read_lock(); local_bh_disable(); } |
diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 8a0222ce3b1..c739d90f5e6 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h | |||
@@ -33,7 +33,6 @@ | |||
33 | extern void rcu_sched_qs(int cpu); | 33 | extern void rcu_sched_qs(int cpu); |
34 | extern void rcu_bh_qs(int cpu); | 34 | extern void rcu_bh_qs(int cpu); |
35 | 35 | ||
36 | extern int rcu_pending(int cpu); | ||
37 | extern int rcu_needs_cpu(int cpu); | 36 | extern int rcu_needs_cpu(int cpu); |
38 | 37 | ||
39 | static inline void __rcu_read_lock(void) | 38 | static inline void __rcu_read_lock(void) |
diff --git a/kernel/rcupreempt.c b/kernel/rcupreempt.c index 7d777c9f394..0053ce56e32 100644 --- a/kernel/rcupreempt.c +++ b/kernel/rcupreempt.c | |||
@@ -159,6 +159,8 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_dyntick_sched, rcu_dyntick_sched | |||
159 | .dynticks = 1, | 159 | .dynticks = 1, |
160 | }; | 160 | }; |
161 | 161 | ||
162 | static int rcu_pending(int cpu); | ||
163 | |||
162 | void rcu_sched_qs(int cpu) | 164 | void rcu_sched_qs(int cpu) |
163 | { | 165 | { |
164 | struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu); | 166 | struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu); |
@@ -961,7 +963,10 @@ static void rcu_check_mb(int cpu) | |||
961 | void rcu_check_callbacks(int cpu, int user) | 963 | void rcu_check_callbacks(int cpu, int user) |
962 | { | 964 | { |
963 | unsigned long flags; | 965 | unsigned long flags; |
964 | struct rcu_data *rdp = RCU_DATA_CPU(cpu); | 966 | struct rcu_data *rdp; |
967 | |||
968 | if (!rcu_pending(cpu)) | ||
969 | return; /* if nothing for RCU to do. */ | ||
965 | 970 | ||
966 | /* | 971 | /* |
967 | * If this CPU took its interrupt from user mode or from the | 972 | * If this CPU took its interrupt from user mode or from the |
@@ -976,6 +981,7 @@ void rcu_check_callbacks(int cpu, int user) | |||
976 | * CPUs to happen after any such write. | 981 | * CPUs to happen after any such write. |
977 | */ | 982 | */ |
978 | 983 | ||
984 | rdp = RCU_DATA_CPU(cpu); | ||
979 | if (user || | 985 | if (user || |
980 | (idle_cpu(cpu) && !in_softirq() && | 986 | (idle_cpu(cpu) && !in_softirq() && |
981 | hardirq_count() <= (1 << HARDIRQ_SHIFT))) { | 987 | hardirq_count() <= (1 << HARDIRQ_SHIFT))) { |
@@ -1382,7 +1388,7 @@ int rcu_needs_cpu(int cpu) | |||
1382 | rdp->waitschedlist != NULL); | 1388 | rdp->waitschedlist != NULL); |
1383 | } | 1389 | } |
1384 | 1390 | ||
1385 | int rcu_pending(int cpu) | 1391 | static int rcu_pending(int cpu) |
1386 | { | 1392 | { |
1387 | struct rcu_data *rdp = RCU_DATA_CPU(cpu); | 1393 | struct rcu_data *rdp = RCU_DATA_CPU(cpu); |
1388 | 1394 | ||
diff --git a/kernel/rcutree.c b/kernel/rcutree.c index 7c515082ae8..4ce3adcfa94 100644 --- a/kernel/rcutree.c +++ b/kernel/rcutree.c | |||
@@ -111,6 +111,7 @@ static int qhimark = 10000; /* If this many pending, ignore blimit. */ | |||
111 | static int qlowmark = 100; /* Once only this many pending, use blimit. */ | 111 | static int qlowmark = 100; /* Once only this many pending, use blimit. */ |
112 | 112 | ||
113 | static void force_quiescent_state(struct rcu_state *rsp, int relaxed); | 113 | static void force_quiescent_state(struct rcu_state *rsp, int relaxed); |
114 | static int rcu_pending(int cpu); | ||
114 | 115 | ||
115 | /* | 116 | /* |
116 | * Return the number of RCU-sched batches processed thus far for debug & stats. | 117 | * Return the number of RCU-sched batches processed thus far for debug & stats. |
@@ -974,6 +975,8 @@ static void rcu_do_batch(struct rcu_data *rdp) | |||
974 | */ | 975 | */ |
975 | void rcu_check_callbacks(int cpu, int user) | 976 | void rcu_check_callbacks(int cpu, int user) |
976 | { | 977 | { |
978 | if (!rcu_pending(cpu)) | ||
979 | return; /* if nothing for RCU to do. */ | ||
977 | if (user || | 980 | if (user || |
978 | (idle_cpu(cpu) && rcu_scheduler_active && | 981 | (idle_cpu(cpu) && rcu_scheduler_active && |
979 | !in_softirq() && hardirq_count() <= (1 << HARDIRQ_SHIFT))) { | 982 | !in_softirq() && hardirq_count() <= (1 << HARDIRQ_SHIFT))) { |
@@ -1329,7 +1332,7 @@ static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp) | |||
1329 | * by the current CPU, returning 1 if so. This function is part of the | 1332 | * by the current CPU, returning 1 if so. This function is part of the |
1330 | * RCU implementation; it is -not- an exported member of the RCU API. | 1333 | * RCU implementation; it is -not- an exported member of the RCU API. |
1331 | */ | 1334 | */ |
1332 | int rcu_pending(int cpu) | 1335 | static int rcu_pending(int cpu) |
1333 | { | 1336 | { |
1334 | return __rcu_pending(&rcu_sched_state, &per_cpu(rcu_sched_data, cpu)) || | 1337 | return __rcu_pending(&rcu_sched_state, &per_cpu(rcu_sched_data, cpu)) || |
1335 | __rcu_pending(&rcu_bh_state, &per_cpu(rcu_bh_data, cpu)); | 1338 | __rcu_pending(&rcu_bh_state, &per_cpu(rcu_bh_data, cpu)); |
diff --git a/kernel/timer.c b/kernel/timer.c index a7f07d5a624..a3d25f41501 100644 --- a/kernel/timer.c +++ b/kernel/timer.c | |||
@@ -1156,8 +1156,7 @@ void update_process_times(int user_tick) | |||
1156 | /* Note: this timer irq context must be accounted for as well. */ | 1156 | /* Note: this timer irq context must be accounted for as well. */ |
1157 | account_process_tick(p, user_tick); | 1157 | account_process_tick(p, user_tick); |
1158 | run_local_timers(); | 1158 | run_local_timers(); |
1159 | if (rcu_pending(cpu)) | 1159 | rcu_check_callbacks(cpu, user_tick); |
1160 | rcu_check_callbacks(cpu, user_tick); | ||
1161 | printk_tick(); | 1160 | printk_tick(); |
1162 | scheduler_tick(); | 1161 | scheduler_tick(); |
1163 | run_posix_cpu_timers(p); | 1162 | run_posix_cpu_timers(p); |