aboutsummaryrefslogtreecommitdiffstats
path: root/include/linux/cpu.h
diff options
context:
space:
mode:
authorSrivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com>2014-03-10 16:34:14 -0400
committerRafael J. Wysocki <rafael.j.wysocki@intel.com>2014-03-20 08:43:40 -0400
commit93ae4f978ca7f26d17df915ac7afc919c1dd0353 (patch)
tree4afd87d18abf25ad1e63af24b8f87a8bc738c9e9 /include/linux/cpu.h
parenta19423b98704aa85e84097be6d1d44a8615c2340 (diff)
CPU hotplug: Provide lockless versions of callback registration functions
The following method of CPU hotplug callback registration is not safe due to the possibility of an ABBA deadlock involving the cpu_add_remove_lock and the cpu_hotplug.lock. get_online_cpus(); for_each_online_cpu(cpu) init_cpu(cpu); register_cpu_notifier(&foobar_cpu_notifier); put_online_cpus(); The deadlock is shown below: CPU 0 CPU 1 ----- ----- Acquire cpu_hotplug.lock [via get_online_cpus()] CPU online/offline operation takes cpu_add_remove_lock [via cpu_maps_update_begin()] Try to acquire cpu_add_remove_lock [via register_cpu_notifier()] CPU online/offline operation tries to acquire cpu_hotplug.lock [via cpu_hotplug_begin()] *** DEADLOCK! *** The problem here is that callback registration takes the locks in one order whereas the CPU hotplug operations take the same locks in the opposite order. To avoid this issue and to provide a race-free method to register CPU hotplug callbacks (along with initialization of already online CPUs), introduce new variants of the callback registration APIs that simply register the callbacks without holding the cpu_add_remove_lock during the registration. That way, we can avoid the ABBA scenario. However, we will need to hold the cpu_add_remove_lock throughout the entire critical section, to protect updates to the callback/notifier chain. This can be achieved by writing the callback registration code as follows: cpu_maps_update_begin(); [ or cpu_notifier_register_begin(); see below ] for_each_online_cpu(cpu) init_cpu(cpu); /* This doesn't take the cpu_add_remove_lock */ __register_cpu_notifier(&foobar_cpu_notifier); cpu_maps_update_done(); [ or cpu_notifier_register_done(); see below ] Note that we can't use get_online_cpus() here instead of cpu_maps_update_begin() because the cpu_hotplug.lock is dropped during the invocation of CPU_POST_DEAD notifiers, and hence get_online_cpus() cannot provide the necessary synchronization to protect the callback/notifier chains against concurrent reads and writes. On the other hand, since the cpu_add_remove_lock protects the entire hotplug operation (including CPU_POST_DEAD), we can use cpu_maps_update_begin/done() to guarantee proper synchronization. Also, since cpu_maps_update_begin/done() is like a super-set of get/put_online_cpus(), the former naturally protects the critical sections from concurrent hotplug operations. Since the names cpu_maps_update_begin/done() don't make much sense in CPU hotplug callback registration scenarios, we'll introduce new APIs named cpu_notifier_register_begin/done() and map them to cpu_maps_update_begin/done(). In summary, introduce the lockless variants of un/register_cpu_notifier() and also export the cpu_notifier_register_begin/done() APIs for use by modules. This way, we provide a race-free way to register hotplug callbacks as well as perform initialization for the CPUs that are already online. Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Ingo Molnar <mingo@kernel.org> Acked-by: Oleg Nesterov <oleg@redhat.com> Acked-by: Toshi Kani <toshi.kani@hp.com> Reviewed-by: Gautham R. Shenoy <ego@linux.vnet.ibm.com> Signed-off-by: Srivatsa S. Bhat <srivatsa.bhat@linux.vnet.ibm.com> Signed-off-by: Rafael J. Wysocki <rafael.j.wysocki@intel.com>
Diffstat (limited to 'include/linux/cpu.h')
-rw-r--r--include/linux/cpu.h47
1 files changed, 47 insertions, 0 deletions
diff --git a/include/linux/cpu.h b/include/linux/cpu.h
index 03e235ad1bba..488d6ebcf6a1 100644
--- a/include/linux/cpu.h
+++ b/include/linux/cpu.h
@@ -122,26 +122,46 @@ enum {
122 { .notifier_call = fn, .priority = pri }; \ 122 { .notifier_call = fn, .priority = pri }; \
123 register_cpu_notifier(&fn##_nb); \ 123 register_cpu_notifier(&fn##_nb); \
124} 124}
125
126#define __cpu_notifier(fn, pri) { \
127 static struct notifier_block fn##_nb = \
128 { .notifier_call = fn, .priority = pri }; \
129 __register_cpu_notifier(&fn##_nb); \
130}
125#else /* #if defined(CONFIG_HOTPLUG_CPU) || !defined(MODULE) */ 131#else /* #if defined(CONFIG_HOTPLUG_CPU) || !defined(MODULE) */
126#define cpu_notifier(fn, pri) do { (void)(fn); } while (0) 132#define cpu_notifier(fn, pri) do { (void)(fn); } while (0)
133#define __cpu_notifier(fn, pri) do { (void)(fn); } while (0)
127#endif /* #else #if defined(CONFIG_HOTPLUG_CPU) || !defined(MODULE) */ 134#endif /* #else #if defined(CONFIG_HOTPLUG_CPU) || !defined(MODULE) */
135
128#ifdef CONFIG_HOTPLUG_CPU 136#ifdef CONFIG_HOTPLUG_CPU
129extern int register_cpu_notifier(struct notifier_block *nb); 137extern int register_cpu_notifier(struct notifier_block *nb);
138extern int __register_cpu_notifier(struct notifier_block *nb);
130extern void unregister_cpu_notifier(struct notifier_block *nb); 139extern void unregister_cpu_notifier(struct notifier_block *nb);
140extern void __unregister_cpu_notifier(struct notifier_block *nb);
131#else 141#else
132 142
133#ifndef MODULE 143#ifndef MODULE
134extern int register_cpu_notifier(struct notifier_block *nb); 144extern int register_cpu_notifier(struct notifier_block *nb);
145extern int __register_cpu_notifier(struct notifier_block *nb);
135#else 146#else
136static inline int register_cpu_notifier(struct notifier_block *nb) 147static inline int register_cpu_notifier(struct notifier_block *nb)
137{ 148{
138 return 0; 149 return 0;
139} 150}
151
152static inline int __register_cpu_notifier(struct notifier_block *nb)
153{
154 return 0;
155}
140#endif 156#endif
141 157
142static inline void unregister_cpu_notifier(struct notifier_block *nb) 158static inline void unregister_cpu_notifier(struct notifier_block *nb)
143{ 159{
144} 160}
161
162static inline void __unregister_cpu_notifier(struct notifier_block *nb)
163{
164}
145#endif 165#endif
146 166
147int cpu_up(unsigned int cpu); 167int cpu_up(unsigned int cpu);
@@ -149,19 +169,32 @@ void notify_cpu_starting(unsigned int cpu);
149extern void cpu_maps_update_begin(void); 169extern void cpu_maps_update_begin(void);
150extern void cpu_maps_update_done(void); 170extern void cpu_maps_update_done(void);
151 171
172#define cpu_notifier_register_begin cpu_maps_update_begin
173#define cpu_notifier_register_done cpu_maps_update_done
174
152#else /* CONFIG_SMP */ 175#else /* CONFIG_SMP */
153 176
154#define cpu_notifier(fn, pri) do { (void)(fn); } while (0) 177#define cpu_notifier(fn, pri) do { (void)(fn); } while (0)
178#define __cpu_notifier(fn, pri) do { (void)(fn); } while (0)
155 179
156static inline int register_cpu_notifier(struct notifier_block *nb) 180static inline int register_cpu_notifier(struct notifier_block *nb)
157{ 181{
158 return 0; 182 return 0;
159} 183}
160 184
185static inline int __register_cpu_notifier(struct notifier_block *nb)
186{
187 return 0;
188}
189
161static inline void unregister_cpu_notifier(struct notifier_block *nb) 190static inline void unregister_cpu_notifier(struct notifier_block *nb)
162{ 191{
163} 192}
164 193
194static inline void __unregister_cpu_notifier(struct notifier_block *nb)
195{
196}
197
165static inline void cpu_maps_update_begin(void) 198static inline void cpu_maps_update_begin(void)
166{ 199{
167} 200}
@@ -170,6 +203,14 @@ static inline void cpu_maps_update_done(void)
170{ 203{
171} 204}
172 205
206static inline void cpu_notifier_register_begin(void)
207{
208}
209
210static inline void cpu_notifier_register_done(void)
211{
212}
213
173#endif /* CONFIG_SMP */ 214#endif /* CONFIG_SMP */
174extern struct bus_type cpu_subsys; 215extern struct bus_type cpu_subsys;
175 216
@@ -183,8 +224,11 @@ extern void put_online_cpus(void);
183extern void cpu_hotplug_disable(void); 224extern void cpu_hotplug_disable(void);
184extern void cpu_hotplug_enable(void); 225extern void cpu_hotplug_enable(void);
185#define hotcpu_notifier(fn, pri) cpu_notifier(fn, pri) 226#define hotcpu_notifier(fn, pri) cpu_notifier(fn, pri)
227#define __hotcpu_notifier(fn, pri) __cpu_notifier(fn, pri)
186#define register_hotcpu_notifier(nb) register_cpu_notifier(nb) 228#define register_hotcpu_notifier(nb) register_cpu_notifier(nb)
229#define __register_hotcpu_notifier(nb) __register_cpu_notifier(nb)
187#define unregister_hotcpu_notifier(nb) unregister_cpu_notifier(nb) 230#define unregister_hotcpu_notifier(nb) unregister_cpu_notifier(nb)
231#define __unregister_hotcpu_notifier(nb) __unregister_cpu_notifier(nb)
188void clear_tasks_mm_cpumask(int cpu); 232void clear_tasks_mm_cpumask(int cpu);
189int cpu_down(unsigned int cpu); 233int cpu_down(unsigned int cpu);
190 234
@@ -197,9 +241,12 @@ static inline void cpu_hotplug_done(void) {}
197#define cpu_hotplug_disable() do { } while (0) 241#define cpu_hotplug_disable() do { } while (0)
198#define cpu_hotplug_enable() do { } while (0) 242#define cpu_hotplug_enable() do { } while (0)
199#define hotcpu_notifier(fn, pri) do { (void)(fn); } while (0) 243#define hotcpu_notifier(fn, pri) do { (void)(fn); } while (0)
244#define __hotcpu_notifier(fn, pri) do { (void)(fn); } while (0)
200/* These aren't inline functions due to a GCC bug. */ 245/* These aren't inline functions due to a GCC bug. */
201#define register_hotcpu_notifier(nb) ({ (void)(nb); 0; }) 246#define register_hotcpu_notifier(nb) ({ (void)(nb); 0; })
247#define __register_hotcpu_notifier(nb) ({ (void)(nb); 0; })
202#define unregister_hotcpu_notifier(nb) ({ (void)(nb); }) 248#define unregister_hotcpu_notifier(nb) ({ (void)(nb); })
249#define __unregister_hotcpu_notifier(nb) ({ (void)(nb); })
203#endif /* CONFIG_HOTPLUG_CPU */ 250#endif /* CONFIG_HOTPLUG_CPU */
204 251
205#ifdef CONFIG_PM_SLEEP_SMP 252#ifdef CONFIG_PM_SLEEP_SMP