aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/Kconfig.preempt25
-rw-r--r--kernel/Makefile6
-rw-r--r--kernel/cpu.c11
-rw-r--r--kernel/cpuset.c4
-rw-r--r--kernel/exit.c4
-rw-r--r--kernel/extable.c16
-rw-r--r--kernel/fork.c4
-rw-r--r--kernel/futex.c351
-rw-r--r--kernel/hrtimer.c332
-rw-r--r--kernel/irq/Makefile1
-rw-r--r--kernel/irq/chip.c18
-rw-r--r--kernel/irq/handle.c205
-rw-r--r--kernel/irq/internals.h5
-rw-r--r--kernel/irq/manage.c49
-rw-r--r--kernel/irq/migration.c14
-rw-r--r--kernel/irq/numa_migrate.c119
-rw-r--r--kernel/irq/proc.c35
-rw-r--r--kernel/lockdep.c60
-rw-r--r--kernel/lockdep_proc.c28
-rw-r--r--kernel/mutex.c10
-rw-r--r--kernel/notifier.c8
-rw-r--r--kernel/panic.c32
-rw-r--r--kernel/posix-cpu-timers.c10
-rw-r--r--kernel/posix-timers.c40
-rw-r--r--kernel/printk.c2
-rw-r--r--kernel/profile.c4
-rw-r--r--kernel/rcuclassic.c6
-rw-r--r--kernel/rcupreempt.c10
-rw-r--r--kernel/rcupreempt_trace.c10
-rw-r--r--kernel/rcutorture.c66
-rw-r--r--kernel/rcutree.c1535
-rw-r--r--kernel/rcutree_trace.c271
-rw-r--r--kernel/resource.c9
-rw-r--r--kernel/sched.c975
-rw-r--r--kernel/sched_cpupri.c39
-rw-r--r--kernel/sched_cpupri.h5
-rw-r--r--kernel/sched_fair.c32
-rw-r--r--kernel/sched_rt.c73
-rw-r--r--kernel/sched_stats.h3
-rw-r--r--kernel/softirq.c39
-rw-r--r--kernel/softlockup.c2
-rw-r--r--kernel/stacktrace.c11
-rw-r--r--kernel/sys.c2
-rw-r--r--kernel/sysctl.c14
-rw-r--r--kernel/sysctl_check.c1
-rw-r--r--kernel/taskstats.c2
-rw-r--r--kernel/time/clockevents.c2
-rw-r--r--kernel/time/ntp.c4
-rw-r--r--kernel/time/tick-broadcast.c2
-rw-r--r--kernel/time/tick-common.c12
-rw-r--r--kernel/time/tick-sched.c54
-rw-r--r--kernel/trace/ring_buffer.c36
-rw-r--r--kernel/trace/trace.c9
-rw-r--r--kernel/trace/trace_sysprof.c1
54 files changed, 3408 insertions, 1210 deletions
diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt
index 9fdba03dc1fc..bf987b95b356 100644
--- a/kernel/Kconfig.preempt
+++ b/kernel/Kconfig.preempt
@@ -52,28 +52,3 @@ config PREEMPT
52 52
53endchoice 53endchoice
54 54
55config PREEMPT_RCU
56 bool "Preemptible RCU"
57 depends on PREEMPT
58 default n
59 help
60 This option reduces the latency of the kernel by making certain
61 RCU sections preemptible. Normally RCU code is non-preemptible, if
62 this option is selected then read-only RCU sections become
63 preemptible. This helps latency, but may expose bugs due to
64 now-naive assumptions about each RCU read-side critical section
65 remaining on a given CPU through its execution.
66
67 Say N if you are unsure.
68
69config RCU_TRACE
70 bool "Enable tracing for RCU - currently stats in debugfs"
71 depends on PREEMPT_RCU
72 select DEBUG_FS
73 default y
74 help
75 This option provides tracing in RCU which presents stats
76 in debugfs for debugging RCU implementation.
77
78 Say Y here if you want to enable RCU tracing
79 Say N if you are unsure.
diff --git a/kernel/Makefile b/kernel/Makefile
index 027edda63511..e1c5bf3365c0 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -73,10 +73,10 @@ obj-$(CONFIG_GENERIC_HARDIRQS) += irq/
73obj-$(CONFIG_SECCOMP) += seccomp.o 73obj-$(CONFIG_SECCOMP) += seccomp.o
74obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o 74obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
75obj-$(CONFIG_CLASSIC_RCU) += rcuclassic.o 75obj-$(CONFIG_CLASSIC_RCU) += rcuclassic.o
76obj-$(CONFIG_TREE_RCU) += rcutree.o
76obj-$(CONFIG_PREEMPT_RCU) += rcupreempt.o 77obj-$(CONFIG_PREEMPT_RCU) += rcupreempt.o
77ifeq ($(CONFIG_PREEMPT_RCU),y) 78obj-$(CONFIG_TREE_RCU_TRACE) += rcutree_trace.o
78obj-$(CONFIG_RCU_TRACE) += rcupreempt_trace.o 79obj-$(CONFIG_PREEMPT_RCU_TRACE) += rcupreempt_trace.o
79endif
80obj-$(CONFIG_RELAY) += relay.o 80obj-$(CONFIG_RELAY) += relay.o
81obj-$(CONFIG_SYSCTL) += utsname_sysctl.o 81obj-$(CONFIG_SYSCTL) += utsname_sysctl.o
82obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o 82obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 8ea32e8d68b0..bae131a1211b 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -24,19 +24,20 @@
24cpumask_t cpu_present_map __read_mostly; 24cpumask_t cpu_present_map __read_mostly;
25EXPORT_SYMBOL(cpu_present_map); 25EXPORT_SYMBOL(cpu_present_map);
26 26
27#ifndef CONFIG_SMP
28
29/* 27/*
30 * Represents all cpu's that are currently online. 28 * Represents all cpu's that are currently online.
31 */ 29 */
32cpumask_t cpu_online_map __read_mostly = CPU_MASK_ALL; 30cpumask_t cpu_online_map __read_mostly;
33EXPORT_SYMBOL(cpu_online_map); 31EXPORT_SYMBOL(cpu_online_map);
34 32
33#ifdef CONFIG_INIT_ALL_POSSIBLE
35cpumask_t cpu_possible_map __read_mostly = CPU_MASK_ALL; 34cpumask_t cpu_possible_map __read_mostly = CPU_MASK_ALL;
35#else
36cpumask_t cpu_possible_map __read_mostly;
37#endif
36EXPORT_SYMBOL(cpu_possible_map); 38EXPORT_SYMBOL(cpu_possible_map);
37 39
38#else /* CONFIG_SMP */ 40#ifdef CONFIG_SMP
39
40/* Serializes the updates to cpu_online_map, cpu_present_map */ 41/* Serializes the updates to cpu_online_map, cpu_present_map */
41static DEFINE_MUTEX(cpu_add_remove_lock); 42static DEFINE_MUTEX(cpu_add_remove_lock);
42 43
diff --git a/kernel/cpuset.c b/kernel/cpuset.c
index 96c0ba13b8cd..39c1a4c1c5a9 100644
--- a/kernel/cpuset.c
+++ b/kernel/cpuset.c
@@ -896,7 +896,7 @@ static int update_cpumask(struct cpuset *cs, const char *buf)
896 if (!*buf) { 896 if (!*buf) {
897 cpus_clear(trialcs.cpus_allowed); 897 cpus_clear(trialcs.cpus_allowed);
898 } else { 898 } else {
899 retval = cpulist_parse(buf, trialcs.cpus_allowed); 899 retval = cpulist_parse(buf, &trialcs.cpus_allowed);
900 if (retval < 0) 900 if (retval < 0)
901 return retval; 901 return retval;
902 902
@@ -1482,7 +1482,7 @@ static int cpuset_sprintf_cpulist(char *page, struct cpuset *cs)
1482 mask = cs->cpus_allowed; 1482 mask = cs->cpus_allowed;
1483 mutex_unlock(&callback_mutex); 1483 mutex_unlock(&callback_mutex);
1484 1484
1485 return cpulist_scnprintf(page, PAGE_SIZE, mask); 1485 return cpulist_scnprintf(page, PAGE_SIZE, &mask);
1486} 1486}
1487 1487
1488static int cpuset_sprintf_memlist(char *page, struct cpuset *cs) 1488static int cpuset_sprintf_memlist(char *page, struct cpuset *cs)
diff --git a/kernel/exit.c b/kernel/exit.c
index c7422ca92038..c9e5a1c14e08 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -1037,8 +1037,6 @@ NORET_TYPE void do_exit(long code)
1037 * task into the wait for ever nirwana as well. 1037 * task into the wait for ever nirwana as well.
1038 */ 1038 */
1039 tsk->flags |= PF_EXITPIDONE; 1039 tsk->flags |= PF_EXITPIDONE;
1040 if (tsk->io_context)
1041 exit_io_context();
1042 set_current_state(TASK_UNINTERRUPTIBLE); 1040 set_current_state(TASK_UNINTERRUPTIBLE);
1043 schedule(); 1041 schedule();
1044 } 1042 }
@@ -1328,10 +1326,10 @@ static int wait_task_zombie(struct task_struct *p, int options,
1328 * group, which consolidates times for all threads in the 1326 * group, which consolidates times for all threads in the
1329 * group including the group leader. 1327 * group including the group leader.
1330 */ 1328 */
1329 thread_group_cputime(p, &cputime);
1331 spin_lock_irq(&p->parent->sighand->siglock); 1330 spin_lock_irq(&p->parent->sighand->siglock);
1332 psig = p->parent->signal; 1331 psig = p->parent->signal;
1333 sig = p->signal; 1332 sig = p->signal;
1334 thread_group_cputime(p, &cputime);
1335 psig->cutime = 1333 psig->cutime =
1336 cputime_add(psig->cutime, 1334 cputime_add(psig->cutime,
1337 cputime_add(cputime.utime, 1335 cputime_add(cputime.utime,
diff --git a/kernel/extable.c b/kernel/extable.c
index feb0317cf09a..e136ed8d82ba 100644
--- a/kernel/extable.c
+++ b/kernel/extable.c
@@ -67,3 +67,19 @@ int kernel_text_address(unsigned long addr)
67 return 1; 67 return 1;
68 return module_text_address(addr) != NULL; 68 return module_text_address(addr) != NULL;
69} 69}
70
71/*
72 * On some architectures (PPC64, IA64) function pointers
73 * are actually only tokens to some data that then holds the
74 * real function address. As a result, to find if a function
75 * pointer is part of the kernel text, we need to do some
76 * special dereferencing first.
77 */
78int func_ptr_is_kernel_text(void *ptr)
79{
80 unsigned long addr;
81 addr = (unsigned long) dereference_function_descriptor(ptr);
82 if (core_kernel_text(addr))
83 return 1;
84 return module_text_address(addr) != NULL;
85}
diff --git a/kernel/fork.c b/kernel/fork.c
index 6144b36cd897..43cbf30669e6 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -415,8 +415,8 @@ static struct mm_struct * mm_init(struct mm_struct * mm, struct task_struct *p)
415 set_mm_counter(mm, file_rss, 0); 415 set_mm_counter(mm, file_rss, 0);
416 set_mm_counter(mm, anon_rss, 0); 416 set_mm_counter(mm, anon_rss, 0);
417 spin_lock_init(&mm->page_table_lock); 417 spin_lock_init(&mm->page_table_lock);
418 rwlock_init(&mm->ioctx_list_lock); 418 spin_lock_init(&mm->ioctx_lock);
419 mm->ioctx_list = NULL; 419 INIT_HLIST_HEAD(&mm->ioctx_list);
420 mm->free_area_cache = TASK_UNMAPPED_BASE; 420 mm->free_area_cache = TASK_UNMAPPED_BASE;
421 mm->cached_hole_size = ~0UL; 421 mm->cached_hole_size = ~0UL;
422 mm_init_owner(mm, p); 422 mm_init_owner(mm, p);
diff --git a/kernel/futex.c b/kernel/futex.c
index 4fe790e89d0f..7c6cbabe52b3 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -92,11 +92,12 @@ struct futex_pi_state {
92 * A futex_q has a woken state, just like tasks have TASK_RUNNING. 92 * A futex_q has a woken state, just like tasks have TASK_RUNNING.
93 * It is considered woken when plist_node_empty(&q->list) || q->lock_ptr == 0. 93 * It is considered woken when plist_node_empty(&q->list) || q->lock_ptr == 0.
94 * The order of wakup is always to make the first condition true, then 94 * The order of wakup is always to make the first condition true, then
95 * wake up q->waiters, then make the second condition true. 95 * wake up q->waiter, then make the second condition true.
96 */ 96 */
97struct futex_q { 97struct futex_q {
98 struct plist_node list; 98 struct plist_node list;
99 wait_queue_head_t waiters; 99 /* There can only be a single waiter */
100 wait_queue_head_t waiter;
100 101
101 /* Which hash list lock to use: */ 102 /* Which hash list lock to use: */
102 spinlock_t *lock_ptr; 103 spinlock_t *lock_ptr;
@@ -123,24 +124,6 @@ struct futex_hash_bucket {
123static struct futex_hash_bucket futex_queues[1<<FUTEX_HASHBITS]; 124static struct futex_hash_bucket futex_queues[1<<FUTEX_HASHBITS];
124 125
125/* 126/*
126 * Take mm->mmap_sem, when futex is shared
127 */
128static inline void futex_lock_mm(struct rw_semaphore *fshared)
129{
130 if (fshared)
131 down_read(fshared);
132}
133
134/*
135 * Release mm->mmap_sem, when the futex is shared
136 */
137static inline void futex_unlock_mm(struct rw_semaphore *fshared)
138{
139 if (fshared)
140 up_read(fshared);
141}
142
143/*
144 * We hash on the keys returned from get_futex_key (see below). 127 * We hash on the keys returned from get_futex_key (see below).
145 */ 128 */
146static struct futex_hash_bucket *hash_futex(union futex_key *key) 129static struct futex_hash_bucket *hash_futex(union futex_key *key)
@@ -161,6 +144,45 @@ static inline int match_futex(union futex_key *key1, union futex_key *key2)
161 && key1->both.offset == key2->both.offset); 144 && key1->both.offset == key2->both.offset);
162} 145}
163 146
147/*
148 * Take a reference to the resource addressed by a key.
149 * Can be called while holding spinlocks.
150 *
151 */
152static void get_futex_key_refs(union futex_key *key)
153{
154 if (!key->both.ptr)
155 return;
156
157 switch (key->both.offset & (FUT_OFF_INODE|FUT_OFF_MMSHARED)) {
158 case FUT_OFF_INODE:
159 atomic_inc(&key->shared.inode->i_count);
160 break;
161 case FUT_OFF_MMSHARED:
162 atomic_inc(&key->private.mm->mm_count);
163 break;
164 }
165}
166
167/*
168 * Drop a reference to the resource addressed by a key.
169 * The hash bucket spinlock must not be held.
170 */
171static void drop_futex_key_refs(union futex_key *key)
172{
173 if (!key->both.ptr)
174 return;
175
176 switch (key->both.offset & (FUT_OFF_INODE|FUT_OFF_MMSHARED)) {
177 case FUT_OFF_INODE:
178 iput(key->shared.inode);
179 break;
180 case FUT_OFF_MMSHARED:
181 mmdrop(key->private.mm);
182 break;
183 }
184}
185
164/** 186/**
165 * get_futex_key - Get parameters which are the keys for a futex. 187 * get_futex_key - Get parameters which are the keys for a futex.
166 * @uaddr: virtual address of the futex 188 * @uaddr: virtual address of the futex
@@ -179,12 +201,10 @@ static inline int match_futex(union futex_key *key1, union futex_key *key2)
179 * For other futexes, it points to &current->mm->mmap_sem and 201 * For other futexes, it points to &current->mm->mmap_sem and
180 * caller must have taken the reader lock. but NOT any spinlocks. 202 * caller must have taken the reader lock. but NOT any spinlocks.
181 */ 203 */
182static int get_futex_key(u32 __user *uaddr, struct rw_semaphore *fshared, 204static int get_futex_key(u32 __user *uaddr, int fshared, union futex_key *key)
183 union futex_key *key)
184{ 205{
185 unsigned long address = (unsigned long)uaddr; 206 unsigned long address = (unsigned long)uaddr;
186 struct mm_struct *mm = current->mm; 207 struct mm_struct *mm = current->mm;
187 struct vm_area_struct *vma;
188 struct page *page; 208 struct page *page;
189 int err; 209 int err;
190 210
@@ -208,100 +228,50 @@ static int get_futex_key(u32 __user *uaddr, struct rw_semaphore *fshared,
208 return -EFAULT; 228 return -EFAULT;
209 key->private.mm = mm; 229 key->private.mm = mm;
210 key->private.address = address; 230 key->private.address = address;
231 get_futex_key_refs(key);
211 return 0; 232 return 0;
212 } 233 }
213 /*
214 * The futex is hashed differently depending on whether
215 * it's in a shared or private mapping. So check vma first.
216 */
217 vma = find_extend_vma(mm, address);
218 if (unlikely(!vma))
219 return -EFAULT;
220 234
221 /* 235again:
222 * Permissions. 236 err = get_user_pages_fast(address, 1, 0, &page);
223 */ 237 if (err < 0)
224 if (unlikely((vma->vm_flags & (VM_IO|VM_READ)) != VM_READ)) 238 return err;
225 return (vma->vm_flags & VM_IO) ? -EPERM : -EACCES; 239
240 lock_page(page);
241 if (!page->mapping) {
242 unlock_page(page);
243 put_page(page);
244 goto again;
245 }
226 246
227 /* 247 /*
228 * Private mappings are handled in a simple way. 248 * Private mappings are handled in a simple way.
229 * 249 *
230 * NOTE: When userspace waits on a MAP_SHARED mapping, even if 250 * NOTE: When userspace waits on a MAP_SHARED mapping, even if
231 * it's a read-only handle, it's expected that futexes attach to 251 * it's a read-only handle, it's expected that futexes attach to
232 * the object not the particular process. Therefore we use 252 * the object not the particular process.
233 * VM_MAYSHARE here, not VM_SHARED which is restricted to shared
234 * mappings of _writable_ handles.
235 */ 253 */
236 if (likely(!(vma->vm_flags & VM_MAYSHARE))) { 254 if (PageAnon(page)) {
237 key->both.offset |= FUT_OFF_MMSHARED; /* reference taken on mm */ 255 key->both.offset |= FUT_OFF_MMSHARED; /* ref taken on mm */
238 key->private.mm = mm; 256 key->private.mm = mm;
239 key->private.address = address; 257 key->private.address = address;
240 return 0; 258 } else {
259 key->both.offset |= FUT_OFF_INODE; /* inode-based key */
260 key->shared.inode = page->mapping->host;
261 key->shared.pgoff = page->index;
241 } 262 }
242 263
243 /* 264 get_futex_key_refs(key);
244 * Linear file mappings are also simple.
245 */
246 key->shared.inode = vma->vm_file->f_path.dentry->d_inode;
247 key->both.offset |= FUT_OFF_INODE; /* inode-based key. */
248 if (likely(!(vma->vm_flags & VM_NONLINEAR))) {
249 key->shared.pgoff = (((address - vma->vm_start) >> PAGE_SHIFT)
250 + vma->vm_pgoff);
251 return 0;
252 }
253 265
254 /* 266 unlock_page(page);
255 * We could walk the page table to read the non-linear 267 put_page(page);
256 * pte, and get the page index without fetching the page 268 return 0;
257 * from swap. But that's a lot of code to duplicate here
258 * for a rare case, so we simply fetch the page.
259 */
260 err = get_user_pages(current, mm, address, 1, 0, 0, &page, NULL);
261 if (err >= 0) {
262 key->shared.pgoff =
263 page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
264 put_page(page);
265 return 0;
266 }
267 return err;
268}
269
270/*
271 * Take a reference to the resource addressed by a key.
272 * Can be called while holding spinlocks.
273 *
274 */
275static void get_futex_key_refs(union futex_key *key)
276{
277 if (key->both.ptr == NULL)
278 return;
279 switch (key->both.offset & (FUT_OFF_INODE|FUT_OFF_MMSHARED)) {
280 case FUT_OFF_INODE:
281 atomic_inc(&key->shared.inode->i_count);
282 break;
283 case FUT_OFF_MMSHARED:
284 atomic_inc(&key->private.mm->mm_count);
285 break;
286 }
287} 269}
288 270
289/* 271static inline
290 * Drop a reference to the resource addressed by a key. 272void put_futex_key(int fshared, union futex_key *key)
291 * The hash bucket spinlock must not be held.
292 */
293static void drop_futex_key_refs(union futex_key *key)
294{ 273{
295 if (!key->both.ptr) 274 drop_futex_key_refs(key);
296 return;
297 switch (key->both.offset & (FUT_OFF_INODE|FUT_OFF_MMSHARED)) {
298 case FUT_OFF_INODE:
299 iput(key->shared.inode);
300 break;
301 case FUT_OFF_MMSHARED:
302 mmdrop(key->private.mm);
303 break;
304 }
305} 275}
306 276
307static u32 cmpxchg_futex_value_locked(u32 __user *uaddr, u32 uval, u32 newval) 277static u32 cmpxchg_futex_value_locked(u32 __user *uaddr, u32 uval, u32 newval)
@@ -328,10 +298,8 @@ static int get_futex_value_locked(u32 *dest, u32 __user *from)
328 298
329/* 299/*
330 * Fault handling. 300 * Fault handling.
331 * if fshared is non NULL, current->mm->mmap_sem is already held
332 */ 301 */
333static int futex_handle_fault(unsigned long address, 302static int futex_handle_fault(unsigned long address, int attempt)
334 struct rw_semaphore *fshared, int attempt)
335{ 303{
336 struct vm_area_struct * vma; 304 struct vm_area_struct * vma;
337 struct mm_struct *mm = current->mm; 305 struct mm_struct *mm = current->mm;
@@ -340,8 +308,7 @@ static int futex_handle_fault(unsigned long address,
340 if (attempt > 2) 308 if (attempt > 2)
341 return ret; 309 return ret;
342 310
343 if (!fshared) 311 down_read(&mm->mmap_sem);
344 down_read(&mm->mmap_sem);
345 vma = find_vma(mm, address); 312 vma = find_vma(mm, address);
346 if (vma && address >= vma->vm_start && 313 if (vma && address >= vma->vm_start &&
347 (vma->vm_flags & VM_WRITE)) { 314 (vma->vm_flags & VM_WRITE)) {
@@ -361,8 +328,7 @@ static int futex_handle_fault(unsigned long address,
361 current->min_flt++; 328 current->min_flt++;
362 } 329 }
363 } 330 }
364 if (!fshared) 331 up_read(&mm->mmap_sem);
365 up_read(&mm->mmap_sem);
366 return ret; 332 return ret;
367} 333}
368 334
@@ -385,6 +351,7 @@ static int refill_pi_state_cache(void)
385 /* pi_mutex gets initialized later */ 351 /* pi_mutex gets initialized later */
386 pi_state->owner = NULL; 352 pi_state->owner = NULL;
387 atomic_set(&pi_state->refcount, 1); 353 atomic_set(&pi_state->refcount, 1);
354 pi_state->key = FUTEX_KEY_INIT;
388 355
389 current->pi_state_cache = pi_state; 356 current->pi_state_cache = pi_state;
390 357
@@ -469,7 +436,7 @@ void exit_pi_state_list(struct task_struct *curr)
469 struct list_head *next, *head = &curr->pi_state_list; 436 struct list_head *next, *head = &curr->pi_state_list;
470 struct futex_pi_state *pi_state; 437 struct futex_pi_state *pi_state;
471 struct futex_hash_bucket *hb; 438 struct futex_hash_bucket *hb;
472 union futex_key key; 439 union futex_key key = FUTEX_KEY_INIT;
473 440
474 if (!futex_cmpxchg_enabled) 441 if (!futex_cmpxchg_enabled)
475 return; 442 return;
@@ -614,7 +581,7 @@ static void wake_futex(struct futex_q *q)
614 * The lock in wake_up_all() is a crucial memory barrier after the 581 * The lock in wake_up_all() is a crucial memory barrier after the
615 * plist_del() and also before assigning to q->lock_ptr. 582 * plist_del() and also before assigning to q->lock_ptr.
616 */ 583 */
617 wake_up_all(&q->waiters); 584 wake_up(&q->waiter);
618 /* 585 /*
619 * The waiting task can free the futex_q as soon as this is written, 586 * The waiting task can free the futex_q as soon as this is written,
620 * without taking any locks. This must come last. 587 * without taking any locks. This must come last.
@@ -726,20 +693,17 @@ double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
726 * Wake up all waiters hashed on the physical page that is mapped 693 * Wake up all waiters hashed on the physical page that is mapped
727 * to this virtual address: 694 * to this virtual address:
728 */ 695 */
729static int futex_wake(u32 __user *uaddr, struct rw_semaphore *fshared, 696static int futex_wake(u32 __user *uaddr, int fshared, int nr_wake, u32 bitset)
730 int nr_wake, u32 bitset)
731{ 697{
732 struct futex_hash_bucket *hb; 698 struct futex_hash_bucket *hb;
733 struct futex_q *this, *next; 699 struct futex_q *this, *next;
734 struct plist_head *head; 700 struct plist_head *head;
735 union futex_key key; 701 union futex_key key = FUTEX_KEY_INIT;
736 int ret; 702 int ret;
737 703
738 if (!bitset) 704 if (!bitset)
739 return -EINVAL; 705 return -EINVAL;
740 706
741 futex_lock_mm(fshared);
742
743 ret = get_futex_key(uaddr, fshared, &key); 707 ret = get_futex_key(uaddr, fshared, &key);
744 if (unlikely(ret != 0)) 708 if (unlikely(ret != 0))
745 goto out; 709 goto out;
@@ -767,7 +731,7 @@ static int futex_wake(u32 __user *uaddr, struct rw_semaphore *fshared,
767 731
768 spin_unlock(&hb->lock); 732 spin_unlock(&hb->lock);
769out: 733out:
770 futex_unlock_mm(fshared); 734 put_futex_key(fshared, &key);
771 return ret; 735 return ret;
772} 736}
773 737
@@ -776,19 +740,16 @@ out:
776 * to this virtual address: 740 * to this virtual address:
777 */ 741 */
778static int 742static int
779futex_wake_op(u32 __user *uaddr1, struct rw_semaphore *fshared, 743futex_wake_op(u32 __user *uaddr1, int fshared, u32 __user *uaddr2,
780 u32 __user *uaddr2,
781 int nr_wake, int nr_wake2, int op) 744 int nr_wake, int nr_wake2, int op)
782{ 745{
783 union futex_key key1, key2; 746 union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
784 struct futex_hash_bucket *hb1, *hb2; 747 struct futex_hash_bucket *hb1, *hb2;
785 struct plist_head *head; 748 struct plist_head *head;
786 struct futex_q *this, *next; 749 struct futex_q *this, *next;
787 int ret, op_ret, attempt = 0; 750 int ret, op_ret, attempt = 0;
788 751
789retryfull: 752retryfull:
790 futex_lock_mm(fshared);
791
792 ret = get_futex_key(uaddr1, fshared, &key1); 753 ret = get_futex_key(uaddr1, fshared, &key1);
793 if (unlikely(ret != 0)) 754 if (unlikely(ret != 0))
794 goto out; 755 goto out;
@@ -833,18 +794,12 @@ retry:
833 */ 794 */
834 if (attempt++) { 795 if (attempt++) {
835 ret = futex_handle_fault((unsigned long)uaddr2, 796 ret = futex_handle_fault((unsigned long)uaddr2,
836 fshared, attempt); 797 attempt);
837 if (ret) 798 if (ret)
838 goto out; 799 goto out;
839 goto retry; 800 goto retry;
840 } 801 }
841 802
842 /*
843 * If we would have faulted, release mmap_sem,
844 * fault it in and start all over again.
845 */
846 futex_unlock_mm(fshared);
847
848 ret = get_user(dummy, uaddr2); 803 ret = get_user(dummy, uaddr2);
849 if (ret) 804 if (ret)
850 return ret; 805 return ret;
@@ -880,7 +835,8 @@ retry:
880 if (hb1 != hb2) 835 if (hb1 != hb2)
881 spin_unlock(&hb2->lock); 836 spin_unlock(&hb2->lock);
882out: 837out:
883 futex_unlock_mm(fshared); 838 put_futex_key(fshared, &key2);
839 put_futex_key(fshared, &key1);
884 840
885 return ret; 841 return ret;
886} 842}
@@ -889,19 +845,16 @@ out:
889 * Requeue all waiters hashed on one physical page to another 845 * Requeue all waiters hashed on one physical page to another
890 * physical page. 846 * physical page.
891 */ 847 */
892static int futex_requeue(u32 __user *uaddr1, struct rw_semaphore *fshared, 848static int futex_requeue(u32 __user *uaddr1, int fshared, u32 __user *uaddr2,
893 u32 __user *uaddr2,
894 int nr_wake, int nr_requeue, u32 *cmpval) 849 int nr_wake, int nr_requeue, u32 *cmpval)
895{ 850{
896 union futex_key key1, key2; 851 union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
897 struct futex_hash_bucket *hb1, *hb2; 852 struct futex_hash_bucket *hb1, *hb2;
898 struct plist_head *head1; 853 struct plist_head *head1;
899 struct futex_q *this, *next; 854 struct futex_q *this, *next;
900 int ret, drop_count = 0; 855 int ret, drop_count = 0;
901 856
902 retry: 857 retry:
903 futex_lock_mm(fshared);
904
905 ret = get_futex_key(uaddr1, fshared, &key1); 858 ret = get_futex_key(uaddr1, fshared, &key1);
906 if (unlikely(ret != 0)) 859 if (unlikely(ret != 0))
907 goto out; 860 goto out;
@@ -924,12 +877,6 @@ static int futex_requeue(u32 __user *uaddr1, struct rw_semaphore *fshared,
924 if (hb1 != hb2) 877 if (hb1 != hb2)
925 spin_unlock(&hb2->lock); 878 spin_unlock(&hb2->lock);
926 879
927 /*
928 * If we would have faulted, release mmap_sem, fault
929 * it in and start all over again.
930 */
931 futex_unlock_mm(fshared);
932
933 ret = get_user(curval, uaddr1); 880 ret = get_user(curval, uaddr1);
934 881
935 if (!ret) 882 if (!ret)
@@ -981,7 +928,8 @@ out_unlock:
981 drop_futex_key_refs(&key1); 928 drop_futex_key_refs(&key1);
982 929
983out: 930out:
984 futex_unlock_mm(fshared); 931 put_futex_key(fshared, &key2);
932 put_futex_key(fshared, &key1);
985 return ret; 933 return ret;
986} 934}
987 935
@@ -990,7 +938,7 @@ static inline struct futex_hash_bucket *queue_lock(struct futex_q *q)
990{ 938{
991 struct futex_hash_bucket *hb; 939 struct futex_hash_bucket *hb;
992 940
993 init_waitqueue_head(&q->waiters); 941 init_waitqueue_head(&q->waiter);
994 942
995 get_futex_key_refs(&q->key); 943 get_futex_key_refs(&q->key);
996 hb = hash_futex(&q->key); 944 hb = hash_futex(&q->key);
@@ -1103,8 +1051,7 @@ static void unqueue_me_pi(struct futex_q *q)
1103 * private futexes. 1051 * private futexes.
1104 */ 1052 */
1105static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q, 1053static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q,
1106 struct task_struct *newowner, 1054 struct task_struct *newowner, int fshared)
1107 struct rw_semaphore *fshared)
1108{ 1055{
1109 u32 newtid = task_pid_vnr(newowner) | FUTEX_WAITERS; 1056 u32 newtid = task_pid_vnr(newowner) | FUTEX_WAITERS;
1110 struct futex_pi_state *pi_state = q->pi_state; 1057 struct futex_pi_state *pi_state = q->pi_state;
@@ -1183,7 +1130,7 @@ retry:
1183handle_fault: 1130handle_fault:
1184 spin_unlock(q->lock_ptr); 1131 spin_unlock(q->lock_ptr);
1185 1132
1186 ret = futex_handle_fault((unsigned long)uaddr, fshared, attempt++); 1133 ret = futex_handle_fault((unsigned long)uaddr, attempt++);
1187 1134
1188 spin_lock(q->lock_ptr); 1135 spin_lock(q->lock_ptr);
1189 1136
@@ -1203,12 +1150,13 @@ handle_fault:
1203 * In case we must use restart_block to restart a futex_wait, 1150 * In case we must use restart_block to restart a futex_wait,
1204 * we encode in the 'flags' shared capability 1151 * we encode in the 'flags' shared capability
1205 */ 1152 */
1206#define FLAGS_SHARED 1 1153#define FLAGS_SHARED 0x01
1154#define FLAGS_CLOCKRT 0x02
1207 1155
1208static long futex_wait_restart(struct restart_block *restart); 1156static long futex_wait_restart(struct restart_block *restart);
1209 1157
1210static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared, 1158static int futex_wait(u32 __user *uaddr, int fshared,
1211 u32 val, ktime_t *abs_time, u32 bitset) 1159 u32 val, ktime_t *abs_time, u32 bitset, int clockrt)
1212{ 1160{
1213 struct task_struct *curr = current; 1161 struct task_struct *curr = current;
1214 DECLARE_WAITQUEUE(wait, curr); 1162 DECLARE_WAITQUEUE(wait, curr);
@@ -1225,8 +1173,7 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared,
1225 q.pi_state = NULL; 1173 q.pi_state = NULL;
1226 q.bitset = bitset; 1174 q.bitset = bitset;
1227 retry: 1175 retry:
1228 futex_lock_mm(fshared); 1176 q.key = FUTEX_KEY_INIT;
1229
1230 ret = get_futex_key(uaddr, fshared, &q.key); 1177 ret = get_futex_key(uaddr, fshared, &q.key);
1231 if (unlikely(ret != 0)) 1178 if (unlikely(ret != 0))
1232 goto out_release_sem; 1179 goto out_release_sem;
@@ -1258,12 +1205,6 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared,
1258 if (unlikely(ret)) { 1205 if (unlikely(ret)) {
1259 queue_unlock(&q, hb); 1206 queue_unlock(&q, hb);
1260 1207
1261 /*
1262 * If we would have faulted, release mmap_sem, fault it in and
1263 * start all over again.
1264 */
1265 futex_unlock_mm(fshared);
1266
1267 ret = get_user(uval, uaddr); 1208 ret = get_user(uval, uaddr);
1268 1209
1269 if (!ret) 1210 if (!ret)
@@ -1278,12 +1219,6 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared,
1278 queue_me(&q, hb); 1219 queue_me(&q, hb);
1279 1220
1280 /* 1221 /*
1281 * Now the futex is queued and we have checked the data, we
1282 * don't want to hold mmap_sem while we sleep.
1283 */
1284 futex_unlock_mm(fshared);
1285
1286 /*
1287 * There might have been scheduling since the queue_me(), as we 1222 * There might have been scheduling since the queue_me(), as we
1288 * cannot hold a spinlock across the get_user() in case it 1223 * cannot hold a spinlock across the get_user() in case it
1289 * faults, and we cannot just set TASK_INTERRUPTIBLE state when 1224 * faults, and we cannot just set TASK_INTERRUPTIBLE state when
@@ -1294,7 +1229,7 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared,
1294 1229
1295 /* add_wait_queue is the barrier after __set_current_state. */ 1230 /* add_wait_queue is the barrier after __set_current_state. */
1296 __set_current_state(TASK_INTERRUPTIBLE); 1231 __set_current_state(TASK_INTERRUPTIBLE);
1297 add_wait_queue(&q.waiters, &wait); 1232 add_wait_queue(&q.waiter, &wait);
1298 /* 1233 /*
1299 * !plist_node_empty() is safe here without any lock. 1234 * !plist_node_empty() is safe here without any lock.
1300 * q.lock_ptr != 0 is not safe, because of ordering against wakeup. 1235 * q.lock_ptr != 0 is not safe, because of ordering against wakeup.
@@ -1307,8 +1242,10 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared,
1307 slack = current->timer_slack_ns; 1242 slack = current->timer_slack_ns;
1308 if (rt_task(current)) 1243 if (rt_task(current))
1309 slack = 0; 1244 slack = 0;
1310 hrtimer_init_on_stack(&t.timer, CLOCK_MONOTONIC, 1245 hrtimer_init_on_stack(&t.timer,
1311 HRTIMER_MODE_ABS); 1246 clockrt ? CLOCK_REALTIME :
1247 CLOCK_MONOTONIC,
1248 HRTIMER_MODE_ABS);
1312 hrtimer_init_sleeper(&t, current); 1249 hrtimer_init_sleeper(&t, current);
1313 hrtimer_set_expires_range_ns(&t.timer, *abs_time, slack); 1250 hrtimer_set_expires_range_ns(&t.timer, *abs_time, slack);
1314 1251
@@ -1363,6 +1300,8 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared,
1363 1300
1364 if (fshared) 1301 if (fshared)
1365 restart->futex.flags |= FLAGS_SHARED; 1302 restart->futex.flags |= FLAGS_SHARED;
1303 if (clockrt)
1304 restart->futex.flags |= FLAGS_CLOCKRT;
1366 return -ERESTART_RESTARTBLOCK; 1305 return -ERESTART_RESTARTBLOCK;
1367 } 1306 }
1368 1307
@@ -1370,7 +1309,7 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared,
1370 queue_unlock(&q, hb); 1309 queue_unlock(&q, hb);
1371 1310
1372 out_release_sem: 1311 out_release_sem:
1373 futex_unlock_mm(fshared); 1312 put_futex_key(fshared, &q.key);
1374 return ret; 1313 return ret;
1375} 1314}
1376 1315
@@ -1378,15 +1317,16 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared,
1378static long futex_wait_restart(struct restart_block *restart) 1317static long futex_wait_restart(struct restart_block *restart)
1379{ 1318{
1380 u32 __user *uaddr = (u32 __user *)restart->futex.uaddr; 1319 u32 __user *uaddr = (u32 __user *)restart->futex.uaddr;
1381 struct rw_semaphore *fshared = NULL; 1320 int fshared = 0;
1382 ktime_t t; 1321 ktime_t t;
1383 1322
1384 t.tv64 = restart->futex.time; 1323 t.tv64 = restart->futex.time;
1385 restart->fn = do_no_restart_syscall; 1324 restart->fn = do_no_restart_syscall;
1386 if (restart->futex.flags & FLAGS_SHARED) 1325 if (restart->futex.flags & FLAGS_SHARED)
1387 fshared = &current->mm->mmap_sem; 1326 fshared = 1;
1388 return (long)futex_wait(uaddr, fshared, restart->futex.val, &t, 1327 return (long)futex_wait(uaddr, fshared, restart->futex.val, &t,
1389 restart->futex.bitset); 1328 restart->futex.bitset,
1329 restart->futex.flags & FLAGS_CLOCKRT);
1390} 1330}
1391 1331
1392 1332
@@ -1396,7 +1336,7 @@ static long futex_wait_restart(struct restart_block *restart)
1396 * if there are waiters then it will block, it does PI, etc. (Due to 1336 * if there are waiters then it will block, it does PI, etc. (Due to
1397 * races the kernel might see a 0 value of the futex too.) 1337 * races the kernel might see a 0 value of the futex too.)
1398 */ 1338 */
1399static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared, 1339static int futex_lock_pi(u32 __user *uaddr, int fshared,
1400 int detect, ktime_t *time, int trylock) 1340 int detect, ktime_t *time, int trylock)
1401{ 1341{
1402 struct hrtimer_sleeper timeout, *to = NULL; 1342 struct hrtimer_sleeper timeout, *to = NULL;
@@ -1419,8 +1359,7 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared,
1419 1359
1420 q.pi_state = NULL; 1360 q.pi_state = NULL;
1421 retry: 1361 retry:
1422 futex_lock_mm(fshared); 1362 q.key = FUTEX_KEY_INIT;
1423
1424 ret = get_futex_key(uaddr, fshared, &q.key); 1363 ret = get_futex_key(uaddr, fshared, &q.key);
1425 if (unlikely(ret != 0)) 1364 if (unlikely(ret != 0))
1426 goto out_release_sem; 1365 goto out_release_sem;
@@ -1509,7 +1448,6 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared,
1509 * exit to complete. 1448 * exit to complete.
1510 */ 1449 */
1511 queue_unlock(&q, hb); 1450 queue_unlock(&q, hb);
1512 futex_unlock_mm(fshared);
1513 cond_resched(); 1451 cond_resched();
1514 goto retry; 1452 goto retry;
1515 1453
@@ -1541,12 +1479,6 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared,
1541 */ 1479 */
1542 queue_me(&q, hb); 1480 queue_me(&q, hb);
1543 1481
1544 /*
1545 * Now the futex is queued and we have checked the data, we
1546 * don't want to hold mmap_sem while we sleep.
1547 */
1548 futex_unlock_mm(fshared);
1549
1550 WARN_ON(!q.pi_state); 1482 WARN_ON(!q.pi_state);
1551 /* 1483 /*
1552 * Block on the PI mutex: 1484 * Block on the PI mutex:
@@ -1559,7 +1491,6 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared,
1559 ret = ret ? 0 : -EWOULDBLOCK; 1491 ret = ret ? 0 : -EWOULDBLOCK;
1560 } 1492 }
1561 1493
1562 futex_lock_mm(fshared);
1563 spin_lock(q.lock_ptr); 1494 spin_lock(q.lock_ptr);
1564 1495
1565 if (!ret) { 1496 if (!ret) {
@@ -1625,7 +1556,6 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared,
1625 1556
1626 /* Unqueue and drop the lock */ 1557 /* Unqueue and drop the lock */
1627 unqueue_me_pi(&q); 1558 unqueue_me_pi(&q);
1628 futex_unlock_mm(fshared);
1629 1559
1630 if (to) 1560 if (to)
1631 destroy_hrtimer_on_stack(&to->timer); 1561 destroy_hrtimer_on_stack(&to->timer);
@@ -1635,34 +1565,30 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared,
1635 queue_unlock(&q, hb); 1565 queue_unlock(&q, hb);
1636 1566
1637 out_release_sem: 1567 out_release_sem:
1638 futex_unlock_mm(fshared); 1568 put_futex_key(fshared, &q.key);
1639 if (to) 1569 if (to)
1640 destroy_hrtimer_on_stack(&to->timer); 1570 destroy_hrtimer_on_stack(&to->timer);
1641 return ret; 1571 return ret;
1642 1572
1643 uaddr_faulted: 1573 uaddr_faulted:
1644 /* 1574 /*
1645 * We have to r/w *(int __user *)uaddr, but we can't modify it 1575 * We have to r/w *(int __user *)uaddr, and we have to modify it
1646 * non-atomically. Therefore, if get_user below is not 1576 * atomically. Therefore, if we continue to fault after get_user()
1647 * enough, we need to handle the fault ourselves, while 1577 * below, we need to handle the fault ourselves, while still holding
1648 * still holding the mmap_sem. 1578 * the mmap_sem. This can occur if the uaddr is under contention as
1649 * 1579 * we have to drop the mmap_sem in order to call get_user().
1650 * ... and hb->lock. :-) --ANK
1651 */ 1580 */
1652 queue_unlock(&q, hb); 1581 queue_unlock(&q, hb);
1653 1582
1654 if (attempt++) { 1583 if (attempt++) {
1655 ret = futex_handle_fault((unsigned long)uaddr, fshared, 1584 ret = futex_handle_fault((unsigned long)uaddr, attempt);
1656 attempt);
1657 if (ret) 1585 if (ret)
1658 goto out_release_sem; 1586 goto out_release_sem;
1659 goto retry_unlocked; 1587 goto retry_unlocked;
1660 } 1588 }
1661 1589
1662 futex_unlock_mm(fshared);
1663
1664 ret = get_user(uval, uaddr); 1590 ret = get_user(uval, uaddr);
1665 if (!ret && (uval != -EFAULT)) 1591 if (!ret)
1666 goto retry; 1592 goto retry;
1667 1593
1668 if (to) 1594 if (to)
@@ -1675,13 +1601,13 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared,
1675 * This is the in-kernel slowpath: we look up the PI state (if any), 1601 * This is the in-kernel slowpath: we look up the PI state (if any),
1676 * and do the rt-mutex unlock. 1602 * and do the rt-mutex unlock.
1677 */ 1603 */
1678static int futex_unlock_pi(u32 __user *uaddr, struct rw_semaphore *fshared) 1604static int futex_unlock_pi(u32 __user *uaddr, int fshared)
1679{ 1605{
1680 struct futex_hash_bucket *hb; 1606 struct futex_hash_bucket *hb;
1681 struct futex_q *this, *next; 1607 struct futex_q *this, *next;
1682 u32 uval; 1608 u32 uval;
1683 struct plist_head *head; 1609 struct plist_head *head;
1684 union futex_key key; 1610 union futex_key key = FUTEX_KEY_INIT;
1685 int ret, attempt = 0; 1611 int ret, attempt = 0;
1686 1612
1687retry: 1613retry:
@@ -1692,10 +1618,6 @@ retry:
1692 */ 1618 */
1693 if ((uval & FUTEX_TID_MASK) != task_pid_vnr(current)) 1619 if ((uval & FUTEX_TID_MASK) != task_pid_vnr(current))
1694 return -EPERM; 1620 return -EPERM;
1695 /*
1696 * First take all the futex related locks:
1697 */
1698 futex_lock_mm(fshared);
1699 1621
1700 ret = get_futex_key(uaddr, fshared, &key); 1622 ret = get_futex_key(uaddr, fshared, &key);
1701 if (unlikely(ret != 0)) 1623 if (unlikely(ret != 0))
@@ -1754,34 +1676,30 @@ retry_unlocked:
1754out_unlock: 1676out_unlock:
1755 spin_unlock(&hb->lock); 1677 spin_unlock(&hb->lock);
1756out: 1678out:
1757 futex_unlock_mm(fshared); 1679 put_futex_key(fshared, &key);
1758 1680
1759 return ret; 1681 return ret;
1760 1682
1761pi_faulted: 1683pi_faulted:
1762 /* 1684 /*
1763 * We have to r/w *(int __user *)uaddr, but we can't modify it 1685 * We have to r/w *(int __user *)uaddr, and we have to modify it
1764 * non-atomically. Therefore, if get_user below is not 1686 * atomically. Therefore, if we continue to fault after get_user()
1765 * enough, we need to handle the fault ourselves, while 1687 * below, we need to handle the fault ourselves, while still holding
1766 * still holding the mmap_sem. 1688 * the mmap_sem. This can occur if the uaddr is under contention as
1767 * 1689 * we have to drop the mmap_sem in order to call get_user().
1768 * ... and hb->lock. --ANK
1769 */ 1690 */
1770 spin_unlock(&hb->lock); 1691 spin_unlock(&hb->lock);
1771 1692
1772 if (attempt++) { 1693 if (attempt++) {
1773 ret = futex_handle_fault((unsigned long)uaddr, fshared, 1694 ret = futex_handle_fault((unsigned long)uaddr, attempt);
1774 attempt);
1775 if (ret) 1695 if (ret)
1776 goto out; 1696 goto out;
1777 uval = 0; 1697 uval = 0;
1778 goto retry_unlocked; 1698 goto retry_unlocked;
1779 } 1699 }
1780 1700
1781 futex_unlock_mm(fshared);
1782
1783 ret = get_user(uval, uaddr); 1701 ret = get_user(uval, uaddr);
1784 if (!ret && (uval != -EFAULT)) 1702 if (!ret)
1785 goto retry; 1703 goto retry;
1786 1704
1787 return ret; 1705 return ret;
@@ -1908,8 +1826,7 @@ retry:
1908 * PI futexes happens in exit_pi_state(): 1826 * PI futexes happens in exit_pi_state():
1909 */ 1827 */
1910 if (!pi && (uval & FUTEX_WAITERS)) 1828 if (!pi && (uval & FUTEX_WAITERS))
1911 futex_wake(uaddr, &curr->mm->mmap_sem, 1, 1829 futex_wake(uaddr, 1, 1, FUTEX_BITSET_MATCH_ANY);
1912 FUTEX_BITSET_MATCH_ANY);
1913 } 1830 }
1914 return 0; 1831 return 0;
1915} 1832}
@@ -2003,18 +1920,22 @@ void exit_robust_list(struct task_struct *curr)
2003long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, 1920long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
2004 u32 __user *uaddr2, u32 val2, u32 val3) 1921 u32 __user *uaddr2, u32 val2, u32 val3)
2005{ 1922{
2006 int ret = -ENOSYS; 1923 int clockrt, ret = -ENOSYS;
2007 int cmd = op & FUTEX_CMD_MASK; 1924 int cmd = op & FUTEX_CMD_MASK;
2008 struct rw_semaphore *fshared = NULL; 1925 int fshared = 0;
2009 1926
2010 if (!(op & FUTEX_PRIVATE_FLAG)) 1927 if (!(op & FUTEX_PRIVATE_FLAG))
2011 fshared = &current->mm->mmap_sem; 1928 fshared = 1;
1929
1930 clockrt = op & FUTEX_CLOCK_REALTIME;
1931 if (clockrt && cmd != FUTEX_WAIT_BITSET)
1932 return -ENOSYS;
2012 1933
2013 switch (cmd) { 1934 switch (cmd) {
2014 case FUTEX_WAIT: 1935 case FUTEX_WAIT:
2015 val3 = FUTEX_BITSET_MATCH_ANY; 1936 val3 = FUTEX_BITSET_MATCH_ANY;
2016 case FUTEX_WAIT_BITSET: 1937 case FUTEX_WAIT_BITSET:
2017 ret = futex_wait(uaddr, fshared, val, timeout, val3); 1938 ret = futex_wait(uaddr, fshared, val, timeout, val3, clockrt);
2018 break; 1939 break;
2019 case FUTEX_WAKE: 1940 case FUTEX_WAKE:
2020 val3 = FUTEX_BITSET_MATCH_ANY; 1941 val3 = FUTEX_BITSET_MATCH_ANY;
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c
index 47e63349d1b2..eb2bfefa6dcc 100644
--- a/kernel/hrtimer.c
+++ b/kernel/hrtimer.c
@@ -32,7 +32,6 @@
32 */ 32 */
33 33
34#include <linux/cpu.h> 34#include <linux/cpu.h>
35#include <linux/irq.h>
36#include <linux/module.h> 35#include <linux/module.h>
37#include <linux/percpu.h> 36#include <linux/percpu.h>
38#include <linux/hrtimer.h> 37#include <linux/hrtimer.h>
@@ -442,22 +441,6 @@ static inline void debug_hrtimer_activate(struct hrtimer *timer) { }
442static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { } 441static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }
443#endif 442#endif
444 443
445/*
446 * Check, whether the timer is on the callback pending list
447 */
448static inline int hrtimer_cb_pending(const struct hrtimer *timer)
449{
450 return timer->state & HRTIMER_STATE_PENDING;
451}
452
453/*
454 * Remove a timer from the callback pending list
455 */
456static inline void hrtimer_remove_cb_pending(struct hrtimer *timer)
457{
458 list_del_init(&timer->cb_entry);
459}
460
461/* High resolution timer related functions */ 444/* High resolution timer related functions */
462#ifdef CONFIG_HIGH_RES_TIMERS 445#ifdef CONFIG_HIGH_RES_TIMERS
463 446
@@ -651,6 +634,8 @@ static inline void hrtimer_init_timer_hres(struct hrtimer *timer)
651{ 634{
652} 635}
653 636
637static void __run_hrtimer(struct hrtimer *timer);
638
654/* 639/*
655 * When High resolution timers are active, try to reprogram. Note, that in case 640 * When High resolution timers are active, try to reprogram. Note, that in case
656 * the state has HRTIMER_STATE_CALLBACK set, no reprogramming and no expiry 641 * the state has HRTIMER_STATE_CALLBACK set, no reprogramming and no expiry
@@ -661,31 +646,14 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
661 struct hrtimer_clock_base *base) 646 struct hrtimer_clock_base *base)
662{ 647{
663 if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) { 648 if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) {
664 649 /*
665 /* Timer is expired, act upon the callback mode */ 650 * XXX: recursion check?
666 switch(timer->cb_mode) { 651 * hrtimer_forward() should round up with timer granularity
667 case HRTIMER_CB_IRQSAFE_PERCPU: 652 * so that we never get into inf recursion here,
668 case HRTIMER_CB_IRQSAFE_UNLOCKED: 653 * it doesn't do that though
669 /* 654 */
670 * This is solely for the sched tick emulation with 655 __run_hrtimer(timer);
671 * dynamic tick support to ensure that we do not 656 return 1;
672 * restart the tick right on the edge and end up with
673 * the tick timer in the softirq ! The calling site
674 * takes care of this. Also used for hrtimer sleeper !
675 */
676 debug_hrtimer_deactivate(timer);
677 return 1;
678 case HRTIMER_CB_SOFTIRQ:
679 /*
680 * Move everything else into the softirq pending list !
681 */
682 list_add_tail(&timer->cb_entry,
683 &base->cpu_base->cb_pending);
684 timer->state = HRTIMER_STATE_PENDING;
685 return 1;
686 default:
687 BUG();
688 }
689 } 657 }
690 return 0; 658 return 0;
691} 659}
@@ -724,11 +692,6 @@ static int hrtimer_switch_to_hres(void)
724 return 1; 692 return 1;
725} 693}
726 694
727static inline void hrtimer_raise_softirq(void)
728{
729 raise_softirq(HRTIMER_SOFTIRQ);
730}
731
732#else 695#else
733 696
734static inline int hrtimer_hres_active(void) { return 0; } 697static inline int hrtimer_hres_active(void) { return 0; }
@@ -747,7 +710,6 @@ static inline int hrtimer_reprogram(struct hrtimer *timer,
747{ 710{
748 return 0; 711 return 0;
749} 712}
750static inline void hrtimer_raise_softirq(void) { }
751 713
752#endif /* CONFIG_HIGH_RES_TIMERS */ 714#endif /* CONFIG_HIGH_RES_TIMERS */
753 715
@@ -890,10 +852,7 @@ static void __remove_hrtimer(struct hrtimer *timer,
890 struct hrtimer_clock_base *base, 852 struct hrtimer_clock_base *base,
891 unsigned long newstate, int reprogram) 853 unsigned long newstate, int reprogram)
892{ 854{
893 /* High res. callback list. NOP for !HIGHRES */ 855 if (timer->state & HRTIMER_STATE_ENQUEUED) {
894 if (hrtimer_cb_pending(timer))
895 hrtimer_remove_cb_pending(timer);
896 else {
897 /* 856 /*
898 * Remove the timer from the rbtree and replace the 857 * Remove the timer from the rbtree and replace the
899 * first entry pointer if necessary. 858 * first entry pointer if necessary.
@@ -953,7 +912,7 @@ hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, unsigned long delta_n
953{ 912{
954 struct hrtimer_clock_base *base, *new_base; 913 struct hrtimer_clock_base *base, *new_base;
955 unsigned long flags; 914 unsigned long flags;
956 int ret, raise; 915 int ret;
957 916
958 base = lock_hrtimer_base(timer, &flags); 917 base = lock_hrtimer_base(timer, &flags);
959 918
@@ -988,26 +947,8 @@ hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, unsigned long delta_n
988 enqueue_hrtimer(timer, new_base, 947 enqueue_hrtimer(timer, new_base,
989 new_base->cpu_base == &__get_cpu_var(hrtimer_bases)); 948 new_base->cpu_base == &__get_cpu_var(hrtimer_bases));
990 949
991 /*
992 * The timer may be expired and moved to the cb_pending
993 * list. We can not raise the softirq with base lock held due
994 * to a possible deadlock with runqueue lock.
995 */
996 raise = timer->state == HRTIMER_STATE_PENDING;
997
998 /*
999 * We use preempt_disable to prevent this task from migrating after
1000 * setting up the softirq and raising it. Otherwise, if me migrate
1001 * we will raise the softirq on the wrong CPU.
1002 */
1003 preempt_disable();
1004
1005 unlock_hrtimer_base(timer, &flags); 950 unlock_hrtimer_base(timer, &flags);
1006 951
1007 if (raise)
1008 hrtimer_raise_softirq();
1009 preempt_enable();
1010
1011 return ret; 952 return ret;
1012} 953}
1013EXPORT_SYMBOL_GPL(hrtimer_start_range_ns); 954EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
@@ -1192,75 +1133,6 @@ int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp)
1192} 1133}
1193EXPORT_SYMBOL_GPL(hrtimer_get_res); 1134EXPORT_SYMBOL_GPL(hrtimer_get_res);
1194 1135
1195static void run_hrtimer_pending(struct hrtimer_cpu_base *cpu_base)
1196{
1197 spin_lock_irq(&cpu_base->lock);
1198
1199 while (!list_empty(&cpu_base->cb_pending)) {
1200 enum hrtimer_restart (*fn)(struct hrtimer *);
1201 struct hrtimer *timer;
1202 int restart;
1203 int emulate_hardirq_ctx = 0;
1204
1205 timer = list_entry(cpu_base->cb_pending.next,
1206 struct hrtimer, cb_entry);
1207
1208 debug_hrtimer_deactivate(timer);
1209 timer_stats_account_hrtimer(timer);
1210
1211 fn = timer->function;
1212 /*
1213 * A timer might have been added to the cb_pending list
1214 * when it was migrated during a cpu-offline operation.
1215 * Emulate hardirq context for such timers.
1216 */
1217 if (timer->cb_mode == HRTIMER_CB_IRQSAFE_PERCPU ||
1218 timer->cb_mode == HRTIMER_CB_IRQSAFE_UNLOCKED)
1219 emulate_hardirq_ctx = 1;
1220
1221 __remove_hrtimer(timer, timer->base, HRTIMER_STATE_CALLBACK, 0);
1222 spin_unlock_irq(&cpu_base->lock);
1223
1224 if (unlikely(emulate_hardirq_ctx)) {
1225 local_irq_disable();
1226 restart = fn(timer);
1227 local_irq_enable();
1228 } else
1229 restart = fn(timer);
1230
1231 spin_lock_irq(&cpu_base->lock);
1232
1233 timer->state &= ~HRTIMER_STATE_CALLBACK;
1234 if (restart == HRTIMER_RESTART) {
1235 BUG_ON(hrtimer_active(timer));
1236 /*
1237 * Enqueue the timer, allow reprogramming of the event
1238 * device
1239 */
1240 enqueue_hrtimer(timer, timer->base, 1);
1241 } else if (hrtimer_active(timer)) {
1242 /*
1243 * If the timer was rearmed on another CPU, reprogram
1244 * the event device.
1245 */
1246 struct hrtimer_clock_base *base = timer->base;
1247
1248 if (base->first == &timer->node &&
1249 hrtimer_reprogram(timer, base)) {
1250 /*
1251 * Timer is expired. Thus move it from tree to
1252 * pending list again.
1253 */
1254 __remove_hrtimer(timer, base,
1255 HRTIMER_STATE_PENDING, 0);
1256 list_add_tail(&timer->cb_entry,
1257 &base->cpu_base->cb_pending);
1258 }
1259 }
1260 }
1261 spin_unlock_irq(&cpu_base->lock);
1262}
1263
1264static void __run_hrtimer(struct hrtimer *timer) 1136static void __run_hrtimer(struct hrtimer *timer)
1265{ 1137{
1266 struct hrtimer_clock_base *base = timer->base; 1138 struct hrtimer_clock_base *base = timer->base;
@@ -1268,25 +1140,21 @@ static void __run_hrtimer(struct hrtimer *timer)
1268 enum hrtimer_restart (*fn)(struct hrtimer *); 1140 enum hrtimer_restart (*fn)(struct hrtimer *);
1269 int restart; 1141 int restart;
1270 1142
1143 WARN_ON(!irqs_disabled());
1144
1271 debug_hrtimer_deactivate(timer); 1145 debug_hrtimer_deactivate(timer);
1272 __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0); 1146 __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0);
1273 timer_stats_account_hrtimer(timer); 1147 timer_stats_account_hrtimer(timer);
1274
1275 fn = timer->function; 1148 fn = timer->function;
1276 if (timer->cb_mode == HRTIMER_CB_IRQSAFE_PERCPU || 1149
1277 timer->cb_mode == HRTIMER_CB_IRQSAFE_UNLOCKED) { 1150 /*
1278 /* 1151 * Because we run timers from hardirq context, there is no chance
1279 * Used for scheduler timers, avoid lock inversion with 1152 * they get migrated to another cpu, therefore its safe to unlock
1280 * rq->lock and tasklist_lock. 1153 * the timer base.
1281 * 1154 */
1282 * These timers are required to deal with enqueue expiry 1155 spin_unlock(&cpu_base->lock);
1283 * themselves and are not allowed to migrate. 1156 restart = fn(timer);
1284 */ 1157 spin_lock(&cpu_base->lock);
1285 spin_unlock(&cpu_base->lock);
1286 restart = fn(timer);
1287 spin_lock(&cpu_base->lock);
1288 } else
1289 restart = fn(timer);
1290 1158
1291 /* 1159 /*
1292 * Note: We clear the CALLBACK bit after enqueue_hrtimer to avoid 1160 * Note: We clear the CALLBACK bit after enqueue_hrtimer to avoid
@@ -1311,7 +1179,7 @@ void hrtimer_interrupt(struct clock_event_device *dev)
1311 struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); 1179 struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases);
1312 struct hrtimer_clock_base *base; 1180 struct hrtimer_clock_base *base;
1313 ktime_t expires_next, now; 1181 ktime_t expires_next, now;
1314 int i, raise = 0; 1182 int i;
1315 1183
1316 BUG_ON(!cpu_base->hres_active); 1184 BUG_ON(!cpu_base->hres_active);
1317 cpu_base->nr_events++; 1185 cpu_base->nr_events++;
@@ -1360,16 +1228,6 @@ void hrtimer_interrupt(struct clock_event_device *dev)
1360 break; 1228 break;
1361 } 1229 }
1362 1230
1363 /* Move softirq callbacks to the pending list */
1364 if (timer->cb_mode == HRTIMER_CB_SOFTIRQ) {
1365 __remove_hrtimer(timer, base,
1366 HRTIMER_STATE_PENDING, 0);
1367 list_add_tail(&timer->cb_entry,
1368 &base->cpu_base->cb_pending);
1369 raise = 1;
1370 continue;
1371 }
1372
1373 __run_hrtimer(timer); 1231 __run_hrtimer(timer);
1374 } 1232 }
1375 spin_unlock(&cpu_base->lock); 1233 spin_unlock(&cpu_base->lock);
@@ -1383,10 +1241,6 @@ void hrtimer_interrupt(struct clock_event_device *dev)
1383 if (tick_program_event(expires_next, 0)) 1241 if (tick_program_event(expires_next, 0))
1384 goto retry; 1242 goto retry;
1385 } 1243 }
1386
1387 /* Raise softirq ? */
1388 if (raise)
1389 raise_softirq(HRTIMER_SOFTIRQ);
1390} 1244}
1391 1245
1392/** 1246/**
@@ -1413,11 +1267,6 @@ void hrtimer_peek_ahead_timers(void)
1413 local_irq_restore(flags); 1267 local_irq_restore(flags);
1414} 1268}
1415 1269
1416static void run_hrtimer_softirq(struct softirq_action *h)
1417{
1418 run_hrtimer_pending(&__get_cpu_var(hrtimer_bases));
1419}
1420
1421#endif /* CONFIG_HIGH_RES_TIMERS */ 1270#endif /* CONFIG_HIGH_RES_TIMERS */
1422 1271
1423/* 1272/*
@@ -1429,8 +1278,6 @@ static void run_hrtimer_softirq(struct softirq_action *h)
1429 */ 1278 */
1430void hrtimer_run_pending(void) 1279void hrtimer_run_pending(void)
1431{ 1280{
1432 struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases);
1433
1434 if (hrtimer_hres_active()) 1281 if (hrtimer_hres_active())
1435 return; 1282 return;
1436 1283
@@ -1444,8 +1291,6 @@ void hrtimer_run_pending(void)
1444 */ 1291 */
1445 if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) 1292 if (tick_check_oneshot_change(!hrtimer_is_hres_enabled()))
1446 hrtimer_switch_to_hres(); 1293 hrtimer_switch_to_hres();
1447
1448 run_hrtimer_pending(cpu_base);
1449} 1294}
1450 1295
1451/* 1296/*
@@ -1482,14 +1327,6 @@ void hrtimer_run_queues(void)
1482 hrtimer_get_expires_tv64(timer)) 1327 hrtimer_get_expires_tv64(timer))
1483 break; 1328 break;
1484 1329
1485 if (timer->cb_mode == HRTIMER_CB_SOFTIRQ) {
1486 __remove_hrtimer(timer, base,
1487 HRTIMER_STATE_PENDING, 0);
1488 list_add_tail(&timer->cb_entry,
1489 &base->cpu_base->cb_pending);
1490 continue;
1491 }
1492
1493 __run_hrtimer(timer); 1330 __run_hrtimer(timer);
1494 } 1331 }
1495 spin_unlock(&cpu_base->lock); 1332 spin_unlock(&cpu_base->lock);
@@ -1516,9 +1353,6 @@ void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task)
1516{ 1353{
1517 sl->timer.function = hrtimer_wakeup; 1354 sl->timer.function = hrtimer_wakeup;
1518 sl->task = task; 1355 sl->task = task;
1519#ifdef CONFIG_HIGH_RES_TIMERS
1520 sl->timer.cb_mode = HRTIMER_CB_IRQSAFE_UNLOCKED;
1521#endif
1522} 1356}
1523 1357
1524static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode) 1358static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode)
@@ -1655,18 +1489,16 @@ static void __cpuinit init_hrtimers_cpu(int cpu)
1655 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) 1489 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++)
1656 cpu_base->clock_base[i].cpu_base = cpu_base; 1490 cpu_base->clock_base[i].cpu_base = cpu_base;
1657 1491
1658 INIT_LIST_HEAD(&cpu_base->cb_pending);
1659 hrtimer_init_hres(cpu_base); 1492 hrtimer_init_hres(cpu_base);
1660} 1493}
1661 1494
1662#ifdef CONFIG_HOTPLUG_CPU 1495#ifdef CONFIG_HOTPLUG_CPU
1663 1496
1664static int migrate_hrtimer_list(struct hrtimer_clock_base *old_base, 1497static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
1665 struct hrtimer_clock_base *new_base, int dcpu) 1498 struct hrtimer_clock_base *new_base)
1666{ 1499{
1667 struct hrtimer *timer; 1500 struct hrtimer *timer;
1668 struct rb_node *node; 1501 struct rb_node *node;
1669 int raise = 0;
1670 1502
1671 while ((node = rb_first(&old_base->active))) { 1503 while ((node = rb_first(&old_base->active))) {
1672 timer = rb_entry(node, struct hrtimer, node); 1504 timer = rb_entry(node, struct hrtimer, node);
@@ -1674,18 +1506,6 @@ static int migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
1674 debug_hrtimer_deactivate(timer); 1506 debug_hrtimer_deactivate(timer);
1675 1507
1676 /* 1508 /*
1677 * Should not happen. Per CPU timers should be
1678 * canceled _before_ the migration code is called
1679 */
1680 if (timer->cb_mode == HRTIMER_CB_IRQSAFE_PERCPU) {
1681 __remove_hrtimer(timer, old_base,
1682 HRTIMER_STATE_INACTIVE, 0);
1683 WARN(1, "hrtimer (%p %p)active but cpu %d dead\n",
1684 timer, timer->function, dcpu);
1685 continue;
1686 }
1687
1688 /*
1689 * Mark it as STATE_MIGRATE not INACTIVE otherwise the 1509 * Mark it as STATE_MIGRATE not INACTIVE otherwise the
1690 * timer could be seen as !active and just vanish away 1510 * timer could be seen as !active and just vanish away
1691 * under us on another CPU 1511 * under us on another CPU
@@ -1693,69 +1513,34 @@ static int migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
1693 __remove_hrtimer(timer, old_base, HRTIMER_STATE_MIGRATE, 0); 1513 __remove_hrtimer(timer, old_base, HRTIMER_STATE_MIGRATE, 0);
1694 timer->base = new_base; 1514 timer->base = new_base;
1695 /* 1515 /*
1696 * Enqueue the timer. Allow reprogramming of the event device 1516 * Enqueue the timers on the new cpu, but do not reprogram
1517 * the timer as that would enable a deadlock between
1518 * hrtimer_enqueue_reprogramm() running the timer and us still
1519 * holding a nested base lock.
1520 *
1521 * Instead we tickle the hrtimer interrupt after the migration
1522 * is done, which will run all expired timers and re-programm
1523 * the timer device.
1697 */ 1524 */
1698 enqueue_hrtimer(timer, new_base, 1); 1525 enqueue_hrtimer(timer, new_base, 0);
1699 1526
1700#ifdef CONFIG_HIGH_RES_TIMERS
1701 /*
1702 * Happens with high res enabled when the timer was
1703 * already expired and the callback mode is
1704 * HRTIMER_CB_IRQSAFE_UNLOCKED (hrtimer_sleeper). The
1705 * enqueue code does not move them to the soft irq
1706 * pending list for performance/latency reasons, but
1707 * in the migration state, we need to do that
1708 * otherwise we end up with a stale timer.
1709 */
1710 if (timer->state == HRTIMER_STATE_MIGRATE) {
1711 timer->state = HRTIMER_STATE_PENDING;
1712 list_add_tail(&timer->cb_entry,
1713 &new_base->cpu_base->cb_pending);
1714 raise = 1;
1715 }
1716#endif
1717 /* Clear the migration state bit */ 1527 /* Clear the migration state bit */
1718 timer->state &= ~HRTIMER_STATE_MIGRATE; 1528 timer->state &= ~HRTIMER_STATE_MIGRATE;
1719 } 1529 }
1720 return raise;
1721}
1722
1723#ifdef CONFIG_HIGH_RES_TIMERS
1724static int migrate_hrtimer_pending(struct hrtimer_cpu_base *old_base,
1725 struct hrtimer_cpu_base *new_base)
1726{
1727 struct hrtimer *timer;
1728 int raise = 0;
1729
1730 while (!list_empty(&old_base->cb_pending)) {
1731 timer = list_entry(old_base->cb_pending.next,
1732 struct hrtimer, cb_entry);
1733
1734 __remove_hrtimer(timer, timer->base, HRTIMER_STATE_PENDING, 0);
1735 timer->base = &new_base->clock_base[timer->base->index];
1736 list_add_tail(&timer->cb_entry, &new_base->cb_pending);
1737 raise = 1;
1738 }
1739 return raise;
1740}
1741#else
1742static int migrate_hrtimer_pending(struct hrtimer_cpu_base *old_base,
1743 struct hrtimer_cpu_base *new_base)
1744{
1745 return 0;
1746} 1530}
1747#endif
1748 1531
1749static void migrate_hrtimers(int cpu) 1532static int migrate_hrtimers(int scpu)
1750{ 1533{
1751 struct hrtimer_cpu_base *old_base, *new_base; 1534 struct hrtimer_cpu_base *old_base, *new_base;
1752 int i, raise = 0; 1535 int dcpu, i;
1753 1536
1754 BUG_ON(cpu_online(cpu)); 1537 BUG_ON(cpu_online(scpu));
1755 old_base = &per_cpu(hrtimer_bases, cpu); 1538 old_base = &per_cpu(hrtimer_bases, scpu);
1756 new_base = &get_cpu_var(hrtimer_bases); 1539 new_base = &get_cpu_var(hrtimer_bases);
1757 1540
1758 tick_cancel_sched_timer(cpu); 1541 dcpu = smp_processor_id();
1542
1543 tick_cancel_sched_timer(scpu);
1759 /* 1544 /*
1760 * The caller is globally serialized and nobody else 1545 * The caller is globally serialized and nobody else
1761 * takes two locks at once, deadlock is not possible. 1546 * takes two locks at once, deadlock is not possible.
@@ -1764,41 +1549,47 @@ static void migrate_hrtimers(int cpu)
1764 spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); 1549 spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
1765 1550
1766 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { 1551 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
1767 if (migrate_hrtimer_list(&old_base->clock_base[i], 1552 migrate_hrtimer_list(&old_base->clock_base[i],
1768 &new_base->clock_base[i], cpu)) 1553 &new_base->clock_base[i]);
1769 raise = 1;
1770 } 1554 }
1771 1555
1772 if (migrate_hrtimer_pending(old_base, new_base))
1773 raise = 1;
1774
1775 spin_unlock(&old_base->lock); 1556 spin_unlock(&old_base->lock);
1776 spin_unlock_irq(&new_base->lock); 1557 spin_unlock_irq(&new_base->lock);
1777 put_cpu_var(hrtimer_bases); 1558 put_cpu_var(hrtimer_bases);
1778 1559
1779 if (raise) 1560 return dcpu;
1780 hrtimer_raise_softirq(); 1561}
1562
1563static void tickle_timers(void *arg)
1564{
1565 hrtimer_peek_ahead_timers();
1781} 1566}
1567
1782#endif /* CONFIG_HOTPLUG_CPU */ 1568#endif /* CONFIG_HOTPLUG_CPU */
1783 1569
1784static int __cpuinit hrtimer_cpu_notify(struct notifier_block *self, 1570static int __cpuinit hrtimer_cpu_notify(struct notifier_block *self,
1785 unsigned long action, void *hcpu) 1571 unsigned long action, void *hcpu)
1786{ 1572{
1787 unsigned int cpu = (long)hcpu; 1573 int scpu = (long)hcpu;
1788 1574
1789 switch (action) { 1575 switch (action) {
1790 1576
1791 case CPU_UP_PREPARE: 1577 case CPU_UP_PREPARE:
1792 case CPU_UP_PREPARE_FROZEN: 1578 case CPU_UP_PREPARE_FROZEN:
1793 init_hrtimers_cpu(cpu); 1579 init_hrtimers_cpu(scpu);
1794 break; 1580 break;
1795 1581
1796#ifdef CONFIG_HOTPLUG_CPU 1582#ifdef CONFIG_HOTPLUG_CPU
1797 case CPU_DEAD: 1583 case CPU_DEAD:
1798 case CPU_DEAD_FROZEN: 1584 case CPU_DEAD_FROZEN:
1799 clockevents_notify(CLOCK_EVT_NOTIFY_CPU_DEAD, &cpu); 1585 {
1800 migrate_hrtimers(cpu); 1586 int dcpu;
1587
1588 clockevents_notify(CLOCK_EVT_NOTIFY_CPU_DEAD, &scpu);
1589 dcpu = migrate_hrtimers(scpu);
1590 smp_call_function_single(dcpu, tickle_timers, NULL, 0);
1801 break; 1591 break;
1592 }
1802#endif 1593#endif
1803 1594
1804 default: 1595 default:
@@ -1817,9 +1608,6 @@ void __init hrtimers_init(void)
1817 hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE, 1608 hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE,
1818 (void *)(long)smp_processor_id()); 1609 (void *)(long)smp_processor_id());
1819 register_cpu_notifier(&hrtimers_nb); 1610 register_cpu_notifier(&hrtimers_nb);
1820#ifdef CONFIG_HIGH_RES_TIMERS
1821 open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq);
1822#endif
1823} 1611}
1824 1612
1825/** 1613/**
diff --git a/kernel/irq/Makefile b/kernel/irq/Makefile
index 681c52dbfe22..4dd5b1edac98 100644
--- a/kernel/irq/Makefile
+++ b/kernel/irq/Makefile
@@ -3,3 +3,4 @@ obj-y := handle.o manage.o spurious.o resend.o chip.o devres.o
3obj-$(CONFIG_GENERIC_IRQ_PROBE) += autoprobe.o 3obj-$(CONFIG_GENERIC_IRQ_PROBE) += autoprobe.o
4obj-$(CONFIG_PROC_FS) += proc.o 4obj-$(CONFIG_PROC_FS) += proc.o
5obj-$(CONFIG_GENERIC_PENDING_IRQ) += migration.o 5obj-$(CONFIG_GENERIC_PENDING_IRQ) += migration.o
6obj-$(CONFIG_NUMA_MIGRATE_IRQ_DESC) += numa_migrate.o
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c
index 10b5092e9bfe..f63c706d25e1 100644
--- a/kernel/irq/chip.c
+++ b/kernel/irq/chip.c
@@ -24,9 +24,10 @@
24 */ 24 */
25void dynamic_irq_init(unsigned int irq) 25void dynamic_irq_init(unsigned int irq)
26{ 26{
27 struct irq_desc *desc = irq_to_desc(irq); 27 struct irq_desc *desc;
28 unsigned long flags; 28 unsigned long flags;
29 29
30 desc = irq_to_desc(irq);
30 if (!desc) { 31 if (!desc) {
31 WARN(1, KERN_ERR "Trying to initialize invalid IRQ%d\n", irq); 32 WARN(1, KERN_ERR "Trying to initialize invalid IRQ%d\n", irq);
32 return; 33 return;
@@ -45,7 +46,7 @@ void dynamic_irq_init(unsigned int irq)
45 desc->irq_count = 0; 46 desc->irq_count = 0;
46 desc->irqs_unhandled = 0; 47 desc->irqs_unhandled = 0;
47#ifdef CONFIG_SMP 48#ifdef CONFIG_SMP
48 cpus_setall(desc->affinity); 49 cpumask_setall(&desc->affinity);
49#endif 50#endif
50 spin_unlock_irqrestore(&desc->lock, flags); 51 spin_unlock_irqrestore(&desc->lock, flags);
51} 52}
@@ -124,6 +125,7 @@ int set_irq_type(unsigned int irq, unsigned int type)
124 return -ENODEV; 125 return -ENODEV;
125 } 126 }
126 127
128 type &= IRQ_TYPE_SENSE_MASK;
127 if (type == IRQ_TYPE_NONE) 129 if (type == IRQ_TYPE_NONE)
128 return 0; 130 return 0;
129 131
@@ -352,6 +354,7 @@ handle_level_irq(unsigned int irq, struct irq_desc *desc)
352 354
353 spin_lock(&desc->lock); 355 spin_lock(&desc->lock);
354 mask_ack_irq(desc, irq); 356 mask_ack_irq(desc, irq);
357 desc = irq_remap_to_desc(irq, desc);
355 358
356 if (unlikely(desc->status & IRQ_INPROGRESS)) 359 if (unlikely(desc->status & IRQ_INPROGRESS))
357 goto out_unlock; 360 goto out_unlock;
@@ -429,6 +432,7 @@ handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc)
429 desc->status &= ~IRQ_INPROGRESS; 432 desc->status &= ~IRQ_INPROGRESS;
430out: 433out:
431 desc->chip->eoi(irq); 434 desc->chip->eoi(irq);
435 desc = irq_remap_to_desc(irq, desc);
432 436
433 spin_unlock(&desc->lock); 437 spin_unlock(&desc->lock);
434} 438}
@@ -465,12 +469,14 @@ handle_edge_irq(unsigned int irq, struct irq_desc *desc)
465 !desc->action)) { 469 !desc->action)) {
466 desc->status |= (IRQ_PENDING | IRQ_MASKED); 470 desc->status |= (IRQ_PENDING | IRQ_MASKED);
467 mask_ack_irq(desc, irq); 471 mask_ack_irq(desc, irq);
472 desc = irq_remap_to_desc(irq, desc);
468 goto out_unlock; 473 goto out_unlock;
469 } 474 }
470 kstat_incr_irqs_this_cpu(irq, desc); 475 kstat_incr_irqs_this_cpu(irq, desc);
471 476
472 /* Start handling the irq */ 477 /* Start handling the irq */
473 desc->chip->ack(irq); 478 desc->chip->ack(irq);
479 desc = irq_remap_to_desc(irq, desc);
474 480
475 /* Mark the IRQ currently in progress.*/ 481 /* Mark the IRQ currently in progress.*/
476 desc->status |= IRQ_INPROGRESS; 482 desc->status |= IRQ_INPROGRESS;
@@ -531,8 +537,10 @@ handle_percpu_irq(unsigned int irq, struct irq_desc *desc)
531 if (!noirqdebug) 537 if (!noirqdebug)
532 note_interrupt(irq, desc, action_ret); 538 note_interrupt(irq, desc, action_ret);
533 539
534 if (desc->chip->eoi) 540 if (desc->chip->eoi) {
535 desc->chip->eoi(irq); 541 desc->chip->eoi(irq);
542 desc = irq_remap_to_desc(irq, desc);
543 }
536} 544}
537 545
538void 546void
@@ -567,8 +575,10 @@ __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
567 575
568 /* Uninstall? */ 576 /* Uninstall? */
569 if (handle == handle_bad_irq) { 577 if (handle == handle_bad_irq) {
570 if (desc->chip != &no_irq_chip) 578 if (desc->chip != &no_irq_chip) {
571 mask_ack_irq(desc, irq); 579 mask_ack_irq(desc, irq);
580 desc = irq_remap_to_desc(irq, desc);
581 }
572 desc->status |= IRQ_DISABLED; 582 desc->status |= IRQ_DISABLED;
573 desc->depth = 1; 583 desc->depth = 1;
574 } 584 }
diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c
index c815b42d0f5b..c20db0be9173 100644
--- a/kernel/irq/handle.c
+++ b/kernel/irq/handle.c
@@ -15,9 +15,16 @@
15#include <linux/random.h> 15#include <linux/random.h>
16#include <linux/interrupt.h> 16#include <linux/interrupt.h>
17#include <linux/kernel_stat.h> 17#include <linux/kernel_stat.h>
18#include <linux/rculist.h>
19#include <linux/hash.h>
18 20
19#include "internals.h" 21#include "internals.h"
20 22
23/*
24 * lockdep: we want to handle all irq_desc locks as a single lock-class:
25 */
26struct lock_class_key irq_desc_lock_class;
27
21/** 28/**
22 * handle_bad_irq - handle spurious and unhandled irqs 29 * handle_bad_irq - handle spurious and unhandled irqs
23 * @irq: the interrupt number 30 * @irq: the interrupt number
@@ -49,6 +56,150 @@ void handle_bad_irq(unsigned int irq, struct irq_desc *desc)
49int nr_irqs = NR_IRQS; 56int nr_irqs = NR_IRQS;
50EXPORT_SYMBOL_GPL(nr_irqs); 57EXPORT_SYMBOL_GPL(nr_irqs);
51 58
59#ifdef CONFIG_SPARSE_IRQ
60static struct irq_desc irq_desc_init = {
61 .irq = -1,
62 .status = IRQ_DISABLED,
63 .chip = &no_irq_chip,
64 .handle_irq = handle_bad_irq,
65 .depth = 1,
66 .lock = __SPIN_LOCK_UNLOCKED(irq_desc_init.lock),
67#ifdef CONFIG_SMP
68 .affinity = CPU_MASK_ALL
69#endif
70};
71
72void init_kstat_irqs(struct irq_desc *desc, int cpu, int nr)
73{
74 unsigned long bytes;
75 char *ptr;
76 int node;
77
78 /* Compute how many bytes we need per irq and allocate them */
79 bytes = nr * sizeof(unsigned int);
80
81 node = cpu_to_node(cpu);
82 ptr = kzalloc_node(bytes, GFP_ATOMIC, node);
83 printk(KERN_DEBUG " alloc kstat_irqs on cpu %d node %d\n", cpu, node);
84
85 if (ptr)
86 desc->kstat_irqs = (unsigned int *)ptr;
87}
88
89static void init_one_irq_desc(int irq, struct irq_desc *desc, int cpu)
90{
91 memcpy(desc, &irq_desc_init, sizeof(struct irq_desc));
92
93 spin_lock_init(&desc->lock);
94 desc->irq = irq;
95#ifdef CONFIG_SMP
96 desc->cpu = cpu;
97#endif
98 lockdep_set_class(&desc->lock, &irq_desc_lock_class);
99 init_kstat_irqs(desc, cpu, nr_cpu_ids);
100 if (!desc->kstat_irqs) {
101 printk(KERN_ERR "can not alloc kstat_irqs\n");
102 BUG_ON(1);
103 }
104 arch_init_chip_data(desc, cpu);
105}
106
107/*
108 * Protect the sparse_irqs:
109 */
110DEFINE_SPINLOCK(sparse_irq_lock);
111
112struct irq_desc *irq_desc_ptrs[NR_IRQS] __read_mostly;
113
114static struct irq_desc irq_desc_legacy[NR_IRQS_LEGACY] __cacheline_aligned_in_smp = {
115 [0 ... NR_IRQS_LEGACY-1] = {
116 .irq = -1,
117 .status = IRQ_DISABLED,
118 .chip = &no_irq_chip,
119 .handle_irq = handle_bad_irq,
120 .depth = 1,
121 .lock = __SPIN_LOCK_UNLOCKED(irq_desc_init.lock),
122#ifdef CONFIG_SMP
123 .affinity = CPU_MASK_ALL
124#endif
125 }
126};
127
128/* FIXME: use bootmem alloc ...*/
129static unsigned int kstat_irqs_legacy[NR_IRQS_LEGACY][NR_CPUS];
130
131int __init early_irq_init(void)
132{
133 struct irq_desc *desc;
134 int legacy_count;
135 int i;
136
137 desc = irq_desc_legacy;
138 legacy_count = ARRAY_SIZE(irq_desc_legacy);
139
140 for (i = 0; i < legacy_count; i++) {
141 desc[i].irq = i;
142 desc[i].kstat_irqs = kstat_irqs_legacy[i];
143 lockdep_set_class(&desc[i].lock, &irq_desc_lock_class);
144
145 irq_desc_ptrs[i] = desc + i;
146 }
147
148 for (i = legacy_count; i < NR_IRQS; i++)
149 irq_desc_ptrs[i] = NULL;
150
151 return arch_early_irq_init();
152}
153
154struct irq_desc *irq_to_desc(unsigned int irq)
155{
156 return (irq < NR_IRQS) ? irq_desc_ptrs[irq] : NULL;
157}
158
159struct irq_desc *irq_to_desc_alloc_cpu(unsigned int irq, int cpu)
160{
161 struct irq_desc *desc;
162 unsigned long flags;
163 int node;
164
165 if (irq >= NR_IRQS) {
166 printk(KERN_WARNING "irq >= NR_IRQS in irq_to_desc_alloc: %d %d\n",
167 irq, NR_IRQS);
168 WARN_ON(1);
169 return NULL;
170 }
171
172 desc = irq_desc_ptrs[irq];
173 if (desc)
174 return desc;
175
176 spin_lock_irqsave(&sparse_irq_lock, flags);
177
178 /* We have to check it to avoid races with another CPU */
179 desc = irq_desc_ptrs[irq];
180 if (desc)
181 goto out_unlock;
182
183 node = cpu_to_node(cpu);
184 desc = kzalloc_node(sizeof(*desc), GFP_ATOMIC, node);
185 printk(KERN_DEBUG " alloc irq_desc for %d on cpu %d node %d\n",
186 irq, cpu, node);
187 if (!desc) {
188 printk(KERN_ERR "can not alloc irq_desc\n");
189 BUG_ON(1);
190 }
191 init_one_irq_desc(irq, desc, cpu);
192
193 irq_desc_ptrs[irq] = desc;
194
195out_unlock:
196 spin_unlock_irqrestore(&sparse_irq_lock, flags);
197
198 return desc;
199}
200
201#else /* !CONFIG_SPARSE_IRQ */
202
52struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = { 203struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = {
53 [0 ... NR_IRQS-1] = { 204 [0 ... NR_IRQS-1] = {
54 .status = IRQ_DISABLED, 205 .status = IRQ_DISABLED,
@@ -62,6 +213,32 @@ struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = {
62 } 213 }
63}; 214};
64 215
216int __init early_irq_init(void)
217{
218 struct irq_desc *desc;
219 int count;
220 int i;
221
222 desc = irq_desc;
223 count = ARRAY_SIZE(irq_desc);
224
225 for (i = 0; i < count; i++)
226 desc[i].irq = i;
227
228 return arch_early_irq_init();
229}
230
231struct irq_desc *irq_to_desc(unsigned int irq)
232{
233 return (irq < NR_IRQS) ? irq_desc + irq : NULL;
234}
235
236struct irq_desc *irq_to_desc_alloc_cpu(unsigned int irq, int cpu)
237{
238 return irq_to_desc(irq);
239}
240#endif /* !CONFIG_SPARSE_IRQ */
241
65/* 242/*
66 * What should we do if we get a hw irq event on an illegal vector? 243 * What should we do if we get a hw irq event on an illegal vector?
67 * Each architecture has to answer this themself. 244 * Each architecture has to answer this themself.
@@ -179,8 +356,11 @@ unsigned int __do_IRQ(unsigned int irq)
179 /* 356 /*
180 * No locking required for CPU-local interrupts: 357 * No locking required for CPU-local interrupts:
181 */ 358 */
182 if (desc->chip->ack) 359 if (desc->chip->ack) {
183 desc->chip->ack(irq); 360 desc->chip->ack(irq);
361 /* get new one */
362 desc = irq_remap_to_desc(irq, desc);
363 }
184 if (likely(!(desc->status & IRQ_DISABLED))) { 364 if (likely(!(desc->status & IRQ_DISABLED))) {
185 action_ret = handle_IRQ_event(irq, desc->action); 365 action_ret = handle_IRQ_event(irq, desc->action);
186 if (!noirqdebug) 366 if (!noirqdebug)
@@ -191,8 +371,10 @@ unsigned int __do_IRQ(unsigned int irq)
191 } 371 }
192 372
193 spin_lock(&desc->lock); 373 spin_lock(&desc->lock);
194 if (desc->chip->ack) 374 if (desc->chip->ack) {
195 desc->chip->ack(irq); 375 desc->chip->ack(irq);
376 desc = irq_remap_to_desc(irq, desc);
377 }
196 /* 378 /*
197 * REPLAY is when Linux resends an IRQ that was dropped earlier 379 * REPLAY is when Linux resends an IRQ that was dropped earlier
198 * WAITING is used by probe to mark irqs that are being tested 380 * WAITING is used by probe to mark irqs that are being tested
@@ -259,19 +441,22 @@ out:
259} 441}
260#endif 442#endif
261 443
262
263#ifdef CONFIG_TRACE_IRQFLAGS
264/*
265 * lockdep: we want to handle all irq_desc locks as a single lock-class:
266 */
267static struct lock_class_key irq_desc_lock_class;
268
269void early_init_irq_lock_class(void) 444void early_init_irq_lock_class(void)
270{ 445{
271 struct irq_desc *desc; 446 struct irq_desc *desc;
272 int i; 447 int i;
273 448
274 for_each_irq_desc(i, desc) 449 for_each_irq_desc(i, desc) {
275 lockdep_set_class(&desc->lock, &irq_desc_lock_class); 450 lockdep_set_class(&desc->lock, &irq_desc_lock_class);
451 }
452}
453
454#ifdef CONFIG_SPARSE_IRQ
455unsigned int kstat_irqs_cpu(unsigned int irq, int cpu)
456{
457 struct irq_desc *desc = irq_to_desc(irq);
458 return desc ? desc->kstat_irqs[cpu] : 0;
276} 459}
277#endif 460#endif
461EXPORT_SYMBOL(kstat_irqs_cpu);
462
diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h
index 64c1c7253dae..e6d0a43cc125 100644
--- a/kernel/irq/internals.h
+++ b/kernel/irq/internals.h
@@ -13,6 +13,11 @@ extern void compat_irq_chip_set_default_handler(struct irq_desc *desc);
13extern int __irq_set_trigger(struct irq_desc *desc, unsigned int irq, 13extern int __irq_set_trigger(struct irq_desc *desc, unsigned int irq,
14 unsigned long flags); 14 unsigned long flags);
15 15
16extern struct lock_class_key irq_desc_lock_class;
17extern void init_kstat_irqs(struct irq_desc *desc, int cpu, int nr);
18extern spinlock_t sparse_irq_lock;
19extern struct irq_desc *irq_desc_ptrs[NR_IRQS];
20
16#ifdef CONFIG_PROC_FS 21#ifdef CONFIG_PROC_FS
17extern void register_irq_proc(unsigned int irq, struct irq_desc *desc); 22extern void register_irq_proc(unsigned int irq, struct irq_desc *desc);
18extern void register_handler_proc(unsigned int irq, struct irqaction *action); 23extern void register_handler_proc(unsigned int irq, struct irqaction *action);
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index 801addda3c43..61c4a9b62165 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -79,7 +79,7 @@ int irq_can_set_affinity(unsigned int irq)
79 * @cpumask: cpumask 79 * @cpumask: cpumask
80 * 80 *
81 */ 81 */
82int irq_set_affinity(unsigned int irq, cpumask_t cpumask) 82int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask)
83{ 83{
84 struct irq_desc *desc = irq_to_desc(irq); 84 struct irq_desc *desc = irq_to_desc(irq);
85 unsigned long flags; 85 unsigned long flags;
@@ -91,14 +91,14 @@ int irq_set_affinity(unsigned int irq, cpumask_t cpumask)
91 91
92#ifdef CONFIG_GENERIC_PENDING_IRQ 92#ifdef CONFIG_GENERIC_PENDING_IRQ
93 if (desc->status & IRQ_MOVE_PCNTXT || desc->status & IRQ_DISABLED) { 93 if (desc->status & IRQ_MOVE_PCNTXT || desc->status & IRQ_DISABLED) {
94 desc->affinity = cpumask; 94 cpumask_copy(&desc->affinity, cpumask);
95 desc->chip->set_affinity(irq, cpumask); 95 desc->chip->set_affinity(irq, cpumask);
96 } else { 96 } else {
97 desc->status |= IRQ_MOVE_PENDING; 97 desc->status |= IRQ_MOVE_PENDING;
98 desc->pending_mask = cpumask; 98 cpumask_copy(&desc->pending_mask, cpumask);
99 } 99 }
100#else 100#else
101 desc->affinity = cpumask; 101 cpumask_copy(&desc->affinity, cpumask);
102 desc->chip->set_affinity(irq, cpumask); 102 desc->chip->set_affinity(irq, cpumask);
103#endif 103#endif
104 desc->status |= IRQ_AFFINITY_SET; 104 desc->status |= IRQ_AFFINITY_SET;
@@ -112,26 +112,24 @@ int irq_set_affinity(unsigned int irq, cpumask_t cpumask)
112 */ 112 */
113int do_irq_select_affinity(unsigned int irq, struct irq_desc *desc) 113int do_irq_select_affinity(unsigned int irq, struct irq_desc *desc)
114{ 114{
115 cpumask_t mask;
116
117 if (!irq_can_set_affinity(irq)) 115 if (!irq_can_set_affinity(irq))
118 return 0; 116 return 0;
119 117
120 cpus_and(mask, cpu_online_map, irq_default_affinity);
121
122 /* 118 /*
123 * Preserve an userspace affinity setup, but make sure that 119 * Preserve an userspace affinity setup, but make sure that
124 * one of the targets is online. 120 * one of the targets is online.
125 */ 121 */
126 if (desc->status & (IRQ_AFFINITY_SET | IRQ_NO_BALANCING)) { 122 if (desc->status & (IRQ_AFFINITY_SET | IRQ_NO_BALANCING)) {
127 if (cpus_intersects(desc->affinity, cpu_online_map)) 123 if (cpumask_any_and(&desc->affinity, cpu_online_mask)
128 mask = desc->affinity; 124 < nr_cpu_ids)
125 goto set_affinity;
129 else 126 else
130 desc->status &= ~IRQ_AFFINITY_SET; 127 desc->status &= ~IRQ_AFFINITY_SET;
131 } 128 }
132 129
133 desc->affinity = mask; 130 cpumask_and(&desc->affinity, cpu_online_mask, &irq_default_affinity);
134 desc->chip->set_affinity(irq, mask); 131set_affinity:
132 desc->chip->set_affinity(irq, &desc->affinity);
135 133
136 return 0; 134 return 0;
137} 135}
@@ -370,16 +368,18 @@ int __irq_set_trigger(struct irq_desc *desc, unsigned int irq,
370 return 0; 368 return 0;
371 } 369 }
372 370
373 ret = chip->set_type(irq, flags & IRQF_TRIGGER_MASK); 371 /* caller masked out all except trigger mode flags */
372 ret = chip->set_type(irq, flags);
374 373
375 if (ret) 374 if (ret)
376 pr_err("setting trigger mode %d for irq %u failed (%pF)\n", 375 pr_err("setting trigger mode %d for irq %u failed (%pF)\n",
377 (int)(flags & IRQF_TRIGGER_MASK), 376 (int)flags, irq, chip->set_type);
378 irq, chip->set_type);
379 else { 377 else {
378 if (flags & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_LEVEL_HIGH))
379 flags |= IRQ_LEVEL;
380 /* note that IRQF_TRIGGER_MASK == IRQ_TYPE_SENSE_MASK */ 380 /* note that IRQF_TRIGGER_MASK == IRQ_TYPE_SENSE_MASK */
381 desc->status &= ~IRQ_TYPE_SENSE_MASK; 381 desc->status &= ~(IRQ_LEVEL | IRQ_TYPE_SENSE_MASK);
382 desc->status |= flags & IRQ_TYPE_SENSE_MASK; 382 desc->status |= flags;
383 } 383 }
384 384
385 return ret; 385 return ret;
@@ -459,7 +459,8 @@ __setup_irq(unsigned int irq, struct irq_desc * desc, struct irqaction *new)
459 459
460 /* Setup the type (level, edge polarity) if configured: */ 460 /* Setup the type (level, edge polarity) if configured: */
461 if (new->flags & IRQF_TRIGGER_MASK) { 461 if (new->flags & IRQF_TRIGGER_MASK) {
462 ret = __irq_set_trigger(desc, irq, new->flags); 462 ret = __irq_set_trigger(desc, irq,
463 new->flags & IRQF_TRIGGER_MASK);
463 464
464 if (ret) { 465 if (ret) {
465 spin_unlock_irqrestore(&desc->lock, flags); 466 spin_unlock_irqrestore(&desc->lock, flags);
@@ -673,6 +674,18 @@ int request_irq(unsigned int irq, irq_handler_t handler,
673 struct irq_desc *desc; 674 struct irq_desc *desc;
674 int retval; 675 int retval;
675 676
677 /*
678 * handle_IRQ_event() always ignores IRQF_DISABLED except for
679 * the _first_ irqaction (sigh). That can cause oopsing, but
680 * the behavior is classified as "will not fix" so we need to
681 * start nudging drivers away from using that idiom.
682 */
683 if ((irqflags & (IRQF_SHARED|IRQF_DISABLED))
684 == (IRQF_SHARED|IRQF_DISABLED))
685 pr_warning("IRQ %d/%s: IRQF_DISABLED is not "
686 "guaranteed on shared IRQs\n",
687 irq, devname);
688
676#ifdef CONFIG_LOCKDEP 689#ifdef CONFIG_LOCKDEP
677 /* 690 /*
678 * Lockdep wants atomic interrupt handlers: 691 * Lockdep wants atomic interrupt handlers:
diff --git a/kernel/irq/migration.c b/kernel/irq/migration.c
index 9db681d95814..bd72329e630c 100644
--- a/kernel/irq/migration.c
+++ b/kernel/irq/migration.c
@@ -4,7 +4,6 @@
4void move_masked_irq(int irq) 4void move_masked_irq(int irq)
5{ 5{
6 struct irq_desc *desc = irq_to_desc(irq); 6 struct irq_desc *desc = irq_to_desc(irq);
7 cpumask_t tmp;
8 7
9 if (likely(!(desc->status & IRQ_MOVE_PENDING))) 8 if (likely(!(desc->status & IRQ_MOVE_PENDING)))
10 return; 9 return;
@@ -19,7 +18,7 @@ void move_masked_irq(int irq)
19 18
20 desc->status &= ~IRQ_MOVE_PENDING; 19 desc->status &= ~IRQ_MOVE_PENDING;
21 20
22 if (unlikely(cpus_empty(desc->pending_mask))) 21 if (unlikely(cpumask_empty(&desc->pending_mask)))
23 return; 22 return;
24 23
25 if (!desc->chip->set_affinity) 24 if (!desc->chip->set_affinity)
@@ -27,8 +26,6 @@ void move_masked_irq(int irq)
27 26
28 assert_spin_locked(&desc->lock); 27 assert_spin_locked(&desc->lock);
29 28
30 cpus_and(tmp, desc->pending_mask, cpu_online_map);
31
32 /* 29 /*
33 * If there was a valid mask to work with, please 30 * If there was a valid mask to work with, please
34 * do the disable, re-program, enable sequence. 31 * do the disable, re-program, enable sequence.
@@ -41,10 +38,13 @@ void move_masked_irq(int irq)
41 * For correct operation this depends on the caller 38 * For correct operation this depends on the caller
42 * masking the irqs. 39 * masking the irqs.
43 */ 40 */
44 if (likely(!cpus_empty(tmp))) { 41 if (likely(cpumask_any_and(&desc->pending_mask, cpu_online_mask)
45 desc->chip->set_affinity(irq,tmp); 42 < nr_cpu_ids)) {
43 cpumask_and(&desc->affinity,
44 &desc->pending_mask, cpu_online_mask);
45 desc->chip->set_affinity(irq, &desc->affinity);
46 } 46 }
47 cpus_clear(desc->pending_mask); 47 cpumask_clear(&desc->pending_mask);
48} 48}
49 49
50void move_native_irq(int irq) 50void move_native_irq(int irq)
diff --git a/kernel/irq/numa_migrate.c b/kernel/irq/numa_migrate.c
new file mode 100644
index 000000000000..ecf765c6a77a
--- /dev/null
+++ b/kernel/irq/numa_migrate.c
@@ -0,0 +1,119 @@
1/*
2 * NUMA irq-desc migration code
3 *
4 * Migrate IRQ data structures (irq_desc, chip_data, etc.) over to
5 * the new "home node" of the IRQ.
6 */
7
8#include <linux/irq.h>
9#include <linux/module.h>
10#include <linux/random.h>
11#include <linux/interrupt.h>
12#include <linux/kernel_stat.h>
13
14#include "internals.h"
15
16static void init_copy_kstat_irqs(struct irq_desc *old_desc,
17 struct irq_desc *desc,
18 int cpu, int nr)
19{
20 unsigned long bytes;
21
22 init_kstat_irqs(desc, cpu, nr);
23
24 if (desc->kstat_irqs != old_desc->kstat_irqs) {
25 /* Compute how many bytes we need per irq and allocate them */
26 bytes = nr * sizeof(unsigned int);
27
28 memcpy(desc->kstat_irqs, old_desc->kstat_irqs, bytes);
29 }
30}
31
32static void free_kstat_irqs(struct irq_desc *old_desc, struct irq_desc *desc)
33{
34 if (old_desc->kstat_irqs == desc->kstat_irqs)
35 return;
36
37 kfree(old_desc->kstat_irqs);
38 old_desc->kstat_irqs = NULL;
39}
40
41static void init_copy_one_irq_desc(int irq, struct irq_desc *old_desc,
42 struct irq_desc *desc, int cpu)
43{
44 memcpy(desc, old_desc, sizeof(struct irq_desc));
45 spin_lock_init(&desc->lock);
46 desc->cpu = cpu;
47 lockdep_set_class(&desc->lock, &irq_desc_lock_class);
48 init_copy_kstat_irqs(old_desc, desc, cpu, nr_cpu_ids);
49 arch_init_copy_chip_data(old_desc, desc, cpu);
50}
51
52static void free_one_irq_desc(struct irq_desc *old_desc, struct irq_desc *desc)
53{
54 free_kstat_irqs(old_desc, desc);
55 arch_free_chip_data(old_desc, desc);
56}
57
58static struct irq_desc *__real_move_irq_desc(struct irq_desc *old_desc,
59 int cpu)
60{
61 struct irq_desc *desc;
62 unsigned int irq;
63 unsigned long flags;
64 int node;
65
66 irq = old_desc->irq;
67
68 spin_lock_irqsave(&sparse_irq_lock, flags);
69
70 /* We have to check it to avoid races with another CPU */
71 desc = irq_desc_ptrs[irq];
72
73 if (desc && old_desc != desc)
74 goto out_unlock;
75
76 node = cpu_to_node(cpu);
77 desc = kzalloc_node(sizeof(*desc), GFP_ATOMIC, node);
78 if (!desc) {
79 printk(KERN_ERR "irq %d: can not get new irq_desc for migration.\n", irq);
80 /* still use old one */
81 desc = old_desc;
82 goto out_unlock;
83 }
84 init_copy_one_irq_desc(irq, old_desc, desc, cpu);
85
86 irq_desc_ptrs[irq] = desc;
87
88 /* free the old one */
89 free_one_irq_desc(old_desc, desc);
90 kfree(old_desc);
91
92out_unlock:
93 spin_unlock_irqrestore(&sparse_irq_lock, flags);
94
95 return desc;
96}
97
98struct irq_desc *move_irq_desc(struct irq_desc *desc, int cpu)
99{
100 int old_cpu;
101 int node, old_node;
102
103 /* those all static, do move them */
104 if (desc->irq < NR_IRQS_LEGACY)
105 return desc;
106
107 old_cpu = desc->cpu;
108 if (old_cpu != cpu) {
109 node = cpu_to_node(cpu);
110 old_node = cpu_to_node(old_cpu);
111 if (old_node != node)
112 desc = __real_move_irq_desc(desc, cpu);
113 else
114 desc->cpu = cpu;
115 }
116
117 return desc;
118}
119
diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c
index d257e7d6a8a4..d2c0e5ee53c5 100644
--- a/kernel/irq/proc.c
+++ b/kernel/irq/proc.c
@@ -40,33 +40,42 @@ static ssize_t irq_affinity_proc_write(struct file *file,
40 const char __user *buffer, size_t count, loff_t *pos) 40 const char __user *buffer, size_t count, loff_t *pos)
41{ 41{
42 unsigned int irq = (int)(long)PDE(file->f_path.dentry->d_inode)->data; 42 unsigned int irq = (int)(long)PDE(file->f_path.dentry->d_inode)->data;
43 cpumask_t new_value; 43 cpumask_var_t new_value;
44 int err; 44 int err;
45 45
46 if (!irq_to_desc(irq)->chip->set_affinity || no_irq_affinity || 46 if (!irq_to_desc(irq)->chip->set_affinity || no_irq_affinity ||
47 irq_balancing_disabled(irq)) 47 irq_balancing_disabled(irq))
48 return -EIO; 48 return -EIO;
49 49
50 if (!alloc_cpumask_var(&new_value, GFP_KERNEL))
51 return -ENOMEM;
52
50 err = cpumask_parse_user(buffer, count, new_value); 53 err = cpumask_parse_user(buffer, count, new_value);
51 if (err) 54 if (err)
52 return err; 55 goto free_cpumask;
53 56
54 if (!is_affinity_mask_valid(new_value)) 57 if (!is_affinity_mask_valid(*new_value)) {
55 return -EINVAL; 58 err = -EINVAL;
59 goto free_cpumask;
60 }
56 61
57 /* 62 /*
58 * Do not allow disabling IRQs completely - it's a too easy 63 * Do not allow disabling IRQs completely - it's a too easy
59 * way to make the system unusable accidentally :-) At least 64 * way to make the system unusable accidentally :-) At least
60 * one online CPU still has to be targeted. 65 * one online CPU still has to be targeted.
61 */ 66 */
62 if (!cpus_intersects(new_value, cpu_online_map)) 67 if (!cpumask_intersects(new_value, cpu_online_mask)) {
63 /* Special case for empty set - allow the architecture 68 /* Special case for empty set - allow the architecture
64 code to set default SMP affinity. */ 69 code to set default SMP affinity. */
65 return irq_select_affinity_usr(irq) ? -EINVAL : count; 70 err = irq_select_affinity_usr(irq) ? -EINVAL : count;
66 71 } else {
67 irq_set_affinity(irq, new_value); 72 irq_set_affinity(irq, new_value);
73 err = count;
74 }
68 75
69 return count; 76free_cpumask:
77 free_cpumask_var(new_value);
78 return err;
70} 79}
71 80
72static int irq_affinity_proc_open(struct inode *inode, struct file *file) 81static int irq_affinity_proc_open(struct inode *inode, struct file *file)
@@ -95,7 +104,7 @@ static ssize_t default_affinity_write(struct file *file,
95 cpumask_t new_value; 104 cpumask_t new_value;
96 int err; 105 int err;
97 106
98 err = cpumask_parse_user(buffer, count, new_value); 107 err = cpumask_parse_user(buffer, count, &new_value);
99 if (err) 108 if (err)
100 return err; 109 return err;
101 110
@@ -243,7 +252,11 @@ void init_irq_proc(void)
243 /* 252 /*
244 * Create entries for all existing IRQs. 253 * Create entries for all existing IRQs.
245 */ 254 */
246 for_each_irq_desc(irq, desc) 255 for_each_irq_desc(irq, desc) {
256 if (!desc)
257 continue;
258
247 register_irq_proc(irq, desc); 259 register_irq_proc(irq, desc);
260 }
248} 261}
249 262
diff --git a/kernel/lockdep.c b/kernel/lockdep.c
index 74b1878b8bb8..06b0c3568f0b 100644
--- a/kernel/lockdep.c
+++ b/kernel/lockdep.c
@@ -137,16 +137,16 @@ static inline struct lock_class *hlock_class(struct held_lock *hlock)
137#ifdef CONFIG_LOCK_STAT 137#ifdef CONFIG_LOCK_STAT
138static DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS], lock_stats); 138static DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS], lock_stats);
139 139
140static int lock_contention_point(struct lock_class *class, unsigned long ip) 140static int lock_point(unsigned long points[], unsigned long ip)
141{ 141{
142 int i; 142 int i;
143 143
144 for (i = 0; i < ARRAY_SIZE(class->contention_point); i++) { 144 for (i = 0; i < LOCKSTAT_POINTS; i++) {
145 if (class->contention_point[i] == 0) { 145 if (points[i] == 0) {
146 class->contention_point[i] = ip; 146 points[i] = ip;
147 break; 147 break;
148 } 148 }
149 if (class->contention_point[i] == ip) 149 if (points[i] == ip)
150 break; 150 break;
151 } 151 }
152 152
@@ -186,6 +186,9 @@ struct lock_class_stats lock_stats(struct lock_class *class)
186 for (i = 0; i < ARRAY_SIZE(stats.contention_point); i++) 186 for (i = 0; i < ARRAY_SIZE(stats.contention_point); i++)
187 stats.contention_point[i] += pcs->contention_point[i]; 187 stats.contention_point[i] += pcs->contention_point[i];
188 188
189 for (i = 0; i < ARRAY_SIZE(stats.contending_point); i++)
190 stats.contending_point[i] += pcs->contending_point[i];
191
189 lock_time_add(&pcs->read_waittime, &stats.read_waittime); 192 lock_time_add(&pcs->read_waittime, &stats.read_waittime);
190 lock_time_add(&pcs->write_waittime, &stats.write_waittime); 193 lock_time_add(&pcs->write_waittime, &stats.write_waittime);
191 194
@@ -210,6 +213,7 @@ void clear_lock_stats(struct lock_class *class)
210 memset(cpu_stats, 0, sizeof(struct lock_class_stats)); 213 memset(cpu_stats, 0, sizeof(struct lock_class_stats));
211 } 214 }
212 memset(class->contention_point, 0, sizeof(class->contention_point)); 215 memset(class->contention_point, 0, sizeof(class->contention_point));
216 memset(class->contending_point, 0, sizeof(class->contending_point));
213} 217}
214 218
215static struct lock_class_stats *get_lock_stats(struct lock_class *class) 219static struct lock_class_stats *get_lock_stats(struct lock_class *class)
@@ -288,14 +292,12 @@ void lockdep_off(void)
288{ 292{
289 current->lockdep_recursion++; 293 current->lockdep_recursion++;
290} 294}
291
292EXPORT_SYMBOL(lockdep_off); 295EXPORT_SYMBOL(lockdep_off);
293 296
294void lockdep_on(void) 297void lockdep_on(void)
295{ 298{
296 current->lockdep_recursion--; 299 current->lockdep_recursion--;
297} 300}
298
299EXPORT_SYMBOL(lockdep_on); 301EXPORT_SYMBOL(lockdep_on);
300 302
301/* 303/*
@@ -577,7 +579,8 @@ static void print_lock_class_header(struct lock_class *class, int depth)
577/* 579/*
578 * printk all lock dependencies starting at <entry>: 580 * printk all lock dependencies starting at <entry>:
579 */ 581 */
580static void print_lock_dependencies(struct lock_class *class, int depth) 582static void __used
583print_lock_dependencies(struct lock_class *class, int depth)
581{ 584{
582 struct lock_list *entry; 585 struct lock_list *entry;
583 586
@@ -2509,7 +2512,6 @@ void lockdep_init_map(struct lockdep_map *lock, const char *name,
2509 if (subclass) 2512 if (subclass)
2510 register_lock_class(lock, subclass, 1); 2513 register_lock_class(lock, subclass, 1);
2511} 2514}
2512
2513EXPORT_SYMBOL_GPL(lockdep_init_map); 2515EXPORT_SYMBOL_GPL(lockdep_init_map);
2514 2516
2515/* 2517/*
@@ -2690,8 +2692,9 @@ static int check_unlock(struct task_struct *curr, struct lockdep_map *lock,
2690} 2692}
2691 2693
2692static int 2694static int
2693__lock_set_subclass(struct lockdep_map *lock, 2695__lock_set_class(struct lockdep_map *lock, const char *name,
2694 unsigned int subclass, unsigned long ip) 2696 struct lock_class_key *key, unsigned int subclass,
2697 unsigned long ip)
2695{ 2698{
2696 struct task_struct *curr = current; 2699 struct task_struct *curr = current;
2697 struct held_lock *hlock, *prev_hlock; 2700 struct held_lock *hlock, *prev_hlock;
@@ -2718,6 +2721,7 @@ __lock_set_subclass(struct lockdep_map *lock,
2718 return print_unlock_inbalance_bug(curr, lock, ip); 2721 return print_unlock_inbalance_bug(curr, lock, ip);
2719 2722
2720found_it: 2723found_it:
2724 lockdep_init_map(lock, name, key, 0);
2721 class = register_lock_class(lock, subclass, 0); 2725 class = register_lock_class(lock, subclass, 0);
2722 hlock->class_idx = class - lock_classes + 1; 2726 hlock->class_idx = class - lock_classes + 1;
2723 2727
@@ -2902,9 +2906,9 @@ static void check_flags(unsigned long flags)
2902#endif 2906#endif
2903} 2907}
2904 2908
2905void 2909void lock_set_class(struct lockdep_map *lock, const char *name,
2906lock_set_subclass(struct lockdep_map *lock, 2910 struct lock_class_key *key, unsigned int subclass,
2907 unsigned int subclass, unsigned long ip) 2911 unsigned long ip)
2908{ 2912{
2909 unsigned long flags; 2913 unsigned long flags;
2910 2914
@@ -2914,13 +2918,12 @@ lock_set_subclass(struct lockdep_map *lock,
2914 raw_local_irq_save(flags); 2918 raw_local_irq_save(flags);
2915 current->lockdep_recursion = 1; 2919 current->lockdep_recursion = 1;
2916 check_flags(flags); 2920 check_flags(flags);
2917 if (__lock_set_subclass(lock, subclass, ip)) 2921 if (__lock_set_class(lock, name, key, subclass, ip))
2918 check_chain_key(current); 2922 check_chain_key(current);
2919 current->lockdep_recursion = 0; 2923 current->lockdep_recursion = 0;
2920 raw_local_irq_restore(flags); 2924 raw_local_irq_restore(flags);
2921} 2925}
2922 2926EXPORT_SYMBOL_GPL(lock_set_class);
2923EXPORT_SYMBOL_GPL(lock_set_subclass);
2924 2927
2925/* 2928/*
2926 * We are not always called with irqs disabled - do that here, 2929 * We are not always called with irqs disabled - do that here,
@@ -2944,7 +2947,6 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
2944 current->lockdep_recursion = 0; 2947 current->lockdep_recursion = 0;
2945 raw_local_irq_restore(flags); 2948 raw_local_irq_restore(flags);
2946} 2949}
2947
2948EXPORT_SYMBOL_GPL(lock_acquire); 2950EXPORT_SYMBOL_GPL(lock_acquire);
2949 2951
2950void lock_release(struct lockdep_map *lock, int nested, 2952void lock_release(struct lockdep_map *lock, int nested,
@@ -2962,7 +2964,6 @@ void lock_release(struct lockdep_map *lock, int nested,
2962 current->lockdep_recursion = 0; 2964 current->lockdep_recursion = 0;
2963 raw_local_irq_restore(flags); 2965 raw_local_irq_restore(flags);
2964} 2966}
2965
2966EXPORT_SYMBOL_GPL(lock_release); 2967EXPORT_SYMBOL_GPL(lock_release);
2967 2968
2968#ifdef CONFIG_LOCK_STAT 2969#ifdef CONFIG_LOCK_STAT
@@ -3000,7 +3001,7 @@ __lock_contended(struct lockdep_map *lock, unsigned long ip)
3000 struct held_lock *hlock, *prev_hlock; 3001 struct held_lock *hlock, *prev_hlock;
3001 struct lock_class_stats *stats; 3002 struct lock_class_stats *stats;
3002 unsigned int depth; 3003 unsigned int depth;
3003 int i, point; 3004 int i, contention_point, contending_point;
3004 3005
3005 depth = curr->lockdep_depth; 3006 depth = curr->lockdep_depth;
3006 if (DEBUG_LOCKS_WARN_ON(!depth)) 3007 if (DEBUG_LOCKS_WARN_ON(!depth))
@@ -3024,18 +3025,22 @@ __lock_contended(struct lockdep_map *lock, unsigned long ip)
3024found_it: 3025found_it:
3025 hlock->waittime_stamp = sched_clock(); 3026 hlock->waittime_stamp = sched_clock();
3026 3027
3027 point = lock_contention_point(hlock_class(hlock), ip); 3028 contention_point = lock_point(hlock_class(hlock)->contention_point, ip);
3029 contending_point = lock_point(hlock_class(hlock)->contending_point,
3030 lock->ip);
3028 3031
3029 stats = get_lock_stats(hlock_class(hlock)); 3032 stats = get_lock_stats(hlock_class(hlock));
3030 if (point < ARRAY_SIZE(stats->contention_point)) 3033 if (contention_point < LOCKSTAT_POINTS)
3031 stats->contention_point[point]++; 3034 stats->contention_point[contention_point]++;
3035 if (contending_point < LOCKSTAT_POINTS)
3036 stats->contending_point[contending_point]++;
3032 if (lock->cpu != smp_processor_id()) 3037 if (lock->cpu != smp_processor_id())
3033 stats->bounces[bounce_contended + !!hlock->read]++; 3038 stats->bounces[bounce_contended + !!hlock->read]++;
3034 put_lock_stats(stats); 3039 put_lock_stats(stats);
3035} 3040}
3036 3041
3037static void 3042static void
3038__lock_acquired(struct lockdep_map *lock) 3043__lock_acquired(struct lockdep_map *lock, unsigned long ip)
3039{ 3044{
3040 struct task_struct *curr = current; 3045 struct task_struct *curr = current;
3041 struct held_lock *hlock, *prev_hlock; 3046 struct held_lock *hlock, *prev_hlock;
@@ -3084,6 +3089,7 @@ found_it:
3084 put_lock_stats(stats); 3089 put_lock_stats(stats);
3085 3090
3086 lock->cpu = cpu; 3091 lock->cpu = cpu;
3092 lock->ip = ip;
3087} 3093}
3088 3094
3089void lock_contended(struct lockdep_map *lock, unsigned long ip) 3095void lock_contended(struct lockdep_map *lock, unsigned long ip)
@@ -3105,7 +3111,7 @@ void lock_contended(struct lockdep_map *lock, unsigned long ip)
3105} 3111}
3106EXPORT_SYMBOL_GPL(lock_contended); 3112EXPORT_SYMBOL_GPL(lock_contended);
3107 3113
3108void lock_acquired(struct lockdep_map *lock) 3114void lock_acquired(struct lockdep_map *lock, unsigned long ip)
3109{ 3115{
3110 unsigned long flags; 3116 unsigned long flags;
3111 3117
@@ -3118,7 +3124,7 @@ void lock_acquired(struct lockdep_map *lock)
3118 raw_local_irq_save(flags); 3124 raw_local_irq_save(flags);
3119 check_flags(flags); 3125 check_flags(flags);
3120 current->lockdep_recursion = 1; 3126 current->lockdep_recursion = 1;
3121 __lock_acquired(lock); 3127 __lock_acquired(lock, ip);
3122 current->lockdep_recursion = 0; 3128 current->lockdep_recursion = 0;
3123 raw_local_irq_restore(flags); 3129 raw_local_irq_restore(flags);
3124} 3130}
@@ -3442,7 +3448,6 @@ retry:
3442 if (unlock) 3448 if (unlock)
3443 read_unlock(&tasklist_lock); 3449 read_unlock(&tasklist_lock);
3444} 3450}
3445
3446EXPORT_SYMBOL_GPL(debug_show_all_locks); 3451EXPORT_SYMBOL_GPL(debug_show_all_locks);
3447 3452
3448/* 3453/*
@@ -3463,7 +3468,6 @@ void debug_show_held_locks(struct task_struct *task)
3463{ 3468{
3464 __debug_show_held_locks(task); 3469 __debug_show_held_locks(task);
3465} 3470}
3466
3467EXPORT_SYMBOL_GPL(debug_show_held_locks); 3471EXPORT_SYMBOL_GPL(debug_show_held_locks);
3468 3472
3469void lockdep_sys_exit(void) 3473void lockdep_sys_exit(void)
diff --git a/kernel/lockdep_proc.c b/kernel/lockdep_proc.c
index 20dbcbf9c7dd..13716b813896 100644
--- a/kernel/lockdep_proc.c
+++ b/kernel/lockdep_proc.c
@@ -470,11 +470,12 @@ static void seq_line(struct seq_file *m, char c, int offset, int length)
470 470
471static void snprint_time(char *buf, size_t bufsiz, s64 nr) 471static void snprint_time(char *buf, size_t bufsiz, s64 nr)
472{ 472{
473 unsigned long rem; 473 s64 div;
474 s32 rem;
474 475
475 nr += 5; /* for display rounding */ 476 nr += 5; /* for display rounding */
476 rem = do_div(nr, 1000); /* XXX: do_div_signed */ 477 div = div_s64_rem(nr, 1000, &rem);
477 snprintf(buf, bufsiz, "%lld.%02d", (long long)nr, (int)rem/10); 478 snprintf(buf, bufsiz, "%lld.%02d", (long long)div, (int)rem/10);
478} 479}
479 480
480static void seq_time(struct seq_file *m, s64 time) 481static void seq_time(struct seq_file *m, s64 time)
@@ -556,7 +557,7 @@ static void seq_stats(struct seq_file *m, struct lock_stat_data *data)
556 if (stats->read_holdtime.nr) 557 if (stats->read_holdtime.nr)
557 namelen += 2; 558 namelen += 2;
558 559
559 for (i = 0; i < ARRAY_SIZE(class->contention_point); i++) { 560 for (i = 0; i < LOCKSTAT_POINTS; i++) {
560 char sym[KSYM_SYMBOL_LEN]; 561 char sym[KSYM_SYMBOL_LEN];
561 char ip[32]; 562 char ip[32];
562 563
@@ -573,6 +574,23 @@ static void seq_stats(struct seq_file *m, struct lock_stat_data *data)
573 stats->contention_point[i], 574 stats->contention_point[i],
574 ip, sym); 575 ip, sym);
575 } 576 }
577 for (i = 0; i < LOCKSTAT_POINTS; i++) {
578 char sym[KSYM_SYMBOL_LEN];
579 char ip[32];
580
581 if (class->contending_point[i] == 0)
582 break;
583
584 if (!i)
585 seq_line(m, '-', 40-namelen, namelen);
586
587 sprint_symbol(sym, class->contending_point[i]);
588 snprintf(ip, sizeof(ip), "[<%p>]",
589 (void *)class->contending_point[i]);
590 seq_printf(m, "%40s %14lu %29s %s\n", name,
591 stats->contending_point[i],
592 ip, sym);
593 }
576 if (i) { 594 if (i) {
577 seq_puts(m, "\n"); 595 seq_puts(m, "\n");
578 seq_line(m, '.', 0, 40 + 1 + 10 * (14 + 1)); 596 seq_line(m, '.', 0, 40 + 1 + 10 * (14 + 1));
@@ -582,7 +600,7 @@ static void seq_stats(struct seq_file *m, struct lock_stat_data *data)
582 600
583static void seq_header(struct seq_file *m) 601static void seq_header(struct seq_file *m)
584{ 602{
585 seq_printf(m, "lock_stat version 0.2\n"); 603 seq_printf(m, "lock_stat version 0.3\n");
586 seq_line(m, '-', 0, 40 + 1 + 10 * (14 + 1)); 604 seq_line(m, '-', 0, 40 + 1 + 10 * (14 + 1));
587 seq_printf(m, "%40s %14s %14s %14s %14s %14s %14s %14s %14s " 605 seq_printf(m, "%40s %14s %14s %14s %14s %14s %14s %14s %14s "
588 "%14s %14s\n", 606 "%14s %14s\n",
diff --git a/kernel/mutex.c b/kernel/mutex.c
index 12c779dc65d4..4f45d4b658ef 100644
--- a/kernel/mutex.c
+++ b/kernel/mutex.c
@@ -59,7 +59,7 @@ EXPORT_SYMBOL(__mutex_init);
59 * We also put the fastpath first in the kernel image, to make sure the 59 * We also put the fastpath first in the kernel image, to make sure the
60 * branch is predicted by the CPU as default-untaken. 60 * branch is predicted by the CPU as default-untaken.
61 */ 61 */
62static void noinline __sched 62static __used noinline void __sched
63__mutex_lock_slowpath(atomic_t *lock_count); 63__mutex_lock_slowpath(atomic_t *lock_count);
64 64
65/*** 65/***
@@ -96,7 +96,7 @@ void inline __sched mutex_lock(struct mutex *lock)
96EXPORT_SYMBOL(mutex_lock); 96EXPORT_SYMBOL(mutex_lock);
97#endif 97#endif
98 98
99static noinline void __sched __mutex_unlock_slowpath(atomic_t *lock_count); 99static __used noinline void __sched __mutex_unlock_slowpath(atomic_t *lock_count);
100 100
101/*** 101/***
102 * mutex_unlock - release the mutex 102 * mutex_unlock - release the mutex
@@ -184,7 +184,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
184 } 184 }
185 185
186done: 186done:
187 lock_acquired(&lock->dep_map); 187 lock_acquired(&lock->dep_map, ip);
188 /* got the lock - rejoice! */ 188 /* got the lock - rejoice! */
189 mutex_remove_waiter(lock, &waiter, task_thread_info(task)); 189 mutex_remove_waiter(lock, &waiter, task_thread_info(task));
190 debug_mutex_set_owner(lock, task_thread_info(task)); 190 debug_mutex_set_owner(lock, task_thread_info(task));
@@ -268,7 +268,7 @@ __mutex_unlock_common_slowpath(atomic_t *lock_count, int nested)
268/* 268/*
269 * Release the lock, slowpath: 269 * Release the lock, slowpath:
270 */ 270 */
271static noinline void 271static __used noinline void
272__mutex_unlock_slowpath(atomic_t *lock_count) 272__mutex_unlock_slowpath(atomic_t *lock_count)
273{ 273{
274 __mutex_unlock_common_slowpath(lock_count, 1); 274 __mutex_unlock_common_slowpath(lock_count, 1);
@@ -313,7 +313,7 @@ int __sched mutex_lock_killable(struct mutex *lock)
313} 313}
314EXPORT_SYMBOL(mutex_lock_killable); 314EXPORT_SYMBOL(mutex_lock_killable);
315 315
316static noinline void __sched 316static __used noinline void __sched
317__mutex_lock_slowpath(atomic_t *lock_count) 317__mutex_lock_slowpath(atomic_t *lock_count)
318{ 318{
319 struct mutex *lock = container_of(lock_count, struct mutex, count); 319 struct mutex *lock = container_of(lock_count, struct mutex, count);
diff --git a/kernel/notifier.c b/kernel/notifier.c
index 4282c0a40a57..61d5aa5eced3 100644
--- a/kernel/notifier.c
+++ b/kernel/notifier.c
@@ -82,6 +82,14 @@ static int __kprobes notifier_call_chain(struct notifier_block **nl,
82 82
83 while (nb && nr_to_call) { 83 while (nb && nr_to_call) {
84 next_nb = rcu_dereference(nb->next); 84 next_nb = rcu_dereference(nb->next);
85
86#ifdef CONFIG_DEBUG_NOTIFIERS
87 if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
88 WARN(1, "Invalid notifier called!");
89 nb = next_nb;
90 continue;
91 }
92#endif
85 ret = nb->notifier_call(nb, val, v); 93 ret = nb->notifier_call(nb, val, v);
86 94
87 if (nr_calls) 95 if (nr_calls)
diff --git a/kernel/panic.c b/kernel/panic.c
index 4d5088355bfe..13f06349a786 100644
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -21,6 +21,7 @@
21#include <linux/debug_locks.h> 21#include <linux/debug_locks.h>
22#include <linux/random.h> 22#include <linux/random.h>
23#include <linux/kallsyms.h> 23#include <linux/kallsyms.h>
24#include <linux/dmi.h>
24 25
25int panic_on_oops; 26int panic_on_oops;
26static unsigned long tainted_mask; 27static unsigned long tainted_mask;
@@ -321,36 +322,27 @@ void oops_exit(void)
321} 322}
322 323
323#ifdef WANT_WARN_ON_SLOWPATH 324#ifdef WANT_WARN_ON_SLOWPATH
324void warn_on_slowpath(const char *file, int line)
325{
326 char function[KSYM_SYMBOL_LEN];
327 unsigned long caller = (unsigned long) __builtin_return_address(0);
328 sprint_symbol(function, caller);
329
330 printk(KERN_WARNING "------------[ cut here ]------------\n");
331 printk(KERN_WARNING "WARNING: at %s:%d %s()\n", file,
332 line, function);
333 print_modules();
334 dump_stack();
335 print_oops_end_marker();
336 add_taint(TAINT_WARN);
337}
338EXPORT_SYMBOL(warn_on_slowpath);
339
340
341void warn_slowpath(const char *file, int line, const char *fmt, ...) 325void warn_slowpath(const char *file, int line, const char *fmt, ...)
342{ 326{
343 va_list args; 327 va_list args;
344 char function[KSYM_SYMBOL_LEN]; 328 char function[KSYM_SYMBOL_LEN];
345 unsigned long caller = (unsigned long)__builtin_return_address(0); 329 unsigned long caller = (unsigned long)__builtin_return_address(0);
330 const char *board;
331
346 sprint_symbol(function, caller); 332 sprint_symbol(function, caller);
347 333
348 printk(KERN_WARNING "------------[ cut here ]------------\n"); 334 printk(KERN_WARNING "------------[ cut here ]------------\n");
349 printk(KERN_WARNING "WARNING: at %s:%d %s()\n", file, 335 printk(KERN_WARNING "WARNING: at %s:%d %s()\n", file,
350 line, function); 336 line, function);
351 va_start(args, fmt); 337 board = dmi_get_system_info(DMI_PRODUCT_NAME);
352 vprintk(fmt, args); 338 if (board)
353 va_end(args); 339 printk(KERN_WARNING "Hardware name: %s\n", board);
340
341 if (fmt) {
342 va_start(args, fmt);
343 vprintk(fmt, args);
344 va_end(args);
345 }
354 346
355 print_modules(); 347 print_modules();
356 dump_stack(); 348 dump_stack();
diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c
index 4e5288a831de..157de3a47832 100644
--- a/kernel/posix-cpu-timers.c
+++ b/kernel/posix-cpu-timers.c
@@ -58,21 +58,21 @@ void thread_group_cputime(
58 struct task_struct *tsk, 58 struct task_struct *tsk,
59 struct task_cputime *times) 59 struct task_cputime *times)
60{ 60{
61 struct signal_struct *sig; 61 struct task_cputime *totals, *tot;
62 int i; 62 int i;
63 struct task_cputime *tot;
64 63
65 sig = tsk->signal; 64 totals = tsk->signal->cputime.totals;
66 if (unlikely(!sig) || !sig->cputime.totals) { 65 if (!totals) {
67 times->utime = tsk->utime; 66 times->utime = tsk->utime;
68 times->stime = tsk->stime; 67 times->stime = tsk->stime;
69 times->sum_exec_runtime = tsk->se.sum_exec_runtime; 68 times->sum_exec_runtime = tsk->se.sum_exec_runtime;
70 return; 69 return;
71 } 70 }
71
72 times->stime = times->utime = cputime_zero; 72 times->stime = times->utime = cputime_zero;
73 times->sum_exec_runtime = 0; 73 times->sum_exec_runtime = 0;
74 for_each_possible_cpu(i) { 74 for_each_possible_cpu(i) {
75 tot = per_cpu_ptr(tsk->signal->cputime.totals, i); 75 tot = per_cpu_ptr(totals, i);
76 times->utime = cputime_add(times->utime, tot->utime); 76 times->utime = cputime_add(times->utime, tot->utime);
77 times->stime = cputime_add(times->stime, tot->stime); 77 times->stime = cputime_add(times->stime, tot->stime);
78 times->sum_exec_runtime += tot->sum_exec_runtime; 78 times->sum_exec_runtime += tot->sum_exec_runtime;
diff --git a/kernel/posix-timers.c b/kernel/posix-timers.c
index a140e44eebba..887c63787de6 100644
--- a/kernel/posix-timers.c
+++ b/kernel/posix-timers.c
@@ -116,7 +116,7 @@ static DEFINE_SPINLOCK(idr_lock);
116 * must supply functions here, even if the function just returns 116 * must supply functions here, even if the function just returns
117 * ENOSYS. The standard POSIX timer management code assumes the 117 * ENOSYS. The standard POSIX timer management code assumes the
118 * following: 1.) The k_itimer struct (sched.h) is used for the 118 * following: 1.) The k_itimer struct (sched.h) is used for the
119 * timer. 2.) The list, it_lock, it_clock, it_id and it_process 119 * timer. 2.) The list, it_lock, it_clock, it_id and it_pid
120 * fields are not modified by timer code. 120 * fields are not modified by timer code.
121 * 121 *
122 * At this time all functions EXCEPT clock_nanosleep can be 122 * At this time all functions EXCEPT clock_nanosleep can be
@@ -319,7 +319,8 @@ void do_schedule_next_timer(struct siginfo *info)
319 319
320int posix_timer_event(struct k_itimer *timr, int si_private) 320int posix_timer_event(struct k_itimer *timr, int si_private)
321{ 321{
322 int shared, ret; 322 struct task_struct *task;
323 int shared, ret = -1;
323 /* 324 /*
324 * FIXME: if ->sigq is queued we can race with 325 * FIXME: if ->sigq is queued we can race with
325 * dequeue_signal()->do_schedule_next_timer(). 326 * dequeue_signal()->do_schedule_next_timer().
@@ -333,8 +334,13 @@ int posix_timer_event(struct k_itimer *timr, int si_private)
333 */ 334 */
334 timr->sigq->info.si_sys_private = si_private; 335 timr->sigq->info.si_sys_private = si_private;
335 336
336 shared = !(timr->it_sigev_notify & SIGEV_THREAD_ID); 337 rcu_read_lock();
337 ret = send_sigqueue(timr->sigq, timr->it_process, shared); 338 task = pid_task(timr->it_pid, PIDTYPE_PID);
339 if (task) {
340 shared = !(timr->it_sigev_notify & SIGEV_THREAD_ID);
341 ret = send_sigqueue(timr->sigq, task, shared);
342 }
343 rcu_read_unlock();
338 /* If we failed to send the signal the timer stops. */ 344 /* If we failed to send the signal the timer stops. */
339 return ret > 0; 345 return ret > 0;
340} 346}
@@ -411,7 +417,7 @@ static enum hrtimer_restart posix_timer_fn(struct hrtimer *timer)
411 return ret; 417 return ret;
412} 418}
413 419
414static struct task_struct * good_sigevent(sigevent_t * event) 420static struct pid *good_sigevent(sigevent_t * event)
415{ 421{
416 struct task_struct *rtn = current->group_leader; 422 struct task_struct *rtn = current->group_leader;
417 423
@@ -425,7 +431,7 @@ static struct task_struct * good_sigevent(sigevent_t * event)
425 ((event->sigev_signo <= 0) || (event->sigev_signo > SIGRTMAX))) 431 ((event->sigev_signo <= 0) || (event->sigev_signo > SIGRTMAX)))
426 return NULL; 432 return NULL;
427 433
428 return rtn; 434 return task_pid(rtn);
429} 435}
430 436
431void register_posix_clock(const clockid_t clock_id, struct k_clock *new_clock) 437void register_posix_clock(const clockid_t clock_id, struct k_clock *new_clock)
@@ -464,6 +470,7 @@ static void release_posix_timer(struct k_itimer *tmr, int it_id_set)
464 idr_remove(&posix_timers_id, tmr->it_id); 470 idr_remove(&posix_timers_id, tmr->it_id);
465 spin_unlock_irqrestore(&idr_lock, flags); 471 spin_unlock_irqrestore(&idr_lock, flags);
466 } 472 }
473 put_pid(tmr->it_pid);
467 sigqueue_free(tmr->sigq); 474 sigqueue_free(tmr->sigq);
468 kmem_cache_free(posix_timers_cache, tmr); 475 kmem_cache_free(posix_timers_cache, tmr);
469} 476}
@@ -477,7 +484,6 @@ sys_timer_create(const clockid_t which_clock,
477{ 484{
478 struct k_itimer *new_timer; 485 struct k_itimer *new_timer;
479 int error, new_timer_id; 486 int error, new_timer_id;
480 struct task_struct *process;
481 sigevent_t event; 487 sigevent_t event;
482 int it_id_set = IT_ID_NOT_SET; 488 int it_id_set = IT_ID_NOT_SET;
483 489
@@ -531,11 +537,9 @@ sys_timer_create(const clockid_t which_clock,
531 goto out; 537 goto out;
532 } 538 }
533 rcu_read_lock(); 539 rcu_read_lock();
534 process = good_sigevent(&event); 540 new_timer->it_pid = get_pid(good_sigevent(&event));
535 if (process)
536 get_task_struct(process);
537 rcu_read_unlock(); 541 rcu_read_unlock();
538 if (!process) { 542 if (!new_timer->it_pid) {
539 error = -EINVAL; 543 error = -EINVAL;
540 goto out; 544 goto out;
541 } 545 }
@@ -543,8 +547,7 @@ sys_timer_create(const clockid_t which_clock,
543 event.sigev_notify = SIGEV_SIGNAL; 547 event.sigev_notify = SIGEV_SIGNAL;
544 event.sigev_signo = SIGALRM; 548 event.sigev_signo = SIGALRM;
545 event.sigev_value.sival_int = new_timer->it_id; 549 event.sigev_value.sival_int = new_timer->it_id;
546 process = current->group_leader; 550 new_timer->it_pid = get_pid(task_tgid(current));
547 get_task_struct(process);
548 } 551 }
549 552
550 new_timer->it_sigev_notify = event.sigev_notify; 553 new_timer->it_sigev_notify = event.sigev_notify;
@@ -554,7 +557,7 @@ sys_timer_create(const clockid_t which_clock,
554 new_timer->sigq->info.si_code = SI_TIMER; 557 new_timer->sigq->info.si_code = SI_TIMER;
555 558
556 spin_lock_irq(&current->sighand->siglock); 559 spin_lock_irq(&current->sighand->siglock);
557 new_timer->it_process = process; 560 new_timer->it_signal = current->signal;
558 list_add(&new_timer->list, &current->signal->posix_timers); 561 list_add(&new_timer->list, &current->signal->posix_timers);
559 spin_unlock_irq(&current->sighand->siglock); 562 spin_unlock_irq(&current->sighand->siglock);
560 563
@@ -589,8 +592,7 @@ static struct k_itimer *lock_timer(timer_t timer_id, unsigned long *flags)
589 timr = idr_find(&posix_timers_id, (int)timer_id); 592 timr = idr_find(&posix_timers_id, (int)timer_id);
590 if (timr) { 593 if (timr) {
591 spin_lock(&timr->it_lock); 594 spin_lock(&timr->it_lock);
592 if (timr->it_process && 595 if (timr->it_signal == current->signal) {
593 same_thread_group(timr->it_process, current)) {
594 spin_unlock(&idr_lock); 596 spin_unlock(&idr_lock);
595 return timr; 597 return timr;
596 } 598 }
@@ -837,8 +839,7 @@ retry_delete:
837 * This keeps any tasks waiting on the spin lock from thinking 839 * This keeps any tasks waiting on the spin lock from thinking
838 * they got something (see the lock code above). 840 * they got something (see the lock code above).
839 */ 841 */
840 put_task_struct(timer->it_process); 842 timer->it_signal = NULL;
841 timer->it_process = NULL;
842 843
843 unlock_timer(timer, flags); 844 unlock_timer(timer, flags);
844 release_posix_timer(timer, IT_ID_SET); 845 release_posix_timer(timer, IT_ID_SET);
@@ -864,8 +865,7 @@ retry_delete:
864 * This keeps any tasks waiting on the spin lock from thinking 865 * This keeps any tasks waiting on the spin lock from thinking
865 * they got something (see the lock code above). 866 * they got something (see the lock code above).
866 */ 867 */
867 put_task_struct(timer->it_process); 868 timer->it_signal = NULL;
868 timer->it_process = NULL;
869 869
870 unlock_timer(timer, flags); 870 unlock_timer(timer, flags);
871 release_posix_timer(timer, IT_ID_SET); 871 release_posix_timer(timer, IT_ID_SET);
diff --git a/kernel/printk.c b/kernel/printk.c
index f492f1583d77..e651ab05655f 100644
--- a/kernel/printk.c
+++ b/kernel/printk.c
@@ -662,7 +662,7 @@ asmlinkage int vprintk(const char *fmt, va_list args)
662 if (recursion_bug) { 662 if (recursion_bug) {
663 recursion_bug = 0; 663 recursion_bug = 0;
664 strcpy(printk_buf, recursion_bug_msg); 664 strcpy(printk_buf, recursion_bug_msg);
665 printed_len = sizeof(recursion_bug_msg); 665 printed_len = strlen(recursion_bug_msg);
666 } 666 }
667 /* Emit the output into the temporary buffer */ 667 /* Emit the output into the temporary buffer */
668 printed_len += vscnprintf(printk_buf + printed_len, 668 printed_len += vscnprintf(printk_buf + printed_len,
diff --git a/kernel/profile.c b/kernel/profile.c
index 60adefb59b5e..4cb7d68fed82 100644
--- a/kernel/profile.c
+++ b/kernel/profile.c
@@ -442,7 +442,7 @@ void profile_tick(int type)
442static int prof_cpu_mask_read_proc(char *page, char **start, off_t off, 442static int prof_cpu_mask_read_proc(char *page, char **start, off_t off,
443 int count, int *eof, void *data) 443 int count, int *eof, void *data)
444{ 444{
445 int len = cpumask_scnprintf(page, count, *(cpumask_t *)data); 445 int len = cpumask_scnprintf(page, count, (cpumask_t *)data);
446 if (count - len < 2) 446 if (count - len < 2)
447 return -EINVAL; 447 return -EINVAL;
448 len += sprintf(page + len, "\n"); 448 len += sprintf(page + len, "\n");
@@ -456,7 +456,7 @@ static int prof_cpu_mask_write_proc(struct file *file,
456 unsigned long full_count = count, err; 456 unsigned long full_count = count, err;
457 cpumask_t new_value; 457 cpumask_t new_value;
458 458
459 err = cpumask_parse_user(buffer, count, new_value); 459 err = cpumask_parse_user(buffer, count, &new_value);
460 if (err) 460 if (err)
461 return err; 461 return err;
462 462
diff --git a/kernel/rcuclassic.c b/kernel/rcuclassic.c
index 37f72e551542..c03ca3e61919 100644
--- a/kernel/rcuclassic.c
+++ b/kernel/rcuclassic.c
@@ -191,7 +191,7 @@ static void print_other_cpu_stall(struct rcu_ctrlblk *rcp)
191 191
192 /* OK, time to rat on our buddy... */ 192 /* OK, time to rat on our buddy... */
193 193
194 printk(KERN_ERR "RCU detected CPU stalls:"); 194 printk(KERN_ERR "INFO: RCU detected CPU stalls:");
195 for_each_possible_cpu(cpu) { 195 for_each_possible_cpu(cpu) {
196 if (cpu_isset(cpu, rcp->cpumask)) 196 if (cpu_isset(cpu, rcp->cpumask))
197 printk(" %d", cpu); 197 printk(" %d", cpu);
@@ -204,7 +204,7 @@ static void print_cpu_stall(struct rcu_ctrlblk *rcp)
204{ 204{
205 unsigned long flags; 205 unsigned long flags;
206 206
207 printk(KERN_ERR "RCU detected CPU %d stall (t=%lu/%lu jiffies)\n", 207 printk(KERN_ERR "INFO: RCU detected CPU %d stall (t=%lu/%lu jiffies)\n",
208 smp_processor_id(), jiffies, 208 smp_processor_id(), jiffies,
209 jiffies - rcp->gp_start); 209 jiffies - rcp->gp_start);
210 dump_stack(); 210 dump_stack();
@@ -393,7 +393,7 @@ static void rcu_start_batch(struct rcu_ctrlblk *rcp)
393 * unnecessarily. 393 * unnecessarily.
394 */ 394 */
395 smp_mb(); 395 smp_mb();
396 cpus_andnot(rcp->cpumask, cpu_online_map, nohz_cpu_mask); 396 cpumask_andnot(&rcp->cpumask, cpu_online_mask, nohz_cpu_mask);
397 397
398 rcp->signaled = 0; 398 rcp->signaled = 0;
399 } 399 }
diff --git a/kernel/rcupreempt.c b/kernel/rcupreempt.c
index 59236e8b9daa..04982659875a 100644
--- a/kernel/rcupreempt.c
+++ b/kernel/rcupreempt.c
@@ -551,6 +551,16 @@ void rcu_irq_exit(void)
551 } 551 }
552} 552}
553 553
554void rcu_nmi_enter(void)
555{
556 rcu_irq_enter();
557}
558
559void rcu_nmi_exit(void)
560{
561 rcu_irq_exit();
562}
563
554static void dyntick_save_progress_counter(int cpu) 564static void dyntick_save_progress_counter(int cpu)
555{ 565{
556 struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu); 566 struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
diff --git a/kernel/rcupreempt_trace.c b/kernel/rcupreempt_trace.c
index 35c2d3360ecf..7c2665cac172 100644
--- a/kernel/rcupreempt_trace.c
+++ b/kernel/rcupreempt_trace.c
@@ -149,12 +149,12 @@ static void rcupreempt_trace_sum(struct rcupreempt_trace *sp)
149 sp->done_length += cp->done_length; 149 sp->done_length += cp->done_length;
150 sp->done_add += cp->done_add; 150 sp->done_add += cp->done_add;
151 sp->done_remove += cp->done_remove; 151 sp->done_remove += cp->done_remove;
152 atomic_set(&sp->done_invoked, atomic_read(&cp->done_invoked)); 152 atomic_add(atomic_read(&cp->done_invoked), &sp->done_invoked);
153 sp->rcu_check_callbacks += cp->rcu_check_callbacks; 153 sp->rcu_check_callbacks += cp->rcu_check_callbacks;
154 atomic_set(&sp->rcu_try_flip_1, 154 atomic_add(atomic_read(&cp->rcu_try_flip_1),
155 atomic_read(&cp->rcu_try_flip_1)); 155 &sp->rcu_try_flip_1);
156 atomic_set(&sp->rcu_try_flip_e1, 156 atomic_add(atomic_read(&cp->rcu_try_flip_e1),
157 atomic_read(&cp->rcu_try_flip_e1)); 157 &sp->rcu_try_flip_e1);
158 sp->rcu_try_flip_i1 += cp->rcu_try_flip_i1; 158 sp->rcu_try_flip_i1 += cp->rcu_try_flip_i1;
159 sp->rcu_try_flip_ie1 += cp->rcu_try_flip_ie1; 159 sp->rcu_try_flip_ie1 += cp->rcu_try_flip_ie1;
160 sp->rcu_try_flip_g1 += cp->rcu_try_flip_g1; 160 sp->rcu_try_flip_g1 += cp->rcu_try_flip_g1;
diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c
index 85cb90588a55..b31065522104 100644
--- a/kernel/rcutorture.c
+++ b/kernel/rcutorture.c
@@ -39,6 +39,7 @@
39#include <linux/moduleparam.h> 39#include <linux/moduleparam.h>
40#include <linux/percpu.h> 40#include <linux/percpu.h>
41#include <linux/notifier.h> 41#include <linux/notifier.h>
42#include <linux/reboot.h>
42#include <linux/freezer.h> 43#include <linux/freezer.h>
43#include <linux/cpu.h> 44#include <linux/cpu.h>
44#include <linux/delay.h> 45#include <linux/delay.h>
@@ -108,7 +109,6 @@ struct rcu_torture {
108 int rtort_mbtest; 109 int rtort_mbtest;
109}; 110};
110 111
111static int fullstop = 0; /* stop generating callbacks at test end. */
112static LIST_HEAD(rcu_torture_freelist); 112static LIST_HEAD(rcu_torture_freelist);
113static struct rcu_torture *rcu_torture_current = NULL; 113static struct rcu_torture *rcu_torture_current = NULL;
114static long rcu_torture_current_version = 0; 114static long rcu_torture_current_version = 0;
@@ -136,6 +136,30 @@ static int stutter_pause_test = 0;
136#endif 136#endif
137int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT; 137int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT;
138 138
139#define FULLSTOP_SIGNALED 1 /* Bail due to signal. */
140#define FULLSTOP_CLEANUP 2 /* Orderly shutdown. */
141static int fullstop; /* stop generating callbacks at test end. */
142DEFINE_MUTEX(fullstop_mutex); /* protect fullstop transitions and */
143 /* spawning of kthreads. */
144
145/*
146 * Detect and respond to a signal-based shutdown.
147 */
148static int
149rcutorture_shutdown_notify(struct notifier_block *unused1,
150 unsigned long unused2, void *unused3)
151{
152 if (fullstop)
153 return NOTIFY_DONE;
154 if (signal_pending(current)) {
155 mutex_lock(&fullstop_mutex);
156 if (!ACCESS_ONCE(fullstop))
157 fullstop = FULLSTOP_SIGNALED;
158 mutex_unlock(&fullstop_mutex);
159 }
160 return NOTIFY_DONE;
161}
162
139/* 163/*
140 * Allocate an element from the rcu_tortures pool. 164 * Allocate an element from the rcu_tortures pool.
141 */ 165 */
@@ -199,11 +223,12 @@ rcu_random(struct rcu_random_state *rrsp)
199static void 223static void
200rcu_stutter_wait(void) 224rcu_stutter_wait(void)
201{ 225{
202 while (stutter_pause_test || !rcutorture_runnable) 226 while ((stutter_pause_test || !rcutorture_runnable) && !fullstop) {
203 if (rcutorture_runnable) 227 if (rcutorture_runnable)
204 schedule_timeout_interruptible(1); 228 schedule_timeout_interruptible(1);
205 else 229 else
206 schedule_timeout_interruptible(round_jiffies_relative(HZ)); 230 schedule_timeout_interruptible(round_jiffies_relative(HZ));
231 }
207} 232}
208 233
209/* 234/*
@@ -599,7 +624,7 @@ rcu_torture_writer(void *arg)
599 rcu_stutter_wait(); 624 rcu_stutter_wait();
600 } while (!kthread_should_stop() && !fullstop); 625 } while (!kthread_should_stop() && !fullstop);
601 VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping"); 626 VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping");
602 while (!kthread_should_stop()) 627 while (!kthread_should_stop() && fullstop != FULLSTOP_SIGNALED)
603 schedule_timeout_uninterruptible(1); 628 schedule_timeout_uninterruptible(1);
604 return 0; 629 return 0;
605} 630}
@@ -624,7 +649,7 @@ rcu_torture_fakewriter(void *arg)
624 } while (!kthread_should_stop() && !fullstop); 649 } while (!kthread_should_stop() && !fullstop);
625 650
626 VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task stopping"); 651 VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task stopping");
627 while (!kthread_should_stop()) 652 while (!kthread_should_stop() && fullstop != FULLSTOP_SIGNALED)
628 schedule_timeout_uninterruptible(1); 653 schedule_timeout_uninterruptible(1);
629 return 0; 654 return 0;
630} 655}
@@ -734,7 +759,7 @@ rcu_torture_reader(void *arg)
734 VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping"); 759 VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping");
735 if (irqreader && cur_ops->irqcapable) 760 if (irqreader && cur_ops->irqcapable)
736 del_timer_sync(&t); 761 del_timer_sync(&t);
737 while (!kthread_should_stop()) 762 while (!kthread_should_stop() && fullstop != FULLSTOP_SIGNALED)
738 schedule_timeout_uninterruptible(1); 763 schedule_timeout_uninterruptible(1);
739 return 0; 764 return 0;
740} 765}
@@ -831,7 +856,7 @@ rcu_torture_stats(void *arg)
831 do { 856 do {
832 schedule_timeout_interruptible(stat_interval * HZ); 857 schedule_timeout_interruptible(stat_interval * HZ);
833 rcu_torture_stats_print(); 858 rcu_torture_stats_print();
834 } while (!kthread_should_stop()); 859 } while (!kthread_should_stop() && !fullstop);
835 VERBOSE_PRINTK_STRING("rcu_torture_stats task stopping"); 860 VERBOSE_PRINTK_STRING("rcu_torture_stats task stopping");
836 return 0; 861 return 0;
837} 862}
@@ -899,7 +924,7 @@ rcu_torture_shuffle(void *arg)
899 do { 924 do {
900 schedule_timeout_interruptible(shuffle_interval * HZ); 925 schedule_timeout_interruptible(shuffle_interval * HZ);
901 rcu_torture_shuffle_tasks(); 926 rcu_torture_shuffle_tasks();
902 } while (!kthread_should_stop()); 927 } while (!kthread_should_stop() && !fullstop);
903 VERBOSE_PRINTK_STRING("rcu_torture_shuffle task stopping"); 928 VERBOSE_PRINTK_STRING("rcu_torture_shuffle task stopping");
904 return 0; 929 return 0;
905} 930}
@@ -914,10 +939,10 @@ rcu_torture_stutter(void *arg)
914 do { 939 do {
915 schedule_timeout_interruptible(stutter * HZ); 940 schedule_timeout_interruptible(stutter * HZ);
916 stutter_pause_test = 1; 941 stutter_pause_test = 1;
917 if (!kthread_should_stop()) 942 if (!kthread_should_stop() && !fullstop)
918 schedule_timeout_interruptible(stutter * HZ); 943 schedule_timeout_interruptible(stutter * HZ);
919 stutter_pause_test = 0; 944 stutter_pause_test = 0;
920 } while (!kthread_should_stop()); 945 } while (!kthread_should_stop() && !fullstop);
921 VERBOSE_PRINTK_STRING("rcu_torture_stutter task stopping"); 946 VERBOSE_PRINTK_STRING("rcu_torture_stutter task stopping");
922 return 0; 947 return 0;
923} 948}
@@ -934,12 +959,27 @@ rcu_torture_print_module_parms(char *tag)
934 stutter, irqreader); 959 stutter, irqreader);
935} 960}
936 961
962static struct notifier_block rcutorture_nb = {
963 .notifier_call = rcutorture_shutdown_notify,
964};
965
937static void 966static void
938rcu_torture_cleanup(void) 967rcu_torture_cleanup(void)
939{ 968{
940 int i; 969 int i;
941 970
942 fullstop = 1; 971 mutex_lock(&fullstop_mutex);
972 if (!fullstop) {
973 /* If being signaled, let it happen, then exit. */
974 mutex_unlock(&fullstop_mutex);
975 schedule_timeout_interruptible(10 * HZ);
976 if (cur_ops->cb_barrier != NULL)
977 cur_ops->cb_barrier();
978 return;
979 }
980 fullstop = FULLSTOP_CLEANUP;
981 mutex_unlock(&fullstop_mutex);
982 unregister_reboot_notifier(&rcutorture_nb);
943 if (stutter_task) { 983 if (stutter_task) {
944 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task"); 984 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task");
945 kthread_stop(stutter_task); 985 kthread_stop(stutter_task);
@@ -1015,6 +1055,8 @@ rcu_torture_init(void)
1015 { &rcu_ops, &rcu_sync_ops, &rcu_bh_ops, &rcu_bh_sync_ops, 1055 { &rcu_ops, &rcu_sync_ops, &rcu_bh_ops, &rcu_bh_sync_ops,
1016 &srcu_ops, &sched_ops, &sched_ops_sync, }; 1056 &srcu_ops, &sched_ops, &sched_ops_sync, };
1017 1057
1058 mutex_lock(&fullstop_mutex);
1059
1018 /* Process args and tell the world that the torturer is on the job. */ 1060 /* Process args and tell the world that the torturer is on the job. */
1019 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 1061 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
1020 cur_ops = torture_ops[i]; 1062 cur_ops = torture_ops[i];
@@ -1024,6 +1066,7 @@ rcu_torture_init(void)
1024 if (i == ARRAY_SIZE(torture_ops)) { 1066 if (i == ARRAY_SIZE(torture_ops)) {
1025 printk(KERN_ALERT "rcutorture: invalid torture type: \"%s\"\n", 1067 printk(KERN_ALERT "rcutorture: invalid torture type: \"%s\"\n",
1026 torture_type); 1068 torture_type);
1069 mutex_unlock(&fullstop_mutex);
1027 return (-EINVAL); 1070 return (-EINVAL);
1028 } 1071 }
1029 if (cur_ops->init) 1072 if (cur_ops->init)
@@ -1146,9 +1189,12 @@ rcu_torture_init(void)
1146 goto unwind; 1189 goto unwind;
1147 } 1190 }
1148 } 1191 }
1192 register_reboot_notifier(&rcutorture_nb);
1193 mutex_unlock(&fullstop_mutex);
1149 return 0; 1194 return 0;
1150 1195
1151unwind: 1196unwind:
1197 mutex_unlock(&fullstop_mutex);
1152 rcu_torture_cleanup(); 1198 rcu_torture_cleanup();
1153 return firsterr; 1199 return firsterr;
1154} 1200}
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
new file mode 100644
index 000000000000..a342b032112c
--- /dev/null
+++ b/kernel/rcutree.c
@@ -0,0 +1,1535 @@
1/*
2 * Read-Copy Update mechanism for mutual exclusion
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * Copyright IBM Corporation, 2008
19 *
20 * Authors: Dipankar Sarma <dipankar@in.ibm.com>
21 * Manfred Spraul <manfred@colorfullife.com>
22 * Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical version
23 *
24 * Based on the original work by Paul McKenney <paulmck@us.ibm.com>
25 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
26 *
27 * For detailed explanation of Read-Copy Update mechanism see -
28 * Documentation/RCU
29 */
30#include <linux/types.h>
31#include <linux/kernel.h>
32#include <linux/init.h>
33#include <linux/spinlock.h>
34#include <linux/smp.h>
35#include <linux/rcupdate.h>
36#include <linux/interrupt.h>
37#include <linux/sched.h>
38#include <asm/atomic.h>
39#include <linux/bitops.h>
40#include <linux/module.h>
41#include <linux/completion.h>
42#include <linux/moduleparam.h>
43#include <linux/percpu.h>
44#include <linux/notifier.h>
45#include <linux/cpu.h>
46#include <linux/mutex.h>
47#include <linux/time.h>
48
49#ifdef CONFIG_DEBUG_LOCK_ALLOC
50static struct lock_class_key rcu_lock_key;
51struct lockdep_map rcu_lock_map =
52 STATIC_LOCKDEP_MAP_INIT("rcu_read_lock", &rcu_lock_key);
53EXPORT_SYMBOL_GPL(rcu_lock_map);
54#endif
55
56/* Data structures. */
57
58#define RCU_STATE_INITIALIZER(name) { \
59 .level = { &name.node[0] }, \
60 .levelcnt = { \
61 NUM_RCU_LVL_0, /* root of hierarchy. */ \
62 NUM_RCU_LVL_1, \
63 NUM_RCU_LVL_2, \
64 NUM_RCU_LVL_3, /* == MAX_RCU_LVLS */ \
65 }, \
66 .signaled = RCU_SIGNAL_INIT, \
67 .gpnum = -300, \
68 .completed = -300, \
69 .onofflock = __SPIN_LOCK_UNLOCKED(&name.onofflock), \
70 .fqslock = __SPIN_LOCK_UNLOCKED(&name.fqslock), \
71 .n_force_qs = 0, \
72 .n_force_qs_ngp = 0, \
73}
74
75struct rcu_state rcu_state = RCU_STATE_INITIALIZER(rcu_state);
76DEFINE_PER_CPU(struct rcu_data, rcu_data);
77
78struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh_state);
79DEFINE_PER_CPU(struct rcu_data, rcu_bh_data);
80
81#ifdef CONFIG_NO_HZ
82DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks);
83#endif /* #ifdef CONFIG_NO_HZ */
84
85static int blimit = 10; /* Maximum callbacks per softirq. */
86static int qhimark = 10000; /* If this many pending, ignore blimit. */
87static int qlowmark = 100; /* Once only this many pending, use blimit. */
88
89static void force_quiescent_state(struct rcu_state *rsp, int relaxed);
90
91/*
92 * Return the number of RCU batches processed thus far for debug & stats.
93 */
94long rcu_batches_completed(void)
95{
96 return rcu_state.completed;
97}
98EXPORT_SYMBOL_GPL(rcu_batches_completed);
99
100/*
101 * Return the number of RCU BH batches processed thus far for debug & stats.
102 */
103long rcu_batches_completed_bh(void)
104{
105 return rcu_bh_state.completed;
106}
107EXPORT_SYMBOL_GPL(rcu_batches_completed_bh);
108
109/*
110 * Does the CPU have callbacks ready to be invoked?
111 */
112static int
113cpu_has_callbacks_ready_to_invoke(struct rcu_data *rdp)
114{
115 return &rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL];
116}
117
118/*
119 * Does the current CPU require a yet-as-unscheduled grace period?
120 */
121static int
122cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp)
123{
124 /* ACCESS_ONCE() because we are accessing outside of lock. */
125 return *rdp->nxttail[RCU_DONE_TAIL] &&
126 ACCESS_ONCE(rsp->completed) == ACCESS_ONCE(rsp->gpnum);
127}
128
129/*
130 * Return the root node of the specified rcu_state structure.
131 */
132static struct rcu_node *rcu_get_root(struct rcu_state *rsp)
133{
134 return &rsp->node[0];
135}
136
137#ifdef CONFIG_SMP
138
139/*
140 * If the specified CPU is offline, tell the caller that it is in
141 * a quiescent state. Otherwise, whack it with a reschedule IPI.
142 * Grace periods can end up waiting on an offline CPU when that
143 * CPU is in the process of coming online -- it will be added to the
144 * rcu_node bitmasks before it actually makes it online. The same thing
145 * can happen while a CPU is in the process of coming online. Because this
146 * race is quite rare, we check for it after detecting that the grace
147 * period has been delayed rather than checking each and every CPU
148 * each and every time we start a new grace period.
149 */
150static int rcu_implicit_offline_qs(struct rcu_data *rdp)
151{
152 /*
153 * If the CPU is offline, it is in a quiescent state. We can
154 * trust its state not to change because interrupts are disabled.
155 */
156 if (cpu_is_offline(rdp->cpu)) {
157 rdp->offline_fqs++;
158 return 1;
159 }
160
161 /* The CPU is online, so send it a reschedule IPI. */
162 if (rdp->cpu != smp_processor_id())
163 smp_send_reschedule(rdp->cpu);
164 else
165 set_need_resched();
166 rdp->resched_ipi++;
167 return 0;
168}
169
170#endif /* #ifdef CONFIG_SMP */
171
172#ifdef CONFIG_NO_HZ
173static DEFINE_RATELIMIT_STATE(rcu_rs, 10 * HZ, 5);
174
175/**
176 * rcu_enter_nohz - inform RCU that current CPU is entering nohz
177 *
178 * Enter nohz mode, in other words, -leave- the mode in which RCU
179 * read-side critical sections can occur. (Though RCU read-side
180 * critical sections can occur in irq handlers in nohz mode, a possibility
181 * handled by rcu_irq_enter() and rcu_irq_exit()).
182 */
183void rcu_enter_nohz(void)
184{
185 unsigned long flags;
186 struct rcu_dynticks *rdtp;
187
188 smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */
189 local_irq_save(flags);
190 rdtp = &__get_cpu_var(rcu_dynticks);
191 rdtp->dynticks++;
192 rdtp->dynticks_nesting--;
193 WARN_ON_RATELIMIT(rdtp->dynticks & 0x1, &rcu_rs);
194 local_irq_restore(flags);
195}
196
197/*
198 * rcu_exit_nohz - inform RCU that current CPU is leaving nohz
199 *
200 * Exit nohz mode, in other words, -enter- the mode in which RCU
201 * read-side critical sections normally occur.
202 */
203void rcu_exit_nohz(void)
204{
205 unsigned long flags;
206 struct rcu_dynticks *rdtp;
207
208 local_irq_save(flags);
209 rdtp = &__get_cpu_var(rcu_dynticks);
210 rdtp->dynticks++;
211 rdtp->dynticks_nesting++;
212 WARN_ON_RATELIMIT(!(rdtp->dynticks & 0x1), &rcu_rs);
213 local_irq_restore(flags);
214 smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
215}
216
217/**
218 * rcu_nmi_enter - inform RCU of entry to NMI context
219 *
220 * If the CPU was idle with dynamic ticks active, and there is no
221 * irq handler running, this updates rdtp->dynticks_nmi to let the
222 * RCU grace-period handling know that the CPU is active.
223 */
224void rcu_nmi_enter(void)
225{
226 struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);
227
228 if (rdtp->dynticks & 0x1)
229 return;
230 rdtp->dynticks_nmi++;
231 WARN_ON_RATELIMIT(!(rdtp->dynticks_nmi & 0x1), &rcu_rs);
232 smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
233}
234
235/**
236 * rcu_nmi_exit - inform RCU of exit from NMI context
237 *
238 * If the CPU was idle with dynamic ticks active, and there is no
239 * irq handler running, this updates rdtp->dynticks_nmi to let the
240 * RCU grace-period handling know that the CPU is no longer active.
241 */
242void rcu_nmi_exit(void)
243{
244 struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);
245
246 if (rdtp->dynticks & 0x1)
247 return;
248 smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */
249 rdtp->dynticks_nmi++;
250 WARN_ON_RATELIMIT(rdtp->dynticks_nmi & 0x1, &rcu_rs);
251}
252
253/**
254 * rcu_irq_enter - inform RCU of entry to hard irq context
255 *
256 * If the CPU was idle with dynamic ticks active, this updates the
257 * rdtp->dynticks to let the RCU handling know that the CPU is active.
258 */
259void rcu_irq_enter(void)
260{
261 struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);
262
263 if (rdtp->dynticks_nesting++)
264 return;
265 rdtp->dynticks++;
266 WARN_ON_RATELIMIT(!(rdtp->dynticks & 0x1), &rcu_rs);
267 smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
268}
269
270/**
271 * rcu_irq_exit - inform RCU of exit from hard irq context
272 *
273 * If the CPU was idle with dynamic ticks active, update the rdp->dynticks
274 * to put let the RCU handling be aware that the CPU is going back to idle
275 * with no ticks.
276 */
277void rcu_irq_exit(void)
278{
279 struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);
280
281 if (--rdtp->dynticks_nesting)
282 return;
283 smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */
284 rdtp->dynticks++;
285 WARN_ON_RATELIMIT(rdtp->dynticks & 0x1, &rcu_rs);
286
287 /* If the interrupt queued a callback, get out of dyntick mode. */
288 if (__get_cpu_var(rcu_data).nxtlist ||
289 __get_cpu_var(rcu_bh_data).nxtlist)
290 set_need_resched();
291}
292
293/*
294 * Record the specified "completed" value, which is later used to validate
295 * dynticks counter manipulations. Specify "rsp->completed - 1" to
296 * unconditionally invalidate any future dynticks manipulations (which is
297 * useful at the beginning of a grace period).
298 */
299static void dyntick_record_completed(struct rcu_state *rsp, long comp)
300{
301 rsp->dynticks_completed = comp;
302}
303
304#ifdef CONFIG_SMP
305
306/*
307 * Recall the previously recorded value of the completion for dynticks.
308 */
309static long dyntick_recall_completed(struct rcu_state *rsp)
310{
311 return rsp->dynticks_completed;
312}
313
314/*
315 * Snapshot the specified CPU's dynticks counter so that we can later
316 * credit them with an implicit quiescent state. Return 1 if this CPU
317 * is already in a quiescent state courtesy of dynticks idle mode.
318 */
319static int dyntick_save_progress_counter(struct rcu_data *rdp)
320{
321 int ret;
322 int snap;
323 int snap_nmi;
324
325 snap = rdp->dynticks->dynticks;
326 snap_nmi = rdp->dynticks->dynticks_nmi;
327 smp_mb(); /* Order sampling of snap with end of grace period. */
328 rdp->dynticks_snap = snap;
329 rdp->dynticks_nmi_snap = snap_nmi;
330 ret = ((snap & 0x1) == 0) && ((snap_nmi & 0x1) == 0);
331 if (ret)
332 rdp->dynticks_fqs++;
333 return ret;
334}
335
336/*
337 * Return true if the specified CPU has passed through a quiescent
338 * state by virtue of being in or having passed through an dynticks
339 * idle state since the last call to dyntick_save_progress_counter()
340 * for this same CPU.
341 */
342static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
343{
344 long curr;
345 long curr_nmi;
346 long snap;
347 long snap_nmi;
348
349 curr = rdp->dynticks->dynticks;
350 snap = rdp->dynticks_snap;
351 curr_nmi = rdp->dynticks->dynticks_nmi;
352 snap_nmi = rdp->dynticks_nmi_snap;
353 smp_mb(); /* force ordering with cpu entering/leaving dynticks. */
354
355 /*
356 * If the CPU passed through or entered a dynticks idle phase with
357 * no active irq/NMI handlers, then we can safely pretend that the CPU
358 * already acknowledged the request to pass through a quiescent
359 * state. Either way, that CPU cannot possibly be in an RCU
360 * read-side critical section that started before the beginning
361 * of the current RCU grace period.
362 */
363 if ((curr != snap || (curr & 0x1) == 0) &&
364 (curr_nmi != snap_nmi || (curr_nmi & 0x1) == 0)) {
365 rdp->dynticks_fqs++;
366 return 1;
367 }
368
369 /* Go check for the CPU being offline. */
370 return rcu_implicit_offline_qs(rdp);
371}
372
373#endif /* #ifdef CONFIG_SMP */
374
375#else /* #ifdef CONFIG_NO_HZ */
376
377static void dyntick_record_completed(struct rcu_state *rsp, long comp)
378{
379}
380
381#ifdef CONFIG_SMP
382
383/*
384 * If there are no dynticks, then the only way that a CPU can passively
385 * be in a quiescent state is to be offline. Unlike dynticks idle, which
386 * is a point in time during the prior (already finished) grace period,
387 * an offline CPU is always in a quiescent state, and thus can be
388 * unconditionally applied. So just return the current value of completed.
389 */
390static long dyntick_recall_completed(struct rcu_state *rsp)
391{
392 return rsp->completed;
393}
394
395static int dyntick_save_progress_counter(struct rcu_data *rdp)
396{
397 return 0;
398}
399
400static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
401{
402 return rcu_implicit_offline_qs(rdp);
403}
404
405#endif /* #ifdef CONFIG_SMP */
406
407#endif /* #else #ifdef CONFIG_NO_HZ */
408
409#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
410
411static void record_gp_stall_check_time(struct rcu_state *rsp)
412{
413 rsp->gp_start = jiffies;
414 rsp->jiffies_stall = jiffies + RCU_SECONDS_TILL_STALL_CHECK;
415}
416
417static void print_other_cpu_stall(struct rcu_state *rsp)
418{
419 int cpu;
420 long delta;
421 unsigned long flags;
422 struct rcu_node *rnp = rcu_get_root(rsp);
423 struct rcu_node *rnp_cur = rsp->level[NUM_RCU_LVLS - 1];
424 struct rcu_node *rnp_end = &rsp->node[NUM_RCU_NODES];
425
426 /* Only let one CPU complain about others per time interval. */
427
428 spin_lock_irqsave(&rnp->lock, flags);
429 delta = jiffies - rsp->jiffies_stall;
430 if (delta < RCU_STALL_RAT_DELAY || rsp->gpnum == rsp->completed) {
431 spin_unlock_irqrestore(&rnp->lock, flags);
432 return;
433 }
434 rsp->jiffies_stall = jiffies + RCU_SECONDS_TILL_STALL_RECHECK;
435 spin_unlock_irqrestore(&rnp->lock, flags);
436
437 /* OK, time to rat on our buddy... */
438
439 printk(KERN_ERR "INFO: RCU detected CPU stalls:");
440 for (; rnp_cur < rnp_end; rnp_cur++) {
441 if (rnp_cur->qsmask == 0)
442 continue;
443 for (cpu = 0; cpu <= rnp_cur->grphi - rnp_cur->grplo; cpu++)
444 if (rnp_cur->qsmask & (1UL << cpu))
445 printk(" %d", rnp_cur->grplo + cpu);
446 }
447 printk(" (detected by %d, t=%ld jiffies)\n",
448 smp_processor_id(), (long)(jiffies - rsp->gp_start));
449 force_quiescent_state(rsp, 0); /* Kick them all. */
450}
451
452static void print_cpu_stall(struct rcu_state *rsp)
453{
454 unsigned long flags;
455 struct rcu_node *rnp = rcu_get_root(rsp);
456
457 printk(KERN_ERR "INFO: RCU detected CPU %d stall (t=%lu jiffies)\n",
458 smp_processor_id(), jiffies - rsp->gp_start);
459 dump_stack();
460 spin_lock_irqsave(&rnp->lock, flags);
461 if ((long)(jiffies - rsp->jiffies_stall) >= 0)
462 rsp->jiffies_stall =
463 jiffies + RCU_SECONDS_TILL_STALL_RECHECK;
464 spin_unlock_irqrestore(&rnp->lock, flags);
465 set_need_resched(); /* kick ourselves to get things going. */
466}
467
468static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp)
469{
470 long delta;
471 struct rcu_node *rnp;
472
473 delta = jiffies - rsp->jiffies_stall;
474 rnp = rdp->mynode;
475 if ((rnp->qsmask & rdp->grpmask) && delta >= 0) {
476
477 /* We haven't checked in, so go dump stack. */
478 print_cpu_stall(rsp);
479
480 } else if (rsp->gpnum != rsp->completed &&
481 delta >= RCU_STALL_RAT_DELAY) {
482
483 /* They had two time units to dump stack, so complain. */
484 print_other_cpu_stall(rsp);
485 }
486}
487
488#else /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
489
490static void record_gp_stall_check_time(struct rcu_state *rsp)
491{
492}
493
494static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp)
495{
496}
497
498#endif /* #else #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
499
500/*
501 * Update CPU-local rcu_data state to record the newly noticed grace period.
502 * This is used both when we started the grace period and when we notice
503 * that someone else started the grace period.
504 */
505static void note_new_gpnum(struct rcu_state *rsp, struct rcu_data *rdp)
506{
507 rdp->qs_pending = 1;
508 rdp->passed_quiesc = 0;
509 rdp->gpnum = rsp->gpnum;
510 rdp->n_rcu_pending_force_qs = rdp->n_rcu_pending +
511 RCU_JIFFIES_TILL_FORCE_QS;
512}
513
514/*
515 * Did someone else start a new RCU grace period start since we last
516 * checked? Update local state appropriately if so. Must be called
517 * on the CPU corresponding to rdp.
518 */
519static int
520check_for_new_grace_period(struct rcu_state *rsp, struct rcu_data *rdp)
521{
522 unsigned long flags;
523 int ret = 0;
524
525 local_irq_save(flags);
526 if (rdp->gpnum != rsp->gpnum) {
527 note_new_gpnum(rsp, rdp);
528 ret = 1;
529 }
530 local_irq_restore(flags);
531 return ret;
532}
533
534/*
535 * Start a new RCU grace period if warranted, re-initializing the hierarchy
536 * in preparation for detecting the next grace period. The caller must hold
537 * the root node's ->lock, which is released before return. Hard irqs must
538 * be disabled.
539 */
540static void
541rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
542 __releases(rcu_get_root(rsp)->lock)
543{
544 struct rcu_data *rdp = rsp->rda[smp_processor_id()];
545 struct rcu_node *rnp = rcu_get_root(rsp);
546 struct rcu_node *rnp_cur;
547 struct rcu_node *rnp_end;
548
549 if (!cpu_needs_another_gp(rsp, rdp)) {
550 spin_unlock_irqrestore(&rnp->lock, flags);
551 return;
552 }
553
554 /* Advance to a new grace period and initialize state. */
555 rsp->gpnum++;
556 rsp->signaled = RCU_GP_INIT; /* Hold off force_quiescent_state. */
557 rsp->jiffies_force_qs = jiffies + RCU_JIFFIES_TILL_FORCE_QS;
558 rdp->n_rcu_pending_force_qs = rdp->n_rcu_pending +
559 RCU_JIFFIES_TILL_FORCE_QS;
560 record_gp_stall_check_time(rsp);
561 dyntick_record_completed(rsp, rsp->completed - 1);
562 note_new_gpnum(rsp, rdp);
563
564 /*
565 * Because we are first, we know that all our callbacks will
566 * be covered by this upcoming grace period, even the ones
567 * that were registered arbitrarily recently.
568 */
569 rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
570 rdp->nxttail[RCU_WAIT_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
571
572 /* Special-case the common single-level case. */
573 if (NUM_RCU_NODES == 1) {
574 rnp->qsmask = rnp->qsmaskinit;
575 spin_unlock_irqrestore(&rnp->lock, flags);
576 return;
577 }
578
579 spin_unlock(&rnp->lock); /* leave irqs disabled. */
580
581
582 /* Exclude any concurrent CPU-hotplug operations. */
583 spin_lock(&rsp->onofflock); /* irqs already disabled. */
584
585 /*
586 * Set the quiescent-state-needed bits in all the non-leaf RCU
587 * nodes for all currently online CPUs. This operation relies
588 * on the layout of the hierarchy within the rsp->node[] array.
589 * Note that other CPUs will access only the leaves of the
590 * hierarchy, which still indicate that no grace period is in
591 * progress. In addition, we have excluded CPU-hotplug operations.
592 *
593 * We therefore do not need to hold any locks. Any required
594 * memory barriers will be supplied by the locks guarding the
595 * leaf rcu_nodes in the hierarchy.
596 */
597
598 rnp_end = rsp->level[NUM_RCU_LVLS - 1];
599 for (rnp_cur = &rsp->node[0]; rnp_cur < rnp_end; rnp_cur++)
600 rnp_cur->qsmask = rnp_cur->qsmaskinit;
601
602 /*
603 * Now set up the leaf nodes. Here we must be careful. First,
604 * we need to hold the lock in order to exclude other CPUs, which
605 * might be contending for the leaf nodes' locks. Second, as
606 * soon as we initialize a given leaf node, its CPUs might run
607 * up the rest of the hierarchy. We must therefore acquire locks
608 * for each node that we touch during this stage. (But we still
609 * are excluding CPU-hotplug operations.)
610 *
611 * Note that the grace period cannot complete until we finish
612 * the initialization process, as there will be at least one
613 * qsmask bit set in the root node until that time, namely the
614 * one corresponding to this CPU.
615 */
616 rnp_end = &rsp->node[NUM_RCU_NODES];
617 rnp_cur = rsp->level[NUM_RCU_LVLS - 1];
618 for (; rnp_cur < rnp_end; rnp_cur++) {
619 spin_lock(&rnp_cur->lock); /* irqs already disabled. */
620 rnp_cur->qsmask = rnp_cur->qsmaskinit;
621 spin_unlock(&rnp_cur->lock); /* irqs already disabled. */
622 }
623
624 rsp->signaled = RCU_SIGNAL_INIT; /* force_quiescent_state now OK. */
625 spin_unlock_irqrestore(&rsp->onofflock, flags);
626}
627
628/*
629 * Advance this CPU's callbacks, but only if the current grace period
630 * has ended. This may be called only from the CPU to whom the rdp
631 * belongs.
632 */
633static void
634rcu_process_gp_end(struct rcu_state *rsp, struct rcu_data *rdp)
635{
636 long completed_snap;
637 unsigned long flags;
638
639 local_irq_save(flags);
640 completed_snap = ACCESS_ONCE(rsp->completed); /* outside of lock. */
641
642 /* Did another grace period end? */
643 if (rdp->completed != completed_snap) {
644
645 /* Advance callbacks. No harm if list empty. */
646 rdp->nxttail[RCU_DONE_TAIL] = rdp->nxttail[RCU_WAIT_TAIL];
647 rdp->nxttail[RCU_WAIT_TAIL] = rdp->nxttail[RCU_NEXT_READY_TAIL];
648 rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
649
650 /* Remember that we saw this grace-period completion. */
651 rdp->completed = completed_snap;
652 }
653 local_irq_restore(flags);
654}
655
656/*
657 * Similar to cpu_quiet(), for which it is a helper function. Allows
658 * a group of CPUs to be quieted at one go, though all the CPUs in the
659 * group must be represented by the same leaf rcu_node structure.
660 * That structure's lock must be held upon entry, and it is released
661 * before return.
662 */
663static void
664cpu_quiet_msk(unsigned long mask, struct rcu_state *rsp, struct rcu_node *rnp,
665 unsigned long flags)
666 __releases(rnp->lock)
667{
668 /* Walk up the rcu_node hierarchy. */
669 for (;;) {
670 if (!(rnp->qsmask & mask)) {
671
672 /* Our bit has already been cleared, so done. */
673 spin_unlock_irqrestore(&rnp->lock, flags);
674 return;
675 }
676 rnp->qsmask &= ~mask;
677 if (rnp->qsmask != 0) {
678
679 /* Other bits still set at this level, so done. */
680 spin_unlock_irqrestore(&rnp->lock, flags);
681 return;
682 }
683 mask = rnp->grpmask;
684 if (rnp->parent == NULL) {
685
686 /* No more levels. Exit loop holding root lock. */
687
688 break;
689 }
690 spin_unlock_irqrestore(&rnp->lock, flags);
691 rnp = rnp->parent;
692 spin_lock_irqsave(&rnp->lock, flags);
693 }
694
695 /*
696 * Get here if we are the last CPU to pass through a quiescent
697 * state for this grace period. Clean up and let rcu_start_gp()
698 * start up the next grace period if one is needed. Note that
699 * we still hold rnp->lock, as required by rcu_start_gp(), which
700 * will release it.
701 */
702 rsp->completed = rsp->gpnum;
703 rcu_process_gp_end(rsp, rsp->rda[smp_processor_id()]);
704 rcu_start_gp(rsp, flags); /* releases rnp->lock. */
705}
706
707/*
708 * Record a quiescent state for the specified CPU, which must either be
709 * the current CPU or an offline CPU. The lastcomp argument is used to
710 * make sure we are still in the grace period of interest. We don't want
711 * to end the current grace period based on quiescent states detected in
712 * an earlier grace period!
713 */
714static void
715cpu_quiet(int cpu, struct rcu_state *rsp, struct rcu_data *rdp, long lastcomp)
716{
717 unsigned long flags;
718 unsigned long mask;
719 struct rcu_node *rnp;
720
721 rnp = rdp->mynode;
722 spin_lock_irqsave(&rnp->lock, flags);
723 if (lastcomp != ACCESS_ONCE(rsp->completed)) {
724
725 /*
726 * Someone beat us to it for this grace period, so leave.
727 * The race with GP start is resolved by the fact that we
728 * hold the leaf rcu_node lock, so that the per-CPU bits
729 * cannot yet be initialized -- so we would simply find our
730 * CPU's bit already cleared in cpu_quiet_msk() if this race
731 * occurred.
732 */
733 rdp->passed_quiesc = 0; /* try again later! */
734 spin_unlock_irqrestore(&rnp->lock, flags);
735 return;
736 }
737 mask = rdp->grpmask;
738 if ((rnp->qsmask & mask) == 0) {
739 spin_unlock_irqrestore(&rnp->lock, flags);
740 } else {
741 rdp->qs_pending = 0;
742
743 /*
744 * This GP can't end until cpu checks in, so all of our
745 * callbacks can be processed during the next GP.
746 */
747 rdp = rsp->rda[smp_processor_id()];
748 rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
749
750 cpu_quiet_msk(mask, rsp, rnp, flags); /* releases rnp->lock */
751 }
752}
753
754/*
755 * Check to see if there is a new grace period of which this CPU
756 * is not yet aware, and if so, set up local rcu_data state for it.
757 * Otherwise, see if this CPU has just passed through its first
758 * quiescent state for this grace period, and record that fact if so.
759 */
760static void
761rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)
762{
763 /* If there is now a new grace period, record and return. */
764 if (check_for_new_grace_period(rsp, rdp))
765 return;
766
767 /*
768 * Does this CPU still need to do its part for current grace period?
769 * If no, return and let the other CPUs do their part as well.
770 */
771 if (!rdp->qs_pending)
772 return;
773
774 /*
775 * Was there a quiescent state since the beginning of the grace
776 * period? If no, then exit and wait for the next call.
777 */
778 if (!rdp->passed_quiesc)
779 return;
780
781 /* Tell RCU we are done (but cpu_quiet() will be the judge of that). */
782 cpu_quiet(rdp->cpu, rsp, rdp, rdp->passed_quiesc_completed);
783}
784
785#ifdef CONFIG_HOTPLUG_CPU
786
787/*
788 * Remove the outgoing CPU from the bitmasks in the rcu_node hierarchy
789 * and move all callbacks from the outgoing CPU to the current one.
790 */
791static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
792{
793 int i;
794 unsigned long flags;
795 long lastcomp;
796 unsigned long mask;
797 struct rcu_data *rdp = rsp->rda[cpu];
798 struct rcu_data *rdp_me;
799 struct rcu_node *rnp;
800
801 /* Exclude any attempts to start a new grace period. */
802 spin_lock_irqsave(&rsp->onofflock, flags);
803
804 /* Remove the outgoing CPU from the masks in the rcu_node hierarchy. */
805 rnp = rdp->mynode;
806 mask = rdp->grpmask; /* rnp->grplo is constant. */
807 do {
808 spin_lock(&rnp->lock); /* irqs already disabled. */
809 rnp->qsmaskinit &= ~mask;
810 if (rnp->qsmaskinit != 0) {
811 spin_unlock(&rnp->lock); /* irqs already disabled. */
812 break;
813 }
814 mask = rnp->grpmask;
815 spin_unlock(&rnp->lock); /* irqs already disabled. */
816 rnp = rnp->parent;
817 } while (rnp != NULL);
818 lastcomp = rsp->completed;
819
820 spin_unlock(&rsp->onofflock); /* irqs remain disabled. */
821
822 /* Being offline is a quiescent state, so go record it. */
823 cpu_quiet(cpu, rsp, rdp, lastcomp);
824
825 /*
826 * Move callbacks from the outgoing CPU to the running CPU.
827 * Note that the outgoing CPU is now quiscent, so it is now
828 * (uncharacteristically) safe to access it rcu_data structure.
829 * Note also that we must carefully retain the order of the
830 * outgoing CPU's callbacks in order for rcu_barrier() to work
831 * correctly. Finally, note that we start all the callbacks
832 * afresh, even those that have passed through a grace period
833 * and are therefore ready to invoke. The theory is that hotplug
834 * events are rare, and that if they are frequent enough to
835 * indefinitely delay callbacks, you have far worse things to
836 * be worrying about.
837 */
838 rdp_me = rsp->rda[smp_processor_id()];
839 if (rdp->nxtlist != NULL) {
840 *rdp_me->nxttail[RCU_NEXT_TAIL] = rdp->nxtlist;
841 rdp_me->nxttail[RCU_NEXT_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
842 rdp->nxtlist = NULL;
843 for (i = 0; i < RCU_NEXT_SIZE; i++)
844 rdp->nxttail[i] = &rdp->nxtlist;
845 rdp_me->qlen += rdp->qlen;
846 rdp->qlen = 0;
847 }
848 local_irq_restore(flags);
849}
850
851/*
852 * Remove the specified CPU from the RCU hierarchy and move any pending
853 * callbacks that it might have to the current CPU. This code assumes
854 * that at least one CPU in the system will remain running at all times.
855 * Any attempt to offline -all- CPUs is likely to strand RCU callbacks.
856 */
857static void rcu_offline_cpu(int cpu)
858{
859 __rcu_offline_cpu(cpu, &rcu_state);
860 __rcu_offline_cpu(cpu, &rcu_bh_state);
861}
862
863#else /* #ifdef CONFIG_HOTPLUG_CPU */
864
865static void rcu_offline_cpu(int cpu)
866{
867}
868
869#endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
870
871/*
872 * Invoke any RCU callbacks that have made it to the end of their grace
873 * period. Thottle as specified by rdp->blimit.
874 */
875static void rcu_do_batch(struct rcu_data *rdp)
876{
877 unsigned long flags;
878 struct rcu_head *next, *list, **tail;
879 int count;
880
881 /* If no callbacks are ready, just return.*/
882 if (!cpu_has_callbacks_ready_to_invoke(rdp))
883 return;
884
885 /*
886 * Extract the list of ready callbacks, disabling to prevent
887 * races with call_rcu() from interrupt handlers.
888 */
889 local_irq_save(flags);
890 list = rdp->nxtlist;
891 rdp->nxtlist = *rdp->nxttail[RCU_DONE_TAIL];
892 *rdp->nxttail[RCU_DONE_TAIL] = NULL;
893 tail = rdp->nxttail[RCU_DONE_TAIL];
894 for (count = RCU_NEXT_SIZE - 1; count >= 0; count--)
895 if (rdp->nxttail[count] == rdp->nxttail[RCU_DONE_TAIL])
896 rdp->nxttail[count] = &rdp->nxtlist;
897 local_irq_restore(flags);
898
899 /* Invoke callbacks. */
900 count = 0;
901 while (list) {
902 next = list->next;
903 prefetch(next);
904 list->func(list);
905 list = next;
906 if (++count >= rdp->blimit)
907 break;
908 }
909
910 local_irq_save(flags);
911
912 /* Update count, and requeue any remaining callbacks. */
913 rdp->qlen -= count;
914 if (list != NULL) {
915 *tail = rdp->nxtlist;
916 rdp->nxtlist = list;
917 for (count = 0; count < RCU_NEXT_SIZE; count++)
918 if (&rdp->nxtlist == rdp->nxttail[count])
919 rdp->nxttail[count] = tail;
920 else
921 break;
922 }
923
924 /* Reinstate batch limit if we have worked down the excess. */
925 if (rdp->blimit == LONG_MAX && rdp->qlen <= qlowmark)
926 rdp->blimit = blimit;
927
928 local_irq_restore(flags);
929
930 /* Re-raise the RCU softirq if there are callbacks remaining. */
931 if (cpu_has_callbacks_ready_to_invoke(rdp))
932 raise_softirq(RCU_SOFTIRQ);
933}
934
935/*
936 * Check to see if this CPU is in a non-context-switch quiescent state
937 * (user mode or idle loop for rcu, non-softirq execution for rcu_bh).
938 * Also schedule the RCU softirq handler.
939 *
940 * This function must be called with hardirqs disabled. It is normally
941 * invoked from the scheduling-clock interrupt. If rcu_pending returns
942 * false, there is no point in invoking rcu_check_callbacks().
943 */
944void rcu_check_callbacks(int cpu, int user)
945{
946 if (user ||
947 (idle_cpu(cpu) && !in_softirq() &&
948 hardirq_count() <= (1 << HARDIRQ_SHIFT))) {
949
950 /*
951 * Get here if this CPU took its interrupt from user
952 * mode or from the idle loop, and if this is not a
953 * nested interrupt. In this case, the CPU is in
954 * a quiescent state, so count it.
955 *
956 * No memory barrier is required here because both
957 * rcu_qsctr_inc() and rcu_bh_qsctr_inc() reference
958 * only CPU-local variables that other CPUs neither
959 * access nor modify, at least not while the corresponding
960 * CPU is online.
961 */
962
963 rcu_qsctr_inc(cpu);
964 rcu_bh_qsctr_inc(cpu);
965
966 } else if (!in_softirq()) {
967
968 /*
969 * Get here if this CPU did not take its interrupt from
970 * softirq, in other words, if it is not interrupting
971 * a rcu_bh read-side critical section. This is an _bh
972 * critical section, so count it.
973 */
974
975 rcu_bh_qsctr_inc(cpu);
976 }
977 raise_softirq(RCU_SOFTIRQ);
978}
979
980#ifdef CONFIG_SMP
981
982/*
983 * Scan the leaf rcu_node structures, processing dyntick state for any that
984 * have not yet encountered a quiescent state, using the function specified.
985 * Returns 1 if the current grace period ends while scanning (possibly
986 * because we made it end).
987 */
988static int rcu_process_dyntick(struct rcu_state *rsp, long lastcomp,
989 int (*f)(struct rcu_data *))
990{
991 unsigned long bit;
992 int cpu;
993 unsigned long flags;
994 unsigned long mask;
995 struct rcu_node *rnp_cur = rsp->level[NUM_RCU_LVLS - 1];
996 struct rcu_node *rnp_end = &rsp->node[NUM_RCU_NODES];
997
998 for (; rnp_cur < rnp_end; rnp_cur++) {
999 mask = 0;
1000 spin_lock_irqsave(&rnp_cur->lock, flags);
1001 if (rsp->completed != lastcomp) {
1002 spin_unlock_irqrestore(&rnp_cur->lock, flags);
1003 return 1;
1004 }
1005 if (rnp_cur->qsmask == 0) {
1006 spin_unlock_irqrestore(&rnp_cur->lock, flags);
1007 continue;
1008 }
1009 cpu = rnp_cur->grplo;
1010 bit = 1;
1011 for (; cpu <= rnp_cur->grphi; cpu++, bit <<= 1) {
1012 if ((rnp_cur->qsmask & bit) != 0 && f(rsp->rda[cpu]))
1013 mask |= bit;
1014 }
1015 if (mask != 0 && rsp->completed == lastcomp) {
1016
1017 /* cpu_quiet_msk() releases rnp_cur->lock. */
1018 cpu_quiet_msk(mask, rsp, rnp_cur, flags);
1019 continue;
1020 }
1021 spin_unlock_irqrestore(&rnp_cur->lock, flags);
1022 }
1023 return 0;
1024}
1025
1026/*
1027 * Force quiescent states on reluctant CPUs, and also detect which
1028 * CPUs are in dyntick-idle mode.
1029 */
1030static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
1031{
1032 unsigned long flags;
1033 long lastcomp;
1034 struct rcu_data *rdp = rsp->rda[smp_processor_id()];
1035 struct rcu_node *rnp = rcu_get_root(rsp);
1036 u8 signaled;
1037
1038 if (ACCESS_ONCE(rsp->completed) == ACCESS_ONCE(rsp->gpnum))
1039 return; /* No grace period in progress, nothing to force. */
1040 if (!spin_trylock_irqsave(&rsp->fqslock, flags)) {
1041 rsp->n_force_qs_lh++; /* Inexact, can lose counts. Tough! */
1042 return; /* Someone else is already on the job. */
1043 }
1044 if (relaxed &&
1045 (long)(rsp->jiffies_force_qs - jiffies) >= 0 &&
1046 (rdp->n_rcu_pending_force_qs - rdp->n_rcu_pending) >= 0)
1047 goto unlock_ret; /* no emergency and done recently. */
1048 rsp->n_force_qs++;
1049 spin_lock(&rnp->lock);
1050 lastcomp = rsp->completed;
1051 signaled = rsp->signaled;
1052 rsp->jiffies_force_qs = jiffies + RCU_JIFFIES_TILL_FORCE_QS;
1053 rdp->n_rcu_pending_force_qs = rdp->n_rcu_pending +
1054 RCU_JIFFIES_TILL_FORCE_QS;
1055 if (lastcomp == rsp->gpnum) {
1056 rsp->n_force_qs_ngp++;
1057 spin_unlock(&rnp->lock);
1058 goto unlock_ret; /* no GP in progress, time updated. */
1059 }
1060 spin_unlock(&rnp->lock);
1061 switch (signaled) {
1062 case RCU_GP_INIT:
1063
1064 break; /* grace period still initializing, ignore. */
1065
1066 case RCU_SAVE_DYNTICK:
1067
1068 if (RCU_SIGNAL_INIT != RCU_SAVE_DYNTICK)
1069 break; /* So gcc recognizes the dead code. */
1070
1071 /* Record dyntick-idle state. */
1072 if (rcu_process_dyntick(rsp, lastcomp,
1073 dyntick_save_progress_counter))
1074 goto unlock_ret;
1075
1076 /* Update state, record completion counter. */
1077 spin_lock(&rnp->lock);
1078 if (lastcomp == rsp->completed) {
1079 rsp->signaled = RCU_FORCE_QS;
1080 dyntick_record_completed(rsp, lastcomp);
1081 }
1082 spin_unlock(&rnp->lock);
1083 break;
1084
1085 case RCU_FORCE_QS:
1086
1087 /* Check dyntick-idle state, send IPI to laggarts. */
1088 if (rcu_process_dyntick(rsp, dyntick_recall_completed(rsp),
1089 rcu_implicit_dynticks_qs))
1090 goto unlock_ret;
1091
1092 /* Leave state in case more forcing is required. */
1093
1094 break;
1095 }
1096unlock_ret:
1097 spin_unlock_irqrestore(&rsp->fqslock, flags);
1098}
1099
1100#else /* #ifdef CONFIG_SMP */
1101
1102static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
1103{
1104 set_need_resched();
1105}
1106
1107#endif /* #else #ifdef CONFIG_SMP */
1108
1109/*
1110 * This does the RCU processing work from softirq context for the
1111 * specified rcu_state and rcu_data structures. This may be called
1112 * only from the CPU to whom the rdp belongs.
1113 */
1114static void
1115__rcu_process_callbacks(struct rcu_state *rsp, struct rcu_data *rdp)
1116{
1117 unsigned long flags;
1118
1119 /*
1120 * If an RCU GP has gone long enough, go check for dyntick
1121 * idle CPUs and, if needed, send resched IPIs.
1122 */
1123 if ((long)(ACCESS_ONCE(rsp->jiffies_force_qs) - jiffies) < 0 ||
1124 (rdp->n_rcu_pending_force_qs - rdp->n_rcu_pending) < 0)
1125 force_quiescent_state(rsp, 1);
1126
1127 /*
1128 * Advance callbacks in response to end of earlier grace
1129 * period that some other CPU ended.
1130 */
1131 rcu_process_gp_end(rsp, rdp);
1132
1133 /* Update RCU state based on any recent quiescent states. */
1134 rcu_check_quiescent_state(rsp, rdp);
1135
1136 /* Does this CPU require a not-yet-started grace period? */
1137 if (cpu_needs_another_gp(rsp, rdp)) {
1138 spin_lock_irqsave(&rcu_get_root(rsp)->lock, flags);
1139 rcu_start_gp(rsp, flags); /* releases above lock */
1140 }
1141
1142 /* If there are callbacks ready, invoke them. */
1143 rcu_do_batch(rdp);
1144}
1145
1146/*
1147 * Do softirq processing for the current CPU.
1148 */
1149static void rcu_process_callbacks(struct softirq_action *unused)
1150{
1151 /*
1152 * Memory references from any prior RCU read-side critical sections
1153 * executed by the interrupted code must be seen before any RCU
1154 * grace-period manipulations below.
1155 */
1156 smp_mb(); /* See above block comment. */
1157
1158 __rcu_process_callbacks(&rcu_state, &__get_cpu_var(rcu_data));
1159 __rcu_process_callbacks(&rcu_bh_state, &__get_cpu_var(rcu_bh_data));
1160
1161 /*
1162 * Memory references from any later RCU read-side critical sections
1163 * executed by the interrupted code must be seen after any RCU
1164 * grace-period manipulations above.
1165 */
1166 smp_mb(); /* See above block comment. */
1167}
1168
1169static void
1170__call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
1171 struct rcu_state *rsp)
1172{
1173 unsigned long flags;
1174 struct rcu_data *rdp;
1175
1176 head->func = func;
1177 head->next = NULL;
1178
1179 smp_mb(); /* Ensure RCU update seen before callback registry. */
1180
1181 /*
1182 * Opportunistically note grace-period endings and beginnings.
1183 * Note that we might see a beginning right after we see an
1184 * end, but never vice versa, since this CPU has to pass through
1185 * a quiescent state betweentimes.
1186 */
1187 local_irq_save(flags);
1188 rdp = rsp->rda[smp_processor_id()];
1189 rcu_process_gp_end(rsp, rdp);
1190 check_for_new_grace_period(rsp, rdp);
1191
1192 /* Add the callback to our list. */
1193 *rdp->nxttail[RCU_NEXT_TAIL] = head;
1194 rdp->nxttail[RCU_NEXT_TAIL] = &head->next;
1195
1196 /* Start a new grace period if one not already started. */
1197 if (ACCESS_ONCE(rsp->completed) == ACCESS_ONCE(rsp->gpnum)) {
1198 unsigned long nestflag;
1199 struct rcu_node *rnp_root = rcu_get_root(rsp);
1200
1201 spin_lock_irqsave(&rnp_root->lock, nestflag);
1202 rcu_start_gp(rsp, nestflag); /* releases rnp_root->lock. */
1203 }
1204
1205 /* Force the grace period if too many callbacks or too long waiting. */
1206 if (unlikely(++rdp->qlen > qhimark)) {
1207 rdp->blimit = LONG_MAX;
1208 force_quiescent_state(rsp, 0);
1209 } else if ((long)(ACCESS_ONCE(rsp->jiffies_force_qs) - jiffies) < 0 ||
1210 (rdp->n_rcu_pending_force_qs - rdp->n_rcu_pending) < 0)
1211 force_quiescent_state(rsp, 1);
1212 local_irq_restore(flags);
1213}
1214
1215/*
1216 * Queue an RCU callback for invocation after a grace period.
1217 */
1218void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
1219{
1220 __call_rcu(head, func, &rcu_state);
1221}
1222EXPORT_SYMBOL_GPL(call_rcu);
1223
1224/*
1225 * Queue an RCU for invocation after a quicker grace period.
1226 */
1227void call_rcu_bh(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
1228{
1229 __call_rcu(head, func, &rcu_bh_state);
1230}
1231EXPORT_SYMBOL_GPL(call_rcu_bh);
1232
1233/*
1234 * Check to see if there is any immediate RCU-related work to be done
1235 * by the current CPU, for the specified type of RCU, returning 1 if so.
1236 * The checks are in order of increasing expense: checks that can be
1237 * carried out against CPU-local state are performed first. However,
1238 * we must check for CPU stalls first, else we might not get a chance.
1239 */
1240static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
1241{
1242 rdp->n_rcu_pending++;
1243
1244 /* Check for CPU stalls, if enabled. */
1245 check_cpu_stall(rsp, rdp);
1246
1247 /* Is the RCU core waiting for a quiescent state from this CPU? */
1248 if (rdp->qs_pending)
1249 return 1;
1250
1251 /* Does this CPU have callbacks ready to invoke? */
1252 if (cpu_has_callbacks_ready_to_invoke(rdp))
1253 return 1;
1254
1255 /* Has RCU gone idle with this CPU needing another grace period? */
1256 if (cpu_needs_another_gp(rsp, rdp))
1257 return 1;
1258
1259 /* Has another RCU grace period completed? */
1260 if (ACCESS_ONCE(rsp->completed) != rdp->completed) /* outside of lock */
1261 return 1;
1262
1263 /* Has a new RCU grace period started? */
1264 if (ACCESS_ONCE(rsp->gpnum) != rdp->gpnum) /* outside of lock */
1265 return 1;
1266
1267 /* Has an RCU GP gone long enough to send resched IPIs &c? */
1268 if (ACCESS_ONCE(rsp->completed) != ACCESS_ONCE(rsp->gpnum) &&
1269 ((long)(ACCESS_ONCE(rsp->jiffies_force_qs) - jiffies) < 0 ||
1270 (rdp->n_rcu_pending_force_qs - rdp->n_rcu_pending) < 0))
1271 return 1;
1272
1273 /* nothing to do */
1274 return 0;
1275}
1276
1277/*
1278 * Check to see if there is any immediate RCU-related work to be done
1279 * by the current CPU, returning 1 if so. This function is part of the
1280 * RCU implementation; it is -not- an exported member of the RCU API.
1281 */
1282int rcu_pending(int cpu)
1283{
1284 return __rcu_pending(&rcu_state, &per_cpu(rcu_data, cpu)) ||
1285 __rcu_pending(&rcu_bh_state, &per_cpu(rcu_bh_data, cpu));
1286}
1287
1288/*
1289 * Check to see if any future RCU-related work will need to be done
1290 * by the current CPU, even if none need be done immediately, returning
1291 * 1 if so. This function is part of the RCU implementation; it is -not-
1292 * an exported member of the RCU API.
1293 */
1294int rcu_needs_cpu(int cpu)
1295{
1296 /* RCU callbacks either ready or pending? */
1297 return per_cpu(rcu_data, cpu).nxtlist ||
1298 per_cpu(rcu_bh_data, cpu).nxtlist;
1299}
1300
1301/*
1302 * Initialize a CPU's per-CPU RCU data. We take this "scorched earth"
1303 * approach so that we don't have to worry about how long the CPU has
1304 * been gone, or whether it ever was online previously. We do trust the
1305 * ->mynode field, as it is constant for a given struct rcu_data and
1306 * initialized during early boot.
1307 *
1308 * Note that only one online or offline event can be happening at a given
1309 * time. Note also that we can accept some slop in the rsp->completed
1310 * access due to the fact that this CPU cannot possibly have any RCU
1311 * callbacks in flight yet.
1312 */
1313static void
1314rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
1315{
1316 unsigned long flags;
1317 int i;
1318 long lastcomp;
1319 unsigned long mask;
1320 struct rcu_data *rdp = rsp->rda[cpu];
1321 struct rcu_node *rnp = rcu_get_root(rsp);
1322
1323 /* Set up local state, ensuring consistent view of global state. */
1324 spin_lock_irqsave(&rnp->lock, flags);
1325 lastcomp = rsp->completed;
1326 rdp->completed = lastcomp;
1327 rdp->gpnum = lastcomp;
1328 rdp->passed_quiesc = 0; /* We could be racing with new GP, */
1329 rdp->qs_pending = 1; /* so set up to respond to current GP. */
1330 rdp->beenonline = 1; /* We have now been online. */
1331 rdp->passed_quiesc_completed = lastcomp - 1;
1332 rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo);
1333 rdp->nxtlist = NULL;
1334 for (i = 0; i < RCU_NEXT_SIZE; i++)
1335 rdp->nxttail[i] = &rdp->nxtlist;
1336 rdp->qlen = 0;
1337 rdp->blimit = blimit;
1338#ifdef CONFIG_NO_HZ
1339 rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
1340#endif /* #ifdef CONFIG_NO_HZ */
1341 rdp->cpu = cpu;
1342 spin_unlock(&rnp->lock); /* irqs remain disabled. */
1343
1344 /*
1345 * A new grace period might start here. If so, we won't be part
1346 * of it, but that is OK, as we are currently in a quiescent state.
1347 */
1348
1349 /* Exclude any attempts to start a new GP on large systems. */
1350 spin_lock(&rsp->onofflock); /* irqs already disabled. */
1351
1352 /* Add CPU to rcu_node bitmasks. */
1353 rnp = rdp->mynode;
1354 mask = rdp->grpmask;
1355 do {
1356 /* Exclude any attempts to start a new GP on small systems. */
1357 spin_lock(&rnp->lock); /* irqs already disabled. */
1358 rnp->qsmaskinit |= mask;
1359 mask = rnp->grpmask;
1360 spin_unlock(&rnp->lock); /* irqs already disabled. */
1361 rnp = rnp->parent;
1362 } while (rnp != NULL && !(rnp->qsmaskinit & mask));
1363
1364 spin_unlock(&rsp->onofflock); /* irqs remain disabled. */
1365
1366 /*
1367 * A new grace period might start here. If so, we will be part of
1368 * it, and its gpnum will be greater than ours, so we will
1369 * participate. It is also possible for the gpnum to have been
1370 * incremented before this function was called, and the bitmasks
1371 * to not be filled out until now, in which case we will also
1372 * participate due to our gpnum being behind.
1373 */
1374
1375 /* Since it is coming online, the CPU is in a quiescent state. */
1376 cpu_quiet(cpu, rsp, rdp, lastcomp);
1377 local_irq_restore(flags);
1378}
1379
1380static void __cpuinit rcu_online_cpu(int cpu)
1381{
1382#ifdef CONFIG_NO_HZ
1383 struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
1384
1385 rdtp->dynticks_nesting = 1;
1386 rdtp->dynticks |= 1; /* need consecutive #s even for hotplug. */
1387 rdtp->dynticks_nmi = (rdtp->dynticks_nmi + 1) & ~0x1;
1388#endif /* #ifdef CONFIG_NO_HZ */
1389 rcu_init_percpu_data(cpu, &rcu_state);
1390 rcu_init_percpu_data(cpu, &rcu_bh_state);
1391 open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
1392}
1393
1394/*
1395 * Handle CPU online/offline notifcation events.
1396 */
1397static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
1398 unsigned long action, void *hcpu)
1399{
1400 long cpu = (long)hcpu;
1401
1402 switch (action) {
1403 case CPU_UP_PREPARE:
1404 case CPU_UP_PREPARE_FROZEN:
1405 rcu_online_cpu(cpu);
1406 break;
1407 case CPU_DEAD:
1408 case CPU_DEAD_FROZEN:
1409 case CPU_UP_CANCELED:
1410 case CPU_UP_CANCELED_FROZEN:
1411 rcu_offline_cpu(cpu);
1412 break;
1413 default:
1414 break;
1415 }
1416 return NOTIFY_OK;
1417}
1418
1419/*
1420 * Compute the per-level fanout, either using the exact fanout specified
1421 * or balancing the tree, depending on CONFIG_RCU_FANOUT_EXACT.
1422 */
1423#ifdef CONFIG_RCU_FANOUT_EXACT
1424static void __init rcu_init_levelspread(struct rcu_state *rsp)
1425{
1426 int i;
1427
1428 for (i = NUM_RCU_LVLS - 1; i >= 0; i--)
1429 rsp->levelspread[i] = CONFIG_RCU_FANOUT;
1430}
1431#else /* #ifdef CONFIG_RCU_FANOUT_EXACT */
1432static void __init rcu_init_levelspread(struct rcu_state *rsp)
1433{
1434 int ccur;
1435 int cprv;
1436 int i;
1437
1438 cprv = NR_CPUS;
1439 for (i = NUM_RCU_LVLS - 1; i >= 0; i--) {
1440 ccur = rsp->levelcnt[i];
1441 rsp->levelspread[i] = (cprv + ccur - 1) / ccur;
1442 cprv = ccur;
1443 }
1444}
1445#endif /* #else #ifdef CONFIG_RCU_FANOUT_EXACT */
1446
1447/*
1448 * Helper function for rcu_init() that initializes one rcu_state structure.
1449 */
1450static void __init rcu_init_one(struct rcu_state *rsp)
1451{
1452 int cpustride = 1;
1453 int i;
1454 int j;
1455 struct rcu_node *rnp;
1456
1457 /* Initialize the level-tracking arrays. */
1458
1459 for (i = 1; i < NUM_RCU_LVLS; i++)
1460 rsp->level[i] = rsp->level[i - 1] + rsp->levelcnt[i - 1];
1461 rcu_init_levelspread(rsp);
1462
1463 /* Initialize the elements themselves, starting from the leaves. */
1464
1465 for (i = NUM_RCU_LVLS - 1; i >= 0; i--) {
1466 cpustride *= rsp->levelspread[i];
1467 rnp = rsp->level[i];
1468 for (j = 0; j < rsp->levelcnt[i]; j++, rnp++) {
1469 spin_lock_init(&rnp->lock);
1470 rnp->qsmask = 0;
1471 rnp->qsmaskinit = 0;
1472 rnp->grplo = j * cpustride;
1473 rnp->grphi = (j + 1) * cpustride - 1;
1474 if (rnp->grphi >= NR_CPUS)
1475 rnp->grphi = NR_CPUS - 1;
1476 if (i == 0) {
1477 rnp->grpnum = 0;
1478 rnp->grpmask = 0;
1479 rnp->parent = NULL;
1480 } else {
1481 rnp->grpnum = j % rsp->levelspread[i - 1];
1482 rnp->grpmask = 1UL << rnp->grpnum;
1483 rnp->parent = rsp->level[i - 1] +
1484 j / rsp->levelspread[i - 1];
1485 }
1486 rnp->level = i;
1487 }
1488 }
1489}
1490
1491/*
1492 * Helper macro for __rcu_init(). To be used nowhere else!
1493 * Assigns leaf node pointers into each CPU's rcu_data structure.
1494 */
1495#define RCU_DATA_PTR_INIT(rsp, rcu_data) \
1496do { \
1497 rnp = (rsp)->level[NUM_RCU_LVLS - 1]; \
1498 j = 0; \
1499 for_each_possible_cpu(i) { \
1500 if (i > rnp[j].grphi) \
1501 j++; \
1502 per_cpu(rcu_data, i).mynode = &rnp[j]; \
1503 (rsp)->rda[i] = &per_cpu(rcu_data, i); \
1504 } \
1505} while (0)
1506
1507static struct notifier_block __cpuinitdata rcu_nb = {
1508 .notifier_call = rcu_cpu_notify,
1509};
1510
1511void __init __rcu_init(void)
1512{
1513 int i; /* All used by RCU_DATA_PTR_INIT(). */
1514 int j;
1515 struct rcu_node *rnp;
1516
1517 printk(KERN_WARNING "Experimental hierarchical RCU implementation.\n");
1518#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
1519 printk(KERN_INFO "RCU-based detection of stalled CPUs is enabled.\n");
1520#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
1521 rcu_init_one(&rcu_state);
1522 RCU_DATA_PTR_INIT(&rcu_state, rcu_data);
1523 rcu_init_one(&rcu_bh_state);
1524 RCU_DATA_PTR_INIT(&rcu_bh_state, rcu_bh_data);
1525
1526 for_each_online_cpu(i)
1527 rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, (void *)(long)i);
1528 /* Register notifier for non-boot CPUs */
1529 register_cpu_notifier(&rcu_nb);
1530 printk(KERN_WARNING "Experimental hierarchical RCU init done.\n");
1531}
1532
1533module_param(blimit, int, 0);
1534module_param(qhimark, int, 0);
1535module_param(qlowmark, int, 0);
diff --git a/kernel/rcutree_trace.c b/kernel/rcutree_trace.c
new file mode 100644
index 000000000000..d6db3e837826
--- /dev/null
+++ b/kernel/rcutree_trace.c
@@ -0,0 +1,271 @@
1/*
2 * Read-Copy Update tracing for classic implementation
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * Copyright IBM Corporation, 2008
19 *
20 * Papers: http://www.rdrop.com/users/paulmck/RCU
21 *
22 * For detailed explanation of Read-Copy Update mechanism see -
23 * Documentation/RCU
24 *
25 */
26#include <linux/types.h>
27#include <linux/kernel.h>
28#include <linux/init.h>
29#include <linux/spinlock.h>
30#include <linux/smp.h>
31#include <linux/rcupdate.h>
32#include <linux/interrupt.h>
33#include <linux/sched.h>
34#include <asm/atomic.h>
35#include <linux/bitops.h>
36#include <linux/module.h>
37#include <linux/completion.h>
38#include <linux/moduleparam.h>
39#include <linux/percpu.h>
40#include <linux/notifier.h>
41#include <linux/cpu.h>
42#include <linux/mutex.h>
43#include <linux/debugfs.h>
44#include <linux/seq_file.h>
45
46static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
47{
48 if (!rdp->beenonline)
49 return;
50 seq_printf(m, "%3d%cc=%ld g=%ld pq=%d pqc=%ld qp=%d rpfq=%ld rp=%x",
51 rdp->cpu,
52 cpu_is_offline(rdp->cpu) ? '!' : ' ',
53 rdp->completed, rdp->gpnum,
54 rdp->passed_quiesc, rdp->passed_quiesc_completed,
55 rdp->qs_pending,
56 rdp->n_rcu_pending_force_qs - rdp->n_rcu_pending,
57 (int)(rdp->n_rcu_pending & 0xffff));
58#ifdef CONFIG_NO_HZ
59 seq_printf(m, " dt=%d/%d dn=%d df=%lu",
60 rdp->dynticks->dynticks,
61 rdp->dynticks->dynticks_nesting,
62 rdp->dynticks->dynticks_nmi,
63 rdp->dynticks_fqs);
64#endif /* #ifdef CONFIG_NO_HZ */
65 seq_printf(m, " of=%lu ri=%lu", rdp->offline_fqs, rdp->resched_ipi);
66 seq_printf(m, " ql=%ld b=%ld\n", rdp->qlen, rdp->blimit);
67}
68
69#define PRINT_RCU_DATA(name, func, m) \
70 do { \
71 int _p_r_d_i; \
72 \
73 for_each_possible_cpu(_p_r_d_i) \
74 func(m, &per_cpu(name, _p_r_d_i)); \
75 } while (0)
76
77static int show_rcudata(struct seq_file *m, void *unused)
78{
79 seq_puts(m, "rcu:\n");
80 PRINT_RCU_DATA(rcu_data, print_one_rcu_data, m);
81 seq_puts(m, "rcu_bh:\n");
82 PRINT_RCU_DATA(rcu_bh_data, print_one_rcu_data, m);
83 return 0;
84}
85
86static int rcudata_open(struct inode *inode, struct file *file)
87{
88 return single_open(file, show_rcudata, NULL);
89}
90
91static struct file_operations rcudata_fops = {
92 .owner = THIS_MODULE,
93 .open = rcudata_open,
94 .read = seq_read,
95 .llseek = seq_lseek,
96 .release = single_release,
97};
98
99static void print_one_rcu_data_csv(struct seq_file *m, struct rcu_data *rdp)
100{
101 if (!rdp->beenonline)
102 return;
103 seq_printf(m, "%d,%s,%ld,%ld,%d,%ld,%d,%ld,%ld",
104 rdp->cpu,
105 cpu_is_offline(rdp->cpu) ? "\"Y\"" : "\"N\"",
106 rdp->completed, rdp->gpnum,
107 rdp->passed_quiesc, rdp->passed_quiesc_completed,
108 rdp->qs_pending,
109 rdp->n_rcu_pending_force_qs - rdp->n_rcu_pending,
110 rdp->n_rcu_pending);
111#ifdef CONFIG_NO_HZ
112 seq_printf(m, ",%d,%d,%d,%lu",
113 rdp->dynticks->dynticks,
114 rdp->dynticks->dynticks_nesting,
115 rdp->dynticks->dynticks_nmi,
116 rdp->dynticks_fqs);
117#endif /* #ifdef CONFIG_NO_HZ */
118 seq_printf(m, ",%lu,%lu", rdp->offline_fqs, rdp->resched_ipi);
119 seq_printf(m, ",%ld,%ld\n", rdp->qlen, rdp->blimit);
120}
121
122static int show_rcudata_csv(struct seq_file *m, void *unused)
123{
124 seq_puts(m, "\"CPU\",\"Online?\",\"c\",\"g\",\"pq\",\"pqc\",\"pq\",\"rpfq\",\"rp\",");
125#ifdef CONFIG_NO_HZ
126 seq_puts(m, "\"dt\",\"dt nesting\",\"dn\",\"df\",");
127#endif /* #ifdef CONFIG_NO_HZ */
128 seq_puts(m, "\"of\",\"ri\",\"ql\",\"b\"\n");
129 seq_puts(m, "\"rcu:\"\n");
130 PRINT_RCU_DATA(rcu_data, print_one_rcu_data_csv, m);
131 seq_puts(m, "\"rcu_bh:\"\n");
132 PRINT_RCU_DATA(rcu_bh_data, print_one_rcu_data_csv, m);
133 return 0;
134}
135
136static int rcudata_csv_open(struct inode *inode, struct file *file)
137{
138 return single_open(file, show_rcudata_csv, NULL);
139}
140
141static struct file_operations rcudata_csv_fops = {
142 .owner = THIS_MODULE,
143 .open = rcudata_csv_open,
144 .read = seq_read,
145 .llseek = seq_lseek,
146 .release = single_release,
147};
148
149static void print_one_rcu_state(struct seq_file *m, struct rcu_state *rsp)
150{
151 int level = 0;
152 struct rcu_node *rnp;
153
154 seq_printf(m, "c=%ld g=%ld s=%d jfq=%ld j=%x "
155 "nfqs=%lu/nfqsng=%lu(%lu) fqlh=%lu\n",
156 rsp->completed, rsp->gpnum, rsp->signaled,
157 (long)(rsp->jiffies_force_qs - jiffies),
158 (int)(jiffies & 0xffff),
159 rsp->n_force_qs, rsp->n_force_qs_ngp,
160 rsp->n_force_qs - rsp->n_force_qs_ngp,
161 rsp->n_force_qs_lh);
162 for (rnp = &rsp->node[0]; rnp - &rsp->node[0] < NUM_RCU_NODES; rnp++) {
163 if (rnp->level != level) {
164 seq_puts(m, "\n");
165 level = rnp->level;
166 }
167 seq_printf(m, "%lx/%lx %d:%d ^%d ",
168 rnp->qsmask, rnp->qsmaskinit,
169 rnp->grplo, rnp->grphi, rnp->grpnum);
170 }
171 seq_puts(m, "\n");
172}
173
174static int show_rcuhier(struct seq_file *m, void *unused)
175{
176 seq_puts(m, "rcu:\n");
177 print_one_rcu_state(m, &rcu_state);
178 seq_puts(m, "rcu_bh:\n");
179 print_one_rcu_state(m, &rcu_bh_state);
180 return 0;
181}
182
183static int rcuhier_open(struct inode *inode, struct file *file)
184{
185 return single_open(file, show_rcuhier, NULL);
186}
187
188static struct file_operations rcuhier_fops = {
189 .owner = THIS_MODULE,
190 .open = rcuhier_open,
191 .read = seq_read,
192 .llseek = seq_lseek,
193 .release = single_release,
194};
195
196static int show_rcugp(struct seq_file *m, void *unused)
197{
198 seq_printf(m, "rcu: completed=%ld gpnum=%ld\n",
199 rcu_state.completed, rcu_state.gpnum);
200 seq_printf(m, "rcu_bh: completed=%ld gpnum=%ld\n",
201 rcu_bh_state.completed, rcu_bh_state.gpnum);
202 return 0;
203}
204
205static int rcugp_open(struct inode *inode, struct file *file)
206{
207 return single_open(file, show_rcugp, NULL);
208}
209
210static struct file_operations rcugp_fops = {
211 .owner = THIS_MODULE,
212 .open = rcugp_open,
213 .read = seq_read,
214 .llseek = seq_lseek,
215 .release = single_release,
216};
217
218static struct dentry *rcudir, *datadir, *datadir_csv, *hierdir, *gpdir;
219static int __init rcuclassic_trace_init(void)
220{
221 rcudir = debugfs_create_dir("rcu", NULL);
222 if (!rcudir)
223 goto out;
224
225 datadir = debugfs_create_file("rcudata", 0444, rcudir,
226 NULL, &rcudata_fops);
227 if (!datadir)
228 goto free_out;
229
230 datadir_csv = debugfs_create_file("rcudata.csv", 0444, rcudir,
231 NULL, &rcudata_csv_fops);
232 if (!datadir_csv)
233 goto free_out;
234
235 gpdir = debugfs_create_file("rcugp", 0444, rcudir, NULL, &rcugp_fops);
236 if (!gpdir)
237 goto free_out;
238
239 hierdir = debugfs_create_file("rcuhier", 0444, rcudir,
240 NULL, &rcuhier_fops);
241 if (!hierdir)
242 goto free_out;
243 return 0;
244free_out:
245 if (datadir)
246 debugfs_remove(datadir);
247 if (datadir_csv)
248 debugfs_remove(datadir_csv);
249 if (gpdir)
250 debugfs_remove(gpdir);
251 debugfs_remove(rcudir);
252out:
253 return 1;
254}
255
256static void __exit rcuclassic_trace_cleanup(void)
257{
258 debugfs_remove(datadir);
259 debugfs_remove(datadir_csv);
260 debugfs_remove(gpdir);
261 debugfs_remove(hierdir);
262 debugfs_remove(rcudir);
263}
264
265
266module_init(rcuclassic_trace_init);
267module_exit(rcuclassic_trace_cleanup);
268
269MODULE_AUTHOR("Paul E. McKenney");
270MODULE_DESCRIPTION("Read-Copy Update tracing for hierarchical implementation");
271MODULE_LICENSE("GPL");
diff --git a/kernel/resource.c b/kernel/resource.c
index 4337063663ef..e633106b12f6 100644
--- a/kernel/resource.c
+++ b/kernel/resource.c
@@ -853,6 +853,15 @@ int iomem_map_sanity_check(resource_size_t addr, unsigned long size)
853 if (PFN_DOWN(p->start) <= PFN_DOWN(addr) && 853 if (PFN_DOWN(p->start) <= PFN_DOWN(addr) &&
854 PFN_DOWN(p->end) >= PFN_DOWN(addr + size - 1)) 854 PFN_DOWN(p->end) >= PFN_DOWN(addr + size - 1))
855 continue; 855 continue;
856 /*
857 * if a resource is "BUSY", it's not a hardware resource
858 * but a driver mapping of such a resource; we don't want
859 * to warn for those; some drivers legitimately map only
860 * partial hardware resources. (example: vesafb)
861 */
862 if (p->flags & IORESOURCE_BUSY)
863 continue;
864
856 printk(KERN_WARNING "resource map sanity check conflict: " 865 printk(KERN_WARNING "resource map sanity check conflict: "
857 "0x%llx 0x%llx 0x%llx 0x%llx %s\n", 866 "0x%llx 0x%llx 0x%llx 0x%llx %s\n",
858 (unsigned long long)addr, 867 (unsigned long long)addr,
diff --git a/kernel/sched.c b/kernel/sched.c
index 748ff924a290..27ba1d642f0f 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -209,7 +209,6 @@ void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
209 hrtimer_init(&rt_b->rt_period_timer, 209 hrtimer_init(&rt_b->rt_period_timer,
210 CLOCK_MONOTONIC, HRTIMER_MODE_REL); 210 CLOCK_MONOTONIC, HRTIMER_MODE_REL);
211 rt_b->rt_period_timer.function = sched_rt_period_timer; 211 rt_b->rt_period_timer.function = sched_rt_period_timer;
212 rt_b->rt_period_timer.cb_mode = HRTIMER_CB_IRQSAFE_UNLOCKED;
213} 212}
214 213
215static inline int rt_bandwidth_enabled(void) 214static inline int rt_bandwidth_enabled(void)
@@ -499,18 +498,26 @@ struct rt_rq {
499 */ 498 */
500struct root_domain { 499struct root_domain {
501 atomic_t refcount; 500 atomic_t refcount;
502 cpumask_t span; 501 cpumask_var_t span;
503 cpumask_t online; 502 cpumask_var_t online;
504 503
505 /* 504 /*
506 * The "RT overload" flag: it gets set if a CPU has more than 505 * The "RT overload" flag: it gets set if a CPU has more than
507 * one runnable RT task. 506 * one runnable RT task.
508 */ 507 */
509 cpumask_t rto_mask; 508 cpumask_var_t rto_mask;
510 atomic_t rto_count; 509 atomic_t rto_count;
511#ifdef CONFIG_SMP 510#ifdef CONFIG_SMP
512 struct cpupri cpupri; 511 struct cpupri cpupri;
513#endif 512#endif
513#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
514 /*
515 * Preferred wake up cpu nominated by sched_mc balance that will be
516 * used when most cpus are idle in the system indicating overall very
517 * low system utilisation. Triggered at POWERSAVINGS_BALANCE_WAKEUP(2)
518 */
519 unsigned int sched_mc_preferred_wakeup_cpu;
520#endif
514}; 521};
515 522
516/* 523/*
@@ -1139,7 +1146,6 @@ static void init_rq_hrtick(struct rq *rq)
1139 1146
1140 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1147 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1141 rq->hrtick_timer.function = hrtick; 1148 rq->hrtick_timer.function = hrtick;
1142 rq->hrtick_timer.cb_mode = HRTIMER_CB_IRQSAFE_PERCPU;
1143} 1149}
1144#else /* CONFIG_SCHED_HRTICK */ 1150#else /* CONFIG_SCHED_HRTICK */
1145static inline void hrtick_clear(struct rq *rq) 1151static inline void hrtick_clear(struct rq *rq)
@@ -1516,7 +1522,7 @@ static int tg_shares_up(struct task_group *tg, void *data)
1516 struct sched_domain *sd = data; 1522 struct sched_domain *sd = data;
1517 int i; 1523 int i;
1518 1524
1519 for_each_cpu_mask(i, sd->span) { 1525 for_each_cpu(i, sched_domain_span(sd)) {
1520 /* 1526 /*
1521 * If there are currently no tasks on the cpu pretend there 1527 * If there are currently no tasks on the cpu pretend there
1522 * is one of average load so that when a new task gets to 1528 * is one of average load so that when a new task gets to
@@ -1537,7 +1543,7 @@ static int tg_shares_up(struct task_group *tg, void *data)
1537 if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE)) 1543 if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE))
1538 shares = tg->shares; 1544 shares = tg->shares;
1539 1545
1540 for_each_cpu_mask(i, sd->span) 1546 for_each_cpu(i, sched_domain_span(sd))
1541 update_group_shares_cpu(tg, i, shares, rq_weight); 1547 update_group_shares_cpu(tg, i, shares, rq_weight);
1542 1548
1543 return 0; 1549 return 0;
@@ -2103,15 +2109,17 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
2103 int i; 2109 int i;
2104 2110
2105 /* Skip over this group if it has no CPUs allowed */ 2111 /* Skip over this group if it has no CPUs allowed */
2106 if (!cpus_intersects(group->cpumask, p->cpus_allowed)) 2112 if (!cpumask_intersects(sched_group_cpus(group),
2113 &p->cpus_allowed))
2107 continue; 2114 continue;
2108 2115
2109 local_group = cpu_isset(this_cpu, group->cpumask); 2116 local_group = cpumask_test_cpu(this_cpu,
2117 sched_group_cpus(group));
2110 2118
2111 /* Tally up the load of all CPUs in the group */ 2119 /* Tally up the load of all CPUs in the group */
2112 avg_load = 0; 2120 avg_load = 0;
2113 2121
2114 for_each_cpu_mask_nr(i, group->cpumask) { 2122 for_each_cpu(i, sched_group_cpus(group)) {
2115 /* Bias balancing toward cpus of our domain */ 2123 /* Bias balancing toward cpus of our domain */
2116 if (local_group) 2124 if (local_group)
2117 load = source_load(i, load_idx); 2125 load = source_load(i, load_idx);
@@ -2143,17 +2151,14 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
2143 * find_idlest_cpu - find the idlest cpu among the cpus in group. 2151 * find_idlest_cpu - find the idlest cpu among the cpus in group.
2144 */ 2152 */
2145static int 2153static int
2146find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu, 2154find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
2147 cpumask_t *tmp)
2148{ 2155{
2149 unsigned long load, min_load = ULONG_MAX; 2156 unsigned long load, min_load = ULONG_MAX;
2150 int idlest = -1; 2157 int idlest = -1;
2151 int i; 2158 int i;
2152 2159
2153 /* Traverse only the allowed CPUs */ 2160 /* Traverse only the allowed CPUs */
2154 cpus_and(*tmp, group->cpumask, p->cpus_allowed); 2161 for_each_cpu_and(i, sched_group_cpus(group), &p->cpus_allowed) {
2155
2156 for_each_cpu_mask_nr(i, *tmp) {
2157 load = weighted_cpuload(i); 2162 load = weighted_cpuload(i);
2158 2163
2159 if (load < min_load || (load == min_load && i == this_cpu)) { 2164 if (load < min_load || (load == min_load && i == this_cpu)) {
@@ -2195,7 +2200,6 @@ static int sched_balance_self(int cpu, int flag)
2195 update_shares(sd); 2200 update_shares(sd);
2196 2201
2197 while (sd) { 2202 while (sd) {
2198 cpumask_t span, tmpmask;
2199 struct sched_group *group; 2203 struct sched_group *group;
2200 int new_cpu, weight; 2204 int new_cpu, weight;
2201 2205
@@ -2204,14 +2208,13 @@ static int sched_balance_self(int cpu, int flag)
2204 continue; 2208 continue;
2205 } 2209 }
2206 2210
2207 span = sd->span;
2208 group = find_idlest_group(sd, t, cpu); 2211 group = find_idlest_group(sd, t, cpu);
2209 if (!group) { 2212 if (!group) {
2210 sd = sd->child; 2213 sd = sd->child;
2211 continue; 2214 continue;
2212 } 2215 }
2213 2216
2214 new_cpu = find_idlest_cpu(group, t, cpu, &tmpmask); 2217 new_cpu = find_idlest_cpu(group, t, cpu);
2215 if (new_cpu == -1 || new_cpu == cpu) { 2218 if (new_cpu == -1 || new_cpu == cpu) {
2216 /* Now try balancing at a lower domain level of cpu */ 2219 /* Now try balancing at a lower domain level of cpu */
2217 sd = sd->child; 2220 sd = sd->child;
@@ -2220,10 +2223,10 @@ static int sched_balance_self(int cpu, int flag)
2220 2223
2221 /* Now try balancing at a lower domain level of new_cpu */ 2224 /* Now try balancing at a lower domain level of new_cpu */
2222 cpu = new_cpu; 2225 cpu = new_cpu;
2226 weight = cpumask_weight(sched_domain_span(sd));
2223 sd = NULL; 2227 sd = NULL;
2224 weight = cpus_weight(span);
2225 for_each_domain(cpu, tmp) { 2228 for_each_domain(cpu, tmp) {
2226 if (weight <= cpus_weight(tmp->span)) 2229 if (weight <= cpumask_weight(sched_domain_span(tmp)))
2227 break; 2230 break;
2228 if (tmp->flags & flag) 2231 if (tmp->flags & flag)
2229 sd = tmp; 2232 sd = tmp;
@@ -2268,7 +2271,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
2268 cpu = task_cpu(p); 2271 cpu = task_cpu(p);
2269 2272
2270 for_each_domain(this_cpu, sd) { 2273 for_each_domain(this_cpu, sd) {
2271 if (cpu_isset(cpu, sd->span)) { 2274 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2272 update_shares(sd); 2275 update_shares(sd);
2273 break; 2276 break;
2274 } 2277 }
@@ -2317,7 +2320,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
2317 else { 2320 else {
2318 struct sched_domain *sd; 2321 struct sched_domain *sd;
2319 for_each_domain(this_cpu, sd) { 2322 for_each_domain(this_cpu, sd) {
2320 if (cpu_isset(cpu, sd->span)) { 2323 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2321 schedstat_inc(sd, ttwu_wake_remote); 2324 schedstat_inc(sd, ttwu_wake_remote);
2322 break; 2325 break;
2323 } 2326 }
@@ -2848,7 +2851,7 @@ static void sched_migrate_task(struct task_struct *p, int dest_cpu)
2848 struct rq *rq; 2851 struct rq *rq;
2849 2852
2850 rq = task_rq_lock(p, &flags); 2853 rq = task_rq_lock(p, &flags);
2851 if (!cpu_isset(dest_cpu, p->cpus_allowed) 2854 if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed)
2852 || unlikely(!cpu_active(dest_cpu))) 2855 || unlikely(!cpu_active(dest_cpu)))
2853 goto out; 2856 goto out;
2854 2857
@@ -2913,7 +2916,7 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
2913 * 2) cannot be migrated to this CPU due to cpus_allowed, or 2916 * 2) cannot be migrated to this CPU due to cpus_allowed, or
2914 * 3) are cache-hot on their current CPU. 2917 * 3) are cache-hot on their current CPU.
2915 */ 2918 */
2916 if (!cpu_isset(this_cpu, p->cpus_allowed)) { 2919 if (!cpumask_test_cpu(this_cpu, &p->cpus_allowed)) {
2917 schedstat_inc(p, se.nr_failed_migrations_affine); 2920 schedstat_inc(p, se.nr_failed_migrations_affine);
2918 return 0; 2921 return 0;
2919 } 2922 }
@@ -3088,7 +3091,7 @@ static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
3088static struct sched_group * 3091static struct sched_group *
3089find_busiest_group(struct sched_domain *sd, int this_cpu, 3092find_busiest_group(struct sched_domain *sd, int this_cpu,
3090 unsigned long *imbalance, enum cpu_idle_type idle, 3093 unsigned long *imbalance, enum cpu_idle_type idle,
3091 int *sd_idle, const cpumask_t *cpus, int *balance) 3094 int *sd_idle, const struct cpumask *cpus, int *balance)
3092{ 3095{
3093 struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups; 3096 struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups;
3094 unsigned long max_load, avg_load, total_load, this_load, total_pwr; 3097 unsigned long max_load, avg_load, total_load, this_load, total_pwr;
@@ -3124,10 +3127,11 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3124 unsigned long sum_avg_load_per_task; 3127 unsigned long sum_avg_load_per_task;
3125 unsigned long avg_load_per_task; 3128 unsigned long avg_load_per_task;
3126 3129
3127 local_group = cpu_isset(this_cpu, group->cpumask); 3130 local_group = cpumask_test_cpu(this_cpu,
3131 sched_group_cpus(group));
3128 3132
3129 if (local_group) 3133 if (local_group)
3130 balance_cpu = first_cpu(group->cpumask); 3134 balance_cpu = cpumask_first(sched_group_cpus(group));
3131 3135
3132 /* Tally up the load of all CPUs in the group */ 3136 /* Tally up the load of all CPUs in the group */
3133 sum_weighted_load = sum_nr_running = avg_load = 0; 3137 sum_weighted_load = sum_nr_running = avg_load = 0;
@@ -3136,13 +3140,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3136 max_cpu_load = 0; 3140 max_cpu_load = 0;
3137 min_cpu_load = ~0UL; 3141 min_cpu_load = ~0UL;
3138 3142
3139 for_each_cpu_mask_nr(i, group->cpumask) { 3143 for_each_cpu_and(i, sched_group_cpus(group), cpus) {
3140 struct rq *rq; 3144 struct rq *rq = cpu_rq(i);
3141
3142 if (!cpu_isset(i, *cpus))
3143 continue;
3144
3145 rq = cpu_rq(i);
3146 3145
3147 if (*sd_idle && rq->nr_running) 3146 if (*sd_idle && rq->nr_running)
3148 *sd_idle = 0; 3147 *sd_idle = 0;
@@ -3253,8 +3252,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3253 */ 3252 */
3254 if ((sum_nr_running < min_nr_running) || 3253 if ((sum_nr_running < min_nr_running) ||
3255 (sum_nr_running == min_nr_running && 3254 (sum_nr_running == min_nr_running &&
3256 first_cpu(group->cpumask) < 3255 cpumask_first(sched_group_cpus(group)) >
3257 first_cpu(group_min->cpumask))) { 3256 cpumask_first(sched_group_cpus(group_min)))) {
3258 group_min = group; 3257 group_min = group;
3259 min_nr_running = sum_nr_running; 3258 min_nr_running = sum_nr_running;
3260 min_load_per_task = sum_weighted_load / 3259 min_load_per_task = sum_weighted_load /
@@ -3269,8 +3268,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3269 if (sum_nr_running <= group_capacity - 1) { 3268 if (sum_nr_running <= group_capacity - 1) {
3270 if (sum_nr_running > leader_nr_running || 3269 if (sum_nr_running > leader_nr_running ||
3271 (sum_nr_running == leader_nr_running && 3270 (sum_nr_running == leader_nr_running &&
3272 first_cpu(group->cpumask) > 3271 cpumask_first(sched_group_cpus(group)) <
3273 first_cpu(group_leader->cpumask))) { 3272 cpumask_first(sched_group_cpus(group_leader)))) {
3274 group_leader = group; 3273 group_leader = group;
3275 leader_nr_running = sum_nr_running; 3274 leader_nr_running = sum_nr_running;
3276 } 3275 }
@@ -3396,6 +3395,10 @@ out_balanced:
3396 3395
3397 if (this == group_leader && group_leader != group_min) { 3396 if (this == group_leader && group_leader != group_min) {
3398 *imbalance = min_load_per_task; 3397 *imbalance = min_load_per_task;
3398 if (sched_mc_power_savings >= POWERSAVINGS_BALANCE_WAKEUP) {
3399 cpu_rq(this_cpu)->rd->sched_mc_preferred_wakeup_cpu =
3400 cpumask_first(sched_group_cpus(group_leader));
3401 }
3399 return group_min; 3402 return group_min;
3400 } 3403 }
3401#endif 3404#endif
@@ -3409,16 +3412,16 @@ ret:
3409 */ 3412 */
3410static struct rq * 3413static struct rq *
3411find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle, 3414find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
3412 unsigned long imbalance, const cpumask_t *cpus) 3415 unsigned long imbalance, const struct cpumask *cpus)
3413{ 3416{
3414 struct rq *busiest = NULL, *rq; 3417 struct rq *busiest = NULL, *rq;
3415 unsigned long max_load = 0; 3418 unsigned long max_load = 0;
3416 int i; 3419 int i;
3417 3420
3418 for_each_cpu_mask_nr(i, group->cpumask) { 3421 for_each_cpu(i, sched_group_cpus(group)) {
3419 unsigned long wl; 3422 unsigned long wl;
3420 3423
3421 if (!cpu_isset(i, *cpus)) 3424 if (!cpumask_test_cpu(i, cpus))
3422 continue; 3425 continue;
3423 3426
3424 rq = cpu_rq(i); 3427 rq = cpu_rq(i);
@@ -3448,7 +3451,7 @@ find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
3448 */ 3451 */
3449static int load_balance(int this_cpu, struct rq *this_rq, 3452static int load_balance(int this_cpu, struct rq *this_rq,
3450 struct sched_domain *sd, enum cpu_idle_type idle, 3453 struct sched_domain *sd, enum cpu_idle_type idle,
3451 int *balance, cpumask_t *cpus) 3454 int *balance, struct cpumask *cpus)
3452{ 3455{
3453 int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0; 3456 int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
3454 struct sched_group *group; 3457 struct sched_group *group;
@@ -3456,7 +3459,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
3456 struct rq *busiest; 3459 struct rq *busiest;
3457 unsigned long flags; 3460 unsigned long flags;
3458 3461
3459 cpus_setall(*cpus); 3462 cpumask_setall(cpus);
3460 3463
3461 /* 3464 /*
3462 * When power savings policy is enabled for the parent domain, idle 3465 * When power savings policy is enabled for the parent domain, idle
@@ -3516,8 +3519,8 @@ redo:
3516 3519
3517 /* All tasks on this runqueue were pinned by CPU affinity */ 3520 /* All tasks on this runqueue were pinned by CPU affinity */
3518 if (unlikely(all_pinned)) { 3521 if (unlikely(all_pinned)) {
3519 cpu_clear(cpu_of(busiest), *cpus); 3522 cpumask_clear_cpu(cpu_of(busiest), cpus);
3520 if (!cpus_empty(*cpus)) 3523 if (!cpumask_empty(cpus))
3521 goto redo; 3524 goto redo;
3522 goto out_balanced; 3525 goto out_balanced;
3523 } 3526 }
@@ -3534,7 +3537,8 @@ redo:
3534 /* don't kick the migration_thread, if the curr 3537 /* don't kick the migration_thread, if the curr
3535 * task on busiest cpu can't be moved to this_cpu 3538 * task on busiest cpu can't be moved to this_cpu
3536 */ 3539 */
3537 if (!cpu_isset(this_cpu, busiest->curr->cpus_allowed)) { 3540 if (!cpumask_test_cpu(this_cpu,
3541 &busiest->curr->cpus_allowed)) {
3538 spin_unlock_irqrestore(&busiest->lock, flags); 3542 spin_unlock_irqrestore(&busiest->lock, flags);
3539 all_pinned = 1; 3543 all_pinned = 1;
3540 goto out_one_pinned; 3544 goto out_one_pinned;
@@ -3609,7 +3613,7 @@ out:
3609 */ 3613 */
3610static int 3614static int
3611load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd, 3615load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
3612 cpumask_t *cpus) 3616 struct cpumask *cpus)
3613{ 3617{
3614 struct sched_group *group; 3618 struct sched_group *group;
3615 struct rq *busiest = NULL; 3619 struct rq *busiest = NULL;
@@ -3618,7 +3622,7 @@ load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
3618 int sd_idle = 0; 3622 int sd_idle = 0;
3619 int all_pinned = 0; 3623 int all_pinned = 0;
3620 3624
3621 cpus_setall(*cpus); 3625 cpumask_setall(cpus);
3622 3626
3623 /* 3627 /*
3624 * When power savings policy is enabled for the parent domain, idle 3628 * When power savings policy is enabled for the parent domain, idle
@@ -3662,17 +3666,71 @@ redo:
3662 double_unlock_balance(this_rq, busiest); 3666 double_unlock_balance(this_rq, busiest);
3663 3667
3664 if (unlikely(all_pinned)) { 3668 if (unlikely(all_pinned)) {
3665 cpu_clear(cpu_of(busiest), *cpus); 3669 cpumask_clear_cpu(cpu_of(busiest), cpus);
3666 if (!cpus_empty(*cpus)) 3670 if (!cpumask_empty(cpus))
3667 goto redo; 3671 goto redo;
3668 } 3672 }
3669 } 3673 }
3670 3674
3671 if (!ld_moved) { 3675 if (!ld_moved) {
3676 int active_balance = 0;
3677
3672 schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]); 3678 schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]);
3673 if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER && 3679 if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
3674 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE)) 3680 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
3675 return -1; 3681 return -1;
3682
3683 if (sched_mc_power_savings < POWERSAVINGS_BALANCE_WAKEUP)
3684 return -1;
3685
3686 if (sd->nr_balance_failed++ < 2)
3687 return -1;
3688
3689 /*
3690 * The only task running in a non-idle cpu can be moved to this
3691 * cpu in an attempt to completely freeup the other CPU
3692 * package. The same method used to move task in load_balance()
3693 * have been extended for load_balance_newidle() to speedup
3694 * consolidation at sched_mc=POWERSAVINGS_BALANCE_WAKEUP (2)
3695 *
3696 * The package power saving logic comes from
3697 * find_busiest_group(). If there are no imbalance, then
3698 * f_b_g() will return NULL. However when sched_mc={1,2} then
3699 * f_b_g() will select a group from which a running task may be
3700 * pulled to this cpu in order to make the other package idle.
3701 * If there is no opportunity to make a package idle and if
3702 * there are no imbalance, then f_b_g() will return NULL and no
3703 * action will be taken in load_balance_newidle().
3704 *
3705 * Under normal task pull operation due to imbalance, there
3706 * will be more than one task in the source run queue and
3707 * move_tasks() will succeed. ld_moved will be true and this
3708 * active balance code will not be triggered.
3709 */
3710
3711 /* Lock busiest in correct order while this_rq is held */
3712 double_lock_balance(this_rq, busiest);
3713
3714 /*
3715 * don't kick the migration_thread, if the curr
3716 * task on busiest cpu can't be moved to this_cpu
3717 */
3718 if (!cpu_isset(this_cpu, busiest->curr->cpus_allowed)) {
3719 double_unlock_balance(this_rq, busiest);
3720 all_pinned = 1;
3721 return ld_moved;
3722 }
3723
3724 if (!busiest->active_balance) {
3725 busiest->active_balance = 1;
3726 busiest->push_cpu = this_cpu;
3727 active_balance = 1;
3728 }
3729
3730 double_unlock_balance(this_rq, busiest);
3731 if (active_balance)
3732 wake_up_process(busiest->migration_thread);
3733
3676 } else 3734 } else
3677 sd->nr_balance_failed = 0; 3735 sd->nr_balance_failed = 0;
3678 3736
@@ -3698,7 +3756,10 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
3698 struct sched_domain *sd; 3756 struct sched_domain *sd;
3699 int pulled_task = 0; 3757 int pulled_task = 0;
3700 unsigned long next_balance = jiffies + HZ; 3758 unsigned long next_balance = jiffies + HZ;
3701 cpumask_t tmpmask; 3759 cpumask_var_t tmpmask;
3760
3761 if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC))
3762 return;
3702 3763
3703 for_each_domain(this_cpu, sd) { 3764 for_each_domain(this_cpu, sd) {
3704 unsigned long interval; 3765 unsigned long interval;
@@ -3709,7 +3770,7 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
3709 if (sd->flags & SD_BALANCE_NEWIDLE) 3770 if (sd->flags & SD_BALANCE_NEWIDLE)
3710 /* If we've pulled tasks over stop searching: */ 3771 /* If we've pulled tasks over stop searching: */
3711 pulled_task = load_balance_newidle(this_cpu, this_rq, 3772 pulled_task = load_balance_newidle(this_cpu, this_rq,
3712 sd, &tmpmask); 3773 sd, tmpmask);
3713 3774
3714 interval = msecs_to_jiffies(sd->balance_interval); 3775 interval = msecs_to_jiffies(sd->balance_interval);
3715 if (time_after(next_balance, sd->last_balance + interval)) 3776 if (time_after(next_balance, sd->last_balance + interval))
@@ -3724,6 +3785,7 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
3724 */ 3785 */
3725 this_rq->next_balance = next_balance; 3786 this_rq->next_balance = next_balance;
3726 } 3787 }
3788 free_cpumask_var(tmpmask);
3727} 3789}
3728 3790
3729/* 3791/*
@@ -3761,7 +3823,7 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
3761 /* Search for an sd spanning us and the target CPU. */ 3823 /* Search for an sd spanning us and the target CPU. */
3762 for_each_domain(target_cpu, sd) { 3824 for_each_domain(target_cpu, sd) {
3763 if ((sd->flags & SD_LOAD_BALANCE) && 3825 if ((sd->flags & SD_LOAD_BALANCE) &&
3764 cpu_isset(busiest_cpu, sd->span)) 3826 cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
3765 break; 3827 break;
3766 } 3828 }
3767 3829
@@ -3780,10 +3842,9 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
3780#ifdef CONFIG_NO_HZ 3842#ifdef CONFIG_NO_HZ
3781static struct { 3843static struct {
3782 atomic_t load_balancer; 3844 atomic_t load_balancer;
3783 cpumask_t cpu_mask; 3845 cpumask_var_t cpu_mask;
3784} nohz ____cacheline_aligned = { 3846} nohz ____cacheline_aligned = {
3785 .load_balancer = ATOMIC_INIT(-1), 3847 .load_balancer = ATOMIC_INIT(-1),
3786 .cpu_mask = CPU_MASK_NONE,
3787}; 3848};
3788 3849
3789/* 3850/*
@@ -3811,7 +3872,7 @@ int select_nohz_load_balancer(int stop_tick)
3811 int cpu = smp_processor_id(); 3872 int cpu = smp_processor_id();
3812 3873
3813 if (stop_tick) { 3874 if (stop_tick) {
3814 cpu_set(cpu, nohz.cpu_mask); 3875 cpumask_set_cpu(cpu, nohz.cpu_mask);
3815 cpu_rq(cpu)->in_nohz_recently = 1; 3876 cpu_rq(cpu)->in_nohz_recently = 1;
3816 3877
3817 /* 3878 /*
@@ -3825,7 +3886,7 @@ int select_nohz_load_balancer(int stop_tick)
3825 } 3886 }
3826 3887
3827 /* time for ilb owner also to sleep */ 3888 /* time for ilb owner also to sleep */
3828 if (cpus_weight(nohz.cpu_mask) == num_online_cpus()) { 3889 if (cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
3829 if (atomic_read(&nohz.load_balancer) == cpu) 3890 if (atomic_read(&nohz.load_balancer) == cpu)
3830 atomic_set(&nohz.load_balancer, -1); 3891 atomic_set(&nohz.load_balancer, -1);
3831 return 0; 3892 return 0;
@@ -3838,10 +3899,10 @@ int select_nohz_load_balancer(int stop_tick)
3838 } else if (atomic_read(&nohz.load_balancer) == cpu) 3899 } else if (atomic_read(&nohz.load_balancer) == cpu)
3839 return 1; 3900 return 1;
3840 } else { 3901 } else {
3841 if (!cpu_isset(cpu, nohz.cpu_mask)) 3902 if (!cpumask_test_cpu(cpu, nohz.cpu_mask))
3842 return 0; 3903 return 0;
3843 3904
3844 cpu_clear(cpu, nohz.cpu_mask); 3905 cpumask_clear_cpu(cpu, nohz.cpu_mask);
3845 3906
3846 if (atomic_read(&nohz.load_balancer) == cpu) 3907 if (atomic_read(&nohz.load_balancer) == cpu)
3847 if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu) 3908 if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
@@ -3869,7 +3930,11 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3869 unsigned long next_balance = jiffies + 60*HZ; 3930 unsigned long next_balance = jiffies + 60*HZ;
3870 int update_next_balance = 0; 3931 int update_next_balance = 0;
3871 int need_serialize; 3932 int need_serialize;
3872 cpumask_t tmp; 3933 cpumask_var_t tmp;
3934
3935 /* Fails alloc? Rebalancing probably not a priority right now. */
3936 if (!alloc_cpumask_var(&tmp, GFP_ATOMIC))
3937 return;
3873 3938
3874 for_each_domain(cpu, sd) { 3939 for_each_domain(cpu, sd) {
3875 if (!(sd->flags & SD_LOAD_BALANCE)) 3940 if (!(sd->flags & SD_LOAD_BALANCE))
@@ -3894,7 +3959,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3894 } 3959 }
3895 3960
3896 if (time_after_eq(jiffies, sd->last_balance + interval)) { 3961 if (time_after_eq(jiffies, sd->last_balance + interval)) {
3897 if (load_balance(cpu, rq, sd, idle, &balance, &tmp)) { 3962 if (load_balance(cpu, rq, sd, idle, &balance, tmp)) {
3898 /* 3963 /*
3899 * We've pulled tasks over so either we're no 3964 * We've pulled tasks over so either we're no
3900 * longer idle, or one of our SMT siblings is 3965 * longer idle, or one of our SMT siblings is
@@ -3928,6 +3993,8 @@ out:
3928 */ 3993 */
3929 if (likely(update_next_balance)) 3994 if (likely(update_next_balance))
3930 rq->next_balance = next_balance; 3995 rq->next_balance = next_balance;
3996
3997 free_cpumask_var(tmp);
3931} 3998}
3932 3999
3933/* 4000/*
@@ -3952,12 +4019,13 @@ static void run_rebalance_domains(struct softirq_action *h)
3952 */ 4019 */
3953 if (this_rq->idle_at_tick && 4020 if (this_rq->idle_at_tick &&
3954 atomic_read(&nohz.load_balancer) == this_cpu) { 4021 atomic_read(&nohz.load_balancer) == this_cpu) {
3955 cpumask_t cpus = nohz.cpu_mask;
3956 struct rq *rq; 4022 struct rq *rq;
3957 int balance_cpu; 4023 int balance_cpu;
3958 4024
3959 cpu_clear(this_cpu, cpus); 4025 for_each_cpu(balance_cpu, nohz.cpu_mask) {
3960 for_each_cpu_mask_nr(balance_cpu, cpus) { 4026 if (balance_cpu == this_cpu)
4027 continue;
4028
3961 /* 4029 /*
3962 * If this cpu gets work to do, stop the load balancing 4030 * If this cpu gets work to do, stop the load balancing
3963 * work being done for other cpus. Next load 4031 * work being done for other cpus. Next load
@@ -3995,7 +4063,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
3995 rq->in_nohz_recently = 0; 4063 rq->in_nohz_recently = 0;
3996 4064
3997 if (atomic_read(&nohz.load_balancer) == cpu) { 4065 if (atomic_read(&nohz.load_balancer) == cpu) {
3998 cpu_clear(cpu, nohz.cpu_mask); 4066 cpumask_clear_cpu(cpu, nohz.cpu_mask);
3999 atomic_set(&nohz.load_balancer, -1); 4067 atomic_set(&nohz.load_balancer, -1);
4000 } 4068 }
4001 4069
@@ -4008,7 +4076,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4008 * TBD: Traverse the sched domains and nominate 4076 * TBD: Traverse the sched domains and nominate
4009 * the nearest cpu in the nohz.cpu_mask. 4077 * the nearest cpu in the nohz.cpu_mask.
4010 */ 4078 */
4011 int ilb = first_cpu(nohz.cpu_mask); 4079 int ilb = cpumask_first(nohz.cpu_mask);
4012 4080
4013 if (ilb < nr_cpu_ids) 4081 if (ilb < nr_cpu_ids)
4014 resched_cpu(ilb); 4082 resched_cpu(ilb);
@@ -4020,7 +4088,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4020 * cpus with ticks stopped, is it time for that to stop? 4088 * cpus with ticks stopped, is it time for that to stop?
4021 */ 4089 */
4022 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu && 4090 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&
4023 cpus_weight(nohz.cpu_mask) == num_online_cpus()) { 4091 cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
4024 resched_cpu(cpu); 4092 resched_cpu(cpu);
4025 return; 4093 return;
4026 } 4094 }
@@ -4030,7 +4098,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4030 * someone else, then no need raise the SCHED_SOFTIRQ 4098 * someone else, then no need raise the SCHED_SOFTIRQ
4031 */ 4099 */
4032 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu && 4100 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&
4033 cpu_isset(cpu, nohz.cpu_mask)) 4101 cpumask_test_cpu(cpu, nohz.cpu_mask))
4034 return; 4102 return;
4035#endif 4103#endif
4036 if (time_after_eq(jiffies, rq->next_balance)) 4104 if (time_after_eq(jiffies, rq->next_balance))
@@ -4192,7 +4260,6 @@ void account_steal_time(struct task_struct *p, cputime_t steal)
4192 4260
4193 if (p == rq->idle) { 4261 if (p == rq->idle) {
4194 p->stime = cputime_add(p->stime, steal); 4262 p->stime = cputime_add(p->stime, steal);
4195 account_group_system_time(p, steal);
4196 if (atomic_read(&rq->nr_iowait) > 0) 4263 if (atomic_read(&rq->nr_iowait) > 0)
4197 cpustat->iowait = cputime64_add(cpustat->iowait, tmp); 4264 cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
4198 else 4265 else
@@ -4328,7 +4395,7 @@ void __kprobes sub_preempt_count(int val)
4328 /* 4395 /*
4329 * Underflow? 4396 * Underflow?
4330 */ 4397 */
4331 if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) 4398 if (DEBUG_LOCKS_WARN_ON(val > preempt_count() - (!!kernel_locked())))
4332 return; 4399 return;
4333 /* 4400 /*
4334 * Is the spinlock portion underflowing? 4401 * Is the spinlock portion underflowing?
@@ -5404,10 +5471,9 @@ out_unlock:
5404 return retval; 5471 return retval;
5405} 5472}
5406 5473
5407long sched_setaffinity(pid_t pid, const cpumask_t *in_mask) 5474long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
5408{ 5475{
5409 cpumask_t cpus_allowed; 5476 cpumask_var_t cpus_allowed, new_mask;
5410 cpumask_t new_mask = *in_mask;
5411 struct task_struct *p; 5477 struct task_struct *p;
5412 int retval; 5478 int retval;
5413 5479
@@ -5429,6 +5495,14 @@ long sched_setaffinity(pid_t pid, const cpumask_t *in_mask)
5429 get_task_struct(p); 5495 get_task_struct(p);
5430 read_unlock(&tasklist_lock); 5496 read_unlock(&tasklist_lock);
5431 5497
5498 if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
5499 retval = -ENOMEM;
5500 goto out_put_task;
5501 }
5502 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
5503 retval = -ENOMEM;
5504 goto out_free_cpus_allowed;
5505 }
5432 retval = -EPERM; 5506 retval = -EPERM;
5433 if (!check_same_owner(p) && !capable(CAP_SYS_NICE)) 5507 if (!check_same_owner(p) && !capable(CAP_SYS_NICE))
5434 goto out_unlock; 5508 goto out_unlock;
@@ -5437,37 +5511,41 @@ long sched_setaffinity(pid_t pid, const cpumask_t *in_mask)
5437 if (retval) 5511 if (retval)
5438 goto out_unlock; 5512 goto out_unlock;
5439 5513
5440 cpuset_cpus_allowed(p, &cpus_allowed); 5514 cpuset_cpus_allowed(p, cpus_allowed);
5441 cpus_and(new_mask, new_mask, cpus_allowed); 5515 cpumask_and(new_mask, in_mask, cpus_allowed);
5442 again: 5516 again:
5443 retval = set_cpus_allowed_ptr(p, &new_mask); 5517 retval = set_cpus_allowed_ptr(p, new_mask);
5444 5518
5445 if (!retval) { 5519 if (!retval) {
5446 cpuset_cpus_allowed(p, &cpus_allowed); 5520 cpuset_cpus_allowed(p, cpus_allowed);
5447 if (!cpus_subset(new_mask, cpus_allowed)) { 5521 if (!cpumask_subset(new_mask, cpus_allowed)) {
5448 /* 5522 /*
5449 * We must have raced with a concurrent cpuset 5523 * We must have raced with a concurrent cpuset
5450 * update. Just reset the cpus_allowed to the 5524 * update. Just reset the cpus_allowed to the
5451 * cpuset's cpus_allowed 5525 * cpuset's cpus_allowed
5452 */ 5526 */
5453 new_mask = cpus_allowed; 5527 cpumask_copy(new_mask, cpus_allowed);
5454 goto again; 5528 goto again;
5455 } 5529 }
5456 } 5530 }
5457out_unlock: 5531out_unlock:
5532 free_cpumask_var(new_mask);
5533out_free_cpus_allowed:
5534 free_cpumask_var(cpus_allowed);
5535out_put_task:
5458 put_task_struct(p); 5536 put_task_struct(p);
5459 put_online_cpus(); 5537 put_online_cpus();
5460 return retval; 5538 return retval;
5461} 5539}
5462 5540
5463static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, 5541static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
5464 cpumask_t *new_mask) 5542 struct cpumask *new_mask)
5465{ 5543{
5466 if (len < sizeof(cpumask_t)) { 5544 if (len < cpumask_size())
5467 memset(new_mask, 0, sizeof(cpumask_t)); 5545 cpumask_clear(new_mask);
5468 } else if (len > sizeof(cpumask_t)) { 5546 else if (len > cpumask_size())
5469 len = sizeof(cpumask_t); 5547 len = cpumask_size();
5470 } 5548
5471 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; 5549 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
5472} 5550}
5473 5551
@@ -5480,17 +5558,20 @@ static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
5480asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len, 5558asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
5481 unsigned long __user *user_mask_ptr) 5559 unsigned long __user *user_mask_ptr)
5482{ 5560{
5483 cpumask_t new_mask; 5561 cpumask_var_t new_mask;
5484 int retval; 5562 int retval;
5485 5563
5486 retval = get_user_cpu_mask(user_mask_ptr, len, &new_mask); 5564 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
5487 if (retval) 5565 return -ENOMEM;
5488 return retval;
5489 5566
5490 return sched_setaffinity(pid, &new_mask); 5567 retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
5568 if (retval == 0)
5569 retval = sched_setaffinity(pid, new_mask);
5570 free_cpumask_var(new_mask);
5571 return retval;
5491} 5572}
5492 5573
5493long sched_getaffinity(pid_t pid, cpumask_t *mask) 5574long sched_getaffinity(pid_t pid, struct cpumask *mask)
5494{ 5575{
5495 struct task_struct *p; 5576 struct task_struct *p;
5496 int retval; 5577 int retval;
@@ -5507,7 +5588,7 @@ long sched_getaffinity(pid_t pid, cpumask_t *mask)
5507 if (retval) 5588 if (retval)
5508 goto out_unlock; 5589 goto out_unlock;
5509 5590
5510 cpus_and(*mask, p->cpus_allowed, cpu_online_map); 5591 cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
5511 5592
5512out_unlock: 5593out_unlock:
5513 read_unlock(&tasklist_lock); 5594 read_unlock(&tasklist_lock);
@@ -5526,19 +5607,24 @@ asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len,
5526 unsigned long __user *user_mask_ptr) 5607 unsigned long __user *user_mask_ptr)
5527{ 5608{
5528 int ret; 5609 int ret;
5529 cpumask_t mask; 5610 cpumask_var_t mask;
5530 5611
5531 if (len < sizeof(cpumask_t)) 5612 if (len < cpumask_size())
5532 return -EINVAL; 5613 return -EINVAL;
5533 5614
5534 ret = sched_getaffinity(pid, &mask); 5615 if (!alloc_cpumask_var(&mask, GFP_KERNEL))
5535 if (ret < 0) 5616 return -ENOMEM;
5536 return ret;
5537 5617
5538 if (copy_to_user(user_mask_ptr, &mask, sizeof(cpumask_t))) 5618 ret = sched_getaffinity(pid, mask);
5539 return -EFAULT; 5619 if (ret == 0) {
5620 if (copy_to_user(user_mask_ptr, mask, cpumask_size()))
5621 ret = -EFAULT;
5622 else
5623 ret = cpumask_size();
5624 }
5625 free_cpumask_var(mask);
5540 5626
5541 return sizeof(cpumask_t); 5627 return ret;
5542} 5628}
5543 5629
5544/** 5630/**
@@ -5880,7 +5966,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
5880 idle->se.exec_start = sched_clock(); 5966 idle->se.exec_start = sched_clock();
5881 5967
5882 idle->prio = idle->normal_prio = MAX_PRIO; 5968 idle->prio = idle->normal_prio = MAX_PRIO;
5883 idle->cpus_allowed = cpumask_of_cpu(cpu); 5969 cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
5884 __set_task_cpu(idle, cpu); 5970 __set_task_cpu(idle, cpu);
5885 5971
5886 rq->curr = rq->idle = idle; 5972 rq->curr = rq->idle = idle;
@@ -5907,9 +5993,9 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
5907 * indicates which cpus entered this state. This is used 5993 * indicates which cpus entered this state. This is used
5908 * in the rcu update to wait only for active cpus. For system 5994 * in the rcu update to wait only for active cpus. For system
5909 * which do not switch off the HZ timer nohz_cpu_mask should 5995 * which do not switch off the HZ timer nohz_cpu_mask should
5910 * always be CPU_MASK_NONE. 5996 * always be CPU_BITS_NONE.
5911 */ 5997 */
5912cpumask_t nohz_cpu_mask = CPU_MASK_NONE; 5998cpumask_var_t nohz_cpu_mask;
5913 5999
5914/* 6000/*
5915 * Increase the granularity value when there are more CPUs, 6001 * Increase the granularity value when there are more CPUs,
@@ -5964,7 +6050,7 @@ static inline void sched_init_granularity(void)
5964 * task must not exit() & deallocate itself prematurely. The 6050 * task must not exit() & deallocate itself prematurely. The
5965 * call is not atomic; no spinlocks may be held. 6051 * call is not atomic; no spinlocks may be held.
5966 */ 6052 */
5967int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask) 6053int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
5968{ 6054{
5969 struct migration_req req; 6055 struct migration_req req;
5970 unsigned long flags; 6056 unsigned long flags;
@@ -5972,13 +6058,13 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
5972 int ret = 0; 6058 int ret = 0;
5973 6059
5974 rq = task_rq_lock(p, &flags); 6060 rq = task_rq_lock(p, &flags);
5975 if (!cpus_intersects(*new_mask, cpu_online_map)) { 6061 if (!cpumask_intersects(new_mask, cpu_online_mask)) {
5976 ret = -EINVAL; 6062 ret = -EINVAL;
5977 goto out; 6063 goto out;
5978 } 6064 }
5979 6065
5980 if (unlikely((p->flags & PF_THREAD_BOUND) && p != current && 6066 if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
5981 !cpus_equal(p->cpus_allowed, *new_mask))) { 6067 !cpumask_equal(&p->cpus_allowed, new_mask))) {
5982 ret = -EINVAL; 6068 ret = -EINVAL;
5983 goto out; 6069 goto out;
5984 } 6070 }
@@ -5986,15 +6072,15 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
5986 if (p->sched_class->set_cpus_allowed) 6072 if (p->sched_class->set_cpus_allowed)
5987 p->sched_class->set_cpus_allowed(p, new_mask); 6073 p->sched_class->set_cpus_allowed(p, new_mask);
5988 else { 6074 else {
5989 p->cpus_allowed = *new_mask; 6075 cpumask_copy(&p->cpus_allowed, new_mask);
5990 p->rt.nr_cpus_allowed = cpus_weight(*new_mask); 6076 p->rt.nr_cpus_allowed = cpumask_weight(new_mask);
5991 } 6077 }
5992 6078
5993 /* Can the task run on the task's current CPU? If so, we're done */ 6079 /* Can the task run on the task's current CPU? If so, we're done */
5994 if (cpu_isset(task_cpu(p), *new_mask)) 6080 if (cpumask_test_cpu(task_cpu(p), new_mask))
5995 goto out; 6081 goto out;
5996 6082
5997 if (migrate_task(p, any_online_cpu(*new_mask), &req)) { 6083 if (migrate_task(p, cpumask_any_and(cpu_online_mask, new_mask), &req)) {
5998 /* Need help from migration thread: drop lock and wait. */ 6084 /* Need help from migration thread: drop lock and wait. */
5999 task_rq_unlock(rq, &flags); 6085 task_rq_unlock(rq, &flags);
6000 wake_up_process(rq->migration_thread); 6086 wake_up_process(rq->migration_thread);
@@ -6036,7 +6122,7 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
6036 if (task_cpu(p) != src_cpu) 6122 if (task_cpu(p) != src_cpu)
6037 goto done; 6123 goto done;
6038 /* Affinity changed (again). */ 6124 /* Affinity changed (again). */
6039 if (!cpu_isset(dest_cpu, p->cpus_allowed)) 6125 if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
6040 goto fail; 6126 goto fail;
6041 6127
6042 on_rq = p->se.on_rq; 6128 on_rq = p->se.on_rq;
@@ -6133,50 +6219,43 @@ static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu)
6133 */ 6219 */
6134static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p) 6220static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
6135{ 6221{
6136 unsigned long flags;
6137 cpumask_t mask;
6138 struct rq *rq;
6139 int dest_cpu; 6222 int dest_cpu;
6223 /* FIXME: Use cpumask_of_node here. */
6224 cpumask_t _nodemask = node_to_cpumask(cpu_to_node(dead_cpu));
6225 const struct cpumask *nodemask = &_nodemask;
6226
6227again:
6228 /* Look for allowed, online CPU in same node. */
6229 for_each_cpu_and(dest_cpu, nodemask, cpu_online_mask)
6230 if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
6231 goto move;
6232
6233 /* Any allowed, online CPU? */
6234 dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_online_mask);
6235 if (dest_cpu < nr_cpu_ids)
6236 goto move;
6237
6238 /* No more Mr. Nice Guy. */
6239 if (dest_cpu >= nr_cpu_ids) {
6240 cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
6241 dest_cpu = cpumask_any_and(cpu_online_mask, &p->cpus_allowed);
6140 6242
6141 do { 6243 /*
6142 /* On same node? */ 6244 * Don't tell them about moving exiting tasks or
6143 mask = node_to_cpumask(cpu_to_node(dead_cpu)); 6245 * kernel threads (both mm NULL), since they never
6144 cpus_and(mask, mask, p->cpus_allowed); 6246 * leave kernel.
6145 dest_cpu = any_online_cpu(mask); 6247 */
6146 6248 if (p->mm && printk_ratelimit()) {
6147 /* On any allowed CPU? */ 6249 printk(KERN_INFO "process %d (%s) no "
6148 if (dest_cpu >= nr_cpu_ids) 6250 "longer affine to cpu%d\n",
6149 dest_cpu = any_online_cpu(p->cpus_allowed); 6251 task_pid_nr(p), p->comm, dead_cpu);
6150
6151 /* No more Mr. Nice Guy. */
6152 if (dest_cpu >= nr_cpu_ids) {
6153 cpumask_t cpus_allowed;
6154
6155 cpuset_cpus_allowed_locked(p, &cpus_allowed);
6156 /*
6157 * Try to stay on the same cpuset, where the
6158 * current cpuset may be a subset of all cpus.
6159 * The cpuset_cpus_allowed_locked() variant of
6160 * cpuset_cpus_allowed() will not block. It must be
6161 * called within calls to cpuset_lock/cpuset_unlock.
6162 */
6163 rq = task_rq_lock(p, &flags);
6164 p->cpus_allowed = cpus_allowed;
6165 dest_cpu = any_online_cpu(p->cpus_allowed);
6166 task_rq_unlock(rq, &flags);
6167
6168 /*
6169 * Don't tell them about moving exiting tasks or
6170 * kernel threads (both mm NULL), since they never
6171 * leave kernel.
6172 */
6173 if (p->mm && printk_ratelimit()) {
6174 printk(KERN_INFO "process %d (%s) no "
6175 "longer affine to cpu%d\n",
6176 task_pid_nr(p), p->comm, dead_cpu);
6177 }
6178 } 6252 }
6179 } while (!__migrate_task_irq(p, dead_cpu, dest_cpu)); 6253 }
6254
6255move:
6256 /* It can have affinity changed while we were choosing. */
6257 if (unlikely(!__migrate_task_irq(p, dead_cpu, dest_cpu)))
6258 goto again;
6180} 6259}
6181 6260
6182/* 6261/*
@@ -6188,7 +6267,7 @@ static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
6188 */ 6267 */
6189static void migrate_nr_uninterruptible(struct rq *rq_src) 6268static void migrate_nr_uninterruptible(struct rq *rq_src)
6190{ 6269{
6191 struct rq *rq_dest = cpu_rq(any_online_cpu(*CPU_MASK_ALL_PTR)); 6270 struct rq *rq_dest = cpu_rq(cpumask_any(cpu_online_mask));
6192 unsigned long flags; 6271 unsigned long flags;
6193 6272
6194 local_irq_save(flags); 6273 local_irq_save(flags);
@@ -6478,7 +6557,7 @@ static void set_rq_online(struct rq *rq)
6478 if (!rq->online) { 6557 if (!rq->online) {
6479 const struct sched_class *class; 6558 const struct sched_class *class;
6480 6559
6481 cpu_set(rq->cpu, rq->rd->online); 6560 cpumask_set_cpu(rq->cpu, rq->rd->online);
6482 rq->online = 1; 6561 rq->online = 1;
6483 6562
6484 for_each_class(class) { 6563 for_each_class(class) {
@@ -6498,7 +6577,7 @@ static void set_rq_offline(struct rq *rq)
6498 class->rq_offline(rq); 6577 class->rq_offline(rq);
6499 } 6578 }
6500 6579
6501 cpu_clear(rq->cpu, rq->rd->online); 6580 cpumask_clear_cpu(rq->cpu, rq->rd->online);
6502 rq->online = 0; 6581 rq->online = 0;
6503 } 6582 }
6504} 6583}
@@ -6539,7 +6618,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6539 rq = cpu_rq(cpu); 6618 rq = cpu_rq(cpu);
6540 spin_lock_irqsave(&rq->lock, flags); 6619 spin_lock_irqsave(&rq->lock, flags);
6541 if (rq->rd) { 6620 if (rq->rd) {
6542 BUG_ON(!cpu_isset(cpu, rq->rd->span)); 6621 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6543 6622
6544 set_rq_online(rq); 6623 set_rq_online(rq);
6545 } 6624 }
@@ -6553,7 +6632,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6553 break; 6632 break;
6554 /* Unbind it from offline cpu so it can run. Fall thru. */ 6633 /* Unbind it from offline cpu so it can run. Fall thru. */
6555 kthread_bind(cpu_rq(cpu)->migration_thread, 6634 kthread_bind(cpu_rq(cpu)->migration_thread,
6556 any_online_cpu(cpu_online_map)); 6635 cpumask_any(cpu_online_mask));
6557 kthread_stop(cpu_rq(cpu)->migration_thread); 6636 kthread_stop(cpu_rq(cpu)->migration_thread);
6558 cpu_rq(cpu)->migration_thread = NULL; 6637 cpu_rq(cpu)->migration_thread = NULL;
6559 break; 6638 break;
@@ -6603,7 +6682,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6603 rq = cpu_rq(cpu); 6682 rq = cpu_rq(cpu);
6604 spin_lock_irqsave(&rq->lock, flags); 6683 spin_lock_irqsave(&rq->lock, flags);
6605 if (rq->rd) { 6684 if (rq->rd) {
6606 BUG_ON(!cpu_isset(cpu, rq->rd->span)); 6685 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6607 set_rq_offline(rq); 6686 set_rq_offline(rq);
6608 } 6687 }
6609 spin_unlock_irqrestore(&rq->lock, flags); 6688 spin_unlock_irqrestore(&rq->lock, flags);
@@ -6642,13 +6721,13 @@ early_initcall(migration_init);
6642#ifdef CONFIG_SCHED_DEBUG 6721#ifdef CONFIG_SCHED_DEBUG
6643 6722
6644static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, 6723static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6645 cpumask_t *groupmask) 6724 struct cpumask *groupmask)
6646{ 6725{
6647 struct sched_group *group = sd->groups; 6726 struct sched_group *group = sd->groups;
6648 char str[256]; 6727 char str[256];
6649 6728
6650 cpulist_scnprintf(str, sizeof(str), sd->span); 6729 cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd));
6651 cpus_clear(*groupmask); 6730 cpumask_clear(groupmask);
6652 6731
6653 printk(KERN_DEBUG "%*s domain %d: ", level, "", level); 6732 printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
6654 6733
@@ -6662,11 +6741,11 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6662 6741
6663 printk(KERN_CONT "span %s level %s\n", str, sd->name); 6742 printk(KERN_CONT "span %s level %s\n", str, sd->name);
6664 6743
6665 if (!cpu_isset(cpu, sd->span)) { 6744 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
6666 printk(KERN_ERR "ERROR: domain->span does not contain " 6745 printk(KERN_ERR "ERROR: domain->span does not contain "
6667 "CPU%d\n", cpu); 6746 "CPU%d\n", cpu);
6668 } 6747 }
6669 if (!cpu_isset(cpu, group->cpumask)) { 6748 if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
6670 printk(KERN_ERR "ERROR: domain->groups does not contain" 6749 printk(KERN_ERR "ERROR: domain->groups does not contain"
6671 " CPU%d\n", cpu); 6750 " CPU%d\n", cpu);
6672 } 6751 }
@@ -6686,31 +6765,32 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6686 break; 6765 break;
6687 } 6766 }
6688 6767
6689 if (!cpus_weight(group->cpumask)) { 6768 if (!cpumask_weight(sched_group_cpus(group))) {
6690 printk(KERN_CONT "\n"); 6769 printk(KERN_CONT "\n");
6691 printk(KERN_ERR "ERROR: empty group\n"); 6770 printk(KERN_ERR "ERROR: empty group\n");
6692 break; 6771 break;
6693 } 6772 }
6694 6773
6695 if (cpus_intersects(*groupmask, group->cpumask)) { 6774 if (cpumask_intersects(groupmask, sched_group_cpus(group))) {
6696 printk(KERN_CONT "\n"); 6775 printk(KERN_CONT "\n");
6697 printk(KERN_ERR "ERROR: repeated CPUs\n"); 6776 printk(KERN_ERR "ERROR: repeated CPUs\n");
6698 break; 6777 break;
6699 } 6778 }
6700 6779
6701 cpus_or(*groupmask, *groupmask, group->cpumask); 6780 cpumask_or(groupmask, groupmask, sched_group_cpus(group));
6702 6781
6703 cpulist_scnprintf(str, sizeof(str), group->cpumask); 6782 cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
6704 printk(KERN_CONT " %s", str); 6783 printk(KERN_CONT " %s", str);
6705 6784
6706 group = group->next; 6785 group = group->next;
6707 } while (group != sd->groups); 6786 } while (group != sd->groups);
6708 printk(KERN_CONT "\n"); 6787 printk(KERN_CONT "\n");
6709 6788
6710 if (!cpus_equal(sd->span, *groupmask)) 6789 if (!cpumask_equal(sched_domain_span(sd), groupmask))
6711 printk(KERN_ERR "ERROR: groups don't span domain->span\n"); 6790 printk(KERN_ERR "ERROR: groups don't span domain->span\n");
6712 6791
6713 if (sd->parent && !cpus_subset(*groupmask, sd->parent->span)) 6792 if (sd->parent &&
6793 !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
6714 printk(KERN_ERR "ERROR: parent span is not a superset " 6794 printk(KERN_ERR "ERROR: parent span is not a superset "
6715 "of domain->span\n"); 6795 "of domain->span\n");
6716 return 0; 6796 return 0;
@@ -6718,7 +6798,7 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6718 6798
6719static void sched_domain_debug(struct sched_domain *sd, int cpu) 6799static void sched_domain_debug(struct sched_domain *sd, int cpu)
6720{ 6800{
6721 cpumask_t *groupmask; 6801 cpumask_var_t groupmask;
6722 int level = 0; 6802 int level = 0;
6723 6803
6724 if (!sd) { 6804 if (!sd) {
@@ -6728,8 +6808,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6728 6808
6729 printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu); 6809 printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
6730 6810
6731 groupmask = kmalloc(sizeof(cpumask_t), GFP_KERNEL); 6811 if (!alloc_cpumask_var(&groupmask, GFP_KERNEL)) {
6732 if (!groupmask) {
6733 printk(KERN_DEBUG "Cannot load-balance (out of memory)\n"); 6812 printk(KERN_DEBUG "Cannot load-balance (out of memory)\n");
6734 return; 6813 return;
6735 } 6814 }
@@ -6742,7 +6821,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6742 if (!sd) 6821 if (!sd)
6743 break; 6822 break;
6744 } 6823 }
6745 kfree(groupmask); 6824 free_cpumask_var(groupmask);
6746} 6825}
6747#else /* !CONFIG_SCHED_DEBUG */ 6826#else /* !CONFIG_SCHED_DEBUG */
6748# define sched_domain_debug(sd, cpu) do { } while (0) 6827# define sched_domain_debug(sd, cpu) do { } while (0)
@@ -6750,7 +6829,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6750 6829
6751static int sd_degenerate(struct sched_domain *sd) 6830static int sd_degenerate(struct sched_domain *sd)
6752{ 6831{
6753 if (cpus_weight(sd->span) == 1) 6832 if (cpumask_weight(sched_domain_span(sd)) == 1)
6754 return 1; 6833 return 1;
6755 6834
6756 /* Following flags need at least 2 groups */ 6835 /* Following flags need at least 2 groups */
@@ -6781,7 +6860,7 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6781 if (sd_degenerate(parent)) 6860 if (sd_degenerate(parent))
6782 return 1; 6861 return 1;
6783 6862
6784 if (!cpus_equal(sd->span, parent->span)) 6863 if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
6785 return 0; 6864 return 0;
6786 6865
6787 /* Does parent contain flags not in child? */ 6866 /* Does parent contain flags not in child? */
@@ -6805,6 +6884,16 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6805 return 1; 6884 return 1;
6806} 6885}
6807 6886
6887static void free_rootdomain(struct root_domain *rd)
6888{
6889 cpupri_cleanup(&rd->cpupri);
6890
6891 free_cpumask_var(rd->rto_mask);
6892 free_cpumask_var(rd->online);
6893 free_cpumask_var(rd->span);
6894 kfree(rd);
6895}
6896
6808static void rq_attach_root(struct rq *rq, struct root_domain *rd) 6897static void rq_attach_root(struct rq *rq, struct root_domain *rd)
6809{ 6898{
6810 unsigned long flags; 6899 unsigned long flags;
@@ -6814,38 +6903,63 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
6814 if (rq->rd) { 6903 if (rq->rd) {
6815 struct root_domain *old_rd = rq->rd; 6904 struct root_domain *old_rd = rq->rd;
6816 6905
6817 if (cpu_isset(rq->cpu, old_rd->online)) 6906 if (cpumask_test_cpu(rq->cpu, old_rd->online))
6818 set_rq_offline(rq); 6907 set_rq_offline(rq);
6819 6908
6820 cpu_clear(rq->cpu, old_rd->span); 6909 cpumask_clear_cpu(rq->cpu, old_rd->span);
6821 6910
6822 if (atomic_dec_and_test(&old_rd->refcount)) 6911 if (atomic_dec_and_test(&old_rd->refcount))
6823 kfree(old_rd); 6912 free_rootdomain(old_rd);
6824 } 6913 }
6825 6914
6826 atomic_inc(&rd->refcount); 6915 atomic_inc(&rd->refcount);
6827 rq->rd = rd; 6916 rq->rd = rd;
6828 6917
6829 cpu_set(rq->cpu, rd->span); 6918 cpumask_set_cpu(rq->cpu, rd->span);
6830 if (cpu_isset(rq->cpu, cpu_online_map)) 6919 if (cpumask_test_cpu(rq->cpu, cpu_online_mask))
6831 set_rq_online(rq); 6920 set_rq_online(rq);
6832 6921
6833 spin_unlock_irqrestore(&rq->lock, flags); 6922 spin_unlock_irqrestore(&rq->lock, flags);
6834} 6923}
6835 6924
6836static void init_rootdomain(struct root_domain *rd) 6925static int init_rootdomain(struct root_domain *rd, bool bootmem)
6837{ 6926{
6838 memset(rd, 0, sizeof(*rd)); 6927 memset(rd, 0, sizeof(*rd));
6839 6928
6840 cpus_clear(rd->span); 6929 if (bootmem) {
6841 cpus_clear(rd->online); 6930 alloc_bootmem_cpumask_var(&def_root_domain.span);
6931 alloc_bootmem_cpumask_var(&def_root_domain.online);
6932 alloc_bootmem_cpumask_var(&def_root_domain.rto_mask);
6933 cpupri_init(&rd->cpupri, true);
6934 return 0;
6935 }
6936
6937 if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
6938 goto free_rd;
6939 if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
6940 goto free_span;
6941 if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
6942 goto free_online;
6943
6944 if (cpupri_init(&rd->cpupri, false) != 0)
6945 goto free_rto_mask;
6946 return 0;
6842 6947
6843 cpupri_init(&rd->cpupri); 6948free_rto_mask:
6949 free_cpumask_var(rd->rto_mask);
6950free_online:
6951 free_cpumask_var(rd->online);
6952free_span:
6953 free_cpumask_var(rd->span);
6954free_rd:
6955 kfree(rd);
6956 return -ENOMEM;
6844} 6957}
6845 6958
6846static void init_defrootdomain(void) 6959static void init_defrootdomain(void)
6847{ 6960{
6848 init_rootdomain(&def_root_domain); 6961 init_rootdomain(&def_root_domain, true);
6962
6849 atomic_set(&def_root_domain.refcount, 1); 6963 atomic_set(&def_root_domain.refcount, 1);
6850} 6964}
6851 6965
@@ -6857,7 +6971,10 @@ static struct root_domain *alloc_rootdomain(void)
6857 if (!rd) 6971 if (!rd)
6858 return NULL; 6972 return NULL;
6859 6973
6860 init_rootdomain(rd); 6974 if (init_rootdomain(rd, false) != 0) {
6975 kfree(rd);
6976 return NULL;
6977 }
6861 6978
6862 return rd; 6979 return rd;
6863} 6980}
@@ -6899,19 +7016,12 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
6899} 7016}
6900 7017
6901/* cpus with isolated domains */ 7018/* cpus with isolated domains */
6902static cpumask_t cpu_isolated_map = CPU_MASK_NONE; 7019static cpumask_var_t cpu_isolated_map;
6903 7020
6904/* Setup the mask of cpus configured for isolated domains */ 7021/* Setup the mask of cpus configured for isolated domains */
6905static int __init isolated_cpu_setup(char *str) 7022static int __init isolated_cpu_setup(char *str)
6906{ 7023{
6907 static int __initdata ints[NR_CPUS]; 7024 cpulist_parse(str, cpu_isolated_map);
6908 int i;
6909
6910 str = get_options(str, ARRAY_SIZE(ints), ints);
6911 cpus_clear(cpu_isolated_map);
6912 for (i = 1; i <= ints[0]; i++)
6913 if (ints[i] < NR_CPUS)
6914 cpu_set(ints[i], cpu_isolated_map);
6915 return 1; 7025 return 1;
6916} 7026}
6917 7027
@@ -6920,42 +7030,43 @@ __setup("isolcpus=", isolated_cpu_setup);
6920/* 7030/*
6921 * init_sched_build_groups takes the cpumask we wish to span, and a pointer 7031 * init_sched_build_groups takes the cpumask we wish to span, and a pointer
6922 * to a function which identifies what group(along with sched group) a CPU 7032 * to a function which identifies what group(along with sched group) a CPU
6923 * belongs to. The return value of group_fn must be a >= 0 and < NR_CPUS 7033 * belongs to. The return value of group_fn must be a >= 0 and < nr_cpu_ids
6924 * (due to the fact that we keep track of groups covered with a cpumask_t). 7034 * (due to the fact that we keep track of groups covered with a struct cpumask).
6925 * 7035 *
6926 * init_sched_build_groups will build a circular linked list of the groups 7036 * init_sched_build_groups will build a circular linked list of the groups
6927 * covered by the given span, and will set each group's ->cpumask correctly, 7037 * covered by the given span, and will set each group's ->cpumask correctly,
6928 * and ->cpu_power to 0. 7038 * and ->cpu_power to 0.
6929 */ 7039 */
6930static void 7040static void
6931init_sched_build_groups(const cpumask_t *span, const cpumask_t *cpu_map, 7041init_sched_build_groups(const struct cpumask *span,
6932 int (*group_fn)(int cpu, const cpumask_t *cpu_map, 7042 const struct cpumask *cpu_map,
7043 int (*group_fn)(int cpu, const struct cpumask *cpu_map,
6933 struct sched_group **sg, 7044 struct sched_group **sg,
6934 cpumask_t *tmpmask), 7045 struct cpumask *tmpmask),
6935 cpumask_t *covered, cpumask_t *tmpmask) 7046 struct cpumask *covered, struct cpumask *tmpmask)
6936{ 7047{
6937 struct sched_group *first = NULL, *last = NULL; 7048 struct sched_group *first = NULL, *last = NULL;
6938 int i; 7049 int i;
6939 7050
6940 cpus_clear(*covered); 7051 cpumask_clear(covered);
6941 7052
6942 for_each_cpu_mask_nr(i, *span) { 7053 for_each_cpu(i, span) {
6943 struct sched_group *sg; 7054 struct sched_group *sg;
6944 int group = group_fn(i, cpu_map, &sg, tmpmask); 7055 int group = group_fn(i, cpu_map, &sg, tmpmask);
6945 int j; 7056 int j;
6946 7057
6947 if (cpu_isset(i, *covered)) 7058 if (cpumask_test_cpu(i, covered))
6948 continue; 7059 continue;
6949 7060
6950 cpus_clear(sg->cpumask); 7061 cpumask_clear(sched_group_cpus(sg));
6951 sg->__cpu_power = 0; 7062 sg->__cpu_power = 0;
6952 7063
6953 for_each_cpu_mask_nr(j, *span) { 7064 for_each_cpu(j, span) {
6954 if (group_fn(j, cpu_map, NULL, tmpmask) != group) 7065 if (group_fn(j, cpu_map, NULL, tmpmask) != group)
6955 continue; 7066 continue;
6956 7067
6957 cpu_set(j, *covered); 7068 cpumask_set_cpu(j, covered);
6958 cpu_set(j, sg->cpumask); 7069 cpumask_set_cpu(j, sched_group_cpus(sg));
6959 } 7070 }
6960 if (!first) 7071 if (!first)
6961 first = sg; 7072 first = sg;
@@ -7019,9 +7130,10 @@ static int find_next_best_node(int node, nodemask_t *used_nodes)
7019 * should be one that prevents unnecessary balancing, but also spreads tasks 7130 * should be one that prevents unnecessary balancing, but also spreads tasks
7020 * out optimally. 7131 * out optimally.
7021 */ 7132 */
7022static void sched_domain_node_span(int node, cpumask_t *span) 7133static void sched_domain_node_span(int node, struct cpumask *span)
7023{ 7134{
7024 nodemask_t used_nodes; 7135 nodemask_t used_nodes;
7136 /* FIXME: use cpumask_of_node() */
7025 node_to_cpumask_ptr(nodemask, node); 7137 node_to_cpumask_ptr(nodemask, node);
7026 int i; 7138 int i;
7027 7139
@@ -7043,18 +7155,33 @@ static void sched_domain_node_span(int node, cpumask_t *span)
7043int sched_smt_power_savings = 0, sched_mc_power_savings = 0; 7155int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
7044 7156
7045/* 7157/*
7158 * The cpus mask in sched_group and sched_domain hangs off the end.
7159 * FIXME: use cpumask_var_t or dynamic percpu alloc to avoid wasting space
7160 * for nr_cpu_ids < CONFIG_NR_CPUS.
7161 */
7162struct static_sched_group {
7163 struct sched_group sg;
7164 DECLARE_BITMAP(cpus, CONFIG_NR_CPUS);
7165};
7166
7167struct static_sched_domain {
7168 struct sched_domain sd;
7169 DECLARE_BITMAP(span, CONFIG_NR_CPUS);
7170};
7171
7172/*
7046 * SMT sched-domains: 7173 * SMT sched-domains:
7047 */ 7174 */
7048#ifdef CONFIG_SCHED_SMT 7175#ifdef CONFIG_SCHED_SMT
7049static DEFINE_PER_CPU(struct sched_domain, cpu_domains); 7176static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains);
7050static DEFINE_PER_CPU(struct sched_group, sched_group_cpus); 7177static DEFINE_PER_CPU(struct static_sched_group, sched_group_cpus);
7051 7178
7052static int 7179static int
7053cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7180cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map,
7054 cpumask_t *unused) 7181 struct sched_group **sg, struct cpumask *unused)
7055{ 7182{
7056 if (sg) 7183 if (sg)
7057 *sg = &per_cpu(sched_group_cpus, cpu); 7184 *sg = &per_cpu(sched_group_cpus, cpu).sg;
7058 return cpu; 7185 return cpu;
7059} 7186}
7060#endif /* CONFIG_SCHED_SMT */ 7187#endif /* CONFIG_SCHED_SMT */
@@ -7063,56 +7190,55 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
7063 * multi-core sched-domains: 7190 * multi-core sched-domains:
7064 */ 7191 */
7065#ifdef CONFIG_SCHED_MC 7192#ifdef CONFIG_SCHED_MC
7066static DEFINE_PER_CPU(struct sched_domain, core_domains); 7193static DEFINE_PER_CPU(struct static_sched_domain, core_domains);
7067static DEFINE_PER_CPU(struct sched_group, sched_group_core); 7194static DEFINE_PER_CPU(struct static_sched_group, sched_group_core);
7068#endif /* CONFIG_SCHED_MC */ 7195#endif /* CONFIG_SCHED_MC */
7069 7196
7070#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT) 7197#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
7071static int 7198static int
7072cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7199cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
7073 cpumask_t *mask) 7200 struct sched_group **sg, struct cpumask *mask)
7074{ 7201{
7075 int group; 7202 int group;
7076 7203
7077 *mask = per_cpu(cpu_sibling_map, cpu); 7204 cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
7078 cpus_and(*mask, *mask, *cpu_map); 7205 group = cpumask_first(mask);
7079 group = first_cpu(*mask);
7080 if (sg) 7206 if (sg)
7081 *sg = &per_cpu(sched_group_core, group); 7207 *sg = &per_cpu(sched_group_core, group).sg;
7082 return group; 7208 return group;
7083} 7209}
7084#elif defined(CONFIG_SCHED_MC) 7210#elif defined(CONFIG_SCHED_MC)
7085static int 7211static int
7086cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7212cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
7087 cpumask_t *unused) 7213 struct sched_group **sg, struct cpumask *unused)
7088{ 7214{
7089 if (sg) 7215 if (sg)
7090 *sg = &per_cpu(sched_group_core, cpu); 7216 *sg = &per_cpu(sched_group_core, cpu).sg;
7091 return cpu; 7217 return cpu;
7092} 7218}
7093#endif 7219#endif
7094 7220
7095static DEFINE_PER_CPU(struct sched_domain, phys_domains); 7221static DEFINE_PER_CPU(struct static_sched_domain, phys_domains);
7096static DEFINE_PER_CPU(struct sched_group, sched_group_phys); 7222static DEFINE_PER_CPU(struct static_sched_group, sched_group_phys);
7097 7223
7098static int 7224static int
7099cpu_to_phys_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7225cpu_to_phys_group(int cpu, const struct cpumask *cpu_map,
7100 cpumask_t *mask) 7226 struct sched_group **sg, struct cpumask *mask)
7101{ 7227{
7102 int group; 7228 int group;
7103#ifdef CONFIG_SCHED_MC 7229#ifdef CONFIG_SCHED_MC
7230 /* FIXME: Use cpu_coregroup_mask. */
7104 *mask = cpu_coregroup_map(cpu); 7231 *mask = cpu_coregroup_map(cpu);
7105 cpus_and(*mask, *mask, *cpu_map); 7232 cpus_and(*mask, *mask, *cpu_map);
7106 group = first_cpu(*mask); 7233 group = cpumask_first(mask);
7107#elif defined(CONFIG_SCHED_SMT) 7234#elif defined(CONFIG_SCHED_SMT)
7108 *mask = per_cpu(cpu_sibling_map, cpu); 7235 cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
7109 cpus_and(*mask, *mask, *cpu_map); 7236 group = cpumask_first(mask);
7110 group = first_cpu(*mask);
7111#else 7237#else
7112 group = cpu; 7238 group = cpu;
7113#endif 7239#endif
7114 if (sg) 7240 if (sg)
7115 *sg = &per_cpu(sched_group_phys, group); 7241 *sg = &per_cpu(sched_group_phys, group).sg;
7116 return group; 7242 return group;
7117} 7243}
7118 7244
@@ -7126,19 +7252,21 @@ static DEFINE_PER_CPU(struct sched_domain, node_domains);
7126static struct sched_group ***sched_group_nodes_bycpu; 7252static struct sched_group ***sched_group_nodes_bycpu;
7127 7253
7128static DEFINE_PER_CPU(struct sched_domain, allnodes_domains); 7254static DEFINE_PER_CPU(struct sched_domain, allnodes_domains);
7129static DEFINE_PER_CPU(struct sched_group, sched_group_allnodes); 7255static DEFINE_PER_CPU(struct static_sched_group, sched_group_allnodes);
7130 7256
7131static int cpu_to_allnodes_group(int cpu, const cpumask_t *cpu_map, 7257static int cpu_to_allnodes_group(int cpu, const struct cpumask *cpu_map,
7132 struct sched_group **sg, cpumask_t *nodemask) 7258 struct sched_group **sg,
7259 struct cpumask *nodemask)
7133{ 7260{
7134 int group; 7261 int group;
7262 /* FIXME: use cpumask_of_node */
7263 node_to_cpumask_ptr(pnodemask, cpu_to_node(cpu));
7135 7264
7136 *nodemask = node_to_cpumask(cpu_to_node(cpu)); 7265 cpumask_and(nodemask, pnodemask, cpu_map);
7137 cpus_and(*nodemask, *nodemask, *cpu_map); 7266 group = cpumask_first(nodemask);
7138 group = first_cpu(*nodemask);
7139 7267
7140 if (sg) 7268 if (sg)
7141 *sg = &per_cpu(sched_group_allnodes, group); 7269 *sg = &per_cpu(sched_group_allnodes, group).sg;
7142 return group; 7270 return group;
7143} 7271}
7144 7272
@@ -7150,11 +7278,11 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
7150 if (!sg) 7278 if (!sg)
7151 return; 7279 return;
7152 do { 7280 do {
7153 for_each_cpu_mask_nr(j, sg->cpumask) { 7281 for_each_cpu(j, sched_group_cpus(sg)) {
7154 struct sched_domain *sd; 7282 struct sched_domain *sd;
7155 7283
7156 sd = &per_cpu(phys_domains, j); 7284 sd = &per_cpu(phys_domains, j).sd;
7157 if (j != first_cpu(sd->groups->cpumask)) { 7285 if (j != cpumask_first(sched_group_cpus(sd->groups))) {
7158 /* 7286 /*
7159 * Only add "power" once for each 7287 * Only add "power" once for each
7160 * physical package. 7288 * physical package.
@@ -7171,11 +7299,12 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
7171 7299
7172#ifdef CONFIG_NUMA 7300#ifdef CONFIG_NUMA
7173/* Free memory allocated for various sched_group structures */ 7301/* Free memory allocated for various sched_group structures */
7174static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask) 7302static void free_sched_groups(const struct cpumask *cpu_map,
7303 struct cpumask *nodemask)
7175{ 7304{
7176 int cpu, i; 7305 int cpu, i;
7177 7306
7178 for_each_cpu_mask_nr(cpu, *cpu_map) { 7307 for_each_cpu(cpu, cpu_map) {
7179 struct sched_group **sched_group_nodes 7308 struct sched_group **sched_group_nodes
7180 = sched_group_nodes_bycpu[cpu]; 7309 = sched_group_nodes_bycpu[cpu];
7181 7310
@@ -7184,10 +7313,11 @@ static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
7184 7313
7185 for (i = 0; i < nr_node_ids; i++) { 7314 for (i = 0; i < nr_node_ids; i++) {
7186 struct sched_group *oldsg, *sg = sched_group_nodes[i]; 7315 struct sched_group *oldsg, *sg = sched_group_nodes[i];
7316 /* FIXME: Use cpumask_of_node */
7317 node_to_cpumask_ptr(pnodemask, i);
7187 7318
7188 *nodemask = node_to_cpumask(i); 7319 cpus_and(*nodemask, *pnodemask, *cpu_map);
7189 cpus_and(*nodemask, *nodemask, *cpu_map); 7320 if (cpumask_empty(nodemask))
7190 if (cpus_empty(*nodemask))
7191 continue; 7321 continue;
7192 7322
7193 if (sg == NULL) 7323 if (sg == NULL)
@@ -7205,7 +7335,8 @@ next_sg:
7205 } 7335 }
7206} 7336}
7207#else /* !CONFIG_NUMA */ 7337#else /* !CONFIG_NUMA */
7208static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask) 7338static void free_sched_groups(const struct cpumask *cpu_map,
7339 struct cpumask *nodemask)
7209{ 7340{
7210} 7341}
7211#endif /* CONFIG_NUMA */ 7342#endif /* CONFIG_NUMA */
@@ -7231,7 +7362,7 @@ static void init_sched_groups_power(int cpu, struct sched_domain *sd)
7231 7362
7232 WARN_ON(!sd || !sd->groups); 7363 WARN_ON(!sd || !sd->groups);
7233 7364
7234 if (cpu != first_cpu(sd->groups->cpumask)) 7365 if (cpu != cpumask_first(sched_group_cpus(sd->groups)))
7235 return; 7366 return;
7236 7367
7237 child = sd->child; 7368 child = sd->child;
@@ -7296,48 +7427,6 @@ SD_INIT_FUNC(CPU)
7296 SD_INIT_FUNC(MC) 7427 SD_INIT_FUNC(MC)
7297#endif 7428#endif
7298 7429
7299/*
7300 * To minimize stack usage kmalloc room for cpumasks and share the
7301 * space as the usage in build_sched_domains() dictates. Used only
7302 * if the amount of space is significant.
7303 */
7304struct allmasks {
7305 cpumask_t tmpmask; /* make this one first */
7306 union {
7307 cpumask_t nodemask;
7308 cpumask_t this_sibling_map;
7309 cpumask_t this_core_map;
7310 };
7311 cpumask_t send_covered;
7312
7313#ifdef CONFIG_NUMA
7314 cpumask_t domainspan;
7315 cpumask_t covered;
7316 cpumask_t notcovered;
7317#endif
7318};
7319
7320#if NR_CPUS > 128
7321#define SCHED_CPUMASK_DECLARE(v) struct allmasks *v
7322static inline void sched_cpumask_alloc(struct allmasks **masks)
7323{
7324 *masks = kmalloc(sizeof(**masks), GFP_KERNEL);
7325}
7326static inline void sched_cpumask_free(struct allmasks *masks)
7327{
7328 kfree(masks);
7329}
7330#else
7331#define SCHED_CPUMASK_DECLARE(v) struct allmasks _v, *v = &_v
7332static inline void sched_cpumask_alloc(struct allmasks **masks)
7333{ }
7334static inline void sched_cpumask_free(struct allmasks *masks)
7335{ }
7336#endif
7337
7338#define SCHED_CPUMASK_VAR(v, a) cpumask_t *v = (cpumask_t *) \
7339 ((unsigned long)(a) + offsetof(struct allmasks, v))
7340
7341static int default_relax_domain_level = -1; 7430static int default_relax_domain_level = -1;
7342 7431
7343static int __init setup_relax_domain_level(char *str) 7432static int __init setup_relax_domain_level(char *str)
@@ -7377,17 +7466,38 @@ static void set_domain_attribute(struct sched_domain *sd,
7377 * Build sched domains for a given set of cpus and attach the sched domains 7466 * Build sched domains for a given set of cpus and attach the sched domains
7378 * to the individual cpus 7467 * to the individual cpus
7379 */ 7468 */
7380static int __build_sched_domains(const cpumask_t *cpu_map, 7469static int __build_sched_domains(const struct cpumask *cpu_map,
7381 struct sched_domain_attr *attr) 7470 struct sched_domain_attr *attr)
7382{ 7471{
7383 int i; 7472 int i, err = -ENOMEM;
7384 struct root_domain *rd; 7473 struct root_domain *rd;
7385 SCHED_CPUMASK_DECLARE(allmasks); 7474 cpumask_var_t nodemask, this_sibling_map, this_core_map, send_covered,
7386 cpumask_t *tmpmask; 7475 tmpmask;
7387#ifdef CONFIG_NUMA 7476#ifdef CONFIG_NUMA
7477 cpumask_var_t domainspan, covered, notcovered;
7388 struct sched_group **sched_group_nodes = NULL; 7478 struct sched_group **sched_group_nodes = NULL;
7389 int sd_allnodes = 0; 7479 int sd_allnodes = 0;
7390 7480
7481 if (!alloc_cpumask_var(&domainspan, GFP_KERNEL))
7482 goto out;
7483 if (!alloc_cpumask_var(&covered, GFP_KERNEL))
7484 goto free_domainspan;
7485 if (!alloc_cpumask_var(&notcovered, GFP_KERNEL))
7486 goto free_covered;
7487#endif
7488
7489 if (!alloc_cpumask_var(&nodemask, GFP_KERNEL))
7490 goto free_notcovered;
7491 if (!alloc_cpumask_var(&this_sibling_map, GFP_KERNEL))
7492 goto free_nodemask;
7493 if (!alloc_cpumask_var(&this_core_map, GFP_KERNEL))
7494 goto free_this_sibling_map;
7495 if (!alloc_cpumask_var(&send_covered, GFP_KERNEL))
7496 goto free_this_core_map;
7497 if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
7498 goto free_send_covered;
7499
7500#ifdef CONFIG_NUMA
7391 /* 7501 /*
7392 * Allocate the per-node list of sched groups 7502 * Allocate the per-node list of sched groups
7393 */ 7503 */
@@ -7395,54 +7505,37 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7395 GFP_KERNEL); 7505 GFP_KERNEL);
7396 if (!sched_group_nodes) { 7506 if (!sched_group_nodes) {
7397 printk(KERN_WARNING "Can not alloc sched group node list\n"); 7507 printk(KERN_WARNING "Can not alloc sched group node list\n");
7398 return -ENOMEM; 7508 goto free_tmpmask;
7399 } 7509 }
7400#endif 7510#endif
7401 7511
7402 rd = alloc_rootdomain(); 7512 rd = alloc_rootdomain();
7403 if (!rd) { 7513 if (!rd) {
7404 printk(KERN_WARNING "Cannot alloc root domain\n"); 7514 printk(KERN_WARNING "Cannot alloc root domain\n");
7405#ifdef CONFIG_NUMA 7515 goto free_sched_groups;
7406 kfree(sched_group_nodes);
7407#endif
7408 return -ENOMEM;
7409 } 7516 }
7410 7517
7411 /* get space for all scratch cpumask variables */
7412 sched_cpumask_alloc(&allmasks);
7413 if (!allmasks) {
7414 printk(KERN_WARNING "Cannot alloc cpumask array\n");
7415 kfree(rd);
7416#ifdef CONFIG_NUMA 7518#ifdef CONFIG_NUMA
7417 kfree(sched_group_nodes); 7519 sched_group_nodes_bycpu[cpumask_first(cpu_map)] = sched_group_nodes;
7418#endif
7419 return -ENOMEM;
7420 }
7421
7422 tmpmask = (cpumask_t *)allmasks;
7423
7424
7425#ifdef CONFIG_NUMA
7426 sched_group_nodes_bycpu[first_cpu(*cpu_map)] = sched_group_nodes;
7427#endif 7520#endif
7428 7521
7429 /* 7522 /*
7430 * Set up domains for cpus specified by the cpu_map. 7523 * Set up domains for cpus specified by the cpu_map.
7431 */ 7524 */
7432 for_each_cpu_mask_nr(i, *cpu_map) { 7525 for_each_cpu(i, cpu_map) {
7433 struct sched_domain *sd = NULL, *p; 7526 struct sched_domain *sd = NULL, *p;
7434 SCHED_CPUMASK_VAR(nodemask, allmasks);
7435 7527
7528 /* FIXME: use cpumask_of_node */
7436 *nodemask = node_to_cpumask(cpu_to_node(i)); 7529 *nodemask = node_to_cpumask(cpu_to_node(i));
7437 cpus_and(*nodemask, *nodemask, *cpu_map); 7530 cpus_and(*nodemask, *nodemask, *cpu_map);
7438 7531
7439#ifdef CONFIG_NUMA 7532#ifdef CONFIG_NUMA
7440 if (cpus_weight(*cpu_map) > 7533 if (cpumask_weight(cpu_map) >
7441 SD_NODES_PER_DOMAIN*cpus_weight(*nodemask)) { 7534 SD_NODES_PER_DOMAIN*cpumask_weight(nodemask)) {
7442 sd = &per_cpu(allnodes_domains, i); 7535 sd = &per_cpu(allnodes_domains, i);
7443 SD_INIT(sd, ALLNODES); 7536 SD_INIT(sd, ALLNODES);
7444 set_domain_attribute(sd, attr); 7537 set_domain_attribute(sd, attr);
7445 sd->span = *cpu_map; 7538 cpumask_copy(sched_domain_span(sd), cpu_map);
7446 cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask); 7539 cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask);
7447 p = sd; 7540 p = sd;
7448 sd_allnodes = 1; 7541 sd_allnodes = 1;
@@ -7452,18 +7545,19 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7452 sd = &per_cpu(node_domains, i); 7545 sd = &per_cpu(node_domains, i);
7453 SD_INIT(sd, NODE); 7546 SD_INIT(sd, NODE);
7454 set_domain_attribute(sd, attr); 7547 set_domain_attribute(sd, attr);
7455 sched_domain_node_span(cpu_to_node(i), &sd->span); 7548 sched_domain_node_span(cpu_to_node(i), sched_domain_span(sd));
7456 sd->parent = p; 7549 sd->parent = p;
7457 if (p) 7550 if (p)
7458 p->child = sd; 7551 p->child = sd;
7459 cpus_and(sd->span, sd->span, *cpu_map); 7552 cpumask_and(sched_domain_span(sd),
7553 sched_domain_span(sd), cpu_map);
7460#endif 7554#endif
7461 7555
7462 p = sd; 7556 p = sd;
7463 sd = &per_cpu(phys_domains, i); 7557 sd = &per_cpu(phys_domains, i).sd;
7464 SD_INIT(sd, CPU); 7558 SD_INIT(sd, CPU);
7465 set_domain_attribute(sd, attr); 7559 set_domain_attribute(sd, attr);
7466 sd->span = *nodemask; 7560 cpumask_copy(sched_domain_span(sd), nodemask);
7467 sd->parent = p; 7561 sd->parent = p;
7468 if (p) 7562 if (p)
7469 p->child = sd; 7563 p->child = sd;
@@ -7471,11 +7565,12 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7471 7565
7472#ifdef CONFIG_SCHED_MC 7566#ifdef CONFIG_SCHED_MC
7473 p = sd; 7567 p = sd;
7474 sd = &per_cpu(core_domains, i); 7568 sd = &per_cpu(core_domains, i).sd;
7475 SD_INIT(sd, MC); 7569 SD_INIT(sd, MC);
7476 set_domain_attribute(sd, attr); 7570 set_domain_attribute(sd, attr);
7477 sd->span = cpu_coregroup_map(i); 7571 *sched_domain_span(sd) = cpu_coregroup_map(i);
7478 cpus_and(sd->span, sd->span, *cpu_map); 7572 cpumask_and(sched_domain_span(sd),
7573 sched_domain_span(sd), cpu_map);
7479 sd->parent = p; 7574 sd->parent = p;
7480 p->child = sd; 7575 p->child = sd;
7481 cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask); 7576 cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask);
@@ -7483,11 +7578,11 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7483 7578
7484#ifdef CONFIG_SCHED_SMT 7579#ifdef CONFIG_SCHED_SMT
7485 p = sd; 7580 p = sd;
7486 sd = &per_cpu(cpu_domains, i); 7581 sd = &per_cpu(cpu_domains, i).sd;
7487 SD_INIT(sd, SIBLING); 7582 SD_INIT(sd, SIBLING);
7488 set_domain_attribute(sd, attr); 7583 set_domain_attribute(sd, attr);
7489 sd->span = per_cpu(cpu_sibling_map, i); 7584 cpumask_and(sched_domain_span(sd),
7490 cpus_and(sd->span, sd->span, *cpu_map); 7585 &per_cpu(cpu_sibling_map, i), cpu_map);
7491 sd->parent = p; 7586 sd->parent = p;
7492 p->child = sd; 7587 p->child = sd;
7493 cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask); 7588 cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask);
@@ -7496,13 +7591,10 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7496 7591
7497#ifdef CONFIG_SCHED_SMT 7592#ifdef CONFIG_SCHED_SMT
7498 /* Set up CPU (sibling) groups */ 7593 /* Set up CPU (sibling) groups */
7499 for_each_cpu_mask_nr(i, *cpu_map) { 7594 for_each_cpu(i, cpu_map) {
7500 SCHED_CPUMASK_VAR(this_sibling_map, allmasks); 7595 cpumask_and(this_sibling_map,
7501 SCHED_CPUMASK_VAR(send_covered, allmasks); 7596 &per_cpu(cpu_sibling_map, i), cpu_map);
7502 7597 if (i != cpumask_first(this_sibling_map))
7503 *this_sibling_map = per_cpu(cpu_sibling_map, i);
7504 cpus_and(*this_sibling_map, *this_sibling_map, *cpu_map);
7505 if (i != first_cpu(*this_sibling_map))
7506 continue; 7598 continue;
7507 7599
7508 init_sched_build_groups(this_sibling_map, cpu_map, 7600 init_sched_build_groups(this_sibling_map, cpu_map,
@@ -7513,13 +7605,11 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7513 7605
7514#ifdef CONFIG_SCHED_MC 7606#ifdef CONFIG_SCHED_MC
7515 /* Set up multi-core groups */ 7607 /* Set up multi-core groups */
7516 for_each_cpu_mask_nr(i, *cpu_map) { 7608 for_each_cpu(i, cpu_map) {
7517 SCHED_CPUMASK_VAR(this_core_map, allmasks); 7609 /* FIXME: Use cpu_coregroup_mask */
7518 SCHED_CPUMASK_VAR(send_covered, allmasks);
7519
7520 *this_core_map = cpu_coregroup_map(i); 7610 *this_core_map = cpu_coregroup_map(i);
7521 cpus_and(*this_core_map, *this_core_map, *cpu_map); 7611 cpus_and(*this_core_map, *this_core_map, *cpu_map);
7522 if (i != first_cpu(*this_core_map)) 7612 if (i != cpumask_first(this_core_map))
7523 continue; 7613 continue;
7524 7614
7525 init_sched_build_groups(this_core_map, cpu_map, 7615 init_sched_build_groups(this_core_map, cpu_map,
@@ -7530,12 +7620,10 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7530 7620
7531 /* Set up physical groups */ 7621 /* Set up physical groups */
7532 for (i = 0; i < nr_node_ids; i++) { 7622 for (i = 0; i < nr_node_ids; i++) {
7533 SCHED_CPUMASK_VAR(nodemask, allmasks); 7623 /* FIXME: Use cpumask_of_node */
7534 SCHED_CPUMASK_VAR(send_covered, allmasks);
7535
7536 *nodemask = node_to_cpumask(i); 7624 *nodemask = node_to_cpumask(i);
7537 cpus_and(*nodemask, *nodemask, *cpu_map); 7625 cpus_and(*nodemask, *nodemask, *cpu_map);
7538 if (cpus_empty(*nodemask)) 7626 if (cpumask_empty(nodemask))
7539 continue; 7627 continue;
7540 7628
7541 init_sched_build_groups(nodemask, cpu_map, 7629 init_sched_build_groups(nodemask, cpu_map,
@@ -7546,8 +7634,6 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7546#ifdef CONFIG_NUMA 7634#ifdef CONFIG_NUMA
7547 /* Set up node groups */ 7635 /* Set up node groups */
7548 if (sd_allnodes) { 7636 if (sd_allnodes) {
7549 SCHED_CPUMASK_VAR(send_covered, allmasks);
7550
7551 init_sched_build_groups(cpu_map, cpu_map, 7637 init_sched_build_groups(cpu_map, cpu_map,
7552 &cpu_to_allnodes_group, 7638 &cpu_to_allnodes_group,
7553 send_covered, tmpmask); 7639 send_covered, tmpmask);
@@ -7556,58 +7642,58 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7556 for (i = 0; i < nr_node_ids; i++) { 7642 for (i = 0; i < nr_node_ids; i++) {
7557 /* Set up node groups */ 7643 /* Set up node groups */
7558 struct sched_group *sg, *prev; 7644 struct sched_group *sg, *prev;
7559 SCHED_CPUMASK_VAR(nodemask, allmasks);
7560 SCHED_CPUMASK_VAR(domainspan, allmasks);
7561 SCHED_CPUMASK_VAR(covered, allmasks);
7562 int j; 7645 int j;
7563 7646
7647 /* FIXME: Use cpumask_of_node */
7564 *nodemask = node_to_cpumask(i); 7648 *nodemask = node_to_cpumask(i);
7565 cpus_clear(*covered); 7649 cpumask_clear(covered);
7566 7650
7567 cpus_and(*nodemask, *nodemask, *cpu_map); 7651 cpus_and(*nodemask, *nodemask, *cpu_map);
7568 if (cpus_empty(*nodemask)) { 7652 if (cpumask_empty(nodemask)) {
7569 sched_group_nodes[i] = NULL; 7653 sched_group_nodes[i] = NULL;
7570 continue; 7654 continue;
7571 } 7655 }
7572 7656
7573 sched_domain_node_span(i, domainspan); 7657 sched_domain_node_span(i, domainspan);
7574 cpus_and(*domainspan, *domainspan, *cpu_map); 7658 cpumask_and(domainspan, domainspan, cpu_map);
7575 7659
7576 sg = kmalloc_node(sizeof(struct sched_group), GFP_KERNEL, i); 7660 sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
7661 GFP_KERNEL, i);
7577 if (!sg) { 7662 if (!sg) {
7578 printk(KERN_WARNING "Can not alloc domain group for " 7663 printk(KERN_WARNING "Can not alloc domain group for "
7579 "node %d\n", i); 7664 "node %d\n", i);
7580 goto error; 7665 goto error;
7581 } 7666 }
7582 sched_group_nodes[i] = sg; 7667 sched_group_nodes[i] = sg;
7583 for_each_cpu_mask_nr(j, *nodemask) { 7668 for_each_cpu(j, nodemask) {
7584 struct sched_domain *sd; 7669 struct sched_domain *sd;
7585 7670
7586 sd = &per_cpu(node_domains, j); 7671 sd = &per_cpu(node_domains, j);
7587 sd->groups = sg; 7672 sd->groups = sg;
7588 } 7673 }
7589 sg->__cpu_power = 0; 7674 sg->__cpu_power = 0;
7590 sg->cpumask = *nodemask; 7675 cpumask_copy(sched_group_cpus(sg), nodemask);
7591 sg->next = sg; 7676 sg->next = sg;
7592 cpus_or(*covered, *covered, *nodemask); 7677 cpumask_or(covered, covered, nodemask);
7593 prev = sg; 7678 prev = sg;
7594 7679
7595 for (j = 0; j < nr_node_ids; j++) { 7680 for (j = 0; j < nr_node_ids; j++) {
7596 SCHED_CPUMASK_VAR(notcovered, allmasks);
7597 int n = (i + j) % nr_node_ids; 7681 int n = (i + j) % nr_node_ids;
7682 /* FIXME: Use cpumask_of_node */
7598 node_to_cpumask_ptr(pnodemask, n); 7683 node_to_cpumask_ptr(pnodemask, n);
7599 7684
7600 cpus_complement(*notcovered, *covered); 7685 cpumask_complement(notcovered, covered);
7601 cpus_and(*tmpmask, *notcovered, *cpu_map); 7686 cpumask_and(tmpmask, notcovered, cpu_map);
7602 cpus_and(*tmpmask, *tmpmask, *domainspan); 7687 cpumask_and(tmpmask, tmpmask, domainspan);
7603 if (cpus_empty(*tmpmask)) 7688 if (cpumask_empty(tmpmask))
7604 break; 7689 break;
7605 7690
7606 cpus_and(*tmpmask, *tmpmask, *pnodemask); 7691 cpumask_and(tmpmask, tmpmask, pnodemask);
7607 if (cpus_empty(*tmpmask)) 7692 if (cpumask_empty(tmpmask))
7608 continue; 7693 continue;
7609 7694
7610 sg = kmalloc_node(sizeof(struct sched_group), 7695 sg = kmalloc_node(sizeof(struct sched_group) +
7696 cpumask_size(),
7611 GFP_KERNEL, i); 7697 GFP_KERNEL, i);
7612 if (!sg) { 7698 if (!sg) {
7613 printk(KERN_WARNING 7699 printk(KERN_WARNING
@@ -7615,9 +7701,9 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7615 goto error; 7701 goto error;
7616 } 7702 }
7617 sg->__cpu_power = 0; 7703 sg->__cpu_power = 0;
7618 sg->cpumask = *tmpmask; 7704 cpumask_copy(sched_group_cpus(sg), tmpmask);
7619 sg->next = prev->next; 7705 sg->next = prev->next;
7620 cpus_or(*covered, *covered, *tmpmask); 7706 cpumask_or(covered, covered, tmpmask);
7621 prev->next = sg; 7707 prev->next = sg;
7622 prev = sg; 7708 prev = sg;
7623 } 7709 }
@@ -7626,22 +7712,22 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7626 7712
7627 /* Calculate CPU power for physical packages and nodes */ 7713 /* Calculate CPU power for physical packages and nodes */
7628#ifdef CONFIG_SCHED_SMT 7714#ifdef CONFIG_SCHED_SMT
7629 for_each_cpu_mask_nr(i, *cpu_map) { 7715 for_each_cpu(i, cpu_map) {
7630 struct sched_domain *sd = &per_cpu(cpu_domains, i); 7716 struct sched_domain *sd = &per_cpu(cpu_domains, i).sd;
7631 7717
7632 init_sched_groups_power(i, sd); 7718 init_sched_groups_power(i, sd);
7633 } 7719 }
7634#endif 7720#endif
7635#ifdef CONFIG_SCHED_MC 7721#ifdef CONFIG_SCHED_MC
7636 for_each_cpu_mask_nr(i, *cpu_map) { 7722 for_each_cpu(i, cpu_map) {
7637 struct sched_domain *sd = &per_cpu(core_domains, i); 7723 struct sched_domain *sd = &per_cpu(core_domains, i).sd;
7638 7724
7639 init_sched_groups_power(i, sd); 7725 init_sched_groups_power(i, sd);
7640 } 7726 }
7641#endif 7727#endif
7642 7728
7643 for_each_cpu_mask_nr(i, *cpu_map) { 7729 for_each_cpu(i, cpu_map) {
7644 struct sched_domain *sd = &per_cpu(phys_domains, i); 7730 struct sched_domain *sd = &per_cpu(phys_domains, i).sd;
7645 7731
7646 init_sched_groups_power(i, sd); 7732 init_sched_groups_power(i, sd);
7647 } 7733 }
@@ -7653,53 +7739,78 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7653 if (sd_allnodes) { 7739 if (sd_allnodes) {
7654 struct sched_group *sg; 7740 struct sched_group *sg;
7655 7741
7656 cpu_to_allnodes_group(first_cpu(*cpu_map), cpu_map, &sg, 7742 cpu_to_allnodes_group(cpumask_first(cpu_map), cpu_map, &sg,
7657 tmpmask); 7743 tmpmask);
7658 init_numa_sched_groups_power(sg); 7744 init_numa_sched_groups_power(sg);
7659 } 7745 }
7660#endif 7746#endif
7661 7747
7662 /* Attach the domains */ 7748 /* Attach the domains */
7663 for_each_cpu_mask_nr(i, *cpu_map) { 7749 for_each_cpu(i, cpu_map) {
7664 struct sched_domain *sd; 7750 struct sched_domain *sd;
7665#ifdef CONFIG_SCHED_SMT 7751#ifdef CONFIG_SCHED_SMT
7666 sd = &per_cpu(cpu_domains, i); 7752 sd = &per_cpu(cpu_domains, i).sd;
7667#elif defined(CONFIG_SCHED_MC) 7753#elif defined(CONFIG_SCHED_MC)
7668 sd = &per_cpu(core_domains, i); 7754 sd = &per_cpu(core_domains, i).sd;
7669#else 7755#else
7670 sd = &per_cpu(phys_domains, i); 7756 sd = &per_cpu(phys_domains, i).sd;
7671#endif 7757#endif
7672 cpu_attach_domain(sd, rd, i); 7758 cpu_attach_domain(sd, rd, i);
7673 } 7759 }
7674 7760
7675 sched_cpumask_free(allmasks); 7761 err = 0;
7676 return 0; 7762
7763free_tmpmask:
7764 free_cpumask_var(tmpmask);
7765free_send_covered:
7766 free_cpumask_var(send_covered);
7767free_this_core_map:
7768 free_cpumask_var(this_core_map);
7769free_this_sibling_map:
7770 free_cpumask_var(this_sibling_map);
7771free_nodemask:
7772 free_cpumask_var(nodemask);
7773free_notcovered:
7774#ifdef CONFIG_NUMA
7775 free_cpumask_var(notcovered);
7776free_covered:
7777 free_cpumask_var(covered);
7778free_domainspan:
7779 free_cpumask_var(domainspan);
7780out:
7781#endif
7782 return err;
7783
7784free_sched_groups:
7785#ifdef CONFIG_NUMA
7786 kfree(sched_group_nodes);
7787#endif
7788 goto free_tmpmask;
7677 7789
7678#ifdef CONFIG_NUMA 7790#ifdef CONFIG_NUMA
7679error: 7791error:
7680 free_sched_groups(cpu_map, tmpmask); 7792 free_sched_groups(cpu_map, tmpmask);
7681 sched_cpumask_free(allmasks); 7793 free_rootdomain(rd);
7682 kfree(rd); 7794 goto free_tmpmask;
7683 return -ENOMEM;
7684#endif 7795#endif
7685} 7796}
7686 7797
7687static int build_sched_domains(const cpumask_t *cpu_map) 7798static int build_sched_domains(const struct cpumask *cpu_map)
7688{ 7799{
7689 return __build_sched_domains(cpu_map, NULL); 7800 return __build_sched_domains(cpu_map, NULL);
7690} 7801}
7691 7802
7692static cpumask_t *doms_cur; /* current sched domains */ 7803static struct cpumask *doms_cur; /* current sched domains */
7693static int ndoms_cur; /* number of sched domains in 'doms_cur' */ 7804static int ndoms_cur; /* number of sched domains in 'doms_cur' */
7694static struct sched_domain_attr *dattr_cur; 7805static struct sched_domain_attr *dattr_cur;
7695 /* attribues of custom domains in 'doms_cur' */ 7806 /* attribues of custom domains in 'doms_cur' */
7696 7807
7697/* 7808/*
7698 * Special case: If a kmalloc of a doms_cur partition (array of 7809 * Special case: If a kmalloc of a doms_cur partition (array of
7699 * cpumask_t) fails, then fallback to a single sched domain, 7810 * cpumask) fails, then fallback to a single sched domain,
7700 * as determined by the single cpumask_t fallback_doms. 7811 * as determined by the single cpumask fallback_doms.
7701 */ 7812 */
7702static cpumask_t fallback_doms; 7813static cpumask_var_t fallback_doms;
7703 7814
7704/* 7815/*
7705 * arch_update_cpu_topology lets virtualized architectures update the 7816 * arch_update_cpu_topology lets virtualized architectures update the
@@ -7716,16 +7827,16 @@ int __attribute__((weak)) arch_update_cpu_topology(void)
7716 * For now this just excludes isolated cpus, but could be used to 7827 * For now this just excludes isolated cpus, but could be used to
7717 * exclude other special cases in the future. 7828 * exclude other special cases in the future.
7718 */ 7829 */
7719static int arch_init_sched_domains(const cpumask_t *cpu_map) 7830static int arch_init_sched_domains(const struct cpumask *cpu_map)
7720{ 7831{
7721 int err; 7832 int err;
7722 7833
7723 arch_update_cpu_topology(); 7834 arch_update_cpu_topology();
7724 ndoms_cur = 1; 7835 ndoms_cur = 1;
7725 doms_cur = kmalloc(sizeof(cpumask_t), GFP_KERNEL); 7836 doms_cur = kmalloc(cpumask_size(), GFP_KERNEL);
7726 if (!doms_cur) 7837 if (!doms_cur)
7727 doms_cur = &fallback_doms; 7838 doms_cur = fallback_doms;
7728 cpus_andnot(*doms_cur, *cpu_map, cpu_isolated_map); 7839 cpumask_andnot(doms_cur, cpu_map, cpu_isolated_map);
7729 dattr_cur = NULL; 7840 dattr_cur = NULL;
7730 err = build_sched_domains(doms_cur); 7841 err = build_sched_domains(doms_cur);
7731 register_sched_domain_sysctl(); 7842 register_sched_domain_sysctl();
@@ -7733,8 +7844,8 @@ static int arch_init_sched_domains(const cpumask_t *cpu_map)
7733 return err; 7844 return err;
7734} 7845}
7735 7846
7736static void arch_destroy_sched_domains(const cpumask_t *cpu_map, 7847static void arch_destroy_sched_domains(const struct cpumask *cpu_map,
7737 cpumask_t *tmpmask) 7848 struct cpumask *tmpmask)
7738{ 7849{
7739 free_sched_groups(cpu_map, tmpmask); 7850 free_sched_groups(cpu_map, tmpmask);
7740} 7851}
@@ -7743,15 +7854,16 @@ static void arch_destroy_sched_domains(const cpumask_t *cpu_map,
7743 * Detach sched domains from a group of cpus specified in cpu_map 7854 * Detach sched domains from a group of cpus specified in cpu_map
7744 * These cpus will now be attached to the NULL domain 7855 * These cpus will now be attached to the NULL domain
7745 */ 7856 */
7746static void detach_destroy_domains(const cpumask_t *cpu_map) 7857static void detach_destroy_domains(const struct cpumask *cpu_map)
7747{ 7858{
7748 cpumask_t tmpmask; 7859 /* Save because hotplug lock held. */
7860 static DECLARE_BITMAP(tmpmask, CONFIG_NR_CPUS);
7749 int i; 7861 int i;
7750 7862
7751 for_each_cpu_mask_nr(i, *cpu_map) 7863 for_each_cpu(i, cpu_map)
7752 cpu_attach_domain(NULL, &def_root_domain, i); 7864 cpu_attach_domain(NULL, &def_root_domain, i);
7753 synchronize_sched(); 7865 synchronize_sched();
7754 arch_destroy_sched_domains(cpu_map, &tmpmask); 7866 arch_destroy_sched_domains(cpu_map, to_cpumask(tmpmask));
7755} 7867}
7756 7868
7757/* handle null as "default" */ 7869/* handle null as "default" */
@@ -7776,7 +7888,7 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
7776 * doms_new[] to the current sched domain partitioning, doms_cur[]. 7888 * doms_new[] to the current sched domain partitioning, doms_cur[].
7777 * It destroys each deleted domain and builds each new domain. 7889 * It destroys each deleted domain and builds each new domain.
7778 * 7890 *
7779 * 'doms_new' is an array of cpumask_t's of length 'ndoms_new'. 7891 * 'doms_new' is an array of cpumask's of length 'ndoms_new'.
7780 * The masks don't intersect (don't overlap.) We should setup one 7892 * The masks don't intersect (don't overlap.) We should setup one
7781 * sched domain for each mask. CPUs not in any of the cpumasks will 7893 * sched domain for each mask. CPUs not in any of the cpumasks will
7782 * not be load balanced. If the same cpumask appears both in the 7894 * not be load balanced. If the same cpumask appears both in the
@@ -7790,13 +7902,14 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
7790 * the single partition 'fallback_doms', it also forces the domains 7902 * the single partition 'fallback_doms', it also forces the domains
7791 * to be rebuilt. 7903 * to be rebuilt.
7792 * 7904 *
7793 * If doms_new == NULL it will be replaced with cpu_online_map. 7905 * If doms_new == NULL it will be replaced with cpu_online_mask.
7794 * ndoms_new == 0 is a special case for destroying existing domains, 7906 * ndoms_new == 0 is a special case for destroying existing domains,
7795 * and it will not create the default domain. 7907 * and it will not create the default domain.
7796 * 7908 *
7797 * Call with hotplug lock held 7909 * Call with hotplug lock held
7798 */ 7910 */
7799void partition_sched_domains(int ndoms_new, cpumask_t *doms_new, 7911/* FIXME: Change to struct cpumask *doms_new[] */
7912void partition_sched_domains(int ndoms_new, struct cpumask *doms_new,
7800 struct sched_domain_attr *dattr_new) 7913 struct sched_domain_attr *dattr_new)
7801{ 7914{
7802 int i, j, n; 7915 int i, j, n;
@@ -7815,7 +7928,7 @@ void partition_sched_domains(int ndoms_new, cpumask_t *doms_new,
7815 /* Destroy deleted domains */ 7928 /* Destroy deleted domains */
7816 for (i = 0; i < ndoms_cur; i++) { 7929 for (i = 0; i < ndoms_cur; i++) {
7817 for (j = 0; j < n && !new_topology; j++) { 7930 for (j = 0; j < n && !new_topology; j++) {
7818 if (cpus_equal(doms_cur[i], doms_new[j]) 7931 if (cpumask_equal(&doms_cur[i], &doms_new[j])
7819 && dattrs_equal(dattr_cur, i, dattr_new, j)) 7932 && dattrs_equal(dattr_cur, i, dattr_new, j))
7820 goto match1; 7933 goto match1;
7821 } 7934 }
@@ -7827,15 +7940,15 @@ match1:
7827 7940
7828 if (doms_new == NULL) { 7941 if (doms_new == NULL) {
7829 ndoms_cur = 0; 7942 ndoms_cur = 0;
7830 doms_new = &fallback_doms; 7943 doms_new = fallback_doms;
7831 cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map); 7944 cpumask_andnot(&doms_new[0], cpu_online_mask, cpu_isolated_map);
7832 WARN_ON_ONCE(dattr_new); 7945 WARN_ON_ONCE(dattr_new);
7833 } 7946 }
7834 7947
7835 /* Build new domains */ 7948 /* Build new domains */
7836 for (i = 0; i < ndoms_new; i++) { 7949 for (i = 0; i < ndoms_new; i++) {
7837 for (j = 0; j < ndoms_cur && !new_topology; j++) { 7950 for (j = 0; j < ndoms_cur && !new_topology; j++) {
7838 if (cpus_equal(doms_new[i], doms_cur[j]) 7951 if (cpumask_equal(&doms_new[i], &doms_cur[j])
7839 && dattrs_equal(dattr_new, i, dattr_cur, j)) 7952 && dattrs_equal(dattr_new, i, dattr_cur, j))
7840 goto match2; 7953 goto match2;
7841 } 7954 }
@@ -7847,7 +7960,7 @@ match2:
7847 } 7960 }
7848 7961
7849 /* Remember the new sched domains */ 7962 /* Remember the new sched domains */
7850 if (doms_cur != &fallback_doms) 7963 if (doms_cur != fallback_doms)
7851 kfree(doms_cur); 7964 kfree(doms_cur);
7852 kfree(dattr_cur); /* kfree(NULL) is safe */ 7965 kfree(dattr_cur); /* kfree(NULL) is safe */
7853 doms_cur = doms_new; 7966 doms_cur = doms_new;
@@ -7876,14 +7989,25 @@ int arch_reinit_sched_domains(void)
7876static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt) 7989static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
7877{ 7990{
7878 int ret; 7991 int ret;
7992 unsigned int level = 0;
7879 7993
7880 if (buf[0] != '0' && buf[0] != '1') 7994 if (sscanf(buf, "%u", &level) != 1)
7995 return -EINVAL;
7996
7997 /*
7998 * level is always be positive so don't check for
7999 * level < POWERSAVINGS_BALANCE_NONE which is 0
8000 * What happens on 0 or 1 byte write,
8001 * need to check for count as well?
8002 */
8003
8004 if (level >= MAX_POWERSAVINGS_BALANCE_LEVELS)
7881 return -EINVAL; 8005 return -EINVAL;
7882 8006
7883 if (smt) 8007 if (smt)
7884 sched_smt_power_savings = (buf[0] == '1'); 8008 sched_smt_power_savings = level;
7885 else 8009 else
7886 sched_mc_power_savings = (buf[0] == '1'); 8010 sched_mc_power_savings = level;
7887 8011
7888 ret = arch_reinit_sched_domains(); 8012 ret = arch_reinit_sched_domains();
7889 8013
@@ -7987,7 +8111,9 @@ static int update_runtime(struct notifier_block *nfb,
7987 8111
7988void __init sched_init_smp(void) 8112void __init sched_init_smp(void)
7989{ 8113{
7990 cpumask_t non_isolated_cpus; 8114 cpumask_var_t non_isolated_cpus;
8115
8116 alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
7991 8117
7992#if defined(CONFIG_NUMA) 8118#if defined(CONFIG_NUMA)
7993 sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **), 8119 sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **),
@@ -7996,10 +8122,10 @@ void __init sched_init_smp(void)
7996#endif 8122#endif
7997 get_online_cpus(); 8123 get_online_cpus();
7998 mutex_lock(&sched_domains_mutex); 8124 mutex_lock(&sched_domains_mutex);
7999 arch_init_sched_domains(&cpu_online_map); 8125 arch_init_sched_domains(cpu_online_mask);
8000 cpus_andnot(non_isolated_cpus, cpu_possible_map, cpu_isolated_map); 8126 cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
8001 if (cpus_empty(non_isolated_cpus)) 8127 if (cpumask_empty(non_isolated_cpus))
8002 cpu_set(smp_processor_id(), non_isolated_cpus); 8128 cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
8003 mutex_unlock(&sched_domains_mutex); 8129 mutex_unlock(&sched_domains_mutex);
8004 put_online_cpus(); 8130 put_online_cpus();
8005 8131
@@ -8014,9 +8140,13 @@ void __init sched_init_smp(void)
8014 init_hrtick(); 8140 init_hrtick();
8015 8141
8016 /* Move init over to a non-isolated CPU */ 8142 /* Move init over to a non-isolated CPU */
8017 if (set_cpus_allowed_ptr(current, &non_isolated_cpus) < 0) 8143 if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
8018 BUG(); 8144 BUG();
8019 sched_init_granularity(); 8145 sched_init_granularity();
8146 free_cpumask_var(non_isolated_cpus);
8147
8148 alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
8149 init_sched_rt_class();
8020} 8150}
8021#else 8151#else
8022void __init sched_init_smp(void) 8152void __init sched_init_smp(void)
@@ -8331,6 +8461,15 @@ void __init sched_init(void)
8331 */ 8461 */
8332 current->sched_class = &fair_sched_class; 8462 current->sched_class = &fair_sched_class;
8333 8463
8464 /* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */
8465 alloc_bootmem_cpumask_var(&nohz_cpu_mask);
8466#ifdef CONFIG_SMP
8467#ifdef CONFIG_NO_HZ
8468 alloc_bootmem_cpumask_var(&nohz.cpu_mask);
8469#endif
8470 alloc_bootmem_cpumask_var(&cpu_isolated_map);
8471#endif /* SMP */
8472
8334 scheduler_running = 1; 8473 scheduler_running = 1;
8335} 8474}
8336 8475
diff --git a/kernel/sched_cpupri.c b/kernel/sched_cpupri.c
index 52154fefab7e..018b7be1db2e 100644
--- a/kernel/sched_cpupri.c
+++ b/kernel/sched_cpupri.c
@@ -67,24 +67,21 @@ static int convert_prio(int prio)
67 * Returns: (int)bool - CPUs were found 67 * Returns: (int)bool - CPUs were found
68 */ 68 */
69int cpupri_find(struct cpupri *cp, struct task_struct *p, 69int cpupri_find(struct cpupri *cp, struct task_struct *p,
70 cpumask_t *lowest_mask) 70 struct cpumask *lowest_mask)
71{ 71{
72 int idx = 0; 72 int idx = 0;
73 int task_pri = convert_prio(p->prio); 73 int task_pri = convert_prio(p->prio);
74 74
75 for_each_cpupri_active(cp->pri_active, idx) { 75 for_each_cpupri_active(cp->pri_active, idx) {
76 struct cpupri_vec *vec = &cp->pri_to_cpu[idx]; 76 struct cpupri_vec *vec = &cp->pri_to_cpu[idx];
77 cpumask_t mask;
78 77
79 if (idx >= task_pri) 78 if (idx >= task_pri)
80 break; 79 break;
81 80
82 cpus_and(mask, p->cpus_allowed, vec->mask); 81 if (cpumask_any_and(&p->cpus_allowed, vec->mask) >= nr_cpu_ids)
83
84 if (cpus_empty(mask))
85 continue; 82 continue;
86 83
87 *lowest_mask = mask; 84 cpumask_and(lowest_mask, &p->cpus_allowed, vec->mask);
88 return 1; 85 return 1;
89 } 86 }
90 87
@@ -126,7 +123,7 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)
126 vec->count--; 123 vec->count--;
127 if (!vec->count) 124 if (!vec->count)
128 clear_bit(oldpri, cp->pri_active); 125 clear_bit(oldpri, cp->pri_active);
129 cpu_clear(cpu, vec->mask); 126 cpumask_clear_cpu(cpu, vec->mask);
130 127
131 spin_unlock_irqrestore(&vec->lock, flags); 128 spin_unlock_irqrestore(&vec->lock, flags);
132 } 129 }
@@ -136,7 +133,7 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)
136 133
137 spin_lock_irqsave(&vec->lock, flags); 134 spin_lock_irqsave(&vec->lock, flags);
138 135
139 cpu_set(cpu, vec->mask); 136 cpumask_set_cpu(cpu, vec->mask);
140 vec->count++; 137 vec->count++;
141 if (vec->count == 1) 138 if (vec->count == 1)
142 set_bit(newpri, cp->pri_active); 139 set_bit(newpri, cp->pri_active);
@@ -150,10 +147,11 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)
150/** 147/**
151 * cpupri_init - initialize the cpupri structure 148 * cpupri_init - initialize the cpupri structure
152 * @cp: The cpupri context 149 * @cp: The cpupri context
150 * @bootmem: true if allocations need to use bootmem
153 * 151 *
154 * Returns: (void) 152 * Returns: -ENOMEM if memory fails.
155 */ 153 */
156void cpupri_init(struct cpupri *cp) 154int cpupri_init(struct cpupri *cp, bool bootmem)
157{ 155{
158 int i; 156 int i;
159 157
@@ -164,11 +162,30 @@ void cpupri_init(struct cpupri *cp)
164 162
165 spin_lock_init(&vec->lock); 163 spin_lock_init(&vec->lock);
166 vec->count = 0; 164 vec->count = 0;
167 cpus_clear(vec->mask); 165 if (bootmem)
166 alloc_bootmem_cpumask_var(&vec->mask);
167 else if (!alloc_cpumask_var(&vec->mask, GFP_KERNEL))
168 goto cleanup;
168 } 169 }
169 170
170 for_each_possible_cpu(i) 171 for_each_possible_cpu(i)
171 cp->cpu_to_pri[i] = CPUPRI_INVALID; 172 cp->cpu_to_pri[i] = CPUPRI_INVALID;
173 return 0;
174
175cleanup:
176 for (i--; i >= 0; i--)
177 free_cpumask_var(cp->pri_to_cpu[i].mask);
178 return -ENOMEM;
172} 179}
173 180
181/**
182 * cpupri_cleanup - clean up the cpupri structure
183 * @cp: The cpupri context
184 */
185void cpupri_cleanup(struct cpupri *cp)
186{
187 int i;
174 188
189 for (i = 0; i < CPUPRI_NR_PRIORITIES; i++)
190 free_cpumask_var(cp->pri_to_cpu[i].mask);
191}
diff --git a/kernel/sched_cpupri.h b/kernel/sched_cpupri.h
index f25811b0f931..642a94ef8a0a 100644
--- a/kernel/sched_cpupri.h
+++ b/kernel/sched_cpupri.h
@@ -14,7 +14,7 @@
14struct cpupri_vec { 14struct cpupri_vec {
15 spinlock_t lock; 15 spinlock_t lock;
16 int count; 16 int count;
17 cpumask_t mask; 17 cpumask_var_t mask;
18}; 18};
19 19
20struct cpupri { 20struct cpupri {
@@ -27,7 +27,8 @@ struct cpupri {
27int cpupri_find(struct cpupri *cp, 27int cpupri_find(struct cpupri *cp,
28 struct task_struct *p, cpumask_t *lowest_mask); 28 struct task_struct *p, cpumask_t *lowest_mask);
29void cpupri_set(struct cpupri *cp, int cpu, int pri); 29void cpupri_set(struct cpupri *cp, int cpu, int pri);
30void cpupri_init(struct cpupri *cp); 30int cpupri_init(struct cpupri *cp, bool bootmem);
31void cpupri_cleanup(struct cpupri *cp);
31#else 32#else
32#define cpupri_set(cp, cpu, pri) do { } while (0) 33#define cpupri_set(cp, cpu, pri) do { } while (0)
33#define cpupri_init() do { } while (0) 34#define cpupri_init() do { } while (0)
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 5ad4440f0fc4..56c0efe902a7 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -1019,16 +1019,33 @@ static void yield_task_fair(struct rq *rq)
1019 * search starts with cpus closest then further out as needed, 1019 * search starts with cpus closest then further out as needed,
1020 * so we always favor a closer, idle cpu. 1020 * so we always favor a closer, idle cpu.
1021 * Domains may include CPUs that are not usable for migration, 1021 * Domains may include CPUs that are not usable for migration,
1022 * hence we need to mask them out (cpu_active_map) 1022 * hence we need to mask them out (cpu_active_mask)
1023 * 1023 *
1024 * Returns the CPU we should wake onto. 1024 * Returns the CPU we should wake onto.
1025 */ 1025 */
1026#if defined(ARCH_HAS_SCHED_WAKE_IDLE) 1026#if defined(ARCH_HAS_SCHED_WAKE_IDLE)
1027static int wake_idle(int cpu, struct task_struct *p) 1027static int wake_idle(int cpu, struct task_struct *p)
1028{ 1028{
1029 cpumask_t tmp;
1030 struct sched_domain *sd; 1029 struct sched_domain *sd;
1031 int i; 1030 int i;
1031 unsigned int chosen_wakeup_cpu;
1032 int this_cpu;
1033
1034 /*
1035 * At POWERSAVINGS_BALANCE_WAKEUP level, if both this_cpu and prev_cpu
1036 * are idle and this is not a kernel thread and this task's affinity
1037 * allows it to be moved to preferred cpu, then just move!
1038 */
1039
1040 this_cpu = smp_processor_id();
1041 chosen_wakeup_cpu =
1042 cpu_rq(this_cpu)->rd->sched_mc_preferred_wakeup_cpu;
1043
1044 if (sched_mc_power_savings >= POWERSAVINGS_BALANCE_WAKEUP &&
1045 idle_cpu(cpu) && idle_cpu(this_cpu) &&
1046 p->mm && !(p->flags & PF_KTHREAD) &&
1047 cpu_isset(chosen_wakeup_cpu, p->cpus_allowed))
1048 return chosen_wakeup_cpu;
1032 1049
1033 /* 1050 /*
1034 * If it is idle, then it is the best cpu to run this task. 1051 * If it is idle, then it is the best cpu to run this task.
@@ -1046,10 +1063,9 @@ static int wake_idle(int cpu, struct task_struct *p)
1046 if ((sd->flags & SD_WAKE_IDLE) 1063 if ((sd->flags & SD_WAKE_IDLE)
1047 || ((sd->flags & SD_WAKE_IDLE_FAR) 1064 || ((sd->flags & SD_WAKE_IDLE_FAR)
1048 && !task_hot(p, task_rq(p)->clock, sd))) { 1065 && !task_hot(p, task_rq(p)->clock, sd))) {
1049 cpus_and(tmp, sd->span, p->cpus_allowed); 1066 for_each_cpu_and(i, sched_domain_span(sd),
1050 cpus_and(tmp, tmp, cpu_active_map); 1067 &p->cpus_allowed) {
1051 for_each_cpu_mask_nr(i, tmp) { 1068 if (cpu_active(i) && idle_cpu(i)) {
1052 if (idle_cpu(i)) {
1053 if (i != task_cpu(p)) { 1069 if (i != task_cpu(p)) {
1054 schedstat_inc(p, 1070 schedstat_inc(p,
1055 se.nr_wakeups_idle); 1071 se.nr_wakeups_idle);
@@ -1242,13 +1258,13 @@ static int select_task_rq_fair(struct task_struct *p, int sync)
1242 * this_cpu and prev_cpu are present in: 1258 * this_cpu and prev_cpu are present in:
1243 */ 1259 */
1244 for_each_domain(this_cpu, sd) { 1260 for_each_domain(this_cpu, sd) {
1245 if (cpu_isset(prev_cpu, sd->span)) { 1261 if (cpumask_test_cpu(prev_cpu, sched_domain_span(sd))) {
1246 this_sd = sd; 1262 this_sd = sd;
1247 break; 1263 break;
1248 } 1264 }
1249 } 1265 }
1250 1266
1251 if (unlikely(!cpu_isset(this_cpu, p->cpus_allowed))) 1267 if (unlikely(!cpumask_test_cpu(this_cpu, &p->cpus_allowed)))
1252 goto out; 1268 goto out;
1253 1269
1254 /* 1270 /*
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c
index 51d2af3e6191..833b6d44483c 100644
--- a/kernel/sched_rt.c
+++ b/kernel/sched_rt.c
@@ -15,7 +15,7 @@ static inline void rt_set_overload(struct rq *rq)
15 if (!rq->online) 15 if (!rq->online)
16 return; 16 return;
17 17
18 cpu_set(rq->cpu, rq->rd->rto_mask); 18 cpumask_set_cpu(rq->cpu, rq->rd->rto_mask);
19 /* 19 /*
20 * Make sure the mask is visible before we set 20 * Make sure the mask is visible before we set
21 * the overload count. That is checked to determine 21 * the overload count. That is checked to determine
@@ -34,7 +34,7 @@ static inline void rt_clear_overload(struct rq *rq)
34 34
35 /* the order here really doesn't matter */ 35 /* the order here really doesn't matter */
36 atomic_dec(&rq->rd->rto_count); 36 atomic_dec(&rq->rd->rto_count);
37 cpu_clear(rq->cpu, rq->rd->rto_mask); 37 cpumask_clear_cpu(rq->cpu, rq->rd->rto_mask);
38} 38}
39 39
40static void update_rt_migration(struct rq *rq) 40static void update_rt_migration(struct rq *rq)
@@ -139,14 +139,14 @@ static int rt_se_boosted(struct sched_rt_entity *rt_se)
139} 139}
140 140
141#ifdef CONFIG_SMP 141#ifdef CONFIG_SMP
142static inline cpumask_t sched_rt_period_mask(void) 142static inline const struct cpumask *sched_rt_period_mask(void)
143{ 143{
144 return cpu_rq(smp_processor_id())->rd->span; 144 return cpu_rq(smp_processor_id())->rd->span;
145} 145}
146#else 146#else
147static inline cpumask_t sched_rt_period_mask(void) 147static inline const struct cpumask *sched_rt_period_mask(void)
148{ 148{
149 return cpu_online_map; 149 return cpu_online_mask;
150} 150}
151#endif 151#endif
152 152
@@ -212,9 +212,9 @@ static inline int rt_rq_throttled(struct rt_rq *rt_rq)
212 return rt_rq->rt_throttled; 212 return rt_rq->rt_throttled;
213} 213}
214 214
215static inline cpumask_t sched_rt_period_mask(void) 215static inline const struct cpumask *sched_rt_period_mask(void)
216{ 216{
217 return cpu_online_map; 217 return cpu_online_mask;
218} 218}
219 219
220static inline 220static inline
@@ -241,11 +241,11 @@ static int do_balance_runtime(struct rt_rq *rt_rq)
241 int i, weight, more = 0; 241 int i, weight, more = 0;
242 u64 rt_period; 242 u64 rt_period;
243 243
244 weight = cpus_weight(rd->span); 244 weight = cpumask_weight(rd->span);
245 245
246 spin_lock(&rt_b->rt_runtime_lock); 246 spin_lock(&rt_b->rt_runtime_lock);
247 rt_period = ktime_to_ns(rt_b->rt_period); 247 rt_period = ktime_to_ns(rt_b->rt_period);
248 for_each_cpu_mask_nr(i, rd->span) { 248 for_each_cpu(i, rd->span) {
249 struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i); 249 struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
250 s64 diff; 250 s64 diff;
251 251
@@ -324,7 +324,7 @@ static void __disable_runtime(struct rq *rq)
324 /* 324 /*
325 * Greedy reclaim, take back as much as we can. 325 * Greedy reclaim, take back as much as we can.
326 */ 326 */
327 for_each_cpu_mask(i, rd->span) { 327 for_each_cpu(i, rd->span) {
328 struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i); 328 struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
329 s64 diff; 329 s64 diff;
330 330
@@ -429,13 +429,13 @@ static inline int balance_runtime(struct rt_rq *rt_rq)
429static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) 429static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)
430{ 430{
431 int i, idle = 1; 431 int i, idle = 1;
432 cpumask_t span; 432 const struct cpumask *span;
433 433
434 if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF) 434 if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)
435 return 1; 435 return 1;
436 436
437 span = sched_rt_period_mask(); 437 span = sched_rt_period_mask();
438 for_each_cpu_mask(i, span) { 438 for_each_cpu(i, span) {
439 int enqueue = 0; 439 int enqueue = 0;
440 struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i); 440 struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);
441 struct rq *rq = rq_of_rt_rq(rt_rq); 441 struct rq *rq = rq_of_rt_rq(rt_rq);
@@ -805,17 +805,20 @@ static int select_task_rq_rt(struct task_struct *p, int sync)
805 805
806static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p) 806static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
807{ 807{
808 cpumask_t mask; 808 cpumask_var_t mask;
809 809
810 if (rq->curr->rt.nr_cpus_allowed == 1) 810 if (rq->curr->rt.nr_cpus_allowed == 1)
811 return; 811 return;
812 812
813 if (p->rt.nr_cpus_allowed != 1 813 if (!alloc_cpumask_var(&mask, GFP_ATOMIC))
814 && cpupri_find(&rq->rd->cpupri, p, &mask))
815 return; 814 return;
816 815
817 if (!cpupri_find(&rq->rd->cpupri, rq->curr, &mask)) 816 if (p->rt.nr_cpus_allowed != 1
818 return; 817 && cpupri_find(&rq->rd->cpupri, p, mask))
818 goto free;
819
820 if (!cpupri_find(&rq->rd->cpupri, rq->curr, mask))
821 goto free;
819 822
820 /* 823 /*
821 * There appears to be other cpus that can accept 824 * There appears to be other cpus that can accept
@@ -824,6 +827,8 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
824 */ 827 */
825 requeue_task_rt(rq, p, 1); 828 requeue_task_rt(rq, p, 1);
826 resched_task(rq->curr); 829 resched_task(rq->curr);
830free:
831 free_cpumask_var(mask);
827} 832}
828 833
829#endif /* CONFIG_SMP */ 834#endif /* CONFIG_SMP */
@@ -914,7 +919,7 @@ static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep);
914static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu) 919static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu)
915{ 920{
916 if (!task_running(rq, p) && 921 if (!task_running(rq, p) &&
917 (cpu < 0 || cpu_isset(cpu, p->cpus_allowed)) && 922 (cpu < 0 || cpumask_test_cpu(cpu, &p->cpus_allowed)) &&
918 (p->rt.nr_cpus_allowed > 1)) 923 (p->rt.nr_cpus_allowed > 1))
919 return 1; 924 return 1;
920 return 0; 925 return 0;
@@ -953,7 +958,7 @@ static struct task_struct *pick_next_highest_task_rt(struct rq *rq, int cpu)
953 return next; 958 return next;
954} 959}
955 960
956static DEFINE_PER_CPU(cpumask_t, local_cpu_mask); 961static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask);
957 962
958static inline int pick_optimal_cpu(int this_cpu, cpumask_t *mask) 963static inline int pick_optimal_cpu(int this_cpu, cpumask_t *mask)
959{ 964{
@@ -973,7 +978,7 @@ static inline int pick_optimal_cpu(int this_cpu, cpumask_t *mask)
973static int find_lowest_rq(struct task_struct *task) 978static int find_lowest_rq(struct task_struct *task)
974{ 979{
975 struct sched_domain *sd; 980 struct sched_domain *sd;
976 cpumask_t *lowest_mask = &__get_cpu_var(local_cpu_mask); 981 struct cpumask *lowest_mask = __get_cpu_var(local_cpu_mask);
977 int this_cpu = smp_processor_id(); 982 int this_cpu = smp_processor_id();
978 int cpu = task_cpu(task); 983 int cpu = task_cpu(task);
979 984
@@ -988,7 +993,7 @@ static int find_lowest_rq(struct task_struct *task)
988 * I guess we might want to change cpupri_find() to ignore those 993 * I guess we might want to change cpupri_find() to ignore those
989 * in the first place. 994 * in the first place.
990 */ 995 */
991 cpus_and(*lowest_mask, *lowest_mask, cpu_active_map); 996 cpumask_and(lowest_mask, lowest_mask, cpu_active_mask);
992 997
993 /* 998 /*
994 * At this point we have built a mask of cpus representing the 999 * At this point we have built a mask of cpus representing the
@@ -998,7 +1003,7 @@ static int find_lowest_rq(struct task_struct *task)
998 * We prioritize the last cpu that the task executed on since 1003 * We prioritize the last cpu that the task executed on since
999 * it is most likely cache-hot in that location. 1004 * it is most likely cache-hot in that location.
1000 */ 1005 */
1001 if (cpu_isset(cpu, *lowest_mask)) 1006 if (cpumask_test_cpu(cpu, lowest_mask))
1002 return cpu; 1007 return cpu;
1003 1008
1004 /* 1009 /*
@@ -1013,7 +1018,8 @@ static int find_lowest_rq(struct task_struct *task)
1013 cpumask_t domain_mask; 1018 cpumask_t domain_mask;
1014 int best_cpu; 1019 int best_cpu;
1015 1020
1016 cpus_and(domain_mask, sd->span, *lowest_mask); 1021 cpumask_and(&domain_mask, sched_domain_span(sd),
1022 lowest_mask);
1017 1023
1018 best_cpu = pick_optimal_cpu(this_cpu, 1024 best_cpu = pick_optimal_cpu(this_cpu,
1019 &domain_mask); 1025 &domain_mask);
@@ -1054,8 +1060,8 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)
1054 * Also make sure that it wasn't scheduled on its rq. 1060 * Also make sure that it wasn't scheduled on its rq.
1055 */ 1061 */
1056 if (unlikely(task_rq(task) != rq || 1062 if (unlikely(task_rq(task) != rq ||
1057 !cpu_isset(lowest_rq->cpu, 1063 !cpumask_test_cpu(lowest_rq->cpu,
1058 task->cpus_allowed) || 1064 &task->cpus_allowed) ||
1059 task_running(rq, task) || 1065 task_running(rq, task) ||
1060 !task->se.on_rq)) { 1066 !task->se.on_rq)) {
1061 1067
@@ -1176,7 +1182,7 @@ static int pull_rt_task(struct rq *this_rq)
1176 1182
1177 next = pick_next_task_rt(this_rq); 1183 next = pick_next_task_rt(this_rq);
1178 1184
1179 for_each_cpu_mask_nr(cpu, this_rq->rd->rto_mask) { 1185 for_each_cpu(cpu, this_rq->rd->rto_mask) {
1180 if (this_cpu == cpu) 1186 if (this_cpu == cpu)
1181 continue; 1187 continue;
1182 1188
@@ -1305,9 +1311,9 @@ move_one_task_rt(struct rq *this_rq, int this_cpu, struct rq *busiest,
1305} 1311}
1306 1312
1307static void set_cpus_allowed_rt(struct task_struct *p, 1313static void set_cpus_allowed_rt(struct task_struct *p,
1308 const cpumask_t *new_mask) 1314 const struct cpumask *new_mask)
1309{ 1315{
1310 int weight = cpus_weight(*new_mask); 1316 int weight = cpumask_weight(new_mask);
1311 1317
1312 BUG_ON(!rt_task(p)); 1318 BUG_ON(!rt_task(p));
1313 1319
@@ -1328,7 +1334,7 @@ static void set_cpus_allowed_rt(struct task_struct *p,
1328 update_rt_migration(rq); 1334 update_rt_migration(rq);
1329 } 1335 }
1330 1336
1331 p->cpus_allowed = *new_mask; 1337 cpumask_copy(&p->cpus_allowed, new_mask);
1332 p->rt.nr_cpus_allowed = weight; 1338 p->rt.nr_cpus_allowed = weight;
1333} 1339}
1334 1340
@@ -1371,6 +1377,14 @@ static void switched_from_rt(struct rq *rq, struct task_struct *p,
1371 if (!rq->rt.rt_nr_running) 1377 if (!rq->rt.rt_nr_running)
1372 pull_rt_task(rq); 1378 pull_rt_task(rq);
1373} 1379}
1380
1381static inline void init_sched_rt_class(void)
1382{
1383 unsigned int i;
1384
1385 for_each_possible_cpu(i)
1386 alloc_cpumask_var(&per_cpu(local_cpu_mask, i), GFP_KERNEL);
1387}
1374#endif /* CONFIG_SMP */ 1388#endif /* CONFIG_SMP */
1375 1389
1376/* 1390/*
@@ -1541,3 +1555,4 @@ static void print_rt_stats(struct seq_file *m, int cpu)
1541 rcu_read_unlock(); 1555 rcu_read_unlock();
1542} 1556}
1543#endif /* CONFIG_SCHED_DEBUG */ 1557#endif /* CONFIG_SCHED_DEBUG */
1558
diff --git a/kernel/sched_stats.h b/kernel/sched_stats.h
index 3b01098164c8..f2773b5d1226 100644
--- a/kernel/sched_stats.h
+++ b/kernel/sched_stats.h
@@ -42,7 +42,8 @@ static int show_schedstat(struct seq_file *seq, void *v)
42 for_each_domain(cpu, sd) { 42 for_each_domain(cpu, sd) {
43 enum cpu_idle_type itype; 43 enum cpu_idle_type itype;
44 44
45 cpumask_scnprintf(mask_str, mask_len, sd->span); 45 cpumask_scnprintf(mask_str, mask_len,
46 sched_domain_span(sd));
46 seq_printf(seq, "domain%d %s", dcount++, mask_str); 47 seq_printf(seq, "domain%d %s", dcount++, mask_str);
47 for (itype = CPU_IDLE; itype < CPU_MAX_IDLE_TYPES; 48 for (itype = CPU_IDLE; itype < CPU_MAX_IDLE_TYPES;
48 itype++) { 49 itype++) {
diff --git a/kernel/softirq.c b/kernel/softirq.c
index e7c69a720d69..670c1eca47ec 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -102,20 +102,6 @@ void local_bh_disable(void)
102 102
103EXPORT_SYMBOL(local_bh_disable); 103EXPORT_SYMBOL(local_bh_disable);
104 104
105void __local_bh_enable(void)
106{
107 WARN_ON_ONCE(in_irq());
108
109 /*
110 * softirqs should never be enabled by __local_bh_enable(),
111 * it always nests inside local_bh_enable() sections:
112 */
113 WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
114
115 sub_preempt_count(SOFTIRQ_OFFSET);
116}
117EXPORT_SYMBOL_GPL(__local_bh_enable);
118
119/* 105/*
120 * Special-case - softirqs can safely be enabled in 106 * Special-case - softirqs can safely be enabled in
121 * cond_resched_softirq(), or by __do_softirq(), 107 * cond_resched_softirq(), or by __do_softirq(),
@@ -269,6 +255,7 @@ void irq_enter(void)
269{ 255{
270 int cpu = smp_processor_id(); 256 int cpu = smp_processor_id();
271 257
258 rcu_irq_enter();
272 if (idle_cpu(cpu) && !in_interrupt()) { 259 if (idle_cpu(cpu) && !in_interrupt()) {
273 __irq_enter(); 260 __irq_enter();
274 tick_check_idle(cpu); 261 tick_check_idle(cpu);
@@ -295,9 +282,9 @@ void irq_exit(void)
295 282
296#ifdef CONFIG_NO_HZ 283#ifdef CONFIG_NO_HZ
297 /* Make sure that timer wheel updates are propagated */ 284 /* Make sure that timer wheel updates are propagated */
298 if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
299 tick_nohz_stop_sched_tick(0);
300 rcu_irq_exit(); 285 rcu_irq_exit();
286 if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
287 tick_nohz_stop_sched_tick(0);
301#endif 288#endif
302 preempt_enable_no_resched(); 289 preempt_enable_no_resched();
303} 290}
@@ -797,3 +784,23 @@ int on_each_cpu(void (*func) (void *info), void *info, int wait)
797} 784}
798EXPORT_SYMBOL(on_each_cpu); 785EXPORT_SYMBOL(on_each_cpu);
799#endif 786#endif
787
788/*
789 * [ These __weak aliases are kept in a separate compilation unit, so that
790 * GCC does not inline them incorrectly. ]
791 */
792
793int __init __weak early_irq_init(void)
794{
795 return 0;
796}
797
798int __init __weak arch_early_irq_init(void)
799{
800 return 0;
801}
802
803int __weak arch_init_chip_data(struct irq_desc *desc, int cpu)
804{
805 return 0;
806}
diff --git a/kernel/softlockup.c b/kernel/softlockup.c
index dc0b3be6b7d5..1ab790c67b17 100644
--- a/kernel/softlockup.c
+++ b/kernel/softlockup.c
@@ -164,7 +164,7 @@ unsigned long __read_mostly sysctl_hung_task_check_count = 1024;
164/* 164/*
165 * Zero means infinite timeout - no checking done: 165 * Zero means infinite timeout - no checking done:
166 */ 166 */
167unsigned long __read_mostly sysctl_hung_task_timeout_secs = 120; 167unsigned long __read_mostly sysctl_hung_task_timeout_secs = 480;
168 168
169unsigned long __read_mostly sysctl_hung_task_warnings = 10; 169unsigned long __read_mostly sysctl_hung_task_warnings = 10;
170 170
diff --git a/kernel/stacktrace.c b/kernel/stacktrace.c
index 94b527ef1d1e..eb212f8f8bc8 100644
--- a/kernel/stacktrace.c
+++ b/kernel/stacktrace.c
@@ -6,6 +6,7 @@
6 * Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> 6 * Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
7 */ 7 */
8#include <linux/sched.h> 8#include <linux/sched.h>
9#include <linux/kernel.h>
9#include <linux/module.h> 10#include <linux/module.h>
10#include <linux/kallsyms.h> 11#include <linux/kallsyms.h>
11#include <linux/stacktrace.h> 12#include <linux/stacktrace.h>
@@ -24,3 +25,13 @@ void print_stack_trace(struct stack_trace *trace, int spaces)
24} 25}
25EXPORT_SYMBOL_GPL(print_stack_trace); 26EXPORT_SYMBOL_GPL(print_stack_trace);
26 27
28/*
29 * Architectures that do not implement save_stack_trace_tsk get this
30 * weak alias and a once-per-bootup warning (whenever this facility
31 * is utilized - for example by procfs):
32 */
33__weak void
34save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace)
35{
36 WARN_ONCE(1, KERN_INFO "save_stack_trace_tsk() not implemented yet.\n");
37}
diff --git a/kernel/sys.c b/kernel/sys.c
index ebe65c2c9873..d356d79e84ac 100644
--- a/kernel/sys.c
+++ b/kernel/sys.c
@@ -907,8 +907,8 @@ void do_sys_times(struct tms *tms)
907 struct task_cputime cputime; 907 struct task_cputime cputime;
908 cputime_t cutime, cstime; 908 cputime_t cutime, cstime;
909 909
910 spin_lock_irq(&current->sighand->siglock);
911 thread_group_cputime(current, &cputime); 910 thread_group_cputime(current, &cputime);
911 spin_lock_irq(&current->sighand->siglock);
912 cutime = current->signal->cutime; 912 cutime = current->signal->cutime;
913 cstime = current->signal->cstime; 913 cstime = current->signal->cstime;
914 spin_unlock_irq(&current->sighand->siglock); 914 spin_unlock_irq(&current->sighand->siglock);
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 0b627d9c93d8..ff6d45c7626f 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -121,6 +121,10 @@ extern int sg_big_buff;
121#include <asm/system.h> 121#include <asm/system.h>
122#endif 122#endif
123 123
124#ifdef CONFIG_SPARC64
125extern int sysctl_tsb_ratio;
126#endif
127
124#ifdef __hppa__ 128#ifdef __hppa__
125extern int pwrsw_enabled; 129extern int pwrsw_enabled;
126extern int unaligned_enabled; 130extern int unaligned_enabled;
@@ -451,6 +455,16 @@ static struct ctl_table kern_table[] = {
451 .proc_handler = &proc_dointvec, 455 .proc_handler = &proc_dointvec,
452 }, 456 },
453#endif 457#endif
458#ifdef CONFIG_SPARC64
459 {
460 .ctl_name = CTL_UNNUMBERED,
461 .procname = "tsb-ratio",
462 .data = &sysctl_tsb_ratio,
463 .maxlen = sizeof (int),
464 .mode = 0644,
465 .proc_handler = &proc_dointvec,
466 },
467#endif
454#ifdef __hppa__ 468#ifdef __hppa__
455 { 469 {
456 .ctl_name = KERN_HPPA_PWRSW, 470 .ctl_name = KERN_HPPA_PWRSW,
diff --git a/kernel/sysctl_check.c b/kernel/sysctl_check.c
index c35da23ab8fb..fafeb48f27c0 100644
--- a/kernel/sysctl_check.c
+++ b/kernel/sysctl_check.c
@@ -730,7 +730,6 @@ static const struct trans_ctl_table trans_fs_quota_table[] = {
730}; 730};
731 731
732static const struct trans_ctl_table trans_fs_xfs_table[] = { 732static const struct trans_ctl_table trans_fs_xfs_table[] = {
733 { XFS_RESTRICT_CHOWN, "restrict_chown" },
734 { XFS_SGID_INHERIT, "irix_sgid_inherit" }, 733 { XFS_SGID_INHERIT, "irix_sgid_inherit" },
735 { XFS_SYMLINK_MODE, "irix_symlink_mode" }, 734 { XFS_SYMLINK_MODE, "irix_symlink_mode" },
736 { XFS_PANIC_MASK, "panic_mask" }, 735 { XFS_PANIC_MASK, "panic_mask" },
diff --git a/kernel/taskstats.c b/kernel/taskstats.c
index bd6be76303cf..6d7dc4ec4aa5 100644
--- a/kernel/taskstats.c
+++ b/kernel/taskstats.c
@@ -352,7 +352,7 @@ static int parse(struct nlattr *na, cpumask_t *mask)
352 if (!data) 352 if (!data)
353 return -ENOMEM; 353 return -ENOMEM;
354 nla_strlcpy(data, na, len); 354 nla_strlcpy(data, na, len);
355 ret = cpulist_parse(data, *mask); 355 ret = cpulist_parse(data, mask);
356 kfree(data); 356 kfree(data);
357 return ret; 357 return ret;
358} 358}
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c
index f8d968063cea..ea2f48af83cf 100644
--- a/kernel/time/clockevents.c
+++ b/kernel/time/clockevents.c
@@ -166,6 +166,8 @@ static void clockevents_notify_released(void)
166void clockevents_register_device(struct clock_event_device *dev) 166void clockevents_register_device(struct clock_event_device *dev)
167{ 167{
168 BUG_ON(dev->mode != CLOCK_EVT_MODE_UNUSED); 168 BUG_ON(dev->mode != CLOCK_EVT_MODE_UNUSED);
169 BUG_ON(!dev->cpumask);
170
169 /* 171 /*
170 * A nsec2cyc multiplicator of 0 is invalid and we'd crash 172 * A nsec2cyc multiplicator of 0 is invalid and we'd crash
171 * on it, so fix it up and emit a warning: 173 * on it, so fix it up and emit a warning:
diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c
index 8ff15e5d486b..f5f793d92415 100644
--- a/kernel/time/ntp.c
+++ b/kernel/time/ntp.c
@@ -131,7 +131,7 @@ static enum hrtimer_restart ntp_leap_second(struct hrtimer *timer)
131{ 131{
132 enum hrtimer_restart res = HRTIMER_NORESTART; 132 enum hrtimer_restart res = HRTIMER_NORESTART;
133 133
134 write_seqlock_irq(&xtime_lock); 134 write_seqlock(&xtime_lock);
135 135
136 switch (time_state) { 136 switch (time_state) {
137 case TIME_OK: 137 case TIME_OK:
@@ -164,7 +164,7 @@ static enum hrtimer_restart ntp_leap_second(struct hrtimer *timer)
164 } 164 }
165 update_vsyscall(&xtime, clock); 165 update_vsyscall(&xtime, clock);
166 166
167 write_sequnlock_irq(&xtime_lock); 167 write_sequnlock(&xtime_lock);
168 168
169 return res; 169 return res;
170} 170}
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c
index f98a1b7b16e9..9590af2327be 100644
--- a/kernel/time/tick-broadcast.c
+++ b/kernel/time/tick-broadcast.c
@@ -150,7 +150,7 @@ static void tick_do_broadcast(cpumask_t mask)
150 */ 150 */
151 cpu = first_cpu(mask); 151 cpu = first_cpu(mask);
152 td = &per_cpu(tick_cpu_device, cpu); 152 td = &per_cpu(tick_cpu_device, cpu);
153 td->evtdev->broadcast(mask); 153 td->evtdev->broadcast(&mask);
154 } 154 }
155} 155}
156 156
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c
index df12434b43ca..f8372be74122 100644
--- a/kernel/time/tick-common.c
+++ b/kernel/time/tick-common.c
@@ -136,7 +136,7 @@ void tick_setup_periodic(struct clock_event_device *dev, int broadcast)
136 */ 136 */
137static void tick_setup_device(struct tick_device *td, 137static void tick_setup_device(struct tick_device *td,
138 struct clock_event_device *newdev, int cpu, 138 struct clock_event_device *newdev, int cpu,
139 const cpumask_t *cpumask) 139 const struct cpumask *cpumask)
140{ 140{
141 ktime_t next_event; 141 ktime_t next_event;
142 void (*handler)(struct clock_event_device *) = NULL; 142 void (*handler)(struct clock_event_device *) = NULL;
@@ -171,8 +171,8 @@ static void tick_setup_device(struct tick_device *td,
171 * When the device is not per cpu, pin the interrupt to the 171 * When the device is not per cpu, pin the interrupt to the
172 * current cpu: 172 * current cpu:
173 */ 173 */
174 if (!cpus_equal(newdev->cpumask, *cpumask)) 174 if (!cpumask_equal(newdev->cpumask, cpumask))
175 irq_set_affinity(newdev->irq, *cpumask); 175 irq_set_affinity(newdev->irq, cpumask);
176 176
177 /* 177 /*
178 * When global broadcasting is active, check if the current 178 * When global broadcasting is active, check if the current
@@ -202,14 +202,14 @@ static int tick_check_new_device(struct clock_event_device *newdev)
202 spin_lock_irqsave(&tick_device_lock, flags); 202 spin_lock_irqsave(&tick_device_lock, flags);
203 203
204 cpu = smp_processor_id(); 204 cpu = smp_processor_id();
205 if (!cpu_isset(cpu, newdev->cpumask)) 205 if (!cpumask_test_cpu(cpu, newdev->cpumask))
206 goto out_bc; 206 goto out_bc;
207 207
208 td = &per_cpu(tick_cpu_device, cpu); 208 td = &per_cpu(tick_cpu_device, cpu);
209 curdev = td->evtdev; 209 curdev = td->evtdev;
210 210
211 /* cpu local device ? */ 211 /* cpu local device ? */
212 if (!cpus_equal(newdev->cpumask, cpumask_of_cpu(cpu))) { 212 if (!cpumask_equal(newdev->cpumask, cpumask_of(cpu))) {
213 213
214 /* 214 /*
215 * If the cpu affinity of the device interrupt can not 215 * If the cpu affinity of the device interrupt can not
@@ -222,7 +222,7 @@ static int tick_check_new_device(struct clock_event_device *newdev)
222 * If we have a cpu local device already, do not replace it 222 * If we have a cpu local device already, do not replace it
223 * by a non cpu local device 223 * by a non cpu local device
224 */ 224 */
225 if (curdev && cpus_equal(curdev->cpumask, cpumask_of_cpu(cpu))) 225 if (curdev && cpumask_equal(curdev->cpumask, cpumask_of(cpu)))
226 goto out_bc; 226 goto out_bc;
227 } 227 }
228 228
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 342fc9ccab46..76a574bbef97 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -144,7 +144,7 @@ void tick_nohz_update_jiffies(void)
144 if (!ts->tick_stopped) 144 if (!ts->tick_stopped)
145 return; 145 return;
146 146
147 cpu_clear(cpu, nohz_cpu_mask); 147 cpumask_clear_cpu(cpu, nohz_cpu_mask);
148 now = ktime_get(); 148 now = ktime_get();
149 ts->idle_waketime = now; 149 ts->idle_waketime = now;
150 150
@@ -247,7 +247,7 @@ void tick_nohz_stop_sched_tick(int inidle)
247 if (need_resched()) 247 if (need_resched())
248 goto end; 248 goto end;
249 249
250 if (unlikely(local_softirq_pending())) { 250 if (unlikely(local_softirq_pending() && cpu_online(cpu))) {
251 static int ratelimit; 251 static int ratelimit;
252 252
253 if (ratelimit < 10) { 253 if (ratelimit < 10) {
@@ -282,8 +282,31 @@ void tick_nohz_stop_sched_tick(int inidle)
282 /* Schedule the tick, if we are at least one jiffie off */ 282 /* Schedule the tick, if we are at least one jiffie off */
283 if ((long)delta_jiffies >= 1) { 283 if ((long)delta_jiffies >= 1) {
284 284
285 /*
286 * calculate the expiry time for the next timer wheel
287 * timer
288 */
289 expires = ktime_add_ns(last_update, tick_period.tv64 *
290 delta_jiffies);
291
292 /*
293 * If this cpu is the one which updates jiffies, then
294 * give up the assignment and let it be taken by the
295 * cpu which runs the tick timer next, which might be
296 * this cpu as well. If we don't drop this here the
297 * jiffies might be stale and do_timer() never
298 * invoked.
299 */
300 if (cpu == tick_do_timer_cpu)
301 tick_do_timer_cpu = TICK_DO_TIMER_NONE;
302
285 if (delta_jiffies > 1) 303 if (delta_jiffies > 1)
286 cpu_set(cpu, nohz_cpu_mask); 304 cpumask_set_cpu(cpu, nohz_cpu_mask);
305
306 /* Skip reprogram of event if its not changed */
307 if (ts->tick_stopped && ktime_equal(expires, dev->next_event))
308 goto out;
309
287 /* 310 /*
288 * nohz_stop_sched_tick can be called several times before 311 * nohz_stop_sched_tick can be called several times before
289 * the nohz_restart_sched_tick is called. This happens when 312 * the nohz_restart_sched_tick is called. This happens when
@@ -296,7 +319,7 @@ void tick_nohz_stop_sched_tick(int inidle)
296 /* 319 /*
297 * sched tick not stopped! 320 * sched tick not stopped!
298 */ 321 */
299 cpu_clear(cpu, nohz_cpu_mask); 322 cpumask_clear_cpu(cpu, nohz_cpu_mask);
300 goto out; 323 goto out;
301 } 324 }
302 325
@@ -306,17 +329,6 @@ void tick_nohz_stop_sched_tick(int inidle)
306 rcu_enter_nohz(); 329 rcu_enter_nohz();
307 } 330 }
308 331
309 /*
310 * If this cpu is the one which updates jiffies, then
311 * give up the assignment and let it be taken by the
312 * cpu which runs the tick timer next, which might be
313 * this cpu as well. If we don't drop this here the
314 * jiffies might be stale and do_timer() never
315 * invoked.
316 */
317 if (cpu == tick_do_timer_cpu)
318 tick_do_timer_cpu = TICK_DO_TIMER_NONE;
319
320 ts->idle_sleeps++; 332 ts->idle_sleeps++;
321 333
322 /* 334 /*
@@ -332,12 +344,7 @@ void tick_nohz_stop_sched_tick(int inidle)
332 goto out; 344 goto out;
333 } 345 }
334 346
335 /* 347 /* Mark expiries */
336 * calculate the expiry time for the next timer wheel
337 * timer
338 */
339 expires = ktime_add_ns(last_update, tick_period.tv64 *
340 delta_jiffies);
341 ts->idle_expires = expires; 348 ts->idle_expires = expires;
342 349
343 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { 350 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
@@ -354,7 +361,7 @@ void tick_nohz_stop_sched_tick(int inidle)
354 * softirq. 361 * softirq.
355 */ 362 */
356 tick_do_update_jiffies64(ktime_get()); 363 tick_do_update_jiffies64(ktime_get());
357 cpu_clear(cpu, nohz_cpu_mask); 364 cpumask_clear_cpu(cpu, nohz_cpu_mask);
358 } 365 }
359 raise_softirq_irqoff(TIMER_SOFTIRQ); 366 raise_softirq_irqoff(TIMER_SOFTIRQ);
360out: 367out:
@@ -432,7 +439,7 @@ void tick_nohz_restart_sched_tick(void)
432 select_nohz_load_balancer(0); 439 select_nohz_load_balancer(0);
433 now = ktime_get(); 440 now = ktime_get();
434 tick_do_update_jiffies64(now); 441 tick_do_update_jiffies64(now);
435 cpu_clear(cpu, nohz_cpu_mask); 442 cpumask_clear_cpu(cpu, nohz_cpu_mask);
436 443
437 /* 444 /*
438 * We stopped the tick in idle. Update process times would miss the 445 * We stopped the tick in idle. Update process times would miss the
@@ -681,7 +688,6 @@ void tick_setup_sched_timer(void)
681 */ 688 */
682 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); 689 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
683 ts->sched_timer.function = tick_sched_timer; 690 ts->sched_timer.function = tick_sched_timer;
684 ts->sched_timer.cb_mode = HRTIMER_CB_IRQSAFE_PERCPU;
685 691
686 /* Get the next period (per cpu) */ 692 /* Get the next period (per cpu) */
687 hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update()); 693 hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index 76f34c0ef29c..1d601a7c4587 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -69,6 +69,7 @@ void tracing_on(void)
69{ 69{
70 set_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags); 70 set_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags);
71} 71}
72EXPORT_SYMBOL_GPL(tracing_on);
72 73
73/** 74/**
74 * tracing_off - turn off all tracing buffers 75 * tracing_off - turn off all tracing buffers
@@ -82,6 +83,7 @@ void tracing_off(void)
82{ 83{
83 clear_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags); 84 clear_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags);
84} 85}
86EXPORT_SYMBOL_GPL(tracing_off);
85 87
86/** 88/**
87 * tracing_off_permanent - permanently disable ring buffers 89 * tracing_off_permanent - permanently disable ring buffers
@@ -111,12 +113,14 @@ u64 ring_buffer_time_stamp(int cpu)
111 113
112 return time; 114 return time;
113} 115}
116EXPORT_SYMBOL_GPL(ring_buffer_time_stamp);
114 117
115void ring_buffer_normalize_time_stamp(int cpu, u64 *ts) 118void ring_buffer_normalize_time_stamp(int cpu, u64 *ts)
116{ 119{
117 /* Just stupid testing the normalize function and deltas */ 120 /* Just stupid testing the normalize function and deltas */
118 *ts >>= DEBUG_SHIFT; 121 *ts >>= DEBUG_SHIFT;
119} 122}
123EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp);
120 124
121#define RB_EVNT_HDR_SIZE (sizeof(struct ring_buffer_event)) 125#define RB_EVNT_HDR_SIZE (sizeof(struct ring_buffer_event))
122#define RB_ALIGNMENT_SHIFT 2 126#define RB_ALIGNMENT_SHIFT 2
@@ -166,6 +170,7 @@ unsigned ring_buffer_event_length(struct ring_buffer_event *event)
166{ 170{
167 return rb_event_length(event); 171 return rb_event_length(event);
168} 172}
173EXPORT_SYMBOL_GPL(ring_buffer_event_length);
169 174
170/* inline for ring buffer fast paths */ 175/* inline for ring buffer fast paths */
171static inline void * 176static inline void *
@@ -187,6 +192,7 @@ void *ring_buffer_event_data(struct ring_buffer_event *event)
187{ 192{
188 return rb_event_data(event); 193 return rb_event_data(event);
189} 194}
195EXPORT_SYMBOL_GPL(ring_buffer_event_data);
190 196
191#define for_each_buffer_cpu(buffer, cpu) \ 197#define for_each_buffer_cpu(buffer, cpu) \
192 for_each_cpu_mask(cpu, buffer->cpumask) 198 for_each_cpu_mask(cpu, buffer->cpumask)
@@ -427,7 +433,7 @@ extern int ring_buffer_page_too_big(void);
427 433
428/** 434/**
429 * ring_buffer_alloc - allocate a new ring_buffer 435 * ring_buffer_alloc - allocate a new ring_buffer
430 * @size: the size in bytes that is needed. 436 * @size: the size in bytes per cpu that is needed.
431 * @flags: attributes to set for the ring buffer. 437 * @flags: attributes to set for the ring buffer.
432 * 438 *
433 * Currently the only flag that is available is the RB_FL_OVERWRITE 439 * Currently the only flag that is available is the RB_FL_OVERWRITE
@@ -490,6 +496,7 @@ struct ring_buffer *ring_buffer_alloc(unsigned long size, unsigned flags)
490 kfree(buffer); 496 kfree(buffer);
491 return NULL; 497 return NULL;
492} 498}
499EXPORT_SYMBOL_GPL(ring_buffer_alloc);
493 500
494/** 501/**
495 * ring_buffer_free - free a ring buffer. 502 * ring_buffer_free - free a ring buffer.
@@ -505,6 +512,7 @@ ring_buffer_free(struct ring_buffer *buffer)
505 512
506 kfree(buffer); 513 kfree(buffer);
507} 514}
515EXPORT_SYMBOL_GPL(ring_buffer_free);
508 516
509static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer); 517static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer);
510 518
@@ -680,6 +688,7 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size)
680 mutex_unlock(&buffer->mutex); 688 mutex_unlock(&buffer->mutex);
681 return -ENOMEM; 689 return -ENOMEM;
682} 690}
691EXPORT_SYMBOL_GPL(ring_buffer_resize);
683 692
684static inline int rb_null_event(struct ring_buffer_event *event) 693static inline int rb_null_event(struct ring_buffer_event *event)
685{ 694{
@@ -1304,6 +1313,7 @@ ring_buffer_lock_reserve(struct ring_buffer *buffer,
1304 ftrace_preempt_enable(resched); 1313 ftrace_preempt_enable(resched);
1305 return NULL; 1314 return NULL;
1306} 1315}
1316EXPORT_SYMBOL_GPL(ring_buffer_lock_reserve);
1307 1317
1308static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer, 1318static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer,
1309 struct ring_buffer_event *event) 1319 struct ring_buffer_event *event)
@@ -1350,6 +1360,7 @@ int ring_buffer_unlock_commit(struct ring_buffer *buffer,
1350 1360
1351 return 0; 1361 return 0;
1352} 1362}
1363EXPORT_SYMBOL_GPL(ring_buffer_unlock_commit);
1353 1364
1354/** 1365/**
1355 * ring_buffer_write - write data to the buffer without reserving 1366 * ring_buffer_write - write data to the buffer without reserving
@@ -1411,6 +1422,7 @@ int ring_buffer_write(struct ring_buffer *buffer,
1411 1422
1412 return ret; 1423 return ret;
1413} 1424}
1425EXPORT_SYMBOL_GPL(ring_buffer_write);
1414 1426
1415static inline int rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer) 1427static inline int rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer)
1416{ 1428{
@@ -1437,6 +1449,7 @@ void ring_buffer_record_disable(struct ring_buffer *buffer)
1437{ 1449{
1438 atomic_inc(&buffer->record_disabled); 1450 atomic_inc(&buffer->record_disabled);
1439} 1451}
1452EXPORT_SYMBOL_GPL(ring_buffer_record_disable);
1440 1453
1441/** 1454/**
1442 * ring_buffer_record_enable - enable writes to the buffer 1455 * ring_buffer_record_enable - enable writes to the buffer
@@ -1449,6 +1462,7 @@ void ring_buffer_record_enable(struct ring_buffer *buffer)
1449{ 1462{
1450 atomic_dec(&buffer->record_disabled); 1463 atomic_dec(&buffer->record_disabled);
1451} 1464}
1465EXPORT_SYMBOL_GPL(ring_buffer_record_enable);
1452 1466
1453/** 1467/**
1454 * ring_buffer_record_disable_cpu - stop all writes into the cpu_buffer 1468 * ring_buffer_record_disable_cpu - stop all writes into the cpu_buffer
@@ -1470,6 +1484,7 @@ void ring_buffer_record_disable_cpu(struct ring_buffer *buffer, int cpu)
1470 cpu_buffer = buffer->buffers[cpu]; 1484 cpu_buffer = buffer->buffers[cpu];
1471 atomic_inc(&cpu_buffer->record_disabled); 1485 atomic_inc(&cpu_buffer->record_disabled);
1472} 1486}
1487EXPORT_SYMBOL_GPL(ring_buffer_record_disable_cpu);
1473 1488
1474/** 1489/**
1475 * ring_buffer_record_enable_cpu - enable writes to the buffer 1490 * ring_buffer_record_enable_cpu - enable writes to the buffer
@@ -1489,6 +1504,7 @@ void ring_buffer_record_enable_cpu(struct ring_buffer *buffer, int cpu)
1489 cpu_buffer = buffer->buffers[cpu]; 1504 cpu_buffer = buffer->buffers[cpu];
1490 atomic_dec(&cpu_buffer->record_disabled); 1505 atomic_dec(&cpu_buffer->record_disabled);
1491} 1506}
1507EXPORT_SYMBOL_GPL(ring_buffer_record_enable_cpu);
1492 1508
1493/** 1509/**
1494 * ring_buffer_entries_cpu - get the number of entries in a cpu buffer 1510 * ring_buffer_entries_cpu - get the number of entries in a cpu buffer
@@ -1505,6 +1521,7 @@ unsigned long ring_buffer_entries_cpu(struct ring_buffer *buffer, int cpu)
1505 cpu_buffer = buffer->buffers[cpu]; 1521 cpu_buffer = buffer->buffers[cpu];
1506 return cpu_buffer->entries; 1522 return cpu_buffer->entries;
1507} 1523}
1524EXPORT_SYMBOL_GPL(ring_buffer_entries_cpu);
1508 1525
1509/** 1526/**
1510 * ring_buffer_overrun_cpu - get the number of overruns in a cpu_buffer 1527 * ring_buffer_overrun_cpu - get the number of overruns in a cpu_buffer
@@ -1521,6 +1538,7 @@ unsigned long ring_buffer_overrun_cpu(struct ring_buffer *buffer, int cpu)
1521 cpu_buffer = buffer->buffers[cpu]; 1538 cpu_buffer = buffer->buffers[cpu];
1522 return cpu_buffer->overrun; 1539 return cpu_buffer->overrun;
1523} 1540}
1541EXPORT_SYMBOL_GPL(ring_buffer_overrun_cpu);
1524 1542
1525/** 1543/**
1526 * ring_buffer_entries - get the number of entries in a buffer 1544 * ring_buffer_entries - get the number of entries in a buffer
@@ -1543,6 +1561,7 @@ unsigned long ring_buffer_entries(struct ring_buffer *buffer)
1543 1561
1544 return entries; 1562 return entries;
1545} 1563}
1564EXPORT_SYMBOL_GPL(ring_buffer_entries);
1546 1565
1547/** 1566/**
1548 * ring_buffer_overrun_cpu - get the number of overruns in buffer 1567 * ring_buffer_overrun_cpu - get the number of overruns in buffer
@@ -1565,6 +1584,7 @@ unsigned long ring_buffer_overruns(struct ring_buffer *buffer)
1565 1584
1566 return overruns; 1585 return overruns;
1567} 1586}
1587EXPORT_SYMBOL_GPL(ring_buffer_overruns);
1568 1588
1569static void rb_iter_reset(struct ring_buffer_iter *iter) 1589static void rb_iter_reset(struct ring_buffer_iter *iter)
1570{ 1590{
@@ -1600,6 +1620,7 @@ void ring_buffer_iter_reset(struct ring_buffer_iter *iter)
1600 rb_iter_reset(iter); 1620 rb_iter_reset(iter);
1601 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); 1621 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
1602} 1622}
1623EXPORT_SYMBOL_GPL(ring_buffer_iter_reset);
1603 1624
1604/** 1625/**
1605 * ring_buffer_iter_empty - check if an iterator has no more to read 1626 * ring_buffer_iter_empty - check if an iterator has no more to read
@@ -1614,6 +1635,7 @@ int ring_buffer_iter_empty(struct ring_buffer_iter *iter)
1614 return iter->head_page == cpu_buffer->commit_page && 1635 return iter->head_page == cpu_buffer->commit_page &&
1615 iter->head == rb_commit_index(cpu_buffer); 1636 iter->head == rb_commit_index(cpu_buffer);
1616} 1637}
1638EXPORT_SYMBOL_GPL(ring_buffer_iter_empty);
1617 1639
1618static void 1640static void
1619rb_update_read_stamp(struct ring_buffer_per_cpu *cpu_buffer, 1641rb_update_read_stamp(struct ring_buffer_per_cpu *cpu_buffer,
@@ -1880,6 +1902,7 @@ rb_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
1880 1902
1881 return NULL; 1903 return NULL;
1882} 1904}
1905EXPORT_SYMBOL_GPL(ring_buffer_peek);
1883 1906
1884static struct ring_buffer_event * 1907static struct ring_buffer_event *
1885rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts) 1908rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
@@ -1940,6 +1963,7 @@ rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
1940 1963
1941 return NULL; 1964 return NULL;
1942} 1965}
1966EXPORT_SYMBOL_GPL(ring_buffer_iter_peek);
1943 1967
1944/** 1968/**
1945 * ring_buffer_peek - peek at the next event to be read 1969 * ring_buffer_peek - peek at the next event to be read
@@ -2017,6 +2041,7 @@ ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts)
2017 2041
2018 return event; 2042 return event;
2019} 2043}
2044EXPORT_SYMBOL_GPL(ring_buffer_consume);
2020 2045
2021/** 2046/**
2022 * ring_buffer_read_start - start a non consuming read of the buffer 2047 * ring_buffer_read_start - start a non consuming read of the buffer
@@ -2059,6 +2084,7 @@ ring_buffer_read_start(struct ring_buffer *buffer, int cpu)
2059 2084
2060 return iter; 2085 return iter;
2061} 2086}
2087EXPORT_SYMBOL_GPL(ring_buffer_read_start);
2062 2088
2063/** 2089/**
2064 * ring_buffer_finish - finish reading the iterator of the buffer 2090 * ring_buffer_finish - finish reading the iterator of the buffer
@@ -2075,6 +2101,7 @@ ring_buffer_read_finish(struct ring_buffer_iter *iter)
2075 atomic_dec(&cpu_buffer->record_disabled); 2101 atomic_dec(&cpu_buffer->record_disabled);
2076 kfree(iter); 2102 kfree(iter);
2077} 2103}
2104EXPORT_SYMBOL_GPL(ring_buffer_read_finish);
2078 2105
2079/** 2106/**
2080 * ring_buffer_read - read the next item in the ring buffer by the iterator 2107 * ring_buffer_read - read the next item in the ring buffer by the iterator
@@ -2101,6 +2128,7 @@ ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts)
2101 2128
2102 return event; 2129 return event;
2103} 2130}
2131EXPORT_SYMBOL_GPL(ring_buffer_read);
2104 2132
2105/** 2133/**
2106 * ring_buffer_size - return the size of the ring buffer (in bytes) 2134 * ring_buffer_size - return the size of the ring buffer (in bytes)
@@ -2110,6 +2138,7 @@ unsigned long ring_buffer_size(struct ring_buffer *buffer)
2110{ 2138{
2111 return BUF_PAGE_SIZE * buffer->pages; 2139 return BUF_PAGE_SIZE * buffer->pages;
2112} 2140}
2141EXPORT_SYMBOL_GPL(ring_buffer_size);
2113 2142
2114static void 2143static void
2115rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer) 2144rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
@@ -2156,6 +2185,7 @@ void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu)
2156 2185
2157 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); 2186 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
2158} 2187}
2188EXPORT_SYMBOL_GPL(ring_buffer_reset_cpu);
2159 2189
2160/** 2190/**
2161 * ring_buffer_reset - reset a ring buffer 2191 * ring_buffer_reset - reset a ring buffer
@@ -2168,6 +2198,7 @@ void ring_buffer_reset(struct ring_buffer *buffer)
2168 for_each_buffer_cpu(buffer, cpu) 2198 for_each_buffer_cpu(buffer, cpu)
2169 ring_buffer_reset_cpu(buffer, cpu); 2199 ring_buffer_reset_cpu(buffer, cpu);
2170} 2200}
2201EXPORT_SYMBOL_GPL(ring_buffer_reset);
2171 2202
2172/** 2203/**
2173 * rind_buffer_empty - is the ring buffer empty? 2204 * rind_buffer_empty - is the ring buffer empty?
@@ -2186,6 +2217,7 @@ int ring_buffer_empty(struct ring_buffer *buffer)
2186 } 2217 }
2187 return 1; 2218 return 1;
2188} 2219}
2220EXPORT_SYMBOL_GPL(ring_buffer_empty);
2189 2221
2190/** 2222/**
2191 * ring_buffer_empty_cpu - is a cpu buffer of a ring buffer empty? 2223 * ring_buffer_empty_cpu - is a cpu buffer of a ring buffer empty?
@@ -2202,6 +2234,7 @@ int ring_buffer_empty_cpu(struct ring_buffer *buffer, int cpu)
2202 cpu_buffer = buffer->buffers[cpu]; 2234 cpu_buffer = buffer->buffers[cpu];
2203 return rb_per_cpu_empty(cpu_buffer); 2235 return rb_per_cpu_empty(cpu_buffer);
2204} 2236}
2237EXPORT_SYMBOL_GPL(ring_buffer_empty_cpu);
2205 2238
2206/** 2239/**
2207 * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers 2240 * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
@@ -2250,6 +2283,7 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
2250 2283
2251 return 0; 2284 return 0;
2252} 2285}
2286EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu);
2253 2287
2254static void rb_remove_entries(struct ring_buffer_per_cpu *cpu_buffer, 2288static void rb_remove_entries(struct ring_buffer_per_cpu *cpu_buffer,
2255 struct buffer_data_page *bpage) 2289 struct buffer_data_page *bpage)
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index f4bb3800318b..0e91f43b6baf 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -30,7 +30,6 @@
30#include <linux/gfp.h> 30#include <linux/gfp.h>
31#include <linux/fs.h> 31#include <linux/fs.h>
32#include <linux/kprobes.h> 32#include <linux/kprobes.h>
33#include <linux/seq_file.h>
34#include <linux/writeback.h> 33#include <linux/writeback.h>
35 34
36#include <linux/stacktrace.h> 35#include <linux/stacktrace.h>
@@ -1310,7 +1309,7 @@ enum trace_file_type {
1310 TRACE_FILE_ANNOTATE = 2, 1309 TRACE_FILE_ANNOTATE = 2,
1311}; 1310};
1312 1311
1313static void trace_iterator_increment(struct trace_iterator *iter, int cpu) 1312static void trace_iterator_increment(struct trace_iterator *iter)
1314{ 1313{
1315 /* Don't allow ftrace to trace into the ring buffers */ 1314 /* Don't allow ftrace to trace into the ring buffers */
1316 ftrace_disable_cpu(); 1315 ftrace_disable_cpu();
@@ -1389,7 +1388,7 @@ static void *find_next_entry_inc(struct trace_iterator *iter)
1389 iter->ent = __find_next_entry(iter, &iter->cpu, &iter->ts); 1388 iter->ent = __find_next_entry(iter, &iter->cpu, &iter->ts);
1390 1389
1391 if (iter->ent) 1390 if (iter->ent)
1392 trace_iterator_increment(iter, iter->cpu); 1391 trace_iterator_increment(iter);
1393 1392
1394 return iter->ent ? iter : NULL; 1393 return iter->ent ? iter : NULL;
1395} 1394}
@@ -2675,7 +2674,7 @@ tracing_cpumask_read(struct file *filp, char __user *ubuf,
2675 2674
2676 mutex_lock(&tracing_cpumask_update_lock); 2675 mutex_lock(&tracing_cpumask_update_lock);
2677 2676
2678 len = cpumask_scnprintf(mask_str, count, tracing_cpumask); 2677 len = cpumask_scnprintf(mask_str, count, &tracing_cpumask);
2679 if (count - len < 2) { 2678 if (count - len < 2) {
2680 count = -EINVAL; 2679 count = -EINVAL;
2681 goto out_err; 2680 goto out_err;
@@ -2696,7 +2695,7 @@ tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2696 int err, cpu; 2695 int err, cpu;
2697 2696
2698 mutex_lock(&tracing_cpumask_update_lock); 2697 mutex_lock(&tracing_cpumask_update_lock);
2699 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new); 2698 err = cpumask_parse_user(ubuf, count, &tracing_cpumask_new);
2700 if (err) 2699 if (err)
2701 goto err_unlock; 2700 goto err_unlock;
2702 2701
diff --git a/kernel/trace/trace_sysprof.c b/kernel/trace/trace_sysprof.c
index 01becf1f19ff..a5779bd975db 100644
--- a/kernel/trace/trace_sysprof.c
+++ b/kernel/trace/trace_sysprof.c
@@ -202,7 +202,6 @@ static void start_stack_timer(int cpu)
202 202
203 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 203 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
204 hrtimer->function = stack_trace_timer_fn; 204 hrtimer->function = stack_trace_timer_fn;
205 hrtimer->cb_mode = HRTIMER_CB_IRQSAFE_PERCPU;
206 205
207 hrtimer_start(hrtimer, ns_to_ktime(sample_period), HRTIMER_MODE_REL); 206 hrtimer_start(hrtimer, ns_to_ktime(sample_period), HRTIMER_MODE_REL);
208} 207}
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-28 07:39:27 -0500