aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/audit_tree.c13
-rw-r--r--kernel/auditsc.c1
-rw-r--r--kernel/bounds.c2
-rw-r--r--kernel/cpu.c24
-rw-r--r--kernel/exit.c36
-rw-r--r--kernel/fork.c2
-rw-r--r--kernel/kexec.c2
-rw-r--r--kernel/kfifo.c361
-rw-r--r--kernel/kthread.c23
-rw-r--r--kernel/module.c13
-rw-r--r--kernel/perf_event.c36
-rw-r--r--kernel/printk.c4
-rw-r--r--kernel/resource.c32
-rw-r--r--kernel/sched.c307
-rw-r--r--kernel/sched_clock.c23
-rw-r--r--kernel/sched_fair.c53
-rw-r--r--kernel/sched_rt.c4
-rw-r--r--kernel/signal.c25
-rw-r--r--kernel/sys.c2
-rw-r--r--kernel/sysctl.c4
-rw-r--r--kernel/sysctl_binary.c31
-rw-r--r--kernel/time.c1
-rw-r--r--kernel/time/clockevents.c18
-rw-r--r--kernel/time/timekeeping.c27
-rw-r--r--kernel/time/timer_list.c4
-rw-r--r--kernel/timer.c2
-rw-r--r--kernel/trace/trace.c2
-rw-r--r--kernel/trace/trace_kprobe.c31
-rw-r--r--kernel/trace/trace_sysprof.c1
29 files changed, 727 insertions, 357 deletions
diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c
index 2451dc6f3282..4b05bd9479db 100644
--- a/kernel/audit_tree.c
+++ b/kernel/audit_tree.c
@@ -277,7 +277,7 @@ static void untag_chunk(struct node *p)
277 owner->root = NULL; 277 owner->root = NULL;
278 } 278 }
279 279
280 for (i = j = 0; i < size; i++, j++) { 280 for (i = j = 0; j <= size; i++, j++) {
281 struct audit_tree *s; 281 struct audit_tree *s;
282 if (&chunk->owners[j] == p) { 282 if (&chunk->owners[j] == p) {
283 list_del_init(&p->list); 283 list_del_init(&p->list);
@@ -290,7 +290,7 @@ static void untag_chunk(struct node *p)
290 if (!s) /* result of earlier fallback */ 290 if (!s) /* result of earlier fallback */
291 continue; 291 continue;
292 get_tree(s); 292 get_tree(s);
293 list_replace_init(&chunk->owners[i].list, &new->owners[j].list); 293 list_replace_init(&chunk->owners[j].list, &new->owners[i].list);
294 } 294 }
295 295
296 list_replace_rcu(&chunk->hash, &new->hash); 296 list_replace_rcu(&chunk->hash, &new->hash);
@@ -373,15 +373,17 @@ static int tag_chunk(struct inode *inode, struct audit_tree *tree)
373 for (n = 0; n < old->count; n++) { 373 for (n = 0; n < old->count; n++) {
374 if (old->owners[n].owner == tree) { 374 if (old->owners[n].owner == tree) {
375 spin_unlock(&hash_lock); 375 spin_unlock(&hash_lock);
376 put_inotify_watch(watch); 376 put_inotify_watch(&old->watch);
377 return 0; 377 return 0;
378 } 378 }
379 } 379 }
380 spin_unlock(&hash_lock); 380 spin_unlock(&hash_lock);
381 381
382 chunk = alloc_chunk(old->count + 1); 382 chunk = alloc_chunk(old->count + 1);
383 if (!chunk) 383 if (!chunk) {
384 put_inotify_watch(&old->watch);
384 return -ENOMEM; 385 return -ENOMEM;
386 }
385 387
386 mutex_lock(&inode->inotify_mutex); 388 mutex_lock(&inode->inotify_mutex);
387 if (inotify_clone_watch(&old->watch, &chunk->watch) < 0) { 389 if (inotify_clone_watch(&old->watch, &chunk->watch) < 0) {
@@ -425,7 +427,8 @@ static int tag_chunk(struct inode *inode, struct audit_tree *tree)
425 spin_unlock(&hash_lock); 427 spin_unlock(&hash_lock);
426 inotify_evict_watch(&old->watch); 428 inotify_evict_watch(&old->watch);
427 mutex_unlock(&inode->inotify_mutex); 429 mutex_unlock(&inode->inotify_mutex);
428 put_inotify_watch(&old->watch); 430 put_inotify_watch(&old->watch); /* pair to inotify_find_watch */
431 put_inotify_watch(&old->watch); /* and kill it */
429 return 0; 432 return 0;
430} 433}
431 434
diff --git a/kernel/auditsc.c b/kernel/auditsc.c
index 267e484f0198..fc0f928167e7 100644
--- a/kernel/auditsc.c
+++ b/kernel/auditsc.c
@@ -250,7 +250,6 @@ struct audit_context {
250#endif 250#endif
251}; 251};
252 252
253#define ACC_MODE(x) ("\004\002\006\006"[(x)&O_ACCMODE])
254static inline int open_arg(int flags, int mask) 253static inline int open_arg(int flags, int mask)
255{ 254{
256 int n = ACC_MODE(flags); 255 int n = ACC_MODE(flags);
diff --git a/kernel/bounds.c b/kernel/bounds.c
index 3c5301381837..98a51f26c136 100644
--- a/kernel/bounds.c
+++ b/kernel/bounds.c
@@ -12,7 +12,7 @@
12 12
13void foo(void) 13void foo(void)
14{ 14{
15 /* The enum constants to put into include/linux/bounds.h */ 15 /* The enum constants to put into include/generated/bounds.h */
16 DEFINE(NR_PAGEFLAGS, __NR_PAGEFLAGS); 16 DEFINE(NR_PAGEFLAGS, __NR_PAGEFLAGS);
17 DEFINE(MAX_NR_ZONES, __MAX_NR_ZONES); 17 DEFINE(MAX_NR_ZONES, __MAX_NR_ZONES);
18 /* End of constants */ 18 /* End of constants */
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 291ac586f37f..1c8ddd6ee940 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -209,6 +209,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
209 return -ENOMEM; 209 return -ENOMEM;
210 210
211 cpu_hotplug_begin(); 211 cpu_hotplug_begin();
212 set_cpu_active(cpu, false);
212 err = __raw_notifier_call_chain(&cpu_chain, CPU_DOWN_PREPARE | mod, 213 err = __raw_notifier_call_chain(&cpu_chain, CPU_DOWN_PREPARE | mod,
213 hcpu, -1, &nr_calls); 214 hcpu, -1, &nr_calls);
214 if (err == NOTIFY_BAD) { 215 if (err == NOTIFY_BAD) {
@@ -280,18 +281,6 @@ int __ref cpu_down(unsigned int cpu)
280 goto out; 281 goto out;
281 } 282 }
282 283
283 set_cpu_active(cpu, false);
284
285 /*
286 * Make sure the all cpus did the reschedule and are not
287 * using stale version of the cpu_active_mask.
288 * This is not strictly necessary becuase stop_machine()
289 * that we run down the line already provides the required
290 * synchronization. But it's really a side effect and we do not
291 * want to depend on the innards of the stop_machine here.
292 */
293 synchronize_sched();
294
295 err = _cpu_down(cpu, 0); 284 err = _cpu_down(cpu, 0);
296 285
297out: 286out:
@@ -382,19 +371,12 @@ int disable_nonboot_cpus(void)
382 return error; 371 return error;
383 cpu_maps_update_begin(); 372 cpu_maps_update_begin();
384 first_cpu = cpumask_first(cpu_online_mask); 373 first_cpu = cpumask_first(cpu_online_mask);
385 /* We take down all of the non-boot CPUs in one shot to avoid races 374 /*
375 * We take down all of the non-boot CPUs in one shot to avoid races
386 * with the userspace trying to use the CPU hotplug at the same time 376 * with the userspace trying to use the CPU hotplug at the same time
387 */ 377 */
388 cpumask_clear(frozen_cpus); 378 cpumask_clear(frozen_cpus);
389 379
390 for_each_online_cpu(cpu) {
391 if (cpu == first_cpu)
392 continue;
393 set_cpu_active(cpu, false);
394 }
395
396 synchronize_sched();
397
398 printk("Disabling non-boot CPUs ...\n"); 380 printk("Disabling non-boot CPUs ...\n");
399 for_each_online_cpu(cpu) { 381 for_each_online_cpu(cpu) {
400 if (cpu == first_cpu) 382 if (cpu == first_cpu)
diff --git a/kernel/exit.c b/kernel/exit.c
index 5962d7ccf243..546774a31a66 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -68,10 +68,10 @@ static void __unhash_process(struct task_struct *p)
68 detach_pid(p, PIDTYPE_SID); 68 detach_pid(p, PIDTYPE_SID);
69 69
70 list_del_rcu(&p->tasks); 70 list_del_rcu(&p->tasks);
71 list_del_init(&p->sibling);
71 __get_cpu_var(process_counts)--; 72 __get_cpu_var(process_counts)--;
72 } 73 }
73 list_del_rcu(&p->thread_group); 74 list_del_rcu(&p->thread_group);
74 list_del_init(&p->sibling);
75} 75}
76 76
77/* 77/*
@@ -736,12 +736,9 @@ static struct task_struct *find_new_reaper(struct task_struct *father)
736/* 736/*
737* Any that need to be release_task'd are put on the @dead list. 737* Any that need to be release_task'd are put on the @dead list.
738 */ 738 */
739static void reparent_thread(struct task_struct *father, struct task_struct *p, 739static void reparent_leader(struct task_struct *father, struct task_struct *p,
740 struct list_head *dead) 740 struct list_head *dead)
741{ 741{
742 if (p->pdeath_signal)
743 group_send_sig_info(p->pdeath_signal, SEND_SIG_NOINFO, p);
744
745 list_move_tail(&p->sibling, &p->real_parent->children); 742 list_move_tail(&p->sibling, &p->real_parent->children);
746 743
747 if (task_detached(p)) 744 if (task_detached(p))
@@ -780,12 +777,18 @@ static void forget_original_parent(struct task_struct *father)
780 reaper = find_new_reaper(father); 777 reaper = find_new_reaper(father);
781 778
782 list_for_each_entry_safe(p, n, &father->children, sibling) { 779 list_for_each_entry_safe(p, n, &father->children, sibling) {
783 p->real_parent = reaper; 780 struct task_struct *t = p;
784 if (p->parent == father) { 781 do {
785 BUG_ON(task_ptrace(p)); 782 t->real_parent = reaper;
786 p->parent = p->real_parent; 783 if (t->parent == father) {
787 } 784 BUG_ON(task_ptrace(t));
788 reparent_thread(father, p, &dead_children); 785 t->parent = t->real_parent;
786 }
787 if (t->pdeath_signal)
788 group_send_sig_info(t->pdeath_signal,
789 SEND_SIG_NOINFO, t);
790 } while_each_thread(p, t);
791 reparent_leader(father, p, &dead_children);
789 } 792 }
790 write_unlock_irq(&tasklist_lock); 793 write_unlock_irq(&tasklist_lock);
791 794
@@ -1551,14 +1554,9 @@ static int do_wait_thread(struct wait_opts *wo, struct task_struct *tsk)
1551 struct task_struct *p; 1554 struct task_struct *p;
1552 1555
1553 list_for_each_entry(p, &tsk->children, sibling) { 1556 list_for_each_entry(p, &tsk->children, sibling) {
1554 /* 1557 int ret = wait_consider_task(wo, 0, p);
1555 * Do not consider detached threads. 1558 if (ret)
1556 */ 1559 return ret;
1557 if (!task_detached(p)) {
1558 int ret = wait_consider_task(wo, 0, p);
1559 if (ret)
1560 return ret;
1561 }
1562 } 1560 }
1563 1561
1564 return 0; 1562 return 0;
diff --git a/kernel/fork.c b/kernel/fork.c
index 202a0ba63d3c..5b2959b3ffc2 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1291,7 +1291,6 @@ static struct task_struct *copy_process(unsigned long clone_flags,
1291 } 1291 }
1292 1292
1293 if (likely(p->pid)) { 1293 if (likely(p->pid)) {
1294 list_add_tail(&p->sibling, &p->real_parent->children);
1295 tracehook_finish_clone(p, clone_flags, trace); 1294 tracehook_finish_clone(p, clone_flags, trace);
1296 1295
1297 if (thread_group_leader(p)) { 1296 if (thread_group_leader(p)) {
@@ -1303,6 +1302,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
1303 p->signal->tty = tty_kref_get(current->signal->tty); 1302 p->signal->tty = tty_kref_get(current->signal->tty);
1304 attach_pid(p, PIDTYPE_PGID, task_pgrp(current)); 1303 attach_pid(p, PIDTYPE_PGID, task_pgrp(current));
1305 attach_pid(p, PIDTYPE_SID, task_session(current)); 1304 attach_pid(p, PIDTYPE_SID, task_session(current));
1305 list_add_tail(&p->sibling, &p->real_parent->children);
1306 list_add_tail_rcu(&p->tasks, &init_task.tasks); 1306 list_add_tail_rcu(&p->tasks, &init_task.tasks);
1307 __get_cpu_var(process_counts)++; 1307 __get_cpu_var(process_counts)++;
1308 } 1308 }
diff --git a/kernel/kexec.c b/kernel/kexec.c
index 433e9fcc1fc5..a9a93d9ee7a7 100644
--- a/kernel/kexec.c
+++ b/kernel/kexec.c
@@ -21,7 +21,7 @@
21#include <linux/hardirq.h> 21#include <linux/hardirq.h>
22#include <linux/elf.h> 22#include <linux/elf.h>
23#include <linux/elfcore.h> 23#include <linux/elfcore.h>
24#include <linux/utsrelease.h> 24#include <generated/utsrelease.h>
25#include <linux/utsname.h> 25#include <linux/utsname.h>
26#include <linux/numa.h> 26#include <linux/numa.h>
27#include <linux/suspend.h> 27#include <linux/suspend.h>
diff --git a/kernel/kfifo.c b/kernel/kfifo.c
index 3765ff3c1bbe..e92d519f93b1 100644
--- a/kernel/kfifo.c
+++ b/kernel/kfifo.c
@@ -1,6 +1,7 @@
1/* 1/*
2 * A simple kernel FIFO implementation. 2 * A generic kernel FIFO implementation.
3 * 3 *
4 * Copyright (C) 2009 Stefani Seibold <stefani@seibold.net>
4 * Copyright (C) 2004 Stelian Pop <stelian@popies.net> 5 * Copyright (C) 2004 Stelian Pop <stelian@popies.net>
5 * 6 *
6 * This program is free software; you can redistribute it and/or modify 7 * This program is free software; you can redistribute it and/or modify
@@ -25,50 +26,48 @@
25#include <linux/err.h> 26#include <linux/err.h>
26#include <linux/kfifo.h> 27#include <linux/kfifo.h>
27#include <linux/log2.h> 28#include <linux/log2.h>
29#include <linux/uaccess.h>
30
31static void _kfifo_init(struct kfifo *fifo, unsigned char *buffer,
32 unsigned int size)
33{
34 fifo->buffer = buffer;
35 fifo->size = size;
36
37 kfifo_reset(fifo);
38}
28 39
29/** 40/**
30 * kfifo_init - allocates a new FIFO using a preallocated buffer 41 * kfifo_init - initialize a FIFO using a preallocated buffer
42 * @fifo: the fifo to assign the buffer
31 * @buffer: the preallocated buffer to be used. 43 * @buffer: the preallocated buffer to be used.
32 * @size: the size of the internal buffer, this have to be a power of 2. 44 * @size: the size of the internal buffer, this have to be a power of 2.
33 * @gfp_mask: get_free_pages mask, passed to kmalloc()
34 * @lock: the lock to be used to protect the fifo buffer
35 * 45 *
36 * Do NOT pass the kfifo to kfifo_free() after use! Simply free the
37 * &struct kfifo with kfree().
38 */ 46 */
39struct kfifo *kfifo_init(unsigned char *buffer, unsigned int size, 47void kfifo_init(struct kfifo *fifo, unsigned char *buffer, unsigned int size)
40 gfp_t gfp_mask, spinlock_t *lock)
41{ 48{
42 struct kfifo *fifo;
43
44 /* size must be a power of 2 */ 49 /* size must be a power of 2 */
45 BUG_ON(!is_power_of_2(size)); 50 BUG_ON(!is_power_of_2(size));
46 51
47 fifo = kmalloc(sizeof(struct kfifo), gfp_mask); 52 _kfifo_init(fifo, buffer, size);
48 if (!fifo)
49 return ERR_PTR(-ENOMEM);
50
51 fifo->buffer = buffer;
52 fifo->size = size;
53 fifo->in = fifo->out = 0;
54 fifo->lock = lock;
55
56 return fifo;
57} 53}
58EXPORT_SYMBOL(kfifo_init); 54EXPORT_SYMBOL(kfifo_init);
59 55
60/** 56/**
61 * kfifo_alloc - allocates a new FIFO and its internal buffer 57 * kfifo_alloc - allocates a new FIFO internal buffer
62 * @size: the size of the internal buffer to be allocated. 58 * @fifo: the fifo to assign then new buffer
59 * @size: the size of the buffer to be allocated, this have to be a power of 2.
63 * @gfp_mask: get_free_pages mask, passed to kmalloc() 60 * @gfp_mask: get_free_pages mask, passed to kmalloc()
64 * @lock: the lock to be used to protect the fifo buffer 61 *
62 * This function dynamically allocates a new fifo internal buffer
65 * 63 *
66 * The size will be rounded-up to a power of 2. 64 * The size will be rounded-up to a power of 2.
65 * The buffer will be release with kfifo_free().
66 * Return 0 if no error, otherwise the an error code
67 */ 67 */
68struct kfifo *kfifo_alloc(unsigned int size, gfp_t gfp_mask, spinlock_t *lock) 68int kfifo_alloc(struct kfifo *fifo, unsigned int size, gfp_t gfp_mask)
69{ 69{
70 unsigned char *buffer; 70 unsigned char *buffer;
71 struct kfifo *ret;
72 71
73 /* 72 /*
74 * round up to the next power of 2, since our 'let the indices 73 * round up to the next power of 2, since our 'let the indices
@@ -80,48 +79,91 @@ struct kfifo *kfifo_alloc(unsigned int size, gfp_t gfp_mask, spinlock_t *lock)
80 } 79 }
81 80
82 buffer = kmalloc(size, gfp_mask); 81 buffer = kmalloc(size, gfp_mask);
83 if (!buffer) 82 if (!buffer) {
84 return ERR_PTR(-ENOMEM); 83 _kfifo_init(fifo, 0, 0);
85 84 return -ENOMEM;
86 ret = kfifo_init(buffer, size, gfp_mask, lock); 85 }
87 86
88 if (IS_ERR(ret)) 87 _kfifo_init(fifo, buffer, size);
89 kfree(buffer);
90 88
91 return ret; 89 return 0;
92} 90}
93EXPORT_SYMBOL(kfifo_alloc); 91EXPORT_SYMBOL(kfifo_alloc);
94 92
95/** 93/**
96 * kfifo_free - frees the FIFO 94 * kfifo_free - frees the FIFO internal buffer
97 * @fifo: the fifo to be freed. 95 * @fifo: the fifo to be freed.
98 */ 96 */
99void kfifo_free(struct kfifo *fifo) 97void kfifo_free(struct kfifo *fifo)
100{ 98{
101 kfree(fifo->buffer); 99 kfree(fifo->buffer);
102 kfree(fifo);
103} 100}
104EXPORT_SYMBOL(kfifo_free); 101EXPORT_SYMBOL(kfifo_free);
105 102
106/** 103/**
107 * __kfifo_put - puts some data into the FIFO, no locking version 104 * kfifo_skip - skip output data
108 * @fifo: the fifo to be used. 105 * @fifo: the fifo to be used.
109 * @buffer: the data to be added. 106 * @len: number of bytes to skip
110 * @len: the length of the data to be added.
111 *
112 * This function copies at most @len bytes from the @buffer into
113 * the FIFO depending on the free space, and returns the number of
114 * bytes copied.
115 *
116 * Note that with only one concurrent reader and one concurrent
117 * writer, you don't need extra locking to use these functions.
118 */ 107 */
119unsigned int __kfifo_put(struct kfifo *fifo, 108void kfifo_skip(struct kfifo *fifo, unsigned int len)
120 const unsigned char *buffer, unsigned int len) 109{
110 if (len < kfifo_len(fifo)) {
111 __kfifo_add_out(fifo, len);
112 return;
113 }
114 kfifo_reset_out(fifo);
115}
116EXPORT_SYMBOL(kfifo_skip);
117
118static inline void __kfifo_in_data(struct kfifo *fifo,
119 const void *from, unsigned int len, unsigned int off)
121{ 120{
122 unsigned int l; 121 unsigned int l;
123 122
124 len = min(len, fifo->size - fifo->in + fifo->out); 123 /*
124 * Ensure that we sample the fifo->out index -before- we
125 * start putting bytes into the kfifo.
126 */
127
128 smp_mb();
129
130 off = __kfifo_off(fifo, fifo->in + off);
131
132 /* first put the data starting from fifo->in to buffer end */
133 l = min(len, fifo->size - off);
134 memcpy(fifo->buffer + off, from, l);
135
136 /* then put the rest (if any) at the beginning of the buffer */
137 memcpy(fifo->buffer, from + l, len - l);
138}
139
140static inline void __kfifo_out_data(struct kfifo *fifo,
141 void *to, unsigned int len, unsigned int off)
142{
143 unsigned int l;
144
145 /*
146 * Ensure that we sample the fifo->in index -before- we
147 * start removing bytes from the kfifo.
148 */
149
150 smp_rmb();
151
152 off = __kfifo_off(fifo, fifo->out + off);
153
154 /* first get the data from fifo->out until the end of the buffer */
155 l = min(len, fifo->size - off);
156 memcpy(to, fifo->buffer + off, l);
157
158 /* then get the rest (if any) from the beginning of the buffer */
159 memcpy(to + l, fifo->buffer, len - l);
160}
161
162static inline unsigned int __kfifo_from_user_data(struct kfifo *fifo,
163 const void __user *from, unsigned int len, unsigned int off)
164{
165 unsigned int l;
166 int ret;
125 167
126 /* 168 /*
127 * Ensure that we sample the fifo->out index -before- we 169 * Ensure that we sample the fifo->out index -before- we
@@ -130,68 +172,229 @@ unsigned int __kfifo_put(struct kfifo *fifo,
130 172
131 smp_mb(); 173 smp_mb();
132 174
175 off = __kfifo_off(fifo, fifo->in + off);
176
133 /* first put the data starting from fifo->in to buffer end */ 177 /* first put the data starting from fifo->in to buffer end */
134 l = min(len, fifo->size - (fifo->in & (fifo->size - 1))); 178 l = min(len, fifo->size - off);
135 memcpy(fifo->buffer + (fifo->in & (fifo->size - 1)), buffer, l); 179 ret = copy_from_user(fifo->buffer + off, from, l);
180
181 if (unlikely(ret))
182 return ret + len - l;
136 183
137 /* then put the rest (if any) at the beginning of the buffer */ 184 /* then put the rest (if any) at the beginning of the buffer */
138 memcpy(fifo->buffer, buffer + l, len - l); 185 return copy_from_user(fifo->buffer, from + l, len - l);
186}
187
188static inline unsigned int __kfifo_to_user_data(struct kfifo *fifo,
189 void __user *to, unsigned int len, unsigned int off)
190{
191 unsigned int l;
192 int ret;
139 193
140 /* 194 /*
141 * Ensure that we add the bytes to the kfifo -before- 195 * Ensure that we sample the fifo->in index -before- we
142 * we update the fifo->in index. 196 * start removing bytes from the kfifo.
143 */ 197 */
144 198
145 smp_wmb(); 199 smp_rmb();
200
201 off = __kfifo_off(fifo, fifo->out + off);
202
203 /* first get the data from fifo->out until the end of the buffer */
204 l = min(len, fifo->size - off);
205 ret = copy_to_user(to, fifo->buffer + off, l);
206
207 if (unlikely(ret))
208 return ret + len - l;
209
210 /* then get the rest (if any) from the beginning of the buffer */
211 return copy_to_user(to + l, fifo->buffer, len - l);
212}
213
214unsigned int __kfifo_in_n(struct kfifo *fifo,
215 const void *from, unsigned int len, unsigned int recsize)
216{
217 if (kfifo_avail(fifo) < len + recsize)
218 return len + 1;
219
220 __kfifo_in_data(fifo, from, len, recsize);
221 return 0;
222}
223EXPORT_SYMBOL(__kfifo_in_n);
146 224
147 fifo->in += len; 225/**
226 * kfifo_in - puts some data into the FIFO
227 * @fifo: the fifo to be used.
228 * @from: the data to be added.
229 * @len: the length of the data to be added.
230 *
231 * This function copies at most @len bytes from the @from buffer into
232 * the FIFO depending on the free space, and returns the number of
233 * bytes copied.
234 *
235 * Note that with only one concurrent reader and one concurrent
236 * writer, you don't need extra locking to use these functions.
237 */
238unsigned int kfifo_in(struct kfifo *fifo, const unsigned char *from,
239 unsigned int len)
240{
241 len = min(kfifo_avail(fifo), len);
148 242
243 __kfifo_in_data(fifo, from, len, 0);
244 __kfifo_add_in(fifo, len);
149 return len; 245 return len;
150} 246}
151EXPORT_SYMBOL(__kfifo_put); 247EXPORT_SYMBOL(kfifo_in);
248
249unsigned int __kfifo_in_generic(struct kfifo *fifo,
250 const void *from, unsigned int len, unsigned int recsize)
251{
252 return __kfifo_in_rec(fifo, from, len, recsize);
253}
254EXPORT_SYMBOL(__kfifo_in_generic);
255
256unsigned int __kfifo_out_n(struct kfifo *fifo,
257 void *to, unsigned int len, unsigned int recsize)
258{
259 if (kfifo_len(fifo) < len + recsize)
260 return len;
261
262 __kfifo_out_data(fifo, to, len, recsize);
263 __kfifo_add_out(fifo, len + recsize);
264 return 0;
265}
266EXPORT_SYMBOL(__kfifo_out_n);
152 267
153/** 268/**
154 * __kfifo_get - gets some data from the FIFO, no locking version 269 * kfifo_out - gets some data from the FIFO
155 * @fifo: the fifo to be used. 270 * @fifo: the fifo to be used.
156 * @buffer: where the data must be copied. 271 * @to: where the data must be copied.
157 * @len: the size of the destination buffer. 272 * @len: the size of the destination buffer.
158 * 273 *
159 * This function copies at most @len bytes from the FIFO into the 274 * This function copies at most @len bytes from the FIFO into the
160 * @buffer and returns the number of copied bytes. 275 * @to buffer and returns the number of copied bytes.
161 * 276 *
162 * Note that with only one concurrent reader and one concurrent 277 * Note that with only one concurrent reader and one concurrent
163 * writer, you don't need extra locking to use these functions. 278 * writer, you don't need extra locking to use these functions.
164 */ 279 */
165unsigned int __kfifo_get(struct kfifo *fifo, 280unsigned int kfifo_out(struct kfifo *fifo, unsigned char *to, unsigned int len)
166 unsigned char *buffer, unsigned int len)
167{ 281{
168 unsigned int l; 282 len = min(kfifo_len(fifo), len);
169 283
170 len = min(len, fifo->in - fifo->out); 284 __kfifo_out_data(fifo, to, len, 0);
285 __kfifo_add_out(fifo, len);
171 286
172 /* 287 return len;
173 * Ensure that we sample the fifo->in index -before- we 288}
174 * start removing bytes from the kfifo. 289EXPORT_SYMBOL(kfifo_out);
175 */
176 290
177 smp_rmb(); 291unsigned int __kfifo_out_generic(struct kfifo *fifo,
292 void *to, unsigned int len, unsigned int recsize,
293 unsigned int *total)
294{
295 return __kfifo_out_rec(fifo, to, len, recsize, total);
296}
297EXPORT_SYMBOL(__kfifo_out_generic);
178 298
179 /* first get the data from fifo->out until the end of the buffer */ 299unsigned int __kfifo_from_user_n(struct kfifo *fifo,
180 l = min(len, fifo->size - (fifo->out & (fifo->size - 1))); 300 const void __user *from, unsigned int len, unsigned int recsize)
181 memcpy(buffer, fifo->buffer + (fifo->out & (fifo->size - 1)), l); 301{
302 if (kfifo_avail(fifo) < len + recsize)
303 return len + 1;
182 304
183 /* then get the rest (if any) from the beginning of the buffer */ 305 return __kfifo_from_user_data(fifo, from, len, recsize);
184 memcpy(buffer + l, fifo->buffer, len - l); 306}
307EXPORT_SYMBOL(__kfifo_from_user_n);
185 308
186 /* 309/**
187 * Ensure that we remove the bytes from the kfifo -before- 310 * kfifo_from_user - puts some data from user space into the FIFO
188 * we update the fifo->out index. 311 * @fifo: the fifo to be used.
189 */ 312 * @from: pointer to the data to be added.
313 * @len: the length of the data to be added.
314 *
315 * This function copies at most @len bytes from the @from into the
316 * FIFO depending and returns the number of copied bytes.
317 *
318 * Note that with only one concurrent reader and one concurrent
319 * writer, you don't need extra locking to use these functions.
320 */
321unsigned int kfifo_from_user(struct kfifo *fifo,
322 const void __user *from, unsigned int len)
323{
324 len = min(kfifo_avail(fifo), len);
325 len -= __kfifo_from_user_data(fifo, from, len, 0);
326 __kfifo_add_in(fifo, len);
327 return len;
328}
329EXPORT_SYMBOL(kfifo_from_user);
190 330
191 smp_mb(); 331unsigned int __kfifo_from_user_generic(struct kfifo *fifo,
332 const void __user *from, unsigned int len, unsigned int recsize)
333{
334 return __kfifo_from_user_rec(fifo, from, len, recsize);
335}
336EXPORT_SYMBOL(__kfifo_from_user_generic);
192 337
193 fifo->out += len; 338unsigned int __kfifo_to_user_n(struct kfifo *fifo,
339 void __user *to, unsigned int len, unsigned int reclen,
340 unsigned int recsize)
341{
342 unsigned int ret;
343
344 if (kfifo_len(fifo) < reclen + recsize)
345 return len;
346
347 ret = __kfifo_to_user_data(fifo, to, reclen, recsize);
194 348
349 if (likely(ret == 0))
350 __kfifo_add_out(fifo, reclen + recsize);
351
352 return ret;
353}
354EXPORT_SYMBOL(__kfifo_to_user_n);
355
356/**
357 * kfifo_to_user - gets data from the FIFO and write it to user space
358 * @fifo: the fifo to be used.
359 * @to: where the data must be copied.
360 * @len: the size of the destination buffer.
361 *
362 * This function copies at most @len bytes from the FIFO into the
363 * @to buffer and returns the number of copied bytes.
364 *
365 * Note that with only one concurrent reader and one concurrent
366 * writer, you don't need extra locking to use these functions.
367 */
368unsigned int kfifo_to_user(struct kfifo *fifo,
369 void __user *to, unsigned int len)
370{
371 len = min(kfifo_len(fifo), len);
372 len -= __kfifo_to_user_data(fifo, to, len, 0);
373 __kfifo_add_out(fifo, len);
195 return len; 374 return len;
196} 375}
197EXPORT_SYMBOL(__kfifo_get); 376EXPORT_SYMBOL(kfifo_to_user);
377
378unsigned int __kfifo_to_user_generic(struct kfifo *fifo,
379 void __user *to, unsigned int len, unsigned int recsize,
380 unsigned int *total)
381{
382 return __kfifo_to_user_rec(fifo, to, len, recsize, total);
383}
384EXPORT_SYMBOL(__kfifo_to_user_generic);
385
386unsigned int __kfifo_peek_generic(struct kfifo *fifo, unsigned int recsize)
387{
388 if (recsize == 0)
389 return kfifo_avail(fifo);
390
391 return __kfifo_peek_n(fifo, recsize);
392}
393EXPORT_SYMBOL(__kfifo_peek_generic);
394
395void __kfifo_skip_generic(struct kfifo *fifo, unsigned int recsize)
396{
397 __kfifo_skip_rec(fifo, recsize);
398}
399EXPORT_SYMBOL(__kfifo_skip_generic);
400
diff --git a/kernel/kthread.c b/kernel/kthread.c
index ab7ae57773e1..fbb6222fe7e0 100644
--- a/kernel/kthread.c
+++ b/kernel/kthread.c
@@ -150,6 +150,29 @@ struct task_struct *kthread_create(int (*threadfn)(void *data),
150EXPORT_SYMBOL(kthread_create); 150EXPORT_SYMBOL(kthread_create);
151 151
152/** 152/**
153 * kthread_bind - bind a just-created kthread to a cpu.
154 * @p: thread created by kthread_create().
155 * @cpu: cpu (might not be online, must be possible) for @k to run on.
156 *
157 * Description: This function is equivalent to set_cpus_allowed(),
158 * except that @cpu doesn't need to be online, and the thread must be
159 * stopped (i.e., just returned from kthread_create()).
160 */
161void kthread_bind(struct task_struct *p, unsigned int cpu)
162{
163 /* Must have done schedule() in kthread() before we set_task_cpu */
164 if (!wait_task_inactive(p, TASK_UNINTERRUPTIBLE)) {
165 WARN_ON(1);
166 return;
167 }
168
169 p->cpus_allowed = cpumask_of_cpu(cpu);
170 p->rt.nr_cpus_allowed = 1;
171 p->flags |= PF_THREAD_BOUND;
172}
173EXPORT_SYMBOL(kthread_bind);
174
175/**
153 * kthread_stop - stop a thread created by kthread_create(). 176 * kthread_stop - stop a thread created by kthread_create().
154 * @k: thread created by kthread_create(). 177 * @k: thread created by kthread_create().
155 * 178 *
diff --git a/kernel/module.c b/kernel/module.c
index a65dc787a27b..e96b8ed1cb6a 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -1910,9 +1910,7 @@ static void kmemleak_load_module(struct module *mod, Elf_Ehdr *hdr,
1910 unsigned int i; 1910 unsigned int i;
1911 1911
1912 /* only scan the sections containing data */ 1912 /* only scan the sections containing data */
1913 kmemleak_scan_area(mod->module_core, (unsigned long)mod - 1913 kmemleak_scan_area(mod, sizeof(struct module), GFP_KERNEL);
1914 (unsigned long)mod->module_core,
1915 sizeof(struct module), GFP_KERNEL);
1916 1914
1917 for (i = 1; i < hdr->e_shnum; i++) { 1915 for (i = 1; i < hdr->e_shnum; i++) {
1918 if (!(sechdrs[i].sh_flags & SHF_ALLOC)) 1916 if (!(sechdrs[i].sh_flags & SHF_ALLOC))
@@ -1921,8 +1919,7 @@ static void kmemleak_load_module(struct module *mod, Elf_Ehdr *hdr,
1921 && strncmp(secstrings + sechdrs[i].sh_name, ".bss", 4) != 0) 1919 && strncmp(secstrings + sechdrs[i].sh_name, ".bss", 4) != 0)
1922 continue; 1920 continue;
1923 1921
1924 kmemleak_scan_area(mod->module_core, sechdrs[i].sh_addr - 1922 kmemleak_scan_area((void *)sechdrs[i].sh_addr,
1925 (unsigned long)mod->module_core,
1926 sechdrs[i].sh_size, GFP_KERNEL); 1923 sechdrs[i].sh_size, GFP_KERNEL);
1927 } 1924 }
1928} 1925}
@@ -2250,6 +2247,12 @@ static noinline struct module *load_module(void __user *umod,
2250 "_ftrace_events", 2247 "_ftrace_events",
2251 sizeof(*mod->trace_events), 2248 sizeof(*mod->trace_events),
2252 &mod->num_trace_events); 2249 &mod->num_trace_events);
2250 /*
2251 * This section contains pointers to allocated objects in the trace
2252 * code and not scanning it leads to false positives.
2253 */
2254 kmemleak_scan_area(mod->trace_events, sizeof(*mod->trace_events) *
2255 mod->num_trace_events, GFP_KERNEL);
2253#endif 2256#endif
2254#ifdef CONFIG_FTRACE_MCOUNT_RECORD 2257#ifdef CONFIG_FTRACE_MCOUNT_RECORD
2255 /* sechdrs[0].sh_size is always zero */ 2258 /* sechdrs[0].sh_size is always zero */
diff --git a/kernel/perf_event.c b/kernel/perf_event.c
index 8ab86988bd24..1f38270f08c7 100644
--- a/kernel/perf_event.c
+++ b/kernel/perf_event.c
@@ -1381,6 +1381,9 @@ static void perf_ctx_adjust_freq(struct perf_event_context *ctx)
1381 if (event->state != PERF_EVENT_STATE_ACTIVE) 1381 if (event->state != PERF_EVENT_STATE_ACTIVE)
1382 continue; 1382 continue;
1383 1383
1384 if (event->cpu != -1 && event->cpu != smp_processor_id())
1385 continue;
1386
1384 hwc = &event->hw; 1387 hwc = &event->hw;
1385 1388
1386 interrupts = hwc->interrupts; 1389 interrupts = hwc->interrupts;
@@ -1614,7 +1617,7 @@ static struct perf_event_context *find_get_context(pid_t pid, int cpu)
1614 * offline CPU and activate it when the CPU comes up, but 1617 * offline CPU and activate it when the CPU comes up, but
1615 * that's for later. 1618 * that's for later.
1616 */ 1619 */
1617 if (!cpu_isset(cpu, cpu_online_map)) 1620 if (!cpu_online(cpu))
1618 return ERR_PTR(-ENODEV); 1621 return ERR_PTR(-ENODEV);
1619 1622
1620 cpuctx = &per_cpu(perf_cpu_context, cpu); 1623 cpuctx = &per_cpu(perf_cpu_context, cpu);
@@ -3265,6 +3268,9 @@ static void perf_event_task_output(struct perf_event *event,
3265 3268
3266static int perf_event_task_match(struct perf_event *event) 3269static int perf_event_task_match(struct perf_event *event)
3267{ 3270{
3271 if (event->cpu != -1 && event->cpu != smp_processor_id())
3272 return 0;
3273
3268 if (event->attr.comm || event->attr.mmap || event->attr.task) 3274 if (event->attr.comm || event->attr.mmap || event->attr.task)
3269 return 1; 3275 return 1;
3270 3276
@@ -3290,12 +3296,11 @@ static void perf_event_task_event(struct perf_task_event *task_event)
3290 rcu_read_lock(); 3296 rcu_read_lock();
3291 cpuctx = &get_cpu_var(perf_cpu_context); 3297 cpuctx = &get_cpu_var(perf_cpu_context);
3292 perf_event_task_ctx(&cpuctx->ctx, task_event); 3298 perf_event_task_ctx(&cpuctx->ctx, task_event);
3293 put_cpu_var(perf_cpu_context);
3294
3295 if (!ctx) 3299 if (!ctx)
3296 ctx = rcu_dereference(task_event->task->perf_event_ctxp); 3300 ctx = rcu_dereference(task_event->task->perf_event_ctxp);
3297 if (ctx) 3301 if (ctx)
3298 perf_event_task_ctx(ctx, task_event); 3302 perf_event_task_ctx(ctx, task_event);
3303 put_cpu_var(perf_cpu_context);
3299 rcu_read_unlock(); 3304 rcu_read_unlock();
3300} 3305}
3301 3306
@@ -3372,6 +3377,9 @@ static void perf_event_comm_output(struct perf_event *event,
3372 3377
3373static int perf_event_comm_match(struct perf_event *event) 3378static int perf_event_comm_match(struct perf_event *event)
3374{ 3379{
3380 if (event->cpu != -1 && event->cpu != smp_processor_id())
3381 return 0;
3382
3375 if (event->attr.comm) 3383 if (event->attr.comm)
3376 return 1; 3384 return 1;
3377 3385
@@ -3408,15 +3416,10 @@ static void perf_event_comm_event(struct perf_comm_event *comm_event)
3408 rcu_read_lock(); 3416 rcu_read_lock();
3409 cpuctx = &get_cpu_var(perf_cpu_context); 3417 cpuctx = &get_cpu_var(perf_cpu_context);
3410 perf_event_comm_ctx(&cpuctx->ctx, comm_event); 3418 perf_event_comm_ctx(&cpuctx->ctx, comm_event);
3411 put_cpu_var(perf_cpu_context);
3412
3413 /*
3414 * doesn't really matter which of the child contexts the
3415 * events ends up in.
3416 */
3417 ctx = rcu_dereference(current->perf_event_ctxp); 3419 ctx = rcu_dereference(current->perf_event_ctxp);
3418 if (ctx) 3420 if (ctx)
3419 perf_event_comm_ctx(ctx, comm_event); 3421 perf_event_comm_ctx(ctx, comm_event);
3422 put_cpu_var(perf_cpu_context);
3420 rcu_read_unlock(); 3423 rcu_read_unlock();
3421} 3424}
3422 3425
@@ -3491,6 +3494,9 @@ static void perf_event_mmap_output(struct perf_event *event,
3491static int perf_event_mmap_match(struct perf_event *event, 3494static int perf_event_mmap_match(struct perf_event *event,
3492 struct perf_mmap_event *mmap_event) 3495 struct perf_mmap_event *mmap_event)
3493{ 3496{
3497 if (event->cpu != -1 && event->cpu != smp_processor_id())
3498 return 0;
3499
3494 if (event->attr.mmap) 3500 if (event->attr.mmap)
3495 return 1; 3501 return 1;
3496 3502
@@ -3564,15 +3570,10 @@ got_name:
3564 rcu_read_lock(); 3570 rcu_read_lock();
3565 cpuctx = &get_cpu_var(perf_cpu_context); 3571 cpuctx = &get_cpu_var(perf_cpu_context);
3566 perf_event_mmap_ctx(&cpuctx->ctx, mmap_event); 3572 perf_event_mmap_ctx(&cpuctx->ctx, mmap_event);
3567 put_cpu_var(perf_cpu_context);
3568
3569 /*
3570 * doesn't really matter which of the child contexts the
3571 * events ends up in.
3572 */
3573 ctx = rcu_dereference(current->perf_event_ctxp); 3573 ctx = rcu_dereference(current->perf_event_ctxp);
3574 if (ctx) 3574 if (ctx)
3575 perf_event_mmap_ctx(ctx, mmap_event); 3575 perf_event_mmap_ctx(ctx, mmap_event);
3576 put_cpu_var(perf_cpu_context);
3576 rcu_read_unlock(); 3577 rcu_read_unlock();
3577 3578
3578 kfree(buf); 3579 kfree(buf);
@@ -3863,6 +3864,9 @@ static int perf_swevent_match(struct perf_event *event,
3863 struct perf_sample_data *data, 3864 struct perf_sample_data *data,
3864 struct pt_regs *regs) 3865 struct pt_regs *regs)
3865{ 3866{
3867 if (event->cpu != -1 && event->cpu != smp_processor_id())
3868 return 0;
3869
3866 if (!perf_swevent_is_counting(event)) 3870 if (!perf_swevent_is_counting(event))
3867 return 0; 3871 return 0;
3868 3872
@@ -4720,7 +4724,7 @@ SYSCALL_DEFINE5(perf_event_open,
4720 if (IS_ERR(event)) 4724 if (IS_ERR(event))
4721 goto err_put_context; 4725 goto err_put_context;
4722 4726
4723 err = anon_inode_getfd("[perf_event]", &perf_fops, event, 0); 4727 err = anon_inode_getfd("[perf_event]", &perf_fops, event, O_RDWR);
4724 if (err < 0) 4728 if (err < 0)
4725 goto err_free_put_context; 4729 goto err_free_put_context;
4726 4730
diff --git a/kernel/printk.c b/kernel/printk.c
index 1ded8e7dd19b..17463ca2e229 100644
--- a/kernel/printk.c
+++ b/kernel/printk.c
@@ -1412,7 +1412,7 @@ static LIST_HEAD(dump_list);
1412 1412
1413/** 1413/**
1414 * kmsg_dump_register - register a kernel log dumper. 1414 * kmsg_dump_register - register a kernel log dumper.
1415 * @dump: pointer to the kmsg_dumper structure 1415 * @dumper: pointer to the kmsg_dumper structure
1416 * 1416 *
1417 * Adds a kernel log dumper to the system. The dump callback in the 1417 * Adds a kernel log dumper to the system. The dump callback in the
1418 * structure will be called when the kernel oopses or panics and must be 1418 * structure will be called when the kernel oopses or panics and must be
@@ -1442,7 +1442,7 @@ EXPORT_SYMBOL_GPL(kmsg_dump_register);
1442 1442
1443/** 1443/**
1444 * kmsg_dump_unregister - unregister a kmsg dumper. 1444 * kmsg_dump_unregister - unregister a kmsg dumper.
1445 * @dump: pointer to the kmsg_dumper structure 1445 * @dumper: pointer to the kmsg_dumper structure
1446 * 1446 *
1447 * Removes a dump device from the system. Returns zero on success and 1447 * Removes a dump device from the system. Returns zero on success and
1448 * %-EINVAL otherwise. 1448 * %-EINVAL otherwise.
diff --git a/kernel/resource.c b/kernel/resource.c
index dc15686b7a77..af96c1e4b54b 100644
--- a/kernel/resource.c
+++ b/kernel/resource.c
@@ -308,37 +308,37 @@ static int find_resource(struct resource *root, struct resource *new,
308 void *alignf_data) 308 void *alignf_data)
309{ 309{
310 struct resource *this = root->child; 310 struct resource *this = root->child;
311 resource_size_t start, end; 311 struct resource tmp = *new;
312 312
313 start = root->start; 313 tmp.start = root->start;
314 /* 314 /*
315 * Skip past an allocated resource that starts at 0, since the assignment 315 * Skip past an allocated resource that starts at 0, since the assignment
316 * of this->start - 1 to new->end below would cause an underflow. 316 * of this->start - 1 to tmp->end below would cause an underflow.
317 */ 317 */
318 if (this && this->start == 0) { 318 if (this && this->start == 0) {
319 start = this->end + 1; 319 tmp.start = this->end + 1;
320 this = this->sibling; 320 this = this->sibling;
321 } 321 }
322 for(;;) { 322 for(;;) {
323 if (this) 323 if (this)
324 end = this->start - 1; 324 tmp.end = this->start - 1;
325 else 325 else
326 end = root->end; 326 tmp.end = root->end;
327 if (start < min) 327 if (tmp.start < min)
328 start = min; 328 tmp.start = min;
329 if (end > max) 329 if (tmp.end > max)
330 end = max; 330 tmp.end = max;
331 start = ALIGN(start, align); 331 tmp.start = ALIGN(tmp.start, align);
332 if (alignf) 332 if (alignf)
333 alignf(alignf_data, new, size, align); 333 alignf(alignf_data, &tmp, size, align);
334 if (start < end && end - start >= size - 1) { 334 if (tmp.start < tmp.end && tmp.end - tmp.start >= size - 1) {
335 new->start = start; 335 new->start = tmp.start;
336 new->end = start + size - 1; 336 new->end = tmp.start + size - 1;
337 return 0; 337 return 0;
338 } 338 }
339 if (!this) 339 if (!this)
340 break; 340 break;
341 start = this->end + 1; 341 tmp.start = this->end + 1;
342 this = this->sibling; 342 this = this->sibling;
343 } 343 }
344 return -EBUSY; 344 return -EBUSY;
diff --git a/kernel/sched.c b/kernel/sched.c
index 18cceeecce35..c535cc4f6428 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -2002,39 +2002,6 @@ static inline void check_class_changed(struct rq *rq, struct task_struct *p,
2002 p->sched_class->prio_changed(rq, p, oldprio, running); 2002 p->sched_class->prio_changed(rq, p, oldprio, running);
2003} 2003}
2004 2004
2005/**
2006 * kthread_bind - bind a just-created kthread to a cpu.
2007 * @p: thread created by kthread_create().
2008 * @cpu: cpu (might not be online, must be possible) for @k to run on.
2009 *
2010 * Description: This function is equivalent to set_cpus_allowed(),
2011 * except that @cpu doesn't need to be online, and the thread must be
2012 * stopped (i.e., just returned from kthread_create()).
2013 *
2014 * Function lives here instead of kthread.c because it messes with
2015 * scheduler internals which require locking.
2016 */
2017void kthread_bind(struct task_struct *p, unsigned int cpu)
2018{
2019 struct rq *rq = cpu_rq(cpu);
2020 unsigned long flags;
2021
2022 /* Must have done schedule() in kthread() before we set_task_cpu */
2023 if (!wait_task_inactive(p, TASK_UNINTERRUPTIBLE)) {
2024 WARN_ON(1);
2025 return;
2026 }
2027
2028 raw_spin_lock_irqsave(&rq->lock, flags);
2029 update_rq_clock(rq);
2030 set_task_cpu(p, cpu);
2031 p->cpus_allowed = cpumask_of_cpu(cpu);
2032 p->rt.nr_cpus_allowed = 1;
2033 p->flags |= PF_THREAD_BOUND;
2034 raw_spin_unlock_irqrestore(&rq->lock, flags);
2035}
2036EXPORT_SYMBOL(kthread_bind);
2037
2038#ifdef CONFIG_SMP 2005#ifdef CONFIG_SMP
2039/* 2006/*
2040 * Is this task likely cache-hot: 2007 * Is this task likely cache-hot:
@@ -2044,6 +2011,9 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
2044{ 2011{
2045 s64 delta; 2012 s64 delta;
2046 2013
2014 if (p->sched_class != &fair_sched_class)
2015 return 0;
2016
2047 /* 2017 /*
2048 * Buddy candidates are cache hot: 2018 * Buddy candidates are cache hot:
2049 */ 2019 */
@@ -2052,9 +2022,6 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
2052 &p->se == cfs_rq_of(&p->se)->last)) 2022 &p->se == cfs_rq_of(&p->se)->last))
2053 return 1; 2023 return 1;
2054 2024
2055 if (p->sched_class != &fair_sched_class)
2056 return 0;
2057
2058 if (sysctl_sched_migration_cost == -1) 2025 if (sysctl_sched_migration_cost == -1)
2059 return 1; 2026 return 1;
2060 if (sysctl_sched_migration_cost == 0) 2027 if (sysctl_sched_migration_cost == 0)
@@ -2065,22 +2032,23 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
2065 return delta < (s64)sysctl_sched_migration_cost; 2032 return delta < (s64)sysctl_sched_migration_cost;
2066} 2033}
2067 2034
2068
2069void set_task_cpu(struct task_struct *p, unsigned int new_cpu) 2035void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
2070{ 2036{
2071 int old_cpu = task_cpu(p); 2037#ifdef CONFIG_SCHED_DEBUG
2072 struct cfs_rq *old_cfsrq = task_cfs_rq(p), 2038 /*
2073 *new_cfsrq = cpu_cfs_rq(old_cfsrq, new_cpu); 2039 * We should never call set_task_cpu() on a blocked task,
2040 * ttwu() will sort out the placement.
2041 */
2042 WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING &&
2043 !(task_thread_info(p)->preempt_count & PREEMPT_ACTIVE));
2044#endif
2074 2045
2075 trace_sched_migrate_task(p, new_cpu); 2046 trace_sched_migrate_task(p, new_cpu);
2076 2047
2077 if (old_cpu != new_cpu) { 2048 if (task_cpu(p) != new_cpu) {
2078 p->se.nr_migrations++; 2049 p->se.nr_migrations++;
2079 perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 2050 perf_sw_event(PERF_COUNT_SW_CPU_MIGRATIONS, 1, 1, NULL, 0);
2080 1, 1, NULL, 0);
2081 } 2051 }
2082 p->se.vruntime -= old_cfsrq->min_vruntime -
2083 new_cfsrq->min_vruntime;
2084 2052
2085 __set_task_cpu(p, new_cpu); 2053 __set_task_cpu(p, new_cpu);
2086} 2054}
@@ -2105,13 +2073,10 @@ migrate_task(struct task_struct *p, int dest_cpu, struct migration_req *req)
2105 2073
2106 /* 2074 /*
2107 * If the task is not on a runqueue (and not running), then 2075 * If the task is not on a runqueue (and not running), then
2108 * it is sufficient to simply update the task's cpu field. 2076 * the next wake-up will properly place the task.
2109 */ 2077 */
2110 if (!p->se.on_rq && !task_running(rq, p)) { 2078 if (!p->se.on_rq && !task_running(rq, p))
2111 update_rq_clock(rq);
2112 set_task_cpu(p, dest_cpu);
2113 return 0; 2079 return 0;
2114 }
2115 2080
2116 init_completion(&req->done); 2081 init_completion(&req->done);
2117 req->task = p; 2082 req->task = p;
@@ -2317,10 +2282,73 @@ void task_oncpu_function_call(struct task_struct *p,
2317} 2282}
2318 2283
2319#ifdef CONFIG_SMP 2284#ifdef CONFIG_SMP
2285static int select_fallback_rq(int cpu, struct task_struct *p)
2286{
2287 int dest_cpu;
2288 const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(cpu));
2289
2290 /* Look for allowed, online CPU in same node. */
2291 for_each_cpu_and(dest_cpu, nodemask, cpu_active_mask)
2292 if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
2293 return dest_cpu;
2294
2295 /* Any allowed, online CPU? */
2296 dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_active_mask);
2297 if (dest_cpu < nr_cpu_ids)
2298 return dest_cpu;
2299
2300 /* No more Mr. Nice Guy. */
2301 if (dest_cpu >= nr_cpu_ids) {
2302 rcu_read_lock();
2303 cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
2304 rcu_read_unlock();
2305 dest_cpu = cpumask_any_and(cpu_active_mask, &p->cpus_allowed);
2306
2307 /*
2308 * Don't tell them about moving exiting tasks or
2309 * kernel threads (both mm NULL), since they never
2310 * leave kernel.
2311 */
2312 if (p->mm && printk_ratelimit()) {
2313 printk(KERN_INFO "process %d (%s) no "
2314 "longer affine to cpu%d\n",
2315 task_pid_nr(p), p->comm, cpu);
2316 }
2317 }
2318
2319 return dest_cpu;
2320}
2321
2322/*
2323 * Called from:
2324 *
2325 * - fork, @p is stable because it isn't on the tasklist yet
2326 *
2327 * - exec, @p is unstable, retry loop
2328 *
2329 * - wake-up, we serialize ->cpus_allowed against TASK_WAKING so
2330 * we should be good.
2331 */
2320static inline 2332static inline
2321int select_task_rq(struct task_struct *p, int sd_flags, int wake_flags) 2333int select_task_rq(struct task_struct *p, int sd_flags, int wake_flags)
2322{ 2334{
2323 return p->sched_class->select_task_rq(p, sd_flags, wake_flags); 2335 int cpu = p->sched_class->select_task_rq(p, sd_flags, wake_flags);
2336
2337 /*
2338 * In order not to call set_task_cpu() on a blocking task we need
2339 * to rely on ttwu() to place the task on a valid ->cpus_allowed
2340 * cpu.
2341 *
2342 * Since this is common to all placement strategies, this lives here.
2343 *
2344 * [ this allows ->select_task() to simply return task_cpu(p) and
2345 * not worry about this generic constraint ]
2346 */
2347 if (unlikely(!cpumask_test_cpu(cpu, &p->cpus_allowed) ||
2348 !cpu_online(cpu)))
2349 cpu = select_fallback_rq(task_cpu(p), p);
2350
2351 return cpu;
2324} 2352}
2325#endif 2353#endif
2326 2354
@@ -2375,6 +2403,10 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state,
2375 if (task_contributes_to_load(p)) 2403 if (task_contributes_to_load(p))
2376 rq->nr_uninterruptible--; 2404 rq->nr_uninterruptible--;
2377 p->state = TASK_WAKING; 2405 p->state = TASK_WAKING;
2406
2407 if (p->sched_class->task_waking)
2408 p->sched_class->task_waking(rq, p);
2409
2378 __task_rq_unlock(rq); 2410 __task_rq_unlock(rq);
2379 2411
2380 cpu = select_task_rq(p, SD_BALANCE_WAKE, wake_flags); 2412 cpu = select_task_rq(p, SD_BALANCE_WAKE, wake_flags);
@@ -2438,8 +2470,8 @@ out_running:
2438 2470
2439 p->state = TASK_RUNNING; 2471 p->state = TASK_RUNNING;
2440#ifdef CONFIG_SMP 2472#ifdef CONFIG_SMP
2441 if (p->sched_class->task_wake_up) 2473 if (p->sched_class->task_woken)
2442 p->sched_class->task_wake_up(rq, p); 2474 p->sched_class->task_woken(rq, p);
2443 2475
2444 if (unlikely(rq->idle_stamp)) { 2476 if (unlikely(rq->idle_stamp)) {
2445 u64 delta = rq->clock - rq->idle_stamp; 2477 u64 delta = rq->clock - rq->idle_stamp;
@@ -2538,14 +2570,6 @@ static void __sched_fork(struct task_struct *p)
2538#ifdef CONFIG_PREEMPT_NOTIFIERS 2570#ifdef CONFIG_PREEMPT_NOTIFIERS
2539 INIT_HLIST_HEAD(&p->preempt_notifiers); 2571 INIT_HLIST_HEAD(&p->preempt_notifiers);
2540#endif 2572#endif
2541
2542 /*
2543 * We mark the process as running here, but have not actually
2544 * inserted it onto the runqueue yet. This guarantees that
2545 * nobody will actually run it, and a signal or other external
2546 * event cannot wake it up and insert it on the runqueue either.
2547 */
2548 p->state = TASK_RUNNING;
2549} 2573}
2550 2574
2551/* 2575/*
@@ -2556,6 +2580,12 @@ void sched_fork(struct task_struct *p, int clone_flags)
2556 int cpu = get_cpu(); 2580 int cpu = get_cpu();
2557 2581
2558 __sched_fork(p); 2582 __sched_fork(p);
2583 /*
2584 * We mark the process as waking here. This guarantees that
2585 * nobody will actually run it, and a signal or other external
2586 * event cannot wake it up and insert it on the runqueue either.
2587 */
2588 p->state = TASK_WAKING;
2559 2589
2560 /* 2590 /*
2561 * Revert to default priority/policy on fork if requested. 2591 * Revert to default priority/policy on fork if requested.
@@ -2624,14 +2654,15 @@ void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
2624 struct rq *rq; 2654 struct rq *rq;
2625 2655
2626 rq = task_rq_lock(p, &flags); 2656 rq = task_rq_lock(p, &flags);
2627 BUG_ON(p->state != TASK_RUNNING); 2657 BUG_ON(p->state != TASK_WAKING);
2658 p->state = TASK_RUNNING;
2628 update_rq_clock(rq); 2659 update_rq_clock(rq);
2629 activate_task(rq, p, 0); 2660 activate_task(rq, p, 0);
2630 trace_sched_wakeup_new(rq, p, 1); 2661 trace_sched_wakeup_new(rq, p, 1);
2631 check_preempt_curr(rq, p, WF_FORK); 2662 check_preempt_curr(rq, p, WF_FORK);
2632#ifdef CONFIG_SMP 2663#ifdef CONFIG_SMP
2633 if (p->sched_class->task_wake_up) 2664 if (p->sched_class->task_woken)
2634 p->sched_class->task_wake_up(rq, p); 2665 p->sched_class->task_woken(rq, p);
2635#endif 2666#endif
2636 task_rq_unlock(rq, &flags); 2667 task_rq_unlock(rq, &flags);
2637} 2668}
@@ -3101,21 +3132,36 @@ static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
3101} 3132}
3102 3133
3103/* 3134/*
3104 * If dest_cpu is allowed for this process, migrate the task to it. 3135 * sched_exec - execve() is a valuable balancing opportunity, because at
3105 * This is accomplished by forcing the cpu_allowed mask to only 3136 * this point the task has the smallest effective memory and cache footprint.
3106 * allow dest_cpu, which will force the cpu onto dest_cpu. Then
3107 * the cpu_allowed mask is restored.
3108 */ 3137 */
3109static void sched_migrate_task(struct task_struct *p, int dest_cpu) 3138void sched_exec(void)
3110{ 3139{
3140 struct task_struct *p = current;
3111 struct migration_req req; 3141 struct migration_req req;
3142 int dest_cpu, this_cpu;
3112 unsigned long flags; 3143 unsigned long flags;
3113 struct rq *rq; 3144 struct rq *rq;
3114 3145
3146again:
3147 this_cpu = get_cpu();
3148 dest_cpu = select_task_rq(p, SD_BALANCE_EXEC, 0);
3149 if (dest_cpu == this_cpu) {
3150 put_cpu();
3151 return;
3152 }
3153
3115 rq = task_rq_lock(p, &flags); 3154 rq = task_rq_lock(p, &flags);
3155 put_cpu();
3156
3157 /*
3158 * select_task_rq() can race against ->cpus_allowed
3159 */
3116 if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed) 3160 if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed)
3117 || unlikely(!cpu_active(dest_cpu))) 3161 || unlikely(!cpu_active(dest_cpu))) {
3118 goto out; 3162 task_rq_unlock(rq, &flags);
3163 goto again;
3164 }
3119 3165
3120 /* force the process onto the specified CPU */ 3166 /* force the process onto the specified CPU */
3121 if (migrate_task(p, dest_cpu, &req)) { 3167 if (migrate_task(p, dest_cpu, &req)) {
@@ -3130,24 +3176,10 @@ static void sched_migrate_task(struct task_struct *p, int dest_cpu)
3130 3176
3131 return; 3177 return;
3132 } 3178 }
3133out:
3134 task_rq_unlock(rq, &flags); 3179 task_rq_unlock(rq, &flags);
3135} 3180}
3136 3181
3137/* 3182/*
3138 * sched_exec - execve() is a valuable balancing opportunity, because at
3139 * this point the task has the smallest effective memory and cache footprint.
3140 */
3141void sched_exec(void)
3142{
3143 int new_cpu, this_cpu = get_cpu();
3144 new_cpu = select_task_rq(current, SD_BALANCE_EXEC, 0);
3145 put_cpu();
3146 if (new_cpu != this_cpu)
3147 sched_migrate_task(current, new_cpu);
3148}
3149
3150/*
3151 * pull_task - move a task from a remote runqueue to the local runqueue. 3183 * pull_task - move a task from a remote runqueue to the local runqueue.
3152 * Both runqueues must be locked. 3184 * Both runqueues must be locked.
3153 */ 3185 */
@@ -5911,14 +5943,15 @@ EXPORT_SYMBOL(wait_for_completion_killable);
5911 */ 5943 */
5912bool try_wait_for_completion(struct completion *x) 5944bool try_wait_for_completion(struct completion *x)
5913{ 5945{
5946 unsigned long flags;
5914 int ret = 1; 5947 int ret = 1;
5915 5948
5916 spin_lock_irq(&x->wait.lock); 5949 spin_lock_irqsave(&x->wait.lock, flags);
5917 if (!x->done) 5950 if (!x->done)
5918 ret = 0; 5951 ret = 0;
5919 else 5952 else
5920 x->done--; 5953 x->done--;
5921 spin_unlock_irq(&x->wait.lock); 5954 spin_unlock_irqrestore(&x->wait.lock, flags);
5922 return ret; 5955 return ret;
5923} 5956}
5924EXPORT_SYMBOL(try_wait_for_completion); 5957EXPORT_SYMBOL(try_wait_for_completion);
@@ -5933,12 +5966,13 @@ EXPORT_SYMBOL(try_wait_for_completion);
5933 */ 5966 */
5934bool completion_done(struct completion *x) 5967bool completion_done(struct completion *x)
5935{ 5968{
5969 unsigned long flags;
5936 int ret = 1; 5970 int ret = 1;
5937 5971
5938 spin_lock_irq(&x->wait.lock); 5972 spin_lock_irqsave(&x->wait.lock, flags);
5939 if (!x->done) 5973 if (!x->done)
5940 ret = 0; 5974 ret = 0;
5941 spin_unlock_irq(&x->wait.lock); 5975 spin_unlock_irqrestore(&x->wait.lock, flags);
5942 return ret; 5976 return ret;
5943} 5977}
5944EXPORT_SYMBOL(completion_done); 5978EXPORT_SYMBOL(completion_done);
@@ -6457,7 +6491,7 @@ SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
6457 return -EINVAL; 6491 return -EINVAL;
6458 6492
6459 retval = -ESRCH; 6493 retval = -ESRCH;
6460 read_lock(&tasklist_lock); 6494 rcu_read_lock();
6461 p = find_process_by_pid(pid); 6495 p = find_process_by_pid(pid);
6462 if (p) { 6496 if (p) {
6463 retval = security_task_getscheduler(p); 6497 retval = security_task_getscheduler(p);
@@ -6465,7 +6499,7 @@ SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid)
6465 retval = p->policy 6499 retval = p->policy
6466 | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0); 6500 | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0);
6467 } 6501 }
6468 read_unlock(&tasklist_lock); 6502 rcu_read_unlock();
6469 return retval; 6503 return retval;
6470} 6504}
6471 6505
@@ -6483,7 +6517,7 @@ SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
6483 if (!param || pid < 0) 6517 if (!param || pid < 0)
6484 return -EINVAL; 6518 return -EINVAL;
6485 6519
6486 read_lock(&tasklist_lock); 6520 rcu_read_lock();
6487 p = find_process_by_pid(pid); 6521 p = find_process_by_pid(pid);
6488 retval = -ESRCH; 6522 retval = -ESRCH;
6489 if (!p) 6523 if (!p)
@@ -6494,7 +6528,7 @@ SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
6494 goto out_unlock; 6528 goto out_unlock;
6495 6529
6496 lp.sched_priority = p->rt_priority; 6530 lp.sched_priority = p->rt_priority;
6497 read_unlock(&tasklist_lock); 6531 rcu_read_unlock();
6498 6532
6499 /* 6533 /*
6500 * This one might sleep, we cannot do it with a spinlock held ... 6534 * This one might sleep, we cannot do it with a spinlock held ...
@@ -6504,7 +6538,7 @@ SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param)
6504 return retval; 6538 return retval;
6505 6539
6506out_unlock: 6540out_unlock:
6507 read_unlock(&tasklist_lock); 6541 rcu_read_unlock();
6508 return retval; 6542 return retval;
6509} 6543}
6510 6544
@@ -6515,22 +6549,18 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
6515 int retval; 6549 int retval;
6516 6550
6517 get_online_cpus(); 6551 get_online_cpus();
6518 read_lock(&tasklist_lock); 6552 rcu_read_lock();
6519 6553
6520 p = find_process_by_pid(pid); 6554 p = find_process_by_pid(pid);
6521 if (!p) { 6555 if (!p) {
6522 read_unlock(&tasklist_lock); 6556 rcu_read_unlock();
6523 put_online_cpus(); 6557 put_online_cpus();
6524 return -ESRCH; 6558 return -ESRCH;
6525 } 6559 }
6526 6560
6527 /* 6561 /* Prevent p going away */
6528 * It is not safe to call set_cpus_allowed with the
6529 * tasklist_lock held. We will bump the task_struct's
6530 * usage count and then drop tasklist_lock.
6531 */
6532 get_task_struct(p); 6562 get_task_struct(p);
6533 read_unlock(&tasklist_lock); 6563 rcu_read_unlock();
6534 6564
6535 if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) { 6565 if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
6536 retval = -ENOMEM; 6566 retval = -ENOMEM;
@@ -6616,7 +6646,7 @@ long sched_getaffinity(pid_t pid, struct cpumask *mask)
6616 int retval; 6646 int retval;
6617 6647
6618 get_online_cpus(); 6648 get_online_cpus();
6619 read_lock(&tasklist_lock); 6649 rcu_read_lock();
6620 6650
6621 retval = -ESRCH; 6651 retval = -ESRCH;
6622 p = find_process_by_pid(pid); 6652 p = find_process_by_pid(pid);
@@ -6632,7 +6662,7 @@ long sched_getaffinity(pid_t pid, struct cpumask *mask)
6632 task_rq_unlock(rq, &flags); 6662 task_rq_unlock(rq, &flags);
6633 6663
6634out_unlock: 6664out_unlock:
6635 read_unlock(&tasklist_lock); 6665 rcu_read_unlock();
6636 put_online_cpus(); 6666 put_online_cpus();
6637 6667
6638 return retval; 6668 return retval;
@@ -6876,7 +6906,7 @@ SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
6876 return -EINVAL; 6906 return -EINVAL;
6877 6907
6878 retval = -ESRCH; 6908 retval = -ESRCH;
6879 read_lock(&tasklist_lock); 6909 rcu_read_lock();
6880 p = find_process_by_pid(pid); 6910 p = find_process_by_pid(pid);
6881 if (!p) 6911 if (!p)
6882 goto out_unlock; 6912 goto out_unlock;
@@ -6889,13 +6919,13 @@ SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid,
6889 time_slice = p->sched_class->get_rr_interval(rq, p); 6919 time_slice = p->sched_class->get_rr_interval(rq, p);
6890 task_rq_unlock(rq, &flags); 6920 task_rq_unlock(rq, &flags);
6891 6921
6892 read_unlock(&tasklist_lock); 6922 rcu_read_unlock();
6893 jiffies_to_timespec(time_slice, &t); 6923 jiffies_to_timespec(time_slice, &t);
6894 retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0; 6924 retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0;
6895 return retval; 6925 return retval;
6896 6926
6897out_unlock: 6927out_unlock:
6898 read_unlock(&tasklist_lock); 6928 rcu_read_unlock();
6899 return retval; 6929 return retval;
6900} 6930}
6901 6931
@@ -6986,6 +7016,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
6986 raw_spin_lock_irqsave(&rq->lock, flags); 7016 raw_spin_lock_irqsave(&rq->lock, flags);
6987 7017
6988 __sched_fork(idle); 7018 __sched_fork(idle);
7019 idle->state = TASK_RUNNING;
6989 idle->se.exec_start = sched_clock(); 7020 idle->se.exec_start = sched_clock();
6990 7021
6991 cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu)); 7022 cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
@@ -7100,7 +7131,23 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
7100 struct rq *rq; 7131 struct rq *rq;
7101 int ret = 0; 7132 int ret = 0;
7102 7133
7134 /*
7135 * Since we rely on wake-ups to migrate sleeping tasks, don't change
7136 * the ->cpus_allowed mask from under waking tasks, which would be
7137 * possible when we change rq->lock in ttwu(), so synchronize against
7138 * TASK_WAKING to avoid that.
7139 */
7140again:
7141 while (p->state == TASK_WAKING)
7142 cpu_relax();
7143
7103 rq = task_rq_lock(p, &flags); 7144 rq = task_rq_lock(p, &flags);
7145
7146 if (p->state == TASK_WAKING) {
7147 task_rq_unlock(rq, &flags);
7148 goto again;
7149 }
7150
7104 if (!cpumask_intersects(new_mask, cpu_active_mask)) { 7151 if (!cpumask_intersects(new_mask, cpu_active_mask)) {
7105 ret = -EINVAL; 7152 ret = -EINVAL;
7106 goto out; 7153 goto out;
@@ -7156,7 +7203,7 @@ EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
7156static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu) 7203static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
7157{ 7204{
7158 struct rq *rq_dest, *rq_src; 7205 struct rq *rq_dest, *rq_src;
7159 int ret = 0, on_rq; 7206 int ret = 0;
7160 7207
7161 if (unlikely(!cpu_active(dest_cpu))) 7208 if (unlikely(!cpu_active(dest_cpu)))
7162 return ret; 7209 return ret;
@@ -7172,12 +7219,13 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
7172 if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed)) 7219 if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
7173 goto fail; 7220 goto fail;
7174 7221
7175 on_rq = p->se.on_rq; 7222 /*
7176 if (on_rq) 7223 * If we're not on a rq, the next wake-up will ensure we're
7224 * placed properly.
7225 */
7226 if (p->se.on_rq) {
7177 deactivate_task(rq_src, p, 0); 7227 deactivate_task(rq_src, p, 0);
7178 7228 set_task_cpu(p, dest_cpu);
7179 set_task_cpu(p, dest_cpu);
7180 if (on_rq) {
7181 activate_task(rq_dest, p, 0); 7229 activate_task(rq_dest, p, 0);
7182 check_preempt_curr(rq_dest, p, 0); 7230 check_preempt_curr(rq_dest, p, 0);
7183 } 7231 }
@@ -7273,37 +7321,10 @@ static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu)
7273static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p) 7321static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
7274{ 7322{
7275 int dest_cpu; 7323 int dest_cpu;
7276 const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(dead_cpu));
7277 7324
7278again: 7325again:
7279 /* Look for allowed, online CPU in same node. */ 7326 dest_cpu = select_fallback_rq(dead_cpu, p);
7280 for_each_cpu_and(dest_cpu, nodemask, cpu_active_mask)
7281 if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
7282 goto move;
7283
7284 /* Any allowed, online CPU? */
7285 dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_active_mask);
7286 if (dest_cpu < nr_cpu_ids)
7287 goto move;
7288
7289 /* No more Mr. Nice Guy. */
7290 if (dest_cpu >= nr_cpu_ids) {
7291 cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
7292 dest_cpu = cpumask_any_and(cpu_active_mask, &p->cpus_allowed);
7293
7294 /*
7295 * Don't tell them about moving exiting tasks or
7296 * kernel threads (both mm NULL), since they never
7297 * leave kernel.
7298 */
7299 if (p->mm && printk_ratelimit()) {
7300 printk(KERN_INFO "process %d (%s) no "
7301 "longer affine to cpu%d\n",
7302 task_pid_nr(p), p->comm, dead_cpu);
7303 }
7304 }
7305 7327
7306move:
7307 /* It can have affinity changed while we were choosing. */ 7328 /* It can have affinity changed while we were choosing. */
7308 if (unlikely(!__migrate_task_irq(p, dead_cpu, dest_cpu))) 7329 if (unlikely(!__migrate_task_irq(p, dead_cpu, dest_cpu)))
7309 goto again; 7330 goto again;
@@ -9668,7 +9689,7 @@ void __init sched_init(void)
9668#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP 9689#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
9669static inline int preempt_count_equals(int preempt_offset) 9690static inline int preempt_count_equals(int preempt_offset)
9670{ 9691{
9671 int nested = preempt_count() & ~PREEMPT_ACTIVE; 9692 int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth();
9672 9693
9673 return (nested == PREEMPT_INATOMIC_BASE + preempt_offset); 9694 return (nested == PREEMPT_INATOMIC_BASE + preempt_offset);
9674} 9695}
@@ -10083,7 +10104,7 @@ void sched_move_task(struct task_struct *tsk)
10083 10104
10084#ifdef CONFIG_FAIR_GROUP_SCHED 10105#ifdef CONFIG_FAIR_GROUP_SCHED
10085 if (tsk->sched_class->moved_group) 10106 if (tsk->sched_class->moved_group)
10086 tsk->sched_class->moved_group(tsk); 10107 tsk->sched_class->moved_group(tsk, on_rq);
10087#endif 10108#endif
10088 10109
10089 if (unlikely(running)) 10110 if (unlikely(running))
diff --git a/kernel/sched_clock.c b/kernel/sched_clock.c
index 479ce5682d7c..5b496132c28a 100644
--- a/kernel/sched_clock.c
+++ b/kernel/sched_clock.c
@@ -236,6 +236,18 @@ void sched_clock_idle_wakeup_event(u64 delta_ns)
236} 236}
237EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event); 237EXPORT_SYMBOL_GPL(sched_clock_idle_wakeup_event);
238 238
239unsigned long long cpu_clock(int cpu)
240{
241 unsigned long long clock;
242 unsigned long flags;
243
244 local_irq_save(flags);
245 clock = sched_clock_cpu(cpu);
246 local_irq_restore(flags);
247
248 return clock;
249}
250
239#else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */ 251#else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
240 252
241void sched_clock_init(void) 253void sched_clock_init(void)
@@ -251,17 +263,12 @@ u64 sched_clock_cpu(int cpu)
251 return sched_clock(); 263 return sched_clock();
252} 264}
253 265
254#endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
255 266
256unsigned long long cpu_clock(int cpu) 267unsigned long long cpu_clock(int cpu)
257{ 268{
258 unsigned long long clock; 269 return sched_clock_cpu(cpu);
259 unsigned long flags; 270}
260 271
261 local_irq_save(flags); 272#endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
262 clock = sched_clock_cpu(cpu);
263 local_irq_restore(flags);
264 273
265 return clock;
266}
267EXPORT_SYMBOL_GPL(cpu_clock); 274EXPORT_SYMBOL_GPL(cpu_clock);
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 5bedf6e3ebf3..42ac3c9f66f6 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -510,6 +510,7 @@ __update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr,
510 curr->sum_exec_runtime += delta_exec; 510 curr->sum_exec_runtime += delta_exec;
511 schedstat_add(cfs_rq, exec_clock, delta_exec); 511 schedstat_add(cfs_rq, exec_clock, delta_exec);
512 delta_exec_weighted = calc_delta_fair(delta_exec, curr); 512 delta_exec_weighted = calc_delta_fair(delta_exec, curr);
513
513 curr->vruntime += delta_exec_weighted; 514 curr->vruntime += delta_exec_weighted;
514 update_min_vruntime(cfs_rq); 515 update_min_vruntime(cfs_rq);
515} 516}
@@ -765,16 +766,26 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
765 se->vruntime = vruntime; 766 se->vruntime = vruntime;
766} 767}
767 768
769#define ENQUEUE_WAKEUP 1
770#define ENQUEUE_MIGRATE 2
771
768static void 772static void
769enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int wakeup) 773enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
770{ 774{
771 /* 775 /*
776 * Update the normalized vruntime before updating min_vruntime
777 * through callig update_curr().
778 */
779 if (!(flags & ENQUEUE_WAKEUP) || (flags & ENQUEUE_MIGRATE))
780 se->vruntime += cfs_rq->min_vruntime;
781
782 /*
772 * Update run-time statistics of the 'current'. 783 * Update run-time statistics of the 'current'.
773 */ 784 */
774 update_curr(cfs_rq); 785 update_curr(cfs_rq);
775 account_entity_enqueue(cfs_rq, se); 786 account_entity_enqueue(cfs_rq, se);
776 787
777 if (wakeup) { 788 if (flags & ENQUEUE_WAKEUP) {
778 place_entity(cfs_rq, se, 0); 789 place_entity(cfs_rq, se, 0);
779 enqueue_sleeper(cfs_rq, se); 790 enqueue_sleeper(cfs_rq, se);
780 } 791 }
@@ -828,6 +839,14 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep)
828 __dequeue_entity(cfs_rq, se); 839 __dequeue_entity(cfs_rq, se);
829 account_entity_dequeue(cfs_rq, se); 840 account_entity_dequeue(cfs_rq, se);
830 update_min_vruntime(cfs_rq); 841 update_min_vruntime(cfs_rq);
842
843 /*
844 * Normalize the entity after updating the min_vruntime because the
845 * update can refer to the ->curr item and we need to reflect this
846 * movement in our normalized position.
847 */
848 if (!sleep)
849 se->vruntime -= cfs_rq->min_vruntime;
831} 850}
832 851
833/* 852/*
@@ -1038,13 +1057,19 @@ static void enqueue_task_fair(struct rq *rq, struct task_struct *p, int wakeup)
1038{ 1057{
1039 struct cfs_rq *cfs_rq; 1058 struct cfs_rq *cfs_rq;
1040 struct sched_entity *se = &p->se; 1059 struct sched_entity *se = &p->se;
1060 int flags = 0;
1061
1062 if (wakeup)
1063 flags |= ENQUEUE_WAKEUP;
1064 if (p->state == TASK_WAKING)
1065 flags |= ENQUEUE_MIGRATE;
1041 1066
1042 for_each_sched_entity(se) { 1067 for_each_sched_entity(se) {
1043 if (se->on_rq) 1068 if (se->on_rq)
1044 break; 1069 break;
1045 cfs_rq = cfs_rq_of(se); 1070 cfs_rq = cfs_rq_of(se);
1046 enqueue_entity(cfs_rq, se, wakeup); 1071 enqueue_entity(cfs_rq, se, flags);
1047 wakeup = 1; 1072 flags = ENQUEUE_WAKEUP;
1048 } 1073 }
1049 1074
1050 hrtick_update(rq); 1075 hrtick_update(rq);
@@ -1120,6 +1145,14 @@ static void yield_task_fair(struct rq *rq)
1120 1145
1121#ifdef CONFIG_SMP 1146#ifdef CONFIG_SMP
1122 1147
1148static void task_waking_fair(struct rq *rq, struct task_struct *p)
1149{
1150 struct sched_entity *se = &p->se;
1151 struct cfs_rq *cfs_rq = cfs_rq_of(se);
1152
1153 se->vruntime -= cfs_rq->min_vruntime;
1154}
1155
1123#ifdef CONFIG_FAIR_GROUP_SCHED 1156#ifdef CONFIG_FAIR_GROUP_SCHED
1124/* 1157/*
1125 * effective_load() calculates the load change as seen from the root_task_group 1158 * effective_load() calculates the load change as seen from the root_task_group
@@ -1429,6 +1462,9 @@ static int select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flag
1429 } 1462 }
1430 1463
1431 for_each_domain(cpu, tmp) { 1464 for_each_domain(cpu, tmp) {
1465 if (!(tmp->flags & SD_LOAD_BALANCE))
1466 continue;
1467
1432 /* 1468 /*
1433 * If power savings logic is enabled for a domain, see if we 1469 * If power savings logic is enabled for a domain, see if we
1434 * are not overloaded, if so, don't balance wider. 1470 * are not overloaded, if so, don't balance wider.
@@ -1975,6 +2011,8 @@ static void task_fork_fair(struct task_struct *p)
1975 resched_task(rq->curr); 2011 resched_task(rq->curr);
1976 } 2012 }
1977 2013
2014 se->vruntime -= cfs_rq->min_vruntime;
2015
1978 raw_spin_unlock_irqrestore(&rq->lock, flags); 2016 raw_spin_unlock_irqrestore(&rq->lock, flags);
1979} 2017}
1980 2018
@@ -2028,12 +2066,13 @@ static void set_curr_task_fair(struct rq *rq)
2028} 2066}
2029 2067
2030#ifdef CONFIG_FAIR_GROUP_SCHED 2068#ifdef CONFIG_FAIR_GROUP_SCHED
2031static void moved_group_fair(struct task_struct *p) 2069static void moved_group_fair(struct task_struct *p, int on_rq)
2032{ 2070{
2033 struct cfs_rq *cfs_rq = task_cfs_rq(p); 2071 struct cfs_rq *cfs_rq = task_cfs_rq(p);
2034 2072
2035 update_curr(cfs_rq); 2073 update_curr(cfs_rq);
2036 place_entity(cfs_rq, &p->se, 1); 2074 if (!on_rq)
2075 place_entity(cfs_rq, &p->se, 1);
2037} 2076}
2038#endif 2077#endif
2039 2078
@@ -2073,6 +2112,8 @@ static const struct sched_class fair_sched_class = {
2073 .move_one_task = move_one_task_fair, 2112 .move_one_task = move_one_task_fair,
2074 .rq_online = rq_online_fair, 2113 .rq_online = rq_online_fair,
2075 .rq_offline = rq_offline_fair, 2114 .rq_offline = rq_offline_fair,
2115
2116 .task_waking = task_waking_fair,
2076#endif 2117#endif
2077 2118
2078 .set_curr_task = set_curr_task_fair, 2119 .set_curr_task = set_curr_task_fair,
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c
index d2ea2828164e..f48328ac216f 100644
--- a/kernel/sched_rt.c
+++ b/kernel/sched_rt.c
@@ -1472,7 +1472,7 @@ static void post_schedule_rt(struct rq *rq)
1472 * If we are not running and we are not going to reschedule soon, we should 1472 * If we are not running and we are not going to reschedule soon, we should
1473 * try to push tasks away now 1473 * try to push tasks away now
1474 */ 1474 */
1475static void task_wake_up_rt(struct rq *rq, struct task_struct *p) 1475static void task_woken_rt(struct rq *rq, struct task_struct *p)
1476{ 1476{
1477 if (!task_running(rq, p) && 1477 if (!task_running(rq, p) &&
1478 !test_tsk_need_resched(rq->curr) && 1478 !test_tsk_need_resched(rq->curr) &&
@@ -1753,7 +1753,7 @@ static const struct sched_class rt_sched_class = {
1753 .rq_offline = rq_offline_rt, 1753 .rq_offline = rq_offline_rt,
1754 .pre_schedule = pre_schedule_rt, 1754 .pre_schedule = pre_schedule_rt,
1755 .post_schedule = post_schedule_rt, 1755 .post_schedule = post_schedule_rt,
1756 .task_wake_up = task_wake_up_rt, 1756 .task_woken = task_woken_rt,
1757 .switched_from = switched_from_rt, 1757 .switched_from = switched_from_rt,
1758#endif 1758#endif
1759 1759
diff --git a/kernel/signal.c b/kernel/signal.c
index 1814e68e4de3..d09692b40376 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -218,13 +218,13 @@ __sigqueue_alloc(int sig, struct task_struct *t, gfp_t flags, int override_rlimi
218 struct user_struct *user; 218 struct user_struct *user;
219 219
220 /* 220 /*
221 * We won't get problems with the target's UID changing under us 221 * Protect access to @t credentials. This can go away when all
222 * because changing it requires RCU be used, and if t != current, the 222 * callers hold rcu read lock.
223 * caller must be holding the RCU readlock (by way of a spinlock) and
224 * we use RCU protection here
225 */ 223 */
224 rcu_read_lock();
226 user = get_uid(__task_cred(t)->user); 225 user = get_uid(__task_cred(t)->user);
227 atomic_inc(&user->sigpending); 226 atomic_inc(&user->sigpending);
227 rcu_read_unlock();
228 228
229 if (override_rlimit || 229 if (override_rlimit ||
230 atomic_read(&user->sigpending) <= 230 atomic_read(&user->sigpending) <=
@@ -1179,11 +1179,12 @@ int kill_pid_info_as_uid(int sig, struct siginfo *info, struct pid *pid,
1179 int ret = -EINVAL; 1179 int ret = -EINVAL;
1180 struct task_struct *p; 1180 struct task_struct *p;
1181 const struct cred *pcred; 1181 const struct cred *pcred;
1182 unsigned long flags;
1182 1183
1183 if (!valid_signal(sig)) 1184 if (!valid_signal(sig))
1184 return ret; 1185 return ret;
1185 1186
1186 read_lock(&tasklist_lock); 1187 rcu_read_lock();
1187 p = pid_task(pid, PIDTYPE_PID); 1188 p = pid_task(pid, PIDTYPE_PID);
1188 if (!p) { 1189 if (!p) {
1189 ret = -ESRCH; 1190 ret = -ESRCH;
@@ -1199,14 +1200,16 @@ int kill_pid_info_as_uid(int sig, struct siginfo *info, struct pid *pid,
1199 ret = security_task_kill(p, info, sig, secid); 1200 ret = security_task_kill(p, info, sig, secid);
1200 if (ret) 1201 if (ret)
1201 goto out_unlock; 1202 goto out_unlock;
1202 if (sig && p->sighand) { 1203
1203 unsigned long flags; 1204 if (sig) {
1204 spin_lock_irqsave(&p->sighand->siglock, flags); 1205 if (lock_task_sighand(p, &flags)) {
1205 ret = __send_signal(sig, info, p, 1, 0); 1206 ret = __send_signal(sig, info, p, 1, 0);
1206 spin_unlock_irqrestore(&p->sighand->siglock, flags); 1207 unlock_task_sighand(p, &flags);
1208 } else
1209 ret = -ESRCH;
1207 } 1210 }
1208out_unlock: 1211out_unlock:
1209 read_unlock(&tasklist_lock); 1212 rcu_read_unlock();
1210 return ret; 1213 return ret;
1211} 1214}
1212EXPORT_SYMBOL_GPL(kill_pid_info_as_uid); 1215EXPORT_SYMBOL_GPL(kill_pid_info_as_uid);
diff --git a/kernel/sys.c b/kernel/sys.c
index 20ccfb5da6af..26a6b73a6b85 100644
--- a/kernel/sys.c
+++ b/kernel/sys.c
@@ -162,6 +162,7 @@ SYSCALL_DEFINE3(setpriority, int, which, int, who, int, niceval)
162 if (niceval > 19) 162 if (niceval > 19)
163 niceval = 19; 163 niceval = 19;
164 164
165 rcu_read_lock();
165 read_lock(&tasklist_lock); 166 read_lock(&tasklist_lock);
166 switch (which) { 167 switch (which) {
167 case PRIO_PROCESS: 168 case PRIO_PROCESS:
@@ -199,6 +200,7 @@ SYSCALL_DEFINE3(setpriority, int, which, int, who, int, niceval)
199 } 200 }
200out_unlock: 201out_unlock:
201 read_unlock(&tasklist_lock); 202 read_unlock(&tasklist_lock);
203 rcu_read_unlock();
202out: 204out:
203 return error; 205 return error;
204} 206}
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 45e4bef0012a..8a68b2448468 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -1131,7 +1131,7 @@ static struct ctl_table vm_table[] = {
1131 .data = &sysctl_max_map_count, 1131 .data = &sysctl_max_map_count,
1132 .maxlen = sizeof(sysctl_max_map_count), 1132 .maxlen = sizeof(sysctl_max_map_count),
1133 .mode = 0644, 1133 .mode = 0644,
1134 .proc_handler = proc_dointvec, 1134 .proc_handler = proc_dointvec_minmax,
1135 .extra1 = &zero, 1135 .extra1 = &zero,
1136 }, 1136 },
1137#else 1137#else
@@ -1214,6 +1214,7 @@ static struct ctl_table vm_table[] = {
1214 .proc_handler = proc_dointvec_jiffies, 1214 .proc_handler = proc_dointvec_jiffies,
1215 }, 1215 },
1216#endif 1216#endif
1217#ifdef CONFIG_MMU
1217 { 1218 {
1218 .procname = "mmap_min_addr", 1219 .procname = "mmap_min_addr",
1219 .data = &dac_mmap_min_addr, 1220 .data = &dac_mmap_min_addr,
@@ -1221,6 +1222,7 @@ static struct ctl_table vm_table[] = {
1221 .mode = 0644, 1222 .mode = 0644,
1222 .proc_handler = mmap_min_addr_handler, 1223 .proc_handler = mmap_min_addr_handler,
1223 }, 1224 },
1225#endif
1224#ifdef CONFIG_NUMA 1226#ifdef CONFIG_NUMA
1225 { 1227 {
1226 .procname = "numa_zonelist_order", 1228 .procname = "numa_zonelist_order",
diff --git a/kernel/sysctl_binary.c b/kernel/sysctl_binary.c
index 112533d5fc08..8f5d16e0707a 100644
--- a/kernel/sysctl_binary.c
+++ b/kernel/sysctl_binary.c
@@ -1417,6 +1417,35 @@ static void deprecated_sysctl_warning(const int *name, int nlen)
1417 return; 1417 return;
1418} 1418}
1419 1419
1420#define WARN_ONCE_HASH_BITS 8
1421#define WARN_ONCE_HASH_SIZE (1<<WARN_ONCE_HASH_BITS)
1422
1423static DECLARE_BITMAP(warn_once_bitmap, WARN_ONCE_HASH_SIZE);
1424
1425#define FNV32_OFFSET 2166136261U
1426#define FNV32_PRIME 0x01000193
1427
1428/*
1429 * Print each legacy sysctl (approximately) only once.
1430 * To avoid making the tables non-const use a external
1431 * hash-table instead.
1432 * Worst case hash collision: 6, but very rarely.
1433 * NOTE! We don't use the SMP-safe bit tests. We simply
1434 * don't care enough.
1435 */
1436static void warn_on_bintable(const int *name, int nlen)
1437{
1438 int i;
1439 u32 hash = FNV32_OFFSET;
1440
1441 for (i = 0; i < nlen; i++)
1442 hash = (hash ^ name[i]) * FNV32_PRIME;
1443 hash %= WARN_ONCE_HASH_SIZE;
1444 if (__test_and_set_bit(hash, warn_once_bitmap))
1445 return;
1446 deprecated_sysctl_warning(name, nlen);
1447}
1448
1420static ssize_t do_sysctl(int __user *args_name, int nlen, 1449static ssize_t do_sysctl(int __user *args_name, int nlen,
1421 void __user *oldval, size_t oldlen, void __user *newval, size_t newlen) 1450 void __user *oldval, size_t oldlen, void __user *newval, size_t newlen)
1422{ 1451{
@@ -1431,7 +1460,7 @@ static ssize_t do_sysctl(int __user *args_name, int nlen,
1431 if (get_user(name[i], args_name + i)) 1460 if (get_user(name[i], args_name + i))
1432 return -EFAULT; 1461 return -EFAULT;
1433 1462
1434 deprecated_sysctl_warning(name, nlen); 1463 warn_on_bintable(name, nlen);
1435 1464
1436 return binary_sysctl(name, nlen, oldval, oldlen, newval, newlen); 1465 return binary_sysctl(name, nlen, oldval, oldlen, newval, newlen);
1437} 1466}
diff --git a/kernel/time.c b/kernel/time.c
index c6324d96009e..804798005d19 100644
--- a/kernel/time.c
+++ b/kernel/time.c
@@ -136,6 +136,7 @@ static inline void warp_clock(void)
136 write_seqlock_irq(&xtime_lock); 136 write_seqlock_irq(&xtime_lock);
137 wall_to_monotonic.tv_sec -= sys_tz.tz_minuteswest * 60; 137 wall_to_monotonic.tv_sec -= sys_tz.tz_minuteswest * 60;
138 xtime.tv_sec += sys_tz.tz_minuteswest * 60; 138 xtime.tv_sec += sys_tz.tz_minuteswest * 60;
139 update_xtime_cache(0);
139 write_sequnlock_irq(&xtime_lock); 140 write_sequnlock_irq(&xtime_lock);
140 clock_was_set(); 141 clock_was_set();
141} 142}
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c
index 3d5fc0fd1cca..6f740d9f0948 100644
--- a/kernel/time/clockevents.c
+++ b/kernel/time/clockevents.c
@@ -238,8 +238,9 @@ void clockevents_exchange_device(struct clock_event_device *old,
238 */ 238 */
239void clockevents_notify(unsigned long reason, void *arg) 239void clockevents_notify(unsigned long reason, void *arg)
240{ 240{
241 struct list_head *node, *tmp; 241 struct clock_event_device *dev, *tmp;
242 unsigned long flags; 242 unsigned long flags;
243 int cpu;
243 244
244 raw_spin_lock_irqsave(&clockevents_lock, flags); 245 raw_spin_lock_irqsave(&clockevents_lock, flags);
245 clockevents_do_notify(reason, arg); 246 clockevents_do_notify(reason, arg);
@@ -250,8 +251,19 @@ void clockevents_notify(unsigned long reason, void *arg)
250 * Unregister the clock event devices which were 251 * Unregister the clock event devices which were
251 * released from the users in the notify chain. 252 * released from the users in the notify chain.
252 */ 253 */
253 list_for_each_safe(node, tmp, &clockevents_released) 254 list_for_each_entry_safe(dev, tmp, &clockevents_released, list)
254 list_del(node); 255 list_del(&dev->list);
256 /*
257 * Now check whether the CPU has left unused per cpu devices
258 */
259 cpu = *((int *)arg);
260 list_for_each_entry_safe(dev, tmp, &clockevent_devices, list) {
261 if (cpumask_test_cpu(cpu, dev->cpumask) &&
262 cpumask_weight(dev->cpumask) == 1) {
263 BUG_ON(dev->mode != CLOCK_EVT_MODE_UNUSED);
264 list_del(&dev->list);
265 }
266 }
255 break; 267 break;
256 default: 268 default:
257 break; 269 break;
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index af4135f05825..7faaa32fbf4f 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -165,6 +165,13 @@ struct timespec raw_time;
165/* flag for if timekeeping is suspended */ 165/* flag for if timekeeping is suspended */
166int __read_mostly timekeeping_suspended; 166int __read_mostly timekeeping_suspended;
167 167
168static struct timespec xtime_cache __attribute__ ((aligned (16)));
169void update_xtime_cache(u64 nsec)
170{
171 xtime_cache = xtime;
172 timespec_add_ns(&xtime_cache, nsec);
173}
174
168/* must hold xtime_lock */ 175/* must hold xtime_lock */
169void timekeeping_leap_insert(int leapsecond) 176void timekeeping_leap_insert(int leapsecond)
170{ 177{
@@ -325,6 +332,8 @@ int do_settimeofday(struct timespec *tv)
325 332
326 xtime = *tv; 333 xtime = *tv;
327 334
335 update_xtime_cache(0);
336
328 timekeeper.ntp_error = 0; 337 timekeeper.ntp_error = 0;
329 ntp_clear(); 338 ntp_clear();
330 339
@@ -550,6 +559,7 @@ void __init timekeeping_init(void)
550 } 559 }
551 set_normalized_timespec(&wall_to_monotonic, 560 set_normalized_timespec(&wall_to_monotonic,
552 -boot.tv_sec, -boot.tv_nsec); 561 -boot.tv_sec, -boot.tv_nsec);
562 update_xtime_cache(0);
553 total_sleep_time.tv_sec = 0; 563 total_sleep_time.tv_sec = 0;
554 total_sleep_time.tv_nsec = 0; 564 total_sleep_time.tv_nsec = 0;
555 write_sequnlock_irqrestore(&xtime_lock, flags); 565 write_sequnlock_irqrestore(&xtime_lock, flags);
@@ -583,6 +593,7 @@ static int timekeeping_resume(struct sys_device *dev)
583 wall_to_monotonic = timespec_sub(wall_to_monotonic, ts); 593 wall_to_monotonic = timespec_sub(wall_to_monotonic, ts);
584 total_sleep_time = timespec_add_safe(total_sleep_time, ts); 594 total_sleep_time = timespec_add_safe(total_sleep_time, ts);
585 } 595 }
596 update_xtime_cache(0);
586 /* re-base the last cycle value */ 597 /* re-base the last cycle value */
587 timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock); 598 timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock);
588 timekeeper.ntp_error = 0; 599 timekeeper.ntp_error = 0;
@@ -722,6 +733,7 @@ static void timekeeping_adjust(s64 offset)
722 timekeeper.ntp_error_shift; 733 timekeeper.ntp_error_shift;
723} 734}
724 735
736
725/** 737/**
726 * logarithmic_accumulation - shifted accumulation of cycles 738 * logarithmic_accumulation - shifted accumulation of cycles
727 * 739 *
@@ -765,6 +777,7 @@ static cycle_t logarithmic_accumulation(cycle_t offset, int shift)
765 return offset; 777 return offset;
766} 778}
767 779
780
768/** 781/**
769 * update_wall_time - Uses the current clocksource to increment the wall time 782 * update_wall_time - Uses the current clocksource to increment the wall time
770 * 783 *
@@ -774,6 +787,7 @@ void update_wall_time(void)
774{ 787{
775 struct clocksource *clock; 788 struct clocksource *clock;
776 cycle_t offset; 789 cycle_t offset;
790 u64 nsecs;
777 int shift = 0, maxshift; 791 int shift = 0, maxshift;
778 792
779 /* Make sure we're fully resumed: */ 793 /* Make sure we're fully resumed: */
@@ -839,6 +853,9 @@ void update_wall_time(void)
839 timekeeper.ntp_error += timekeeper.xtime_nsec << 853 timekeeper.ntp_error += timekeeper.xtime_nsec <<
840 timekeeper.ntp_error_shift; 854 timekeeper.ntp_error_shift;
841 855
856 nsecs = clocksource_cyc2ns(offset, timekeeper.mult, timekeeper.shift);
857 update_xtime_cache(nsecs);
858
842 /* check to see if there is a new clocksource to use */ 859 /* check to see if there is a new clocksource to use */
843 update_vsyscall(&xtime, timekeeper.clock, timekeeper.mult); 860 update_vsyscall(&xtime, timekeeper.clock, timekeeper.mult);
844} 861}
@@ -875,13 +892,13 @@ void monotonic_to_bootbased(struct timespec *ts)
875 892
876unsigned long get_seconds(void) 893unsigned long get_seconds(void)
877{ 894{
878 return xtime.tv_sec; 895 return xtime_cache.tv_sec;
879} 896}
880EXPORT_SYMBOL(get_seconds); 897EXPORT_SYMBOL(get_seconds);
881 898
882struct timespec __current_kernel_time(void) 899struct timespec __current_kernel_time(void)
883{ 900{
884 return xtime; 901 return xtime_cache;
885} 902}
886 903
887struct timespec current_kernel_time(void) 904struct timespec current_kernel_time(void)
@@ -891,7 +908,8 @@ struct timespec current_kernel_time(void)
891 908
892 do { 909 do {
893 seq = read_seqbegin(&xtime_lock); 910 seq = read_seqbegin(&xtime_lock);
894 now = xtime; 911
912 now = xtime_cache;
895 } while (read_seqretry(&xtime_lock, seq)); 913 } while (read_seqretry(&xtime_lock, seq));
896 914
897 return now; 915 return now;
@@ -905,7 +923,8 @@ struct timespec get_monotonic_coarse(void)
905 923
906 do { 924 do {
907 seq = read_seqbegin(&xtime_lock); 925 seq = read_seqbegin(&xtime_lock);
908 now = xtime; 926
927 now = xtime_cache;
909 mono = wall_to_monotonic; 928 mono = wall_to_monotonic;
910 } while (read_seqretry(&xtime_lock, seq)); 929 } while (read_seqretry(&xtime_lock, seq));
911 930
diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c
index 28265636b6c2..bdfb8dd1050c 100644
--- a/kernel/time/timer_list.c
+++ b/kernel/time/timer_list.c
@@ -237,10 +237,10 @@ static void timer_list_show_tickdevices(struct seq_file *m)
237#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST 237#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
238 print_tickdevice(m, tick_get_broadcast_device(), -1); 238 print_tickdevice(m, tick_get_broadcast_device(), -1);
239 SEQ_printf(m, "tick_broadcast_mask: %08lx\n", 239 SEQ_printf(m, "tick_broadcast_mask: %08lx\n",
240 tick_get_broadcast_mask()->bits[0]); 240 cpumask_bits(tick_get_broadcast_mask())[0]);
241#ifdef CONFIG_TICK_ONESHOT 241#ifdef CONFIG_TICK_ONESHOT
242 SEQ_printf(m, "tick_broadcast_oneshot_mask: %08lx\n", 242 SEQ_printf(m, "tick_broadcast_oneshot_mask: %08lx\n",
243 tick_get_broadcast_oneshot_mask()->bits[0]); 243 cpumask_bits(tick_get_broadcast_oneshot_mask())[0]);
244#endif 244#endif
245 SEQ_printf(m, "\n"); 245 SEQ_printf(m, "\n");
246#endif 246#endif
diff --git a/kernel/timer.c b/kernel/timer.c
index 5db5a8d26811..15533b792397 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -656,8 +656,6 @@ __mod_timer(struct timer_list *timer, unsigned long expires,
656 656
657 debug_activate(timer, expires); 657 debug_activate(timer, expires);
658 658
659 new_base = __get_cpu_var(tvec_bases);
660
661 cpu = smp_processor_id(); 659 cpu = smp_processor_id();
662 660
663#if defined(CONFIG_NO_HZ) && defined(CONFIG_SMP) 661#if defined(CONFIG_NO_HZ) && defined(CONFIG_SMP)
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 06ba26747d7e..8b9f20ab8eed 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -12,7 +12,7 @@
12 * Copyright (C) 2004 William Lee Irwin III 12 * Copyright (C) 2004 William Lee Irwin III
13 */ 13 */
14#include <linux/ring_buffer.h> 14#include <linux/ring_buffer.h>
15#include <linux/utsrelease.h> 15#include <generated/utsrelease.h>
16#include <linux/stacktrace.h> 16#include <linux/stacktrace.h>
17#include <linux/writeback.h> 17#include <linux/writeback.h>
18#include <linux/kallsyms.h> 18#include <linux/kallsyms.h>
diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c
index 7ecab06547a5..375f81a568dc 100644
--- a/kernel/trace/trace_kprobe.c
+++ b/kernel/trace/trace_kprobe.c
@@ -282,6 +282,18 @@ static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);
282static int kretprobe_dispatcher(struct kretprobe_instance *ri, 282static int kretprobe_dispatcher(struct kretprobe_instance *ri,
283 struct pt_regs *regs); 283 struct pt_regs *regs);
284 284
285/* Check the name is good for event/group */
286static int check_event_name(const char *name)
287{
288 if (!isalpha(*name) && *name != '_')
289 return 0;
290 while (*++name != '\0') {
291 if (!isalpha(*name) && !isdigit(*name) && *name != '_')
292 return 0;
293 }
294 return 1;
295}
296
285/* 297/*
286 * Allocate new trace_probe and initialize it (including kprobes). 298 * Allocate new trace_probe and initialize it (including kprobes).
287 */ 299 */
@@ -293,10 +305,11 @@ static struct trace_probe *alloc_trace_probe(const char *group,
293 int nargs, int is_return) 305 int nargs, int is_return)
294{ 306{
295 struct trace_probe *tp; 307 struct trace_probe *tp;
308 int ret = -ENOMEM;
296 309
297 tp = kzalloc(SIZEOF_TRACE_PROBE(nargs), GFP_KERNEL); 310 tp = kzalloc(SIZEOF_TRACE_PROBE(nargs), GFP_KERNEL);
298 if (!tp) 311 if (!tp)
299 return ERR_PTR(-ENOMEM); 312 return ERR_PTR(ret);
300 313
301 if (symbol) { 314 if (symbol) {
302 tp->symbol = kstrdup(symbol, GFP_KERNEL); 315 tp->symbol = kstrdup(symbol, GFP_KERNEL);
@@ -312,14 +325,20 @@ static struct trace_probe *alloc_trace_probe(const char *group,
312 else 325 else
313 tp->rp.kp.pre_handler = kprobe_dispatcher; 326 tp->rp.kp.pre_handler = kprobe_dispatcher;
314 327
315 if (!event) 328 if (!event || !check_event_name(event)) {
329 ret = -EINVAL;
316 goto error; 330 goto error;
331 }
332
317 tp->call.name = kstrdup(event, GFP_KERNEL); 333 tp->call.name = kstrdup(event, GFP_KERNEL);
318 if (!tp->call.name) 334 if (!tp->call.name)
319 goto error; 335 goto error;
320 336
321 if (!group) 337 if (!group || !check_event_name(group)) {
338 ret = -EINVAL;
322 goto error; 339 goto error;
340 }
341
323 tp->call.system = kstrdup(group, GFP_KERNEL); 342 tp->call.system = kstrdup(group, GFP_KERNEL);
324 if (!tp->call.system) 343 if (!tp->call.system)
325 goto error; 344 goto error;
@@ -330,7 +349,7 @@ error:
330 kfree(tp->call.name); 349 kfree(tp->call.name);
331 kfree(tp->symbol); 350 kfree(tp->symbol);
332 kfree(tp); 351 kfree(tp);
333 return ERR_PTR(-ENOMEM); 352 return ERR_PTR(ret);
334} 353}
335 354
336static void free_probe_arg(struct probe_arg *arg) 355static void free_probe_arg(struct probe_arg *arg)
@@ -695,10 +714,10 @@ static int create_trace_probe(int argc, char **argv)
695 if (!event) { 714 if (!event) {
696 /* Make a new event name */ 715 /* Make a new event name */
697 if (symbol) 716 if (symbol)
698 snprintf(buf, MAX_EVENT_NAME_LEN, "%c@%s%+ld", 717 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld",
699 is_return ? 'r' : 'p', symbol, offset); 718 is_return ? 'r' : 'p', symbol, offset);
700 else 719 else
701 snprintf(buf, MAX_EVENT_NAME_LEN, "%c@0x%p", 720 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_0x%p",
702 is_return ? 'r' : 'p', addr); 721 is_return ? 'r' : 'p', addr);
703 event = buf; 722 event = buf;
704 } 723 }
diff --git a/kernel/trace/trace_sysprof.c b/kernel/trace/trace_sysprof.c
index f6693969287d..a7974a552ca9 100644
--- a/kernel/trace/trace_sysprof.c
+++ b/kernel/trace/trace_sysprof.c
@@ -93,6 +93,7 @@ static const struct stacktrace_ops backtrace_ops = {
93 .warning_symbol = backtrace_warning_symbol, 93 .warning_symbol = backtrace_warning_symbol,
94 .stack = backtrace_stack, 94 .stack = backtrace_stack,
95 .address = backtrace_address, 95 .address = backtrace_address,
96 .walk_stack = print_context_stack,
96}; 97};
97 98
98static int 99static int