aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/Kconfig.hz2
-rw-r--r--kernel/Makefile24
-rw-r--r--kernel/acct.c222
-rw-r--r--kernel/auditsc.c3
-rw-r--r--kernel/backtracetest.c65
-rw-r--r--kernel/capability.c338
-rw-r--r--kernel/cgroup.c312
-rw-r--r--kernel/cpu.c72
-rw-r--r--kernel/cpuset.c377
-rw-r--r--kernel/delayacct.c16
-rw-r--r--kernel/exec_domain.c3
-rw-r--r--kernel/exit.c520
-rw-r--r--kernel/fork.c135
-rw-r--r--kernel/hrtimer.c13
-rw-r--r--kernel/irq/chip.c12
-rw-r--r--kernel/irq/manage.c139
-rw-r--r--kernel/irq/proc.c59
-rw-r--r--kernel/kallsyms.c2
-rw-r--r--kernel/kexec.c104
-rw-r--r--kernel/kmod.c15
-rw-r--r--kernel/kprobes.c132
-rw-r--r--kernel/kthread.c9
-rw-r--r--kernel/lockdep.c80
-rw-r--r--kernel/lockdep_internals.h6
-rw-r--r--kernel/lockdep_proc.c97
-rw-r--r--kernel/marker.c55
-rw-r--r--kernel/module.c336
-rw-r--r--kernel/mutex-debug.c2
-rw-r--r--kernel/mutex.c5
-rw-r--r--kernel/ns_cgroup.c8
-rw-r--r--kernel/nsproxy.c8
-rw-r--r--kernel/panic.c22
-rw-r--r--kernel/pid.c11
-rw-r--r--kernel/pid_namespace.c10
-rw-r--r--kernel/pm_qos_params.c7
-rw-r--r--kernel/posix-cpu-timers.c3
-rw-r--r--kernel/posix-timers.c21
-rw-r--r--kernel/power/Kconfig13
-rw-r--r--kernel/power/disk.c50
-rw-r--r--kernel/power/main.c217
-rw-r--r--kernel/power/power.h2
-rw-r--r--kernel/power/poweroff.c4
-rw-r--r--kernel/power/process.c99
-rw-r--r--kernel/power/snapshot.c88
-rw-r--r--kernel/power/user.c71
-rw-r--r--kernel/printk.c131
-rw-r--r--kernel/profile.c10
-rw-r--r--kernel/ptrace.c54
-rw-r--r--kernel/rcuclassic.c38
-rw-r--r--kernel/rcupdate.c71
-rw-r--r--kernel/rcupreempt.c430
-rw-r--r--kernel/rcupreempt_trace.c1
-rw-r--r--kernel/rcutorture.c174
-rw-r--r--kernel/relay.c170
-rw-r--r--kernel/res_counter.c48
-rw-r--r--kernel/rtmutex-tester.c7
-rw-r--r--kernel/sched.c1215
-rw-r--r--kernel/sched_clock.c137
-rw-r--r--kernel/sched_cpupri.c174
-rw-r--r--kernel/sched_cpupri.h36
-rw-r--r--kernel/sched_debug.c64
-rw-r--r--kernel/sched_fair.c423
-rw-r--r--kernel/sched_features.h7
-rw-r--r--kernel/sched_rt.c436
-rw-r--r--kernel/sched_stats.h42
-rw-r--r--kernel/semaphore.c1
-rw-r--r--kernel/signal.c179
-rw-r--r--kernel/smp.c385
-rw-r--r--kernel/softirq.c72
-rw-r--r--kernel/softlockup.c70
-rw-r--r--kernel/spinlock.c2
-rw-r--r--kernel/stacktrace.c14
-rw-r--r--kernel/stop_machine.c5
-rw-r--r--kernel/sys.c35
-rw-r--r--kernel/sys_ni.c7
-rw-r--r--kernel/sysctl.c250
-rw-r--r--kernel/sysctl_check.c2
-rw-r--r--kernel/taskstats.c6
-rw-r--r--kernel/time/clocksource.c12
-rw-r--r--kernel/time/tick-broadcast.c11
-rw-r--r--kernel/time/tick-common.c14
-rw-r--r--kernel/time/tick-sched.c26
-rw-r--r--kernel/timer.c12
-rw-r--r--kernel/trace/Kconfig135
-rw-r--r--kernel/trace/Makefile24
-rw-r--r--kernel/trace/ftrace.c1727
-rw-r--r--kernel/trace/trace.c3157
-rw-r--r--kernel/trace/trace.h339
-rw-r--r--kernel/trace/trace_functions.c81
-rw-r--r--kernel/trace/trace_irqsoff.c490
-rw-r--r--kernel/trace/trace_mmiotrace.c295
-rw-r--r--kernel/trace/trace_sched_switch.c286
-rw-r--r--kernel/trace/trace_sched_wakeup.c453
-rw-r--r--kernel/trace/trace_selftest.c563
-rw-r--r--kernel/trace/trace_selftest_dynamic.c7
-rw-r--r--kernel/trace/trace_sysprof.c365
-rw-r--r--kernel/tsacct.c39
-rw-r--r--kernel/workqueue.c149
98 files changed, 13972 insertions, 2698 deletions
diff --git a/kernel/Kconfig.hz b/kernel/Kconfig.hz
index 526128a2e622..382dd5a8b2d7 100644
--- a/kernel/Kconfig.hz
+++ b/kernel/Kconfig.hz
@@ -55,4 +55,4 @@ config HZ
55 default 1000 if HZ_1000 55 default 1000 if HZ_1000
56 56
57config SCHED_HRTICK 57config SCHED_HRTICK
58 def_bool HIGH_RES_TIMERS && X86 58 def_bool HIGH_RES_TIMERS && USE_GENERIC_SMP_HELPERS
diff --git a/kernel/Makefile b/kernel/Makefile
index 1c9938addb9d..54f69837d35a 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -2,8 +2,8 @@
2# Makefile for the linux kernel. 2# Makefile for the linux kernel.
3# 3#
4 4
5obj-y = sched.o fork.o exec_domain.o panic.o printk.o profile.o \ 5obj-y = sched.o fork.o exec_domain.o panic.o printk.o \
6 exit.o itimer.o time.o softirq.o resource.o \ 6 cpu.o exit.o itimer.o time.o softirq.o resource.o \
7 sysctl.o capability.o ptrace.o timer.o user.o \ 7 sysctl.o capability.o ptrace.o timer.o user.o \
8 signal.o sys.o kmod.o workqueue.o pid.o \ 8 signal.o sys.o kmod.o workqueue.o pid.o \
9 rcupdate.o extable.o params.o posix-timers.o \ 9 rcupdate.o extable.o params.o posix-timers.o \
@@ -11,6 +11,20 @@ obj-y = sched.o fork.o exec_domain.o panic.o printk.o profile.o \
11 hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \ 11 hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \
12 notifier.o ksysfs.o pm_qos_params.o sched_clock.o 12 notifier.o ksysfs.o pm_qos_params.o sched_clock.o
13 13
14CFLAGS_REMOVE_sched.o = -mno-spe
15
16ifdef CONFIG_FTRACE
17# Do not trace debug files and internal ftrace files
18CFLAGS_REMOVE_lockdep.o = -pg
19CFLAGS_REMOVE_lockdep_proc.o = -pg
20CFLAGS_REMOVE_mutex-debug.o = -pg
21CFLAGS_REMOVE_rtmutex-debug.o = -pg
22CFLAGS_REMOVE_cgroup-debug.o = -pg
23CFLAGS_REMOVE_sched_clock.o = -pg
24CFLAGS_REMOVE_sched.o = -mno-spe -pg
25endif
26
27obj-$(CONFIG_PROFILING) += profile.o
14obj-$(CONFIG_SYSCTL_SYSCALL_CHECK) += sysctl_check.o 28obj-$(CONFIG_SYSCTL_SYSCALL_CHECK) += sysctl_check.o
15obj-$(CONFIG_STACKTRACE) += stacktrace.o 29obj-$(CONFIG_STACKTRACE) += stacktrace.o
16obj-y += time/ 30obj-y += time/
@@ -27,7 +41,8 @@ obj-$(CONFIG_RT_MUTEXES) += rtmutex.o
27obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o 41obj-$(CONFIG_DEBUG_RT_MUTEXES) += rtmutex-debug.o
28obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o 42obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o
29obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o 43obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o
30obj-$(CONFIG_SMP) += cpu.o spinlock.o 44obj-$(CONFIG_USE_GENERIC_SMP_HELPERS) += smp.o
45obj-$(CONFIG_SMP) += spinlock.o
31obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock.o 46obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock.o
32obj-$(CONFIG_PROVE_LOCKING) += spinlock.o 47obj-$(CONFIG_PROVE_LOCKING) += spinlock.o
33obj-$(CONFIG_UID16) += uid16.o 48obj-$(CONFIG_UID16) += uid16.o
@@ -69,6 +84,9 @@ obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o
69obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o 84obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o
70obj-$(CONFIG_MARKERS) += marker.o 85obj-$(CONFIG_MARKERS) += marker.o
71obj-$(CONFIG_LATENCYTOP) += latencytop.o 86obj-$(CONFIG_LATENCYTOP) += latencytop.o
87obj-$(CONFIG_FTRACE) += trace/
88obj-$(CONFIG_TRACING) += trace/
89obj-$(CONFIG_SMP) += sched_cpupri.o
72 90
73ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y) 91ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y)
74# According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is 92# According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is
diff --git a/kernel/acct.c b/kernel/acct.c
index 91e1cfd734d2..dd68b9059418 100644
--- a/kernel/acct.c
+++ b/kernel/acct.c
@@ -75,37 +75,39 @@ int acct_parm[3] = {4, 2, 30};
75/* 75/*
76 * External references and all of the globals. 76 * External references and all of the globals.
77 */ 77 */
78static void do_acct_process(struct pid_namespace *ns, struct file *); 78static void do_acct_process(struct bsd_acct_struct *acct,
79 struct pid_namespace *ns, struct file *);
79 80
80/* 81/*
81 * This structure is used so that all the data protected by lock 82 * This structure is used so that all the data protected by lock
82 * can be placed in the same cache line as the lock. This primes 83 * can be placed in the same cache line as the lock. This primes
83 * the cache line to have the data after getting the lock. 84 * the cache line to have the data after getting the lock.
84 */ 85 */
85struct acct_glbs { 86struct bsd_acct_struct {
86 spinlock_t lock;
87 volatile int active; 87 volatile int active;
88 volatile int needcheck; 88 volatile int needcheck;
89 struct file *file; 89 struct file *file;
90 struct pid_namespace *ns; 90 struct pid_namespace *ns;
91 struct timer_list timer; 91 struct timer_list timer;
92 struct list_head list;
92}; 93};
93 94
94static struct acct_glbs acct_globals __cacheline_aligned = 95static DEFINE_SPINLOCK(acct_lock);
95 {__SPIN_LOCK_UNLOCKED(acct_globals.lock)}; 96static LIST_HEAD(acct_list);
96 97
97/* 98/*
98 * Called whenever the timer says to check the free space. 99 * Called whenever the timer says to check the free space.
99 */ 100 */
100static void acct_timeout(unsigned long unused) 101static void acct_timeout(unsigned long x)
101{ 102{
102 acct_globals.needcheck = 1; 103 struct bsd_acct_struct *acct = (struct bsd_acct_struct *)x;
104 acct->needcheck = 1;
103} 105}
104 106
105/* 107/*
106 * Check the amount of free space and suspend/resume accordingly. 108 * Check the amount of free space and suspend/resume accordingly.
107 */ 109 */
108static int check_free_space(struct file *file) 110static int check_free_space(struct bsd_acct_struct *acct, struct file *file)
109{ 111{
110 struct kstatfs sbuf; 112 struct kstatfs sbuf;
111 int res; 113 int res;
@@ -113,11 +115,11 @@ static int check_free_space(struct file *file)
113 sector_t resume; 115 sector_t resume;
114 sector_t suspend; 116 sector_t suspend;
115 117
116 spin_lock(&acct_globals.lock); 118 spin_lock(&acct_lock);
117 res = acct_globals.active; 119 res = acct->active;
118 if (!file || !acct_globals.needcheck) 120 if (!file || !acct->needcheck)
119 goto out; 121 goto out;
120 spin_unlock(&acct_globals.lock); 122 spin_unlock(&acct_lock);
121 123
122 /* May block */ 124 /* May block */
123 if (vfs_statfs(file->f_path.dentry, &sbuf)) 125 if (vfs_statfs(file->f_path.dentry, &sbuf))
@@ -136,35 +138,35 @@ static int check_free_space(struct file *file)
136 act = 0; 138 act = 0;
137 139
138 /* 140 /*
139 * If some joker switched acct_globals.file under us we'ld better be 141 * If some joker switched acct->file under us we'ld better be
140 * silent and _not_ touch anything. 142 * silent and _not_ touch anything.
141 */ 143 */
142 spin_lock(&acct_globals.lock); 144 spin_lock(&acct_lock);
143 if (file != acct_globals.file) { 145 if (file != acct->file) {
144 if (act) 146 if (act)
145 res = act>0; 147 res = act>0;
146 goto out; 148 goto out;
147 } 149 }
148 150
149 if (acct_globals.active) { 151 if (acct->active) {
150 if (act < 0) { 152 if (act < 0) {
151 acct_globals.active = 0; 153 acct->active = 0;
152 printk(KERN_INFO "Process accounting paused\n"); 154 printk(KERN_INFO "Process accounting paused\n");
153 } 155 }
154 } else { 156 } else {
155 if (act > 0) { 157 if (act > 0) {
156 acct_globals.active = 1; 158 acct->active = 1;
157 printk(KERN_INFO "Process accounting resumed\n"); 159 printk(KERN_INFO "Process accounting resumed\n");
158 } 160 }
159 } 161 }
160 162
161 del_timer(&acct_globals.timer); 163 del_timer(&acct->timer);
162 acct_globals.needcheck = 0; 164 acct->needcheck = 0;
163 acct_globals.timer.expires = jiffies + ACCT_TIMEOUT*HZ; 165 acct->timer.expires = jiffies + ACCT_TIMEOUT*HZ;
164 add_timer(&acct_globals.timer); 166 add_timer(&acct->timer);
165 res = acct_globals.active; 167 res = acct->active;
166out: 168out:
167 spin_unlock(&acct_globals.lock); 169 spin_unlock(&acct_lock);
168 return res; 170 return res;
169} 171}
170 172
@@ -172,39 +174,41 @@ out:
172 * Close the old accounting file (if currently open) and then replace 174 * Close the old accounting file (if currently open) and then replace
173 * it with file (if non-NULL). 175 * it with file (if non-NULL).
174 * 176 *
175 * NOTE: acct_globals.lock MUST be held on entry and exit. 177 * NOTE: acct_lock MUST be held on entry and exit.
176 */ 178 */
177static void acct_file_reopen(struct file *file) 179static void acct_file_reopen(struct bsd_acct_struct *acct, struct file *file,
180 struct pid_namespace *ns)
178{ 181{
179 struct file *old_acct = NULL; 182 struct file *old_acct = NULL;
180 struct pid_namespace *old_ns = NULL; 183 struct pid_namespace *old_ns = NULL;
181 184
182 if (acct_globals.file) { 185 if (acct->file) {
183 old_acct = acct_globals.file; 186 old_acct = acct->file;
184 old_ns = acct_globals.ns; 187 old_ns = acct->ns;
185 del_timer(&acct_globals.timer); 188 del_timer(&acct->timer);
186 acct_globals.active = 0; 189 acct->active = 0;
187 acct_globals.needcheck = 0; 190 acct->needcheck = 0;
188 acct_globals.file = NULL; 191 acct->file = NULL;
192 acct->ns = NULL;
193 list_del(&acct->list);
189 } 194 }
190 if (file) { 195 if (file) {
191 acct_globals.file = file; 196 acct->file = file;
192 acct_globals.ns = get_pid_ns(task_active_pid_ns(current)); 197 acct->ns = ns;
193 acct_globals.needcheck = 0; 198 acct->needcheck = 0;
194 acct_globals.active = 1; 199 acct->active = 1;
200 list_add(&acct->list, &acct_list);
195 /* It's been deleted if it was used before so this is safe */ 201 /* It's been deleted if it was used before so this is safe */
196 init_timer(&acct_globals.timer); 202 setup_timer(&acct->timer, acct_timeout, (unsigned long)acct);
197 acct_globals.timer.function = acct_timeout; 203 acct->timer.expires = jiffies + ACCT_TIMEOUT*HZ;
198 acct_globals.timer.expires = jiffies + ACCT_TIMEOUT*HZ; 204 add_timer(&acct->timer);
199 add_timer(&acct_globals.timer);
200 } 205 }
201 if (old_acct) { 206 if (old_acct) {
202 mnt_unpin(old_acct->f_path.mnt); 207 mnt_unpin(old_acct->f_path.mnt);
203 spin_unlock(&acct_globals.lock); 208 spin_unlock(&acct_lock);
204 do_acct_process(old_ns, old_acct); 209 do_acct_process(acct, old_ns, old_acct);
205 filp_close(old_acct, NULL); 210 filp_close(old_acct, NULL);
206 put_pid_ns(old_ns); 211 spin_lock(&acct_lock);
207 spin_lock(&acct_globals.lock);
208 } 212 }
209} 213}
210 214
@@ -212,6 +216,8 @@ static int acct_on(char *name)
212{ 216{
213 struct file *file; 217 struct file *file;
214 int error; 218 int error;
219 struct pid_namespace *ns;
220 struct bsd_acct_struct *acct = NULL;
215 221
216 /* Difference from BSD - they don't do O_APPEND */ 222 /* Difference from BSD - they don't do O_APPEND */
217 file = filp_open(name, O_WRONLY|O_APPEND|O_LARGEFILE, 0); 223 file = filp_open(name, O_WRONLY|O_APPEND|O_LARGEFILE, 0);
@@ -228,18 +234,34 @@ static int acct_on(char *name)
228 return -EIO; 234 return -EIO;
229 } 235 }
230 236
237 ns = task_active_pid_ns(current);
238 if (ns->bacct == NULL) {
239 acct = kzalloc(sizeof(struct bsd_acct_struct), GFP_KERNEL);
240 if (acct == NULL) {
241 filp_close(file, NULL);
242 return -ENOMEM;
243 }
244 }
245
231 error = security_acct(file); 246 error = security_acct(file);
232 if (error) { 247 if (error) {
248 kfree(acct);
233 filp_close(file, NULL); 249 filp_close(file, NULL);
234 return error; 250 return error;
235 } 251 }
236 252
237 spin_lock(&acct_globals.lock); 253 spin_lock(&acct_lock);
254 if (ns->bacct == NULL) {
255 ns->bacct = acct;
256 acct = NULL;
257 }
258
238 mnt_pin(file->f_path.mnt); 259 mnt_pin(file->f_path.mnt);
239 acct_file_reopen(file); 260 acct_file_reopen(ns->bacct, file, ns);
240 spin_unlock(&acct_globals.lock); 261 spin_unlock(&acct_lock);
241 262
242 mntput(file->f_path.mnt); /* it's pinned, now give up active reference */ 263 mntput(file->f_path.mnt); /* it's pinned, now give up active reference */
264 kfree(acct);
243 265
244 return 0; 266 return 0;
245} 267}
@@ -269,11 +291,17 @@ asmlinkage long sys_acct(const char __user *name)
269 error = acct_on(tmp); 291 error = acct_on(tmp);
270 putname(tmp); 292 putname(tmp);
271 } else { 293 } else {
294 struct bsd_acct_struct *acct;
295
296 acct = task_active_pid_ns(current)->bacct;
297 if (acct == NULL)
298 return 0;
299
272 error = security_acct(NULL); 300 error = security_acct(NULL);
273 if (!error) { 301 if (!error) {
274 spin_lock(&acct_globals.lock); 302 spin_lock(&acct_lock);
275 acct_file_reopen(NULL); 303 acct_file_reopen(acct, NULL, NULL);
276 spin_unlock(&acct_globals.lock); 304 spin_unlock(&acct_lock);
277 } 305 }
278 } 306 }
279 return error; 307 return error;
@@ -288,10 +316,16 @@ asmlinkage long sys_acct(const char __user *name)
288 */ 316 */
289void acct_auto_close_mnt(struct vfsmount *m) 317void acct_auto_close_mnt(struct vfsmount *m)
290{ 318{
291 spin_lock(&acct_globals.lock); 319 struct bsd_acct_struct *acct;
292 if (acct_globals.file && acct_globals.file->f_path.mnt == m) 320
293 acct_file_reopen(NULL); 321 spin_lock(&acct_lock);
294 spin_unlock(&acct_globals.lock); 322restart:
323 list_for_each_entry(acct, &acct_list, list)
324 if (acct->file && acct->file->f_path.mnt == m) {
325 acct_file_reopen(acct, NULL, NULL);
326 goto restart;
327 }
328 spin_unlock(&acct_lock);
295} 329}
296 330
297/** 331/**
@@ -303,12 +337,31 @@ void acct_auto_close_mnt(struct vfsmount *m)
303 */ 337 */
304void acct_auto_close(struct super_block *sb) 338void acct_auto_close(struct super_block *sb)
305{ 339{
306 spin_lock(&acct_globals.lock); 340 struct bsd_acct_struct *acct;
307 if (acct_globals.file && 341
308 acct_globals.file->f_path.mnt->mnt_sb == sb) { 342 spin_lock(&acct_lock);
309 acct_file_reopen(NULL); 343restart:
344 list_for_each_entry(acct, &acct_list, list)
345 if (acct->file && acct->file->f_path.mnt->mnt_sb == sb) {
346 acct_file_reopen(acct, NULL, NULL);
347 goto restart;
348 }
349 spin_unlock(&acct_lock);
350}
351
352void acct_exit_ns(struct pid_namespace *ns)
353{
354 struct bsd_acct_struct *acct;
355
356 spin_lock(&acct_lock);
357 acct = ns->bacct;
358 if (acct != NULL) {
359 if (acct->file != NULL)
360 acct_file_reopen(acct, NULL, NULL);
361
362 kfree(acct);
310 } 363 }
311 spin_unlock(&acct_globals.lock); 364 spin_unlock(&acct_lock);
312} 365}
313 366
314/* 367/*
@@ -425,7 +478,8 @@ static u32 encode_float(u64 value)
425/* 478/*
426 * do_acct_process does all actual work. Caller holds the reference to file. 479 * do_acct_process does all actual work. Caller holds the reference to file.
427 */ 480 */
428static void do_acct_process(struct pid_namespace *ns, struct file *file) 481static void do_acct_process(struct bsd_acct_struct *acct,
482 struct pid_namespace *ns, struct file *file)
429{ 483{
430 struct pacct_struct *pacct = &current->signal->pacct; 484 struct pacct_struct *pacct = &current->signal->pacct;
431 acct_t ac; 485 acct_t ac;
@@ -440,7 +494,7 @@ static void do_acct_process(struct pid_namespace *ns, struct file *file)
440 * First check to see if there is enough free_space to continue 494 * First check to see if there is enough free_space to continue
441 * the process accounting system. 495 * the process accounting system.
442 */ 496 */
443 if (!check_free_space(file)) 497 if (!check_free_space(acct, file))
444 return; 498 return;
445 499
446 /* 500 /*
@@ -577,34 +631,46 @@ void acct_collect(long exitcode, int group_dead)
577 spin_unlock_irq(&current->sighand->siglock); 631 spin_unlock_irq(&current->sighand->siglock);
578} 632}
579 633
580/** 634static void acct_process_in_ns(struct pid_namespace *ns)
581 * acct_process - now just a wrapper around do_acct_process
582 * @exitcode: task exit code
583 *
584 * handles process accounting for an exiting task
585 */
586void acct_process(void)
587{ 635{
588 struct file *file = NULL; 636 struct file *file = NULL;
589 struct pid_namespace *ns; 637 struct bsd_acct_struct *acct;
590 638
639 acct = ns->bacct;
591 /* 640 /*
592 * accelerate the common fastpath: 641 * accelerate the common fastpath:
593 */ 642 */
594 if (!acct_globals.file) 643 if (!acct || !acct->file)
595 return; 644 return;
596 645
597 spin_lock(&acct_globals.lock); 646 spin_lock(&acct_lock);
598 file = acct_globals.file; 647 file = acct->file;
599 if (unlikely(!file)) { 648 if (unlikely(!file)) {
600 spin_unlock(&acct_globals.lock); 649 spin_unlock(&acct_lock);
601 return; 650 return;
602 } 651 }
603 get_file(file); 652 get_file(file);
604 ns = get_pid_ns(acct_globals.ns); 653 spin_unlock(&acct_lock);
605 spin_unlock(&acct_globals.lock);
606 654
607 do_acct_process(ns, file); 655 do_acct_process(acct, ns, file);
608 fput(file); 656 fput(file);
609 put_pid_ns(ns); 657}
658
659/**
660 * acct_process - now just a wrapper around acct_process_in_ns,
661 * which in turn is a wrapper around do_acct_process.
662 *
663 * handles process accounting for an exiting task
664 */
665void acct_process(void)
666{
667 struct pid_namespace *ns;
668
669 /*
670 * This loop is safe lockless, since current is still
671 * alive and holds its namespace, which in turn holds
672 * its parent.
673 */
674 for (ns = task_active_pid_ns(current); ns != NULL; ns = ns->parent)
675 acct_process_in_ns(ns);
610} 676}
diff --git a/kernel/auditsc.c b/kernel/auditsc.c
index c10e7aae04d7..4699950e65bd 100644
--- a/kernel/auditsc.c
+++ b/kernel/auditsc.c
@@ -1476,7 +1476,8 @@ void audit_syscall_entry(int arch, int major,
1476 struct audit_context *context = tsk->audit_context; 1476 struct audit_context *context = tsk->audit_context;
1477 enum audit_state state; 1477 enum audit_state state;
1478 1478
1479 BUG_ON(!context); 1479 if (unlikely(!context))
1480 return;
1480 1481
1481 /* 1482 /*
1482 * This happens only on certain architectures that make system 1483 * This happens only on certain architectures that make system
diff --git a/kernel/backtracetest.c b/kernel/backtracetest.c
index d1a7605c5b8f..a5e026bc45c4 100644
--- a/kernel/backtracetest.c
+++ b/kernel/backtracetest.c
@@ -10,30 +10,73 @@
10 * of the License. 10 * of the License.
11 */ 11 */
12 12
13#include <linux/completion.h>
14#include <linux/delay.h>
15#include <linux/interrupt.h>
13#include <linux/module.h> 16#include <linux/module.h>
14#include <linux/sched.h> 17#include <linux/sched.h>
15#include <linux/delay.h> 18#include <linux/stacktrace.h>
19
20static void backtrace_test_normal(void)
21{
22 printk("Testing a backtrace from process context.\n");
23 printk("The following trace is a kernel self test and not a bug!\n");
16 24
17static struct timer_list backtrace_timer; 25 dump_stack();
26}
18 27
19static void backtrace_test_timer(unsigned long data) 28static DECLARE_COMPLETION(backtrace_work);
29
30static void backtrace_test_irq_callback(unsigned long data)
31{
32 dump_stack();
33 complete(&backtrace_work);
34}
35
36static DECLARE_TASKLET(backtrace_tasklet, &backtrace_test_irq_callback, 0);
37
38static void backtrace_test_irq(void)
20{ 39{
21 printk("Testing a backtrace from irq context.\n"); 40 printk("Testing a backtrace from irq context.\n");
22 printk("The following trace is a kernel self test and not a bug!\n"); 41 printk("The following trace is a kernel self test and not a bug!\n");
23 dump_stack(); 42
43 init_completion(&backtrace_work);
44 tasklet_schedule(&backtrace_tasklet);
45 wait_for_completion(&backtrace_work);
46}
47
48#ifdef CONFIG_STACKTRACE
49static void backtrace_test_saved(void)
50{
51 struct stack_trace trace;
52 unsigned long entries[8];
53
54 printk("Testing a saved backtrace.\n");
55 printk("The following trace is a kernel self test and not a bug!\n");
56
57 trace.nr_entries = 0;
58 trace.max_entries = ARRAY_SIZE(entries);
59 trace.entries = entries;
60 trace.skip = 0;
61
62 save_stack_trace(&trace);
63 print_stack_trace(&trace, 0);
64}
65#else
66static void backtrace_test_saved(void)
67{
68 printk("Saved backtrace test skipped.\n");
24} 69}
70#endif
71
25static int backtrace_regression_test(void) 72static int backtrace_regression_test(void)
26{ 73{
27 printk("====[ backtrace testing ]===========\n"); 74 printk("====[ backtrace testing ]===========\n");
28 printk("Testing a backtrace from process context.\n");
29 printk("The following trace is a kernel self test and not a bug!\n");
30 dump_stack();
31 75
32 init_timer(&backtrace_timer); 76 backtrace_test_normal();
33 backtrace_timer.function = backtrace_test_timer; 77 backtrace_test_irq();
34 mod_timer(&backtrace_timer, jiffies + 10); 78 backtrace_test_saved();
35 79
36 msleep(10);
37 printk("====[ end of backtrace testing ]====\n"); 80 printk("====[ end of backtrace testing ]====\n");
38 return 0; 81 return 0;
39} 82}
diff --git a/kernel/capability.c b/kernel/capability.c
index 901e0fdc3fff..0101e847603e 100644
--- a/kernel/capability.c
+++ b/kernel/capability.c
@@ -115,11 +115,208 @@ static int cap_validate_magic(cap_user_header_t header, unsigned *tocopy)
115 return 0; 115 return 0;
116} 116}
117 117
118#ifndef CONFIG_SECURITY_FILE_CAPABILITIES
119
120/*
121 * Without filesystem capability support, we nominally support one process
122 * setting the capabilities of another
123 */
124static inline int cap_get_target_pid(pid_t pid, kernel_cap_t *pEp,
125 kernel_cap_t *pIp, kernel_cap_t *pPp)
126{
127 struct task_struct *target;
128 int ret;
129
130 spin_lock(&task_capability_lock);
131 read_lock(&tasklist_lock);
132
133 if (pid && pid != task_pid_vnr(current)) {
134 target = find_task_by_vpid(pid);
135 if (!target) {
136 ret = -ESRCH;
137 goto out;
138 }
139 } else
140 target = current;
141
142 ret = security_capget(target, pEp, pIp, pPp);
143
144out:
145 read_unlock(&tasklist_lock);
146 spin_unlock(&task_capability_lock);
147
148 return ret;
149}
150
151/*
152 * cap_set_pg - set capabilities for all processes in a given process
153 * group. We call this holding task_capability_lock and tasklist_lock.
154 */
155static inline int cap_set_pg(int pgrp_nr, kernel_cap_t *effective,
156 kernel_cap_t *inheritable,
157 kernel_cap_t *permitted)
158{
159 struct task_struct *g, *target;
160 int ret = -EPERM;
161 int found = 0;
162 struct pid *pgrp;
163
164 spin_lock(&task_capability_lock);
165 read_lock(&tasklist_lock);
166
167 pgrp = find_vpid(pgrp_nr);
168 do_each_pid_task(pgrp, PIDTYPE_PGID, g) {
169 target = g;
170 while_each_thread(g, target) {
171 if (!security_capset_check(target, effective,
172 inheritable, permitted)) {
173 security_capset_set(target, effective,
174 inheritable, permitted);
175 ret = 0;
176 }
177 found = 1;
178 }
179 } while_each_pid_task(pgrp, PIDTYPE_PGID, g);
180
181 read_unlock(&tasklist_lock);
182 spin_unlock(&task_capability_lock);
183
184 if (!found)
185 ret = 0;
186 return ret;
187}
188
118/* 189/*
119 * For sys_getproccap() and sys_setproccap(), any of the three 190 * cap_set_all - set capabilities for all processes other than init
120 * capability set pointers may be NULL -- indicating that that set is 191 * and self. We call this holding task_capability_lock and tasklist_lock.
121 * uninteresting and/or not to be changed.
122 */ 192 */
193static inline int cap_set_all(kernel_cap_t *effective,
194 kernel_cap_t *inheritable,
195 kernel_cap_t *permitted)
196{
197 struct task_struct *g, *target;
198 int ret = -EPERM;
199 int found = 0;
200
201 spin_lock(&task_capability_lock);
202 read_lock(&tasklist_lock);
203
204 do_each_thread(g, target) {
205 if (target == current
206 || is_container_init(target->group_leader))
207 continue;
208 found = 1;
209 if (security_capset_check(target, effective, inheritable,
210 permitted))
211 continue;
212 ret = 0;
213 security_capset_set(target, effective, inheritable, permitted);
214 } while_each_thread(g, target);
215
216 read_unlock(&tasklist_lock);
217 spin_unlock(&task_capability_lock);
218
219 if (!found)
220 ret = 0;
221
222 return ret;
223}
224
225/*
226 * Given the target pid does not refer to the current process we
227 * need more elaborate support... (This support is not present when
228 * filesystem capabilities are configured.)
229 */
230static inline int do_sys_capset_other_tasks(pid_t pid, kernel_cap_t *effective,
231 kernel_cap_t *inheritable,
232 kernel_cap_t *permitted)
233{
234 struct task_struct *target;
235 int ret;
236
237 if (!capable(CAP_SETPCAP))
238 return -EPERM;
239
240 if (pid == -1) /* all procs other than current and init */
241 return cap_set_all(effective, inheritable, permitted);
242
243 else if (pid < 0) /* all procs in process group */
244 return cap_set_pg(-pid, effective, inheritable, permitted);
245
246 /* target != current */
247 spin_lock(&task_capability_lock);
248 read_lock(&tasklist_lock);
249
250 target = find_task_by_vpid(pid);
251 if (!target)
252 ret = -ESRCH;
253 else {
254 ret = security_capset_check(target, effective, inheritable,
255 permitted);
256
257 /* having verified that the proposed changes are legal,
258 we now put them into effect. */
259 if (!ret)
260 security_capset_set(target, effective, inheritable,
261 permitted);
262 }
263
264 read_unlock(&tasklist_lock);
265 spin_unlock(&task_capability_lock);
266
267 return ret;
268}
269
270#else /* ie., def CONFIG_SECURITY_FILE_CAPABILITIES */
271
272/*
273 * If we have configured with filesystem capability support, then the
274 * only thing that can change the capabilities of the current process
275 * is the current process. As such, we can't be in this code at the
276 * same time as we are in the process of setting capabilities in this
277 * process. The net result is that we can limit our use of locks to
278 * when we are reading the caps of another process.
279 */
280static inline int cap_get_target_pid(pid_t pid, kernel_cap_t *pEp,
281 kernel_cap_t *pIp, kernel_cap_t *pPp)
282{
283 int ret;
284
285 if (pid && (pid != task_pid_vnr(current))) {
286 struct task_struct *target;
287
288 spin_lock(&task_capability_lock);
289 read_lock(&tasklist_lock);
290
291 target = find_task_by_vpid(pid);
292 if (!target)
293 ret = -ESRCH;
294 else
295 ret = security_capget(target, pEp, pIp, pPp);
296
297 read_unlock(&tasklist_lock);
298 spin_unlock(&task_capability_lock);
299 } else
300 ret = security_capget(current, pEp, pIp, pPp);
301
302 return ret;
303}
304
305/*
306 * With filesystem capability support configured, the kernel does not
307 * permit the changing of capabilities in one process by another
308 * process. (CAP_SETPCAP has much less broad semantics when configured
309 * this way.)
310 */
311static inline int do_sys_capset_other_tasks(pid_t pid,
312 kernel_cap_t *effective,
313 kernel_cap_t *inheritable,
314 kernel_cap_t *permitted)
315{
316 return -EPERM;
317}
318
319#endif /* ie., ndef CONFIG_SECURITY_FILE_CAPABILITIES */
123 320
124/* 321/*
125 * Atomically modify the effective capabilities returning the original 322 * Atomically modify the effective capabilities returning the original
@@ -155,7 +352,6 @@ asmlinkage long sys_capget(cap_user_header_t header, cap_user_data_t dataptr)
155{ 352{
156 int ret = 0; 353 int ret = 0;
157 pid_t pid; 354 pid_t pid;
158 struct task_struct *target;
159 unsigned tocopy; 355 unsigned tocopy;
160 kernel_cap_t pE, pI, pP; 356 kernel_cap_t pE, pI, pP;
161 357
@@ -169,23 +365,7 @@ asmlinkage long sys_capget(cap_user_header_t header, cap_user_data_t dataptr)
169 if (pid < 0) 365 if (pid < 0)
170 return -EINVAL; 366 return -EINVAL;
171 367
172 spin_lock(&task_capability_lock); 368 ret = cap_get_target_pid(pid, &pE, &pI, &pP);
173 read_lock(&tasklist_lock);
174
175 if (pid && pid != task_pid_vnr(current)) {
176 target = find_task_by_vpid(pid);
177 if (!target) {
178 ret = -ESRCH;
179 goto out;
180 }
181 } else
182 target = current;
183
184 ret = security_capget(target, &pE, &pI, &pP);
185
186out:
187 read_unlock(&tasklist_lock);
188 spin_unlock(&task_capability_lock);
189 369
190 if (!ret) { 370 if (!ret) {
191 struct __user_cap_data_struct kdata[_KERNEL_CAPABILITY_U32S]; 371 struct __user_cap_data_struct kdata[_KERNEL_CAPABILITY_U32S];
@@ -216,7 +396,6 @@ out:
216 * before modification is attempted and the application 396 * before modification is attempted and the application
217 * fails. 397 * fails.
218 */ 398 */
219
220 if (copy_to_user(dataptr, kdata, tocopy 399 if (copy_to_user(dataptr, kdata, tocopy
221 * sizeof(struct __user_cap_data_struct))) { 400 * sizeof(struct __user_cap_data_struct))) {
222 return -EFAULT; 401 return -EFAULT;
@@ -226,70 +405,8 @@ out:
226 return ret; 405 return ret;
227} 406}
228 407
229/*
230 * cap_set_pg - set capabilities for all processes in a given process
231 * group. We call this holding task_capability_lock and tasklist_lock.
232 */
233static inline int cap_set_pg(int pgrp_nr, kernel_cap_t *effective,
234 kernel_cap_t *inheritable,
235 kernel_cap_t *permitted)
236{
237 struct task_struct *g, *target;
238 int ret = -EPERM;
239 int found = 0;
240 struct pid *pgrp;
241
242 pgrp = find_vpid(pgrp_nr);
243 do_each_pid_task(pgrp, PIDTYPE_PGID, g) {
244 target = g;
245 while_each_thread(g, target) {
246 if (!security_capset_check(target, effective,
247 inheritable,
248 permitted)) {
249 security_capset_set(target, effective,
250 inheritable,
251 permitted);
252 ret = 0;
253 }
254 found = 1;
255 }
256 } while_each_pid_task(pgrp, PIDTYPE_PGID, g);
257
258 if (!found)
259 ret = 0;
260 return ret;
261}
262
263/*
264 * cap_set_all - set capabilities for all processes other than init
265 * and self. We call this holding task_capability_lock and tasklist_lock.
266 */
267static inline int cap_set_all(kernel_cap_t *effective,
268 kernel_cap_t *inheritable,
269 kernel_cap_t *permitted)
270{
271 struct task_struct *g, *target;
272 int ret = -EPERM;
273 int found = 0;
274
275 do_each_thread(g, target) {
276 if (target == current || is_container_init(target->group_leader))
277 continue;
278 found = 1;
279 if (security_capset_check(target, effective, inheritable,
280 permitted))
281 continue;
282 ret = 0;
283 security_capset_set(target, effective, inheritable, permitted);
284 } while_each_thread(g, target);
285
286 if (!found)
287 ret = 0;
288 return ret;
289}
290
291/** 408/**
292 * sys_capset - set capabilities for a process or a group of processes 409 * sys_capset - set capabilities for a process or (*) a group of processes
293 * @header: pointer to struct that contains capability version and 410 * @header: pointer to struct that contains capability version and
294 * target pid data 411 * target pid data
295 * @data: pointer to struct that contains the effective, permitted, 412 * @data: pointer to struct that contains the effective, permitted,
@@ -313,7 +430,6 @@ asmlinkage long sys_capset(cap_user_header_t header, const cap_user_data_t data)
313 struct __user_cap_data_struct kdata[_KERNEL_CAPABILITY_U32S]; 430 struct __user_cap_data_struct kdata[_KERNEL_CAPABILITY_U32S];
314 unsigned i, tocopy; 431 unsigned i, tocopy;
315 kernel_cap_t inheritable, permitted, effective; 432 kernel_cap_t inheritable, permitted, effective;
316 struct task_struct *target;
317 int ret; 433 int ret;
318 pid_t pid; 434 pid_t pid;
319 435
@@ -324,9 +440,6 @@ asmlinkage long sys_capset(cap_user_header_t header, const cap_user_data_t data)
324 if (get_user(pid, &header->pid)) 440 if (get_user(pid, &header->pid))
325 return -EFAULT; 441 return -EFAULT;
326 442
327 if (pid && pid != task_pid_vnr(current) && !capable(CAP_SETPCAP))
328 return -EPERM;
329
330 if (copy_from_user(&kdata, data, tocopy 443 if (copy_from_user(&kdata, data, tocopy
331 * sizeof(struct __user_cap_data_struct))) { 444 * sizeof(struct __user_cap_data_struct))) {
332 return -EFAULT; 445 return -EFAULT;
@@ -344,40 +457,31 @@ asmlinkage long sys_capset(cap_user_header_t header, const cap_user_data_t data)
344 i++; 457 i++;
345 } 458 }
346 459
347 spin_lock(&task_capability_lock); 460 if (pid && (pid != task_pid_vnr(current)))
348 read_lock(&tasklist_lock); 461 ret = do_sys_capset_other_tasks(pid, &effective, &inheritable,
349 462 &permitted);
350 if (pid > 0 && pid != task_pid_vnr(current)) { 463 else {
351 target = find_task_by_vpid(pid); 464 /*
352 if (!target) { 465 * This lock is required even when filesystem
353 ret = -ESRCH; 466 * capability support is configured - it protects the
354 goto out; 467 * sys_capget() call from returning incorrect data in
355 } 468 * the case that the targeted process is not the
356 } else 469 * current one.
357 target = current; 470 */
358 471 spin_lock(&task_capability_lock);
359 ret = 0;
360
361 /* having verified that the proposed changes are legal,
362 we now put them into effect. */
363 if (pid < 0) {
364 if (pid == -1) /* all procs other than current and init */
365 ret = cap_set_all(&effective, &inheritable, &permitted);
366 472
367 else /* all procs in process group */ 473 ret = security_capset_check(current, &effective, &inheritable,
368 ret = cap_set_pg(-pid, &effective, &inheritable,
369 &permitted);
370 } else {
371 ret = security_capset_check(target, &effective, &inheritable,
372 &permitted); 474 &permitted);
475 /*
476 * Having verified that the proposed changes are
477 * legal, we now put them into effect.
478 */
373 if (!ret) 479 if (!ret)
374 security_capset_set(target, &effective, &inheritable, 480 security_capset_set(current, &effective, &inheritable,
375 &permitted); 481 &permitted);
482 spin_unlock(&task_capability_lock);
376 } 483 }
377 484
378out:
379 read_unlock(&tasklist_lock);
380 spin_unlock(&task_capability_lock);
381 485
382 return ret; 486 return ret;
383} 487}
diff --git a/kernel/cgroup.c b/kernel/cgroup.c
index 15ac0e1e4f4d..657f8f8d93a5 100644
--- a/kernel/cgroup.c
+++ b/kernel/cgroup.c
@@ -45,6 +45,7 @@
45#include <linux/delayacct.h> 45#include <linux/delayacct.h>
46#include <linux/cgroupstats.h> 46#include <linux/cgroupstats.h>
47#include <linux/hash.h> 47#include <linux/hash.h>
48#include <linux/namei.h>
48 49
49#include <asm/atomic.h> 50#include <asm/atomic.h>
50 51
@@ -89,11 +90,7 @@ struct cgroupfs_root {
89 /* Hierarchy-specific flags */ 90 /* Hierarchy-specific flags */
90 unsigned long flags; 91 unsigned long flags;
91 92
92 /* The path to use for release notifications. No locking 93 /* The path to use for release notifications. */
93 * between setting and use - so if userspace updates this
94 * while child cgroups exist, you could miss a
95 * notification. We ensure that it's always a valid
96 * NUL-terminated string */
97 char release_agent_path[PATH_MAX]; 94 char release_agent_path[PATH_MAX];
98}; 95};
99 96
@@ -118,7 +115,7 @@ static int root_count;
118 * extra work in the fork/exit path if none of the subsystems need to 115 * extra work in the fork/exit path if none of the subsystems need to
119 * be called. 116 * be called.
120 */ 117 */
121static int need_forkexit_callback; 118static int need_forkexit_callback __read_mostly;
122static int need_mm_owner_callback __read_mostly; 119static int need_mm_owner_callback __read_mostly;
123 120
124/* convenient tests for these bits */ 121/* convenient tests for these bits */
@@ -220,7 +217,7 @@ static struct hlist_head *css_set_hash(struct cgroup_subsys_state *css[])
220 * task until after the first call to cgroup_iter_start(). This 217 * task until after the first call to cgroup_iter_start(). This
221 * reduces the fork()/exit() overhead for people who have cgroups 218 * reduces the fork()/exit() overhead for people who have cgroups
222 * compiled into their kernel but not actually in use */ 219 * compiled into their kernel but not actually in use */
223static int use_task_css_set_links; 220static int use_task_css_set_links __read_mostly;
224 221
225/* When we create or destroy a css_set, the operation simply 222/* When we create or destroy a css_set, the operation simply
226 * takes/releases a reference count on all the cgroups referenced 223 * takes/releases a reference count on all the cgroups referenced
@@ -241,17 +238,20 @@ static int use_task_css_set_links;
241 */ 238 */
242static void unlink_css_set(struct css_set *cg) 239static void unlink_css_set(struct css_set *cg)
243{ 240{
241 struct cg_cgroup_link *link;
242 struct cg_cgroup_link *saved_link;
243
244 write_lock(&css_set_lock); 244 write_lock(&css_set_lock);
245 hlist_del(&cg->hlist); 245 hlist_del(&cg->hlist);
246 css_set_count--; 246 css_set_count--;
247 while (!list_empty(&cg->cg_links)) { 247
248 struct cg_cgroup_link *link; 248 list_for_each_entry_safe(link, saved_link, &cg->cg_links,
249 link = list_entry(cg->cg_links.next, 249 cg_link_list) {
250 struct cg_cgroup_link, cg_link_list);
251 list_del(&link->cg_link_list); 250 list_del(&link->cg_link_list);
252 list_del(&link->cgrp_link_list); 251 list_del(&link->cgrp_link_list);
253 kfree(link); 252 kfree(link);
254 } 253 }
254
255 write_unlock(&css_set_lock); 255 write_unlock(&css_set_lock);
256} 256}
257 257
@@ -363,15 +363,14 @@ static struct css_set *find_existing_css_set(
363static int allocate_cg_links(int count, struct list_head *tmp) 363static int allocate_cg_links(int count, struct list_head *tmp)
364{ 364{
365 struct cg_cgroup_link *link; 365 struct cg_cgroup_link *link;
366 struct cg_cgroup_link *saved_link;
366 int i; 367 int i;
367 INIT_LIST_HEAD(tmp); 368 INIT_LIST_HEAD(tmp);
368 for (i = 0; i < count; i++) { 369 for (i = 0; i < count; i++) {
369 link = kmalloc(sizeof(*link), GFP_KERNEL); 370 link = kmalloc(sizeof(*link), GFP_KERNEL);
370 if (!link) { 371 if (!link) {
371 while (!list_empty(tmp)) { 372 list_for_each_entry_safe(link, saved_link, tmp,
372 link = list_entry(tmp->next, 373 cgrp_link_list) {
373 struct cg_cgroup_link,
374 cgrp_link_list);
375 list_del(&link->cgrp_link_list); 374 list_del(&link->cgrp_link_list);
376 kfree(link); 375 kfree(link);
377 } 376 }
@@ -384,11 +383,10 @@ static int allocate_cg_links(int count, struct list_head *tmp)
384 383
385static void free_cg_links(struct list_head *tmp) 384static void free_cg_links(struct list_head *tmp)
386{ 385{
387 while (!list_empty(tmp)) { 386 struct cg_cgroup_link *link;
388 struct cg_cgroup_link *link; 387 struct cg_cgroup_link *saved_link;
389 link = list_entry(tmp->next, 388
390 struct cg_cgroup_link, 389 list_for_each_entry_safe(link, saved_link, tmp, cgrp_link_list) {
391 cgrp_link_list);
392 list_del(&link->cgrp_link_list); 390 list_del(&link->cgrp_link_list);
393 kfree(link); 391 kfree(link);
394 } 392 }
@@ -415,11 +413,11 @@ static struct css_set *find_css_set(
415 413
416 /* First see if we already have a cgroup group that matches 414 /* First see if we already have a cgroup group that matches
417 * the desired set */ 415 * the desired set */
418 write_lock(&css_set_lock); 416 read_lock(&css_set_lock);
419 res = find_existing_css_set(oldcg, cgrp, template); 417 res = find_existing_css_set(oldcg, cgrp, template);
420 if (res) 418 if (res)
421 get_css_set(res); 419 get_css_set(res);
422 write_unlock(&css_set_lock); 420 read_unlock(&css_set_lock);
423 421
424 if (res) 422 if (res)
425 return res; 423 return res;
@@ -507,10 +505,6 @@ static struct css_set *find_css_set(
507 * knows that the cgroup won't be removed, as cgroup_rmdir() 505 * knows that the cgroup won't be removed, as cgroup_rmdir()
508 * needs that mutex. 506 * needs that mutex.
509 * 507 *
510 * The cgroup_common_file_write handler for operations that modify
511 * the cgroup hierarchy holds cgroup_mutex across the entire operation,
512 * single threading all such cgroup modifications across the system.
513 *
514 * The fork and exit callbacks cgroup_fork() and cgroup_exit(), don't 508 * The fork and exit callbacks cgroup_fork() and cgroup_exit(), don't
515 * (usually) take cgroup_mutex. These are the two most performance 509 * (usually) take cgroup_mutex. These are the two most performance
516 * critical pieces of code here. The exception occurs on cgroup_exit(), 510 * critical pieces of code here. The exception occurs on cgroup_exit(),
@@ -1093,6 +1087,8 @@ static void cgroup_kill_sb(struct super_block *sb) {
1093 struct cgroupfs_root *root = sb->s_fs_info; 1087 struct cgroupfs_root *root = sb->s_fs_info;
1094 struct cgroup *cgrp = &root->top_cgroup; 1088 struct cgroup *cgrp = &root->top_cgroup;
1095 int ret; 1089 int ret;
1090 struct cg_cgroup_link *link;
1091 struct cg_cgroup_link *saved_link;
1096 1092
1097 BUG_ON(!root); 1093 BUG_ON(!root);
1098 1094
@@ -1112,10 +1108,9 @@ static void cgroup_kill_sb(struct super_block *sb) {
1112 * root cgroup 1108 * root cgroup
1113 */ 1109 */
1114 write_lock(&css_set_lock); 1110 write_lock(&css_set_lock);
1115 while (!list_empty(&cgrp->css_sets)) { 1111
1116 struct cg_cgroup_link *link; 1112 list_for_each_entry_safe(link, saved_link, &cgrp->css_sets,
1117 link = list_entry(cgrp->css_sets.next, 1113 cgrp_link_list) {
1118 struct cg_cgroup_link, cgrp_link_list);
1119 list_del(&link->cg_link_list); 1114 list_del(&link->cg_link_list);
1120 list_del(&link->cgrp_link_list); 1115 list_del(&link->cgrp_link_list);
1121 kfree(link); 1116 kfree(link);
@@ -1281,18 +1276,14 @@ int cgroup_attach_task(struct cgroup *cgrp, struct task_struct *tsk)
1281} 1276}
1282 1277
1283/* 1278/*
1284 * Attach task with pid 'pid' to cgroup 'cgrp'. Call with 1279 * Attach task with pid 'pid' to cgroup 'cgrp'. Call with cgroup_mutex
1285 * cgroup_mutex, may take task_lock of task 1280 * held. May take task_lock of task
1286 */ 1281 */
1287static int attach_task_by_pid(struct cgroup *cgrp, char *pidbuf) 1282static int attach_task_by_pid(struct cgroup *cgrp, u64 pid)
1288{ 1283{
1289 pid_t pid;
1290 struct task_struct *tsk; 1284 struct task_struct *tsk;
1291 int ret; 1285 int ret;
1292 1286
1293 if (sscanf(pidbuf, "%d", &pid) != 1)
1294 return -EIO;
1295
1296 if (pid) { 1287 if (pid) {
1297 rcu_read_lock(); 1288 rcu_read_lock();
1298 tsk = find_task_by_vpid(pid); 1289 tsk = find_task_by_vpid(pid);
@@ -1318,6 +1309,16 @@ static int attach_task_by_pid(struct cgroup *cgrp, char *pidbuf)
1318 return ret; 1309 return ret;
1319} 1310}
1320 1311
1312static int cgroup_tasks_write(struct cgroup *cgrp, struct cftype *cft, u64 pid)
1313{
1314 int ret;
1315 if (!cgroup_lock_live_group(cgrp))
1316 return -ENODEV;
1317 ret = attach_task_by_pid(cgrp, pid);
1318 cgroup_unlock();
1319 return ret;
1320}
1321
1321/* The various types of files and directories in a cgroup file system */ 1322/* The various types of files and directories in a cgroup file system */
1322enum cgroup_filetype { 1323enum cgroup_filetype {
1323 FILE_ROOT, 1324 FILE_ROOT,
@@ -1327,12 +1328,54 @@ enum cgroup_filetype {
1327 FILE_RELEASE_AGENT, 1328 FILE_RELEASE_AGENT,
1328}; 1329};
1329 1330
1331/**
1332 * cgroup_lock_live_group - take cgroup_mutex and check that cgrp is alive.
1333 * @cgrp: the cgroup to be checked for liveness
1334 *
1335 * On success, returns true; the lock should be later released with
1336 * cgroup_unlock(). On failure returns false with no lock held.
1337 */
1338bool cgroup_lock_live_group(struct cgroup *cgrp)
1339{
1340 mutex_lock(&cgroup_mutex);
1341 if (cgroup_is_removed(cgrp)) {
1342 mutex_unlock(&cgroup_mutex);
1343 return false;
1344 }
1345 return true;
1346}
1347
1348static int cgroup_release_agent_write(struct cgroup *cgrp, struct cftype *cft,
1349 const char *buffer)
1350{
1351 BUILD_BUG_ON(sizeof(cgrp->root->release_agent_path) < PATH_MAX);
1352 if (!cgroup_lock_live_group(cgrp))
1353 return -ENODEV;
1354 strcpy(cgrp->root->release_agent_path, buffer);
1355 cgroup_unlock();
1356 return 0;
1357}
1358
1359static int cgroup_release_agent_show(struct cgroup *cgrp, struct cftype *cft,
1360 struct seq_file *seq)
1361{
1362 if (!cgroup_lock_live_group(cgrp))
1363 return -ENODEV;
1364 seq_puts(seq, cgrp->root->release_agent_path);
1365 seq_putc(seq, '\n');
1366 cgroup_unlock();
1367 return 0;
1368}
1369
1370/* A buffer size big enough for numbers or short strings */
1371#define CGROUP_LOCAL_BUFFER_SIZE 64
1372
1330static ssize_t cgroup_write_X64(struct cgroup *cgrp, struct cftype *cft, 1373static ssize_t cgroup_write_X64(struct cgroup *cgrp, struct cftype *cft,
1331 struct file *file, 1374 struct file *file,
1332 const char __user *userbuf, 1375 const char __user *userbuf,
1333 size_t nbytes, loff_t *unused_ppos) 1376 size_t nbytes, loff_t *unused_ppos)
1334{ 1377{
1335 char buffer[64]; 1378 char buffer[CGROUP_LOCAL_BUFFER_SIZE];
1336 int retval = 0; 1379 int retval = 0;
1337 char *end; 1380 char *end;
1338 1381
@@ -1361,68 +1404,36 @@ static ssize_t cgroup_write_X64(struct cgroup *cgrp, struct cftype *cft,
1361 return retval; 1404 return retval;
1362} 1405}
1363 1406
1364static ssize_t cgroup_common_file_write(struct cgroup *cgrp, 1407static ssize_t cgroup_write_string(struct cgroup *cgrp, struct cftype *cft,
1365 struct cftype *cft, 1408 struct file *file,
1366 struct file *file, 1409 const char __user *userbuf,
1367 const char __user *userbuf, 1410 size_t nbytes, loff_t *unused_ppos)
1368 size_t nbytes, loff_t *unused_ppos)
1369{ 1411{
1370 enum cgroup_filetype type = cft->private; 1412 char local_buffer[CGROUP_LOCAL_BUFFER_SIZE];
1371 char *buffer;
1372 int retval = 0; 1413 int retval = 0;
1414 size_t max_bytes = cft->max_write_len;
1415 char *buffer = local_buffer;
1373 1416
1374 if (nbytes >= PATH_MAX) 1417 if (!max_bytes)
1418 max_bytes = sizeof(local_buffer) - 1;
1419 if (nbytes >= max_bytes)
1375 return -E2BIG; 1420 return -E2BIG;
1376 1421 /* Allocate a dynamic buffer if we need one */
1377 /* +1 for nul-terminator */ 1422 if (nbytes >= sizeof(local_buffer)) {
1378 buffer = kmalloc(nbytes + 1, GFP_KERNEL); 1423 buffer = kmalloc(nbytes + 1, GFP_KERNEL);
1379 if (buffer == NULL) 1424 if (buffer == NULL)
1380 return -ENOMEM; 1425 return -ENOMEM;
1381
1382 if (copy_from_user(buffer, userbuf, nbytes)) {
1383 retval = -EFAULT;
1384 goto out1;
1385 } 1426 }
1386 buffer[nbytes] = 0; /* nul-terminate */ 1427 if (nbytes && copy_from_user(buffer, userbuf, nbytes))
1387 strstrip(buffer); /* strip -just- trailing whitespace */ 1428 return -EFAULT;
1388
1389 mutex_lock(&cgroup_mutex);
1390 1429
1391 /* 1430 buffer[nbytes] = 0; /* nul-terminate */
1392 * This was already checked for in cgroup_file_write(), but 1431 strstrip(buffer);
1393 * check again now we're holding cgroup_mutex. 1432 retval = cft->write_string(cgrp, cft, buffer);
1394 */ 1433 if (!retval)
1395 if (cgroup_is_removed(cgrp)) {
1396 retval = -ENODEV;
1397 goto out2;
1398 }
1399
1400 switch (type) {
1401 case FILE_TASKLIST:
1402 retval = attach_task_by_pid(cgrp, buffer);
1403 break;
1404 case FILE_NOTIFY_ON_RELEASE:
1405 clear_bit(CGRP_RELEASABLE, &cgrp->flags);
1406 if (simple_strtoul(buffer, NULL, 10) != 0)
1407 set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
1408 else
1409 clear_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
1410 break;
1411 case FILE_RELEASE_AGENT:
1412 BUILD_BUG_ON(sizeof(cgrp->root->release_agent_path) < PATH_MAX);
1413 strcpy(cgrp->root->release_agent_path, buffer);
1414 break;
1415 default:
1416 retval = -EINVAL;
1417 goto out2;
1418 }
1419
1420 if (retval == 0)
1421 retval = nbytes; 1434 retval = nbytes;
1422out2: 1435 if (buffer != local_buffer)
1423 mutex_unlock(&cgroup_mutex); 1436 kfree(buffer);
1424out1:
1425 kfree(buffer);
1426 return retval; 1437 return retval;
1427} 1438}
1428 1439
@@ -1438,6 +1449,8 @@ static ssize_t cgroup_file_write(struct file *file, const char __user *buf,
1438 return cft->write(cgrp, cft, file, buf, nbytes, ppos); 1449 return cft->write(cgrp, cft, file, buf, nbytes, ppos);
1439 if (cft->write_u64 || cft->write_s64) 1450 if (cft->write_u64 || cft->write_s64)
1440 return cgroup_write_X64(cgrp, cft, file, buf, nbytes, ppos); 1451 return cgroup_write_X64(cgrp, cft, file, buf, nbytes, ppos);
1452 if (cft->write_string)
1453 return cgroup_write_string(cgrp, cft, file, buf, nbytes, ppos);
1441 if (cft->trigger) { 1454 if (cft->trigger) {
1442 int ret = cft->trigger(cgrp, (unsigned int)cft->private); 1455 int ret = cft->trigger(cgrp, (unsigned int)cft->private);
1443 return ret ? ret : nbytes; 1456 return ret ? ret : nbytes;
@@ -1450,7 +1463,7 @@ static ssize_t cgroup_read_u64(struct cgroup *cgrp, struct cftype *cft,
1450 char __user *buf, size_t nbytes, 1463 char __user *buf, size_t nbytes,
1451 loff_t *ppos) 1464 loff_t *ppos)
1452{ 1465{
1453 char tmp[64]; 1466 char tmp[CGROUP_LOCAL_BUFFER_SIZE];
1454 u64 val = cft->read_u64(cgrp, cft); 1467 u64 val = cft->read_u64(cgrp, cft);
1455 int len = sprintf(tmp, "%llu\n", (unsigned long long) val); 1468 int len = sprintf(tmp, "%llu\n", (unsigned long long) val);
1456 1469
@@ -1462,56 +1475,13 @@ static ssize_t cgroup_read_s64(struct cgroup *cgrp, struct cftype *cft,
1462 char __user *buf, size_t nbytes, 1475 char __user *buf, size_t nbytes,
1463 loff_t *ppos) 1476 loff_t *ppos)
1464{ 1477{
1465 char tmp[64]; 1478 char tmp[CGROUP_LOCAL_BUFFER_SIZE];
1466 s64 val = cft->read_s64(cgrp, cft); 1479 s64 val = cft->read_s64(cgrp, cft);
1467 int len = sprintf(tmp, "%lld\n", (long long) val); 1480 int len = sprintf(tmp, "%lld\n", (long long) val);
1468 1481
1469 return simple_read_from_buffer(buf, nbytes, ppos, tmp, len); 1482 return simple_read_from_buffer(buf, nbytes, ppos, tmp, len);
1470} 1483}
1471 1484
1472static ssize_t cgroup_common_file_read(struct cgroup *cgrp,
1473 struct cftype *cft,
1474 struct file *file,
1475 char __user *buf,
1476 size_t nbytes, loff_t *ppos)
1477{
1478 enum cgroup_filetype type = cft->private;
1479 char *page;
1480 ssize_t retval = 0;
1481 char *s;
1482
1483 if (!(page = (char *)__get_free_page(GFP_KERNEL)))
1484 return -ENOMEM;
1485
1486 s = page;
1487
1488 switch (type) {
1489 case FILE_RELEASE_AGENT:
1490 {
1491 struct cgroupfs_root *root;
1492 size_t n;
1493 mutex_lock(&cgroup_mutex);
1494 root = cgrp->root;
1495 n = strnlen(root->release_agent_path,
1496 sizeof(root->release_agent_path));
1497 n = min(n, (size_t) PAGE_SIZE);
1498 strncpy(s, root->release_agent_path, n);
1499 mutex_unlock(&cgroup_mutex);
1500 s += n;
1501 break;
1502 }
1503 default:
1504 retval = -EINVAL;
1505 goto out;
1506 }
1507 *s++ = '\n';
1508
1509 retval = simple_read_from_buffer(buf, nbytes, ppos, page, s - page);
1510out:
1511 free_page((unsigned long)page);
1512 return retval;
1513}
1514
1515static ssize_t cgroup_file_read(struct file *file, char __user *buf, 1485static ssize_t cgroup_file_read(struct file *file, char __user *buf,
1516 size_t nbytes, loff_t *ppos) 1486 size_t nbytes, loff_t *ppos)
1517{ 1487{
@@ -1560,7 +1530,7 @@ static int cgroup_seqfile_show(struct seq_file *m, void *arg)
1560 return cft->read_seq_string(state->cgroup, cft, m); 1530 return cft->read_seq_string(state->cgroup, cft, m);
1561} 1531}
1562 1532
1563int cgroup_seqfile_release(struct inode *inode, struct file *file) 1533static int cgroup_seqfile_release(struct inode *inode, struct file *file)
1564{ 1534{
1565 struct seq_file *seq = file->private_data; 1535 struct seq_file *seq = file->private_data;
1566 kfree(seq->private); 1536 kfree(seq->private);
@@ -1569,6 +1539,7 @@ int cgroup_seqfile_release(struct inode *inode, struct file *file)
1569 1539
1570static struct file_operations cgroup_seqfile_operations = { 1540static struct file_operations cgroup_seqfile_operations = {
1571 .read = seq_read, 1541 .read = seq_read,
1542 .write = cgroup_file_write,
1572 .llseek = seq_lseek, 1543 .llseek = seq_lseek,
1573 .release = cgroup_seqfile_release, 1544 .release = cgroup_seqfile_release,
1574}; 1545};
@@ -1756,15 +1727,11 @@ int cgroup_add_files(struct cgroup *cgrp,
1756int cgroup_task_count(const struct cgroup *cgrp) 1727int cgroup_task_count(const struct cgroup *cgrp)
1757{ 1728{
1758 int count = 0; 1729 int count = 0;
1759 struct list_head *l; 1730 struct cg_cgroup_link *link;
1760 1731
1761 read_lock(&css_set_lock); 1732 read_lock(&css_set_lock);
1762 l = cgrp->css_sets.next; 1733 list_for_each_entry(link, &cgrp->css_sets, cgrp_link_list) {
1763 while (l != &cgrp->css_sets) {
1764 struct cg_cgroup_link *link =
1765 list_entry(l, struct cg_cgroup_link, cgrp_link_list);
1766 count += atomic_read(&link->cg->ref.refcount); 1734 count += atomic_read(&link->cg->ref.refcount);
1767 l = l->next;
1768 } 1735 }
1769 read_unlock(&css_set_lock); 1736 read_unlock(&css_set_lock);
1770 return count; 1737 return count;
@@ -2227,6 +2194,18 @@ static u64 cgroup_read_notify_on_release(struct cgroup *cgrp,
2227 return notify_on_release(cgrp); 2194 return notify_on_release(cgrp);
2228} 2195}
2229 2196
2197static int cgroup_write_notify_on_release(struct cgroup *cgrp,
2198 struct cftype *cft,
2199 u64 val)
2200{
2201 clear_bit(CGRP_RELEASABLE, &cgrp->flags);
2202 if (val)
2203 set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
2204 else
2205 clear_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags);
2206 return 0;
2207}
2208
2230/* 2209/*
2231 * for the common functions, 'private' gives the type of file 2210 * for the common functions, 'private' gives the type of file
2232 */ 2211 */
@@ -2235,7 +2214,7 @@ static struct cftype files[] = {
2235 .name = "tasks", 2214 .name = "tasks",
2236 .open = cgroup_tasks_open, 2215 .open = cgroup_tasks_open,
2237 .read = cgroup_tasks_read, 2216 .read = cgroup_tasks_read,
2238 .write = cgroup_common_file_write, 2217 .write_u64 = cgroup_tasks_write,
2239 .release = cgroup_tasks_release, 2218 .release = cgroup_tasks_release,
2240 .private = FILE_TASKLIST, 2219 .private = FILE_TASKLIST,
2241 }, 2220 },
@@ -2243,15 +2222,16 @@ static struct cftype files[] = {
2243 { 2222 {
2244 .name = "notify_on_release", 2223 .name = "notify_on_release",
2245 .read_u64 = cgroup_read_notify_on_release, 2224 .read_u64 = cgroup_read_notify_on_release,
2246 .write = cgroup_common_file_write, 2225 .write_u64 = cgroup_write_notify_on_release,
2247 .private = FILE_NOTIFY_ON_RELEASE, 2226 .private = FILE_NOTIFY_ON_RELEASE,
2248 }, 2227 },
2249}; 2228};
2250 2229
2251static struct cftype cft_release_agent = { 2230static struct cftype cft_release_agent = {
2252 .name = "release_agent", 2231 .name = "release_agent",
2253 .read = cgroup_common_file_read, 2232 .read_seq_string = cgroup_release_agent_show,
2254 .write = cgroup_common_file_write, 2233 .write_string = cgroup_release_agent_write,
2234 .max_write_len = PATH_MAX,
2255 .private = FILE_RELEASE_AGENT, 2235 .private = FILE_RELEASE_AGENT,
2256}; 2236};
2257 2237
@@ -2869,16 +2849,17 @@ void cgroup_exit(struct task_struct *tsk, int run_callbacks)
2869 * cgroup_clone - clone the cgroup the given subsystem is attached to 2849 * cgroup_clone - clone the cgroup the given subsystem is attached to
2870 * @tsk: the task to be moved 2850 * @tsk: the task to be moved
2871 * @subsys: the given subsystem 2851 * @subsys: the given subsystem
2852 * @nodename: the name for the new cgroup
2872 * 2853 *
2873 * Duplicate the current cgroup in the hierarchy that the given 2854 * Duplicate the current cgroup in the hierarchy that the given
2874 * subsystem is attached to, and move this task into the new 2855 * subsystem is attached to, and move this task into the new
2875 * child. 2856 * child.
2876 */ 2857 */
2877int cgroup_clone(struct task_struct *tsk, struct cgroup_subsys *subsys) 2858int cgroup_clone(struct task_struct *tsk, struct cgroup_subsys *subsys,
2859 char *nodename)
2878{ 2860{
2879 struct dentry *dentry; 2861 struct dentry *dentry;
2880 int ret = 0; 2862 int ret = 0;
2881 char nodename[MAX_CGROUP_TYPE_NAMELEN];
2882 struct cgroup *parent, *child; 2863 struct cgroup *parent, *child;
2883 struct inode *inode; 2864 struct inode *inode;
2884 struct css_set *cg; 2865 struct css_set *cg;
@@ -2903,8 +2884,6 @@ int cgroup_clone(struct task_struct *tsk, struct cgroup_subsys *subsys)
2903 cg = tsk->cgroups; 2884 cg = tsk->cgroups;
2904 parent = task_cgroup(tsk, subsys->subsys_id); 2885 parent = task_cgroup(tsk, subsys->subsys_id);
2905 2886
2906 snprintf(nodename, MAX_CGROUP_TYPE_NAMELEN, "%d", tsk->pid);
2907
2908 /* Pin the hierarchy */ 2887 /* Pin the hierarchy */
2909 atomic_inc(&parent->root->sb->s_active); 2888 atomic_inc(&parent->root->sb->s_active);
2910 2889
@@ -3078,27 +3057,24 @@ static void cgroup_release_agent(struct work_struct *work)
3078 while (!list_empty(&release_list)) { 3057 while (!list_empty(&release_list)) {
3079 char *argv[3], *envp[3]; 3058 char *argv[3], *envp[3];
3080 int i; 3059 int i;
3081 char *pathbuf; 3060 char *pathbuf = NULL, *agentbuf = NULL;
3082 struct cgroup *cgrp = list_entry(release_list.next, 3061 struct cgroup *cgrp = list_entry(release_list.next,
3083 struct cgroup, 3062 struct cgroup,
3084 release_list); 3063 release_list);
3085 list_del_init(&cgrp->release_list); 3064 list_del_init(&cgrp->release_list);
3086 spin_unlock(&release_list_lock); 3065 spin_unlock(&release_list_lock);
3087 pathbuf = kmalloc(PAGE_SIZE, GFP_KERNEL); 3066 pathbuf = kmalloc(PAGE_SIZE, GFP_KERNEL);
3088 if (!pathbuf) { 3067 if (!pathbuf)
3089 spin_lock(&release_list_lock); 3068 goto continue_free;
3090 continue; 3069 if (cgroup_path(cgrp, pathbuf, PAGE_SIZE) < 0)
3091 } 3070 goto continue_free;
3092 3071 agentbuf = kstrdup(cgrp->root->release_agent_path, GFP_KERNEL);
3093 if (cgroup_path(cgrp, pathbuf, PAGE_SIZE) < 0) { 3072 if (!agentbuf)
3094 kfree(pathbuf); 3073 goto continue_free;
3095 spin_lock(&release_list_lock);
3096 continue;
3097 }
3098 3074
3099 i = 0; 3075 i = 0;
3100 argv[i++] = cgrp->root->release_agent_path; 3076 argv[i++] = agentbuf;
3101 argv[i++] = (char *)pathbuf; 3077 argv[i++] = pathbuf;
3102 argv[i] = NULL; 3078 argv[i] = NULL;
3103 3079
3104 i = 0; 3080 i = 0;
@@ -3112,8 +3088,10 @@ static void cgroup_release_agent(struct work_struct *work)
3112 * be a slow process */ 3088 * be a slow process */
3113 mutex_unlock(&cgroup_mutex); 3089 mutex_unlock(&cgroup_mutex);
3114 call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC); 3090 call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC);
3115 kfree(pathbuf);
3116 mutex_lock(&cgroup_mutex); 3091 mutex_lock(&cgroup_mutex);
3092 continue_free:
3093 kfree(pathbuf);
3094 kfree(agentbuf);
3117 spin_lock(&release_list_lock); 3095 spin_lock(&release_list_lock);
3118 } 3096 }
3119 spin_unlock(&release_list_lock); 3097 spin_unlock(&release_list_lock);
diff --git a/kernel/cpu.c b/kernel/cpu.c
index c77bc3a1c722..10ba5f1004a5 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -15,6 +15,28 @@
15#include <linux/stop_machine.h> 15#include <linux/stop_machine.h>
16#include <linux/mutex.h> 16#include <linux/mutex.h>
17 17
18/*
19 * Represents all cpu's present in the system
20 * In systems capable of hotplug, this map could dynamically grow
21 * as new cpu's are detected in the system via any platform specific
22 * method, such as ACPI for e.g.
23 */
24cpumask_t cpu_present_map __read_mostly;
25EXPORT_SYMBOL(cpu_present_map);
26
27#ifndef CONFIG_SMP
28
29/*
30 * Represents all cpu's that are currently online.
31 */
32cpumask_t cpu_online_map __read_mostly = CPU_MASK_ALL;
33EXPORT_SYMBOL(cpu_online_map);
34
35cpumask_t cpu_possible_map __read_mostly = CPU_MASK_ALL;
36EXPORT_SYMBOL(cpu_possible_map);
37
38#else /* CONFIG_SMP */
39
18/* Serializes the updates to cpu_online_map, cpu_present_map */ 40/* Serializes the updates to cpu_online_map, cpu_present_map */
19static DEFINE_MUTEX(cpu_add_remove_lock); 41static DEFINE_MUTEX(cpu_add_remove_lock);
20 42
@@ -42,6 +64,8 @@ void __init cpu_hotplug_init(void)
42 cpu_hotplug.refcount = 0; 64 cpu_hotplug.refcount = 0;
43} 65}
44 66
67cpumask_t cpu_active_map;
68
45#ifdef CONFIG_HOTPLUG_CPU 69#ifdef CONFIG_HOTPLUG_CPU
46 70
47void get_online_cpus(void) 71void get_online_cpus(void)
@@ -261,6 +285,11 @@ out_allowed:
261 set_cpus_allowed_ptr(current, &old_allowed); 285 set_cpus_allowed_ptr(current, &old_allowed);
262out_release: 286out_release:
263 cpu_hotplug_done(); 287 cpu_hotplug_done();
288 if (!err) {
289 if (raw_notifier_call_chain(&cpu_chain, CPU_POST_DEAD | mod,
290 hcpu) == NOTIFY_BAD)
291 BUG();
292 }
264 return err; 293 return err;
265} 294}
266 295
@@ -269,14 +298,34 @@ int __ref cpu_down(unsigned int cpu)
269 int err = 0; 298 int err = 0;
270 299
271 cpu_maps_update_begin(); 300 cpu_maps_update_begin();
272 if (cpu_hotplug_disabled) 301
302 if (cpu_hotplug_disabled) {
273 err = -EBUSY; 303 err = -EBUSY;
274 else 304 goto out;
275 err = _cpu_down(cpu, 0); 305 }
306
307 cpu_clear(cpu, cpu_active_map);
308
309 /*
310 * Make sure the all cpus did the reschedule and are not
311 * using stale version of the cpu_active_map.
312 * This is not strictly necessary becuase stop_machine()
313 * that we run down the line already provides the required
314 * synchronization. But it's really a side effect and we do not
315 * want to depend on the innards of the stop_machine here.
316 */
317 synchronize_sched();
318
319 err = _cpu_down(cpu, 0);
276 320
321 if (cpu_online(cpu))
322 cpu_set(cpu, cpu_active_map);
323
324out:
277 cpu_maps_update_done(); 325 cpu_maps_update_done();
278 return err; 326 return err;
279} 327}
328EXPORT_SYMBOL(cpu_down);
280#endif /*CONFIG_HOTPLUG_CPU*/ 329#endif /*CONFIG_HOTPLUG_CPU*/
281 330
282/* Requires cpu_add_remove_lock to be held */ 331/* Requires cpu_add_remove_lock to be held */
@@ -332,11 +381,18 @@ int __cpuinit cpu_up(unsigned int cpu)
332 } 381 }
333 382
334 cpu_maps_update_begin(); 383 cpu_maps_update_begin();
335 if (cpu_hotplug_disabled) 384
385 if (cpu_hotplug_disabled) {
336 err = -EBUSY; 386 err = -EBUSY;
337 else 387 goto out;
338 err = _cpu_up(cpu, 0); 388 }
389
390 err = _cpu_up(cpu, 0);
339 391
392 if (cpu_online(cpu))
393 cpu_set(cpu, cpu_active_map);
394
395out:
340 cpu_maps_update_done(); 396 cpu_maps_update_done();
341 return err; 397 return err;
342} 398}
@@ -390,7 +446,7 @@ void __ref enable_nonboot_cpus(void)
390 goto out; 446 goto out;
391 447
392 printk("Enabling non-boot CPUs ...\n"); 448 printk("Enabling non-boot CPUs ...\n");
393 for_each_cpu_mask(cpu, frozen_cpus) { 449 for_each_cpu_mask_nr(cpu, frozen_cpus) {
394 error = _cpu_up(cpu, 1); 450 error = _cpu_up(cpu, 1);
395 if (!error) { 451 if (!error) {
396 printk("CPU%d is up\n", cpu); 452 printk("CPU%d is up\n", cpu);
@@ -403,3 +459,5 @@ out:
403 cpu_maps_update_done(); 459 cpu_maps_update_done();
404} 460}
405#endif /* CONFIG_PM_SLEEP_SMP */ 461#endif /* CONFIG_PM_SLEEP_SMP */
462
463#endif /* CONFIG_SMP */
diff --git a/kernel/cpuset.c b/kernel/cpuset.c
index 798b3ab054eb..91cf85b36dd5 100644
--- a/kernel/cpuset.c
+++ b/kernel/cpuset.c
@@ -227,10 +227,6 @@ static struct cpuset top_cpuset = {
227 * The task_struct fields mems_allowed and mems_generation may only 227 * The task_struct fields mems_allowed and mems_generation may only
228 * be accessed in the context of that task, so require no locks. 228 * be accessed in the context of that task, so require no locks.
229 * 229 *
230 * The cpuset_common_file_write handler for operations that modify
231 * the cpuset hierarchy holds cgroup_mutex across the entire operation,
232 * single threading all such cpuset modifications across the system.
233 *
234 * The cpuset_common_file_read() handlers only hold callback_mutex across 230 * The cpuset_common_file_read() handlers only hold callback_mutex across
235 * small pieces of code, such as when reading out possibly multi-word 231 * small pieces of code, such as when reading out possibly multi-word
236 * cpumasks and nodemasks. 232 * cpumasks and nodemasks.
@@ -369,7 +365,7 @@ void cpuset_update_task_memory_state(void)
369 my_cpusets_mem_gen = top_cpuset.mems_generation; 365 my_cpusets_mem_gen = top_cpuset.mems_generation;
370 } else { 366 } else {
371 rcu_read_lock(); 367 rcu_read_lock();
372 my_cpusets_mem_gen = task_cs(current)->mems_generation; 368 my_cpusets_mem_gen = task_cs(tsk)->mems_generation;
373 rcu_read_unlock(); 369 rcu_read_unlock();
374 } 370 }
375 371
@@ -500,11 +496,16 @@ update_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c)
500/* 496/*
501 * rebuild_sched_domains() 497 * rebuild_sched_domains()
502 * 498 *
503 * If the flag 'sched_load_balance' of any cpuset with non-empty 499 * This routine will be called to rebuild the scheduler's dynamic
504 * 'cpus' changes, or if the 'cpus' allowed changes in any cpuset 500 * sched domains:
505 * which has that flag enabled, or if any cpuset with a non-empty 501 * - if the flag 'sched_load_balance' of any cpuset with non-empty
506 * 'cpus' is removed, then call this routine to rebuild the 502 * 'cpus' changes,
507 * scheduler's dynamic sched domains. 503 * - or if the 'cpus' allowed changes in any cpuset which has that
504 * flag enabled,
505 * - or if the 'sched_relax_domain_level' of any cpuset which has
506 * that flag enabled and with non-empty 'cpus' changes,
507 * - or if any cpuset with non-empty 'cpus' is removed,
508 * - or if a cpu gets offlined.
508 * 509 *
509 * This routine builds a partial partition of the systems CPUs 510 * This routine builds a partial partition of the systems CPUs
510 * (the set of non-overlappping cpumask_t's in the array 'part' 511 * (the set of non-overlappping cpumask_t's in the array 'part'
@@ -564,7 +565,7 @@ update_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c)
564 * partition_sched_domains(). 565 * partition_sched_domains().
565 */ 566 */
566 567
567static void rebuild_sched_domains(void) 568void rebuild_sched_domains(void)
568{ 569{
569 struct kfifo *q; /* queue of cpusets to be scanned */ 570 struct kfifo *q; /* queue of cpusets to be scanned */
570 struct cpuset *cp; /* scans q */ 571 struct cpuset *cp; /* scans q */
@@ -609,8 +610,13 @@ static void rebuild_sched_domains(void)
609 while (__kfifo_get(q, (void *)&cp, sizeof(cp))) { 610 while (__kfifo_get(q, (void *)&cp, sizeof(cp))) {
610 struct cgroup *cont; 611 struct cgroup *cont;
611 struct cpuset *child; /* scans child cpusets of cp */ 612 struct cpuset *child; /* scans child cpusets of cp */
613
614 if (cpus_empty(cp->cpus_allowed))
615 continue;
616
612 if (is_sched_load_balance(cp)) 617 if (is_sched_load_balance(cp))
613 csa[csn++] = cp; 618 csa[csn++] = cp;
619
614 list_for_each_entry(cont, &cp->css.cgroup->children, sibling) { 620 list_for_each_entry(cont, &cp->css.cgroup->children, sibling) {
615 child = cgroup_cs(cont); 621 child = cgroup_cs(cont);
616 __kfifo_put(q, (void *)&child, sizeof(cp)); 622 __kfifo_put(q, (void *)&child, sizeof(cp));
@@ -679,7 +685,9 @@ restart:
679 if (apn == b->pn) { 685 if (apn == b->pn) {
680 cpus_or(*dp, *dp, b->cpus_allowed); 686 cpus_or(*dp, *dp, b->cpus_allowed);
681 b->pn = -1; 687 b->pn = -1;
682 update_domain_attr(dattr, b); 688 if (dattr)
689 update_domain_attr(dattr
690 + nslot, b);
683 } 691 }
684 } 692 }
685 nslot++; 693 nslot++;
@@ -701,36 +709,6 @@ done:
701 /* Don't kfree(dattr) -- partition_sched_domains() does that. */ 709 /* Don't kfree(dattr) -- partition_sched_domains() does that. */
702} 710}
703 711
704static inline int started_after_time(struct task_struct *t1,
705 struct timespec *time,
706 struct task_struct *t2)
707{
708 int start_diff = timespec_compare(&t1->start_time, time);
709 if (start_diff > 0) {
710 return 1;
711 } else if (start_diff < 0) {
712 return 0;
713 } else {
714 /*
715 * Arbitrarily, if two processes started at the same
716 * time, we'll say that the lower pointer value
717 * started first. Note that t2 may have exited by now
718 * so this may not be a valid pointer any longer, but
719 * that's fine - it still serves to distinguish
720 * between two tasks started (effectively)
721 * simultaneously.
722 */
723 return t1 > t2;
724 }
725}
726
727static inline int started_after(void *p1, void *p2)
728{
729 struct task_struct *t1 = p1;
730 struct task_struct *t2 = p2;
731 return started_after_time(t1, &t2->start_time, t2);
732}
733
734/** 712/**
735 * cpuset_test_cpumask - test a task's cpus_allowed versus its cpuset's 713 * cpuset_test_cpumask - test a task's cpus_allowed versus its cpuset's
736 * @tsk: task to test 714 * @tsk: task to test
@@ -766,15 +744,49 @@ static void cpuset_change_cpumask(struct task_struct *tsk,
766} 744}
767 745
768/** 746/**
747 * update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
748 * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
749 *
750 * Called with cgroup_mutex held
751 *
752 * The cgroup_scan_tasks() function will scan all the tasks in a cgroup,
753 * calling callback functions for each.
754 *
755 * Return 0 if successful, -errno if not.
756 */
757static int update_tasks_cpumask(struct cpuset *cs)
758{
759 struct cgroup_scanner scan;
760 struct ptr_heap heap;
761 int retval;
762
763 /*
764 * cgroup_scan_tasks() will initialize heap->gt for us.
765 * heap_init() is still needed here for we should not change
766 * cs->cpus_allowed when heap_init() fails.
767 */
768 retval = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, NULL);
769 if (retval)
770 return retval;
771
772 scan.cg = cs->css.cgroup;
773 scan.test_task = cpuset_test_cpumask;
774 scan.process_task = cpuset_change_cpumask;
775 scan.heap = &heap;
776 retval = cgroup_scan_tasks(&scan);
777
778 heap_free(&heap);
779 return retval;
780}
781
782/**
769 * update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it 783 * update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it
770 * @cs: the cpuset to consider 784 * @cs: the cpuset to consider
771 * @buf: buffer of cpu numbers written to this cpuset 785 * @buf: buffer of cpu numbers written to this cpuset
772 */ 786 */
773static int update_cpumask(struct cpuset *cs, char *buf) 787static int update_cpumask(struct cpuset *cs, const char *buf)
774{ 788{
775 struct cpuset trialcs; 789 struct cpuset trialcs;
776 struct cgroup_scanner scan;
777 struct ptr_heap heap;
778 int retval; 790 int retval;
779 int is_load_balanced; 791 int is_load_balanced;
780 792
@@ -790,7 +802,6 @@ static int update_cpumask(struct cpuset *cs, char *buf)
790 * that parsing. The validate_change() call ensures that cpusets 802 * that parsing. The validate_change() call ensures that cpusets
791 * with tasks have cpus. 803 * with tasks have cpus.
792 */ 804 */
793 buf = strstrip(buf);
794 if (!*buf) { 805 if (!*buf) {
795 cpus_clear(trialcs.cpus_allowed); 806 cpus_clear(trialcs.cpus_allowed);
796 } else { 807 } else {
@@ -809,10 +820,6 @@ static int update_cpumask(struct cpuset *cs, char *buf)
809 if (cpus_equal(cs->cpus_allowed, trialcs.cpus_allowed)) 820 if (cpus_equal(cs->cpus_allowed, trialcs.cpus_allowed))
810 return 0; 821 return 0;
811 822
812 retval = heap_init(&heap, PAGE_SIZE, GFP_KERNEL, &started_after);
813 if (retval)
814 return retval;
815
816 is_load_balanced = is_sched_load_balance(&trialcs); 823 is_load_balanced = is_sched_load_balance(&trialcs);
817 824
818 mutex_lock(&callback_mutex); 825 mutex_lock(&callback_mutex);
@@ -823,12 +830,9 @@ static int update_cpumask(struct cpuset *cs, char *buf)
823 * Scan tasks in the cpuset, and update the cpumasks of any 830 * Scan tasks in the cpuset, and update the cpumasks of any
824 * that need an update. 831 * that need an update.
825 */ 832 */
826 scan.cg = cs->css.cgroup; 833 retval = update_tasks_cpumask(cs);
827 scan.test_task = cpuset_test_cpumask; 834 if (retval < 0)
828 scan.process_task = cpuset_change_cpumask; 835 return retval;
829 scan.heap = &heap;
830 cgroup_scan_tasks(&scan);
831 heap_free(&heap);
832 836
833 if (is_load_balanced) 837 if (is_load_balanced)
834 rebuild_sched_domains(); 838 rebuild_sched_domains();
@@ -884,74 +888,25 @@ static void cpuset_migrate_mm(struct mm_struct *mm, const nodemask_t *from,
884 mutex_unlock(&callback_mutex); 888 mutex_unlock(&callback_mutex);
885} 889}
886 890
887/*
888 * Handle user request to change the 'mems' memory placement
889 * of a cpuset. Needs to validate the request, update the
890 * cpusets mems_allowed and mems_generation, and for each
891 * task in the cpuset, rebind any vma mempolicies and if
892 * the cpuset is marked 'memory_migrate', migrate the tasks
893 * pages to the new memory.
894 *
895 * Call with cgroup_mutex held. May take callback_mutex during call.
896 * Will take tasklist_lock, scan tasklist for tasks in cpuset cs,
897 * lock each such tasks mm->mmap_sem, scan its vma's and rebind
898 * their mempolicies to the cpusets new mems_allowed.
899 */
900
901static void *cpuset_being_rebound; 891static void *cpuset_being_rebound;
902 892
903static int update_nodemask(struct cpuset *cs, char *buf) 893/**
894 * update_tasks_nodemask - Update the nodemasks of tasks in the cpuset.
895 * @cs: the cpuset in which each task's mems_allowed mask needs to be changed
896 * @oldmem: old mems_allowed of cpuset cs
897 *
898 * Called with cgroup_mutex held
899 * Return 0 if successful, -errno if not.
900 */
901static int update_tasks_nodemask(struct cpuset *cs, const nodemask_t *oldmem)
904{ 902{
905 struct cpuset trialcs;
906 nodemask_t oldmem;
907 struct task_struct *p; 903 struct task_struct *p;
908 struct mm_struct **mmarray; 904 struct mm_struct **mmarray;
909 int i, n, ntasks; 905 int i, n, ntasks;
910 int migrate; 906 int migrate;
911 int fudge; 907 int fudge;
912 int retval;
913 struct cgroup_iter it; 908 struct cgroup_iter it;
914 909 int retval;
915 /*
916 * top_cpuset.mems_allowed tracks node_stats[N_HIGH_MEMORY];
917 * it's read-only
918 */
919 if (cs == &top_cpuset)
920 return -EACCES;
921
922 trialcs = *cs;
923
924 /*
925 * An empty mems_allowed is ok iff there are no tasks in the cpuset.
926 * Since nodelist_parse() fails on an empty mask, we special case
927 * that parsing. The validate_change() call ensures that cpusets
928 * with tasks have memory.
929 */
930 buf = strstrip(buf);
931 if (!*buf) {
932 nodes_clear(trialcs.mems_allowed);
933 } else {
934 retval = nodelist_parse(buf, trialcs.mems_allowed);
935 if (retval < 0)
936 goto done;
937
938 if (!nodes_subset(trialcs.mems_allowed,
939 node_states[N_HIGH_MEMORY]))
940 return -EINVAL;
941 }
942 oldmem = cs->mems_allowed;
943 if (nodes_equal(oldmem, trialcs.mems_allowed)) {
944 retval = 0; /* Too easy - nothing to do */
945 goto done;
946 }
947 retval = validate_change(cs, &trialcs);
948 if (retval < 0)
949 goto done;
950
951 mutex_lock(&callback_mutex);
952 cs->mems_allowed = trialcs.mems_allowed;
953 cs->mems_generation = cpuset_mems_generation++;
954 mutex_unlock(&callback_mutex);
955 910
956 cpuset_being_rebound = cs; /* causes mpol_dup() rebind */ 911 cpuset_being_rebound = cs; /* causes mpol_dup() rebind */
957 912
@@ -1018,7 +973,7 @@ static int update_nodemask(struct cpuset *cs, char *buf)
1018 973
1019 mpol_rebind_mm(mm, &cs->mems_allowed); 974 mpol_rebind_mm(mm, &cs->mems_allowed);
1020 if (migrate) 975 if (migrate)
1021 cpuset_migrate_mm(mm, &oldmem, &cs->mems_allowed); 976 cpuset_migrate_mm(mm, oldmem, &cs->mems_allowed);
1022 mmput(mm); 977 mmput(mm);
1023 } 978 }
1024 979
@@ -1030,6 +985,70 @@ done:
1030 return retval; 985 return retval;
1031} 986}
1032 987
988/*
989 * Handle user request to change the 'mems' memory placement
990 * of a cpuset. Needs to validate the request, update the
991 * cpusets mems_allowed and mems_generation, and for each
992 * task in the cpuset, rebind any vma mempolicies and if
993 * the cpuset is marked 'memory_migrate', migrate the tasks
994 * pages to the new memory.
995 *
996 * Call with cgroup_mutex held. May take callback_mutex during call.
997 * Will take tasklist_lock, scan tasklist for tasks in cpuset cs,
998 * lock each such tasks mm->mmap_sem, scan its vma's and rebind
999 * their mempolicies to the cpusets new mems_allowed.
1000 */
1001static int update_nodemask(struct cpuset *cs, const char *buf)
1002{
1003 struct cpuset trialcs;
1004 nodemask_t oldmem;
1005 int retval;
1006
1007 /*
1008 * top_cpuset.mems_allowed tracks node_stats[N_HIGH_MEMORY];
1009 * it's read-only
1010 */
1011 if (cs == &top_cpuset)
1012 return -EACCES;
1013
1014 trialcs = *cs;
1015
1016 /*
1017 * An empty mems_allowed is ok iff there are no tasks in the cpuset.
1018 * Since nodelist_parse() fails on an empty mask, we special case
1019 * that parsing. The validate_change() call ensures that cpusets
1020 * with tasks have memory.
1021 */
1022 if (!*buf) {
1023 nodes_clear(trialcs.mems_allowed);
1024 } else {
1025 retval = nodelist_parse(buf, trialcs.mems_allowed);
1026 if (retval < 0)
1027 goto done;
1028
1029 if (!nodes_subset(trialcs.mems_allowed,
1030 node_states[N_HIGH_MEMORY]))
1031 return -EINVAL;
1032 }
1033 oldmem = cs->mems_allowed;
1034 if (nodes_equal(oldmem, trialcs.mems_allowed)) {
1035 retval = 0; /* Too easy - nothing to do */
1036 goto done;
1037 }
1038 retval = validate_change(cs, &trialcs);
1039 if (retval < 0)
1040 goto done;
1041
1042 mutex_lock(&callback_mutex);
1043 cs->mems_allowed = trialcs.mems_allowed;
1044 cs->mems_generation = cpuset_mems_generation++;
1045 mutex_unlock(&callback_mutex);
1046
1047 retval = update_tasks_nodemask(cs, &oldmem);
1048done:
1049 return retval;
1050}
1051
1033int current_cpuset_is_being_rebound(void) 1052int current_cpuset_is_being_rebound(void)
1034{ 1053{
1035 return task_cs(current) == cpuset_being_rebound; 1054 return task_cs(current) == cpuset_being_rebound;
@@ -1042,7 +1061,8 @@ static int update_relax_domain_level(struct cpuset *cs, s64 val)
1042 1061
1043 if (val != cs->relax_domain_level) { 1062 if (val != cs->relax_domain_level) {
1044 cs->relax_domain_level = val; 1063 cs->relax_domain_level = val;
1045 rebuild_sched_domains(); 1064 if (!cpus_empty(cs->cpus_allowed) && is_sched_load_balance(cs))
1065 rebuild_sched_domains();
1046 } 1066 }
1047 1067
1048 return 0; 1068 return 0;
@@ -1194,6 +1214,15 @@ static int cpuset_can_attach(struct cgroup_subsys *ss,
1194 1214
1195 if (cpus_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed)) 1215 if (cpus_empty(cs->cpus_allowed) || nodes_empty(cs->mems_allowed))
1196 return -ENOSPC; 1216 return -ENOSPC;
1217 if (tsk->flags & PF_THREAD_BOUND) {
1218 cpumask_t mask;
1219
1220 mutex_lock(&callback_mutex);
1221 mask = cs->cpus_allowed;
1222 mutex_unlock(&callback_mutex);
1223 if (!cpus_equal(tsk->cpus_allowed, mask))
1224 return -EINVAL;
1225 }
1197 1226
1198 return security_task_setscheduler(tsk, 0, NULL); 1227 return security_task_setscheduler(tsk, 0, NULL);
1199} 1228}
@@ -1207,11 +1236,14 @@ static void cpuset_attach(struct cgroup_subsys *ss,
1207 struct mm_struct *mm; 1236 struct mm_struct *mm;
1208 struct cpuset *cs = cgroup_cs(cont); 1237 struct cpuset *cs = cgroup_cs(cont);
1209 struct cpuset *oldcs = cgroup_cs(oldcont); 1238 struct cpuset *oldcs = cgroup_cs(oldcont);
1239 int err;
1210 1240
1211 mutex_lock(&callback_mutex); 1241 mutex_lock(&callback_mutex);
1212 guarantee_online_cpus(cs, &cpus); 1242 guarantee_online_cpus(cs, &cpus);
1213 set_cpus_allowed_ptr(tsk, &cpus); 1243 err = set_cpus_allowed_ptr(tsk, &cpus);
1214 mutex_unlock(&callback_mutex); 1244 mutex_unlock(&callback_mutex);
1245 if (err)
1246 return;
1215 1247
1216 from = oldcs->mems_allowed; 1248 from = oldcs->mems_allowed;
1217 to = cs->mems_allowed; 1249 to = cs->mems_allowed;
@@ -1242,72 +1274,14 @@ typedef enum {
1242 FILE_SPREAD_SLAB, 1274 FILE_SPREAD_SLAB,
1243} cpuset_filetype_t; 1275} cpuset_filetype_t;
1244 1276
1245static ssize_t cpuset_common_file_write(struct cgroup *cont,
1246 struct cftype *cft,
1247 struct file *file,
1248 const char __user *userbuf,
1249 size_t nbytes, loff_t *unused_ppos)
1250{
1251 struct cpuset *cs = cgroup_cs(cont);
1252 cpuset_filetype_t type = cft->private;
1253 char *buffer;
1254 int retval = 0;
1255
1256 /* Crude upper limit on largest legitimate cpulist user might write. */
1257 if (nbytes > 100U + 6 * max(NR_CPUS, MAX_NUMNODES))
1258 return -E2BIG;
1259
1260 /* +1 for nul-terminator */
1261 buffer = kmalloc(nbytes + 1, GFP_KERNEL);
1262 if (!buffer)
1263 return -ENOMEM;
1264
1265 if (copy_from_user(buffer, userbuf, nbytes)) {
1266 retval = -EFAULT;
1267 goto out1;
1268 }
1269 buffer[nbytes] = 0; /* nul-terminate */
1270
1271 cgroup_lock();
1272
1273 if (cgroup_is_removed(cont)) {
1274 retval = -ENODEV;
1275 goto out2;
1276 }
1277
1278 switch (type) {
1279 case FILE_CPULIST:
1280 retval = update_cpumask(cs, buffer);
1281 break;
1282 case FILE_MEMLIST:
1283 retval = update_nodemask(cs, buffer);
1284 break;
1285 default:
1286 retval = -EINVAL;
1287 goto out2;
1288 }
1289
1290 if (retval == 0)
1291 retval = nbytes;
1292out2:
1293 cgroup_unlock();
1294out1:
1295 kfree(buffer);
1296 return retval;
1297}
1298
1299static int cpuset_write_u64(struct cgroup *cgrp, struct cftype *cft, u64 val) 1277static int cpuset_write_u64(struct cgroup *cgrp, struct cftype *cft, u64 val)
1300{ 1278{
1301 int retval = 0; 1279 int retval = 0;
1302 struct cpuset *cs = cgroup_cs(cgrp); 1280 struct cpuset *cs = cgroup_cs(cgrp);
1303 cpuset_filetype_t type = cft->private; 1281 cpuset_filetype_t type = cft->private;
1304 1282
1305 cgroup_lock(); 1283 if (!cgroup_lock_live_group(cgrp))
1306
1307 if (cgroup_is_removed(cgrp)) {
1308 cgroup_unlock();
1309 return -ENODEV; 1284 return -ENODEV;
1310 }
1311 1285
1312 switch (type) { 1286 switch (type) {
1313 case FILE_CPU_EXCLUSIVE: 1287 case FILE_CPU_EXCLUSIVE:
@@ -1353,12 +1327,9 @@ static int cpuset_write_s64(struct cgroup *cgrp, struct cftype *cft, s64 val)
1353 struct cpuset *cs = cgroup_cs(cgrp); 1327 struct cpuset *cs = cgroup_cs(cgrp);
1354 cpuset_filetype_t type = cft->private; 1328 cpuset_filetype_t type = cft->private;
1355 1329
1356 cgroup_lock(); 1330 if (!cgroup_lock_live_group(cgrp))
1357
1358 if (cgroup_is_removed(cgrp)) {
1359 cgroup_unlock();
1360 return -ENODEV; 1331 return -ENODEV;
1361 } 1332
1362 switch (type) { 1333 switch (type) {
1363 case FILE_SCHED_RELAX_DOMAIN_LEVEL: 1334 case FILE_SCHED_RELAX_DOMAIN_LEVEL:
1364 retval = update_relax_domain_level(cs, val); 1335 retval = update_relax_domain_level(cs, val);
@@ -1372,6 +1343,32 @@ static int cpuset_write_s64(struct cgroup *cgrp, struct cftype *cft, s64 val)
1372} 1343}
1373 1344
1374/* 1345/*
1346 * Common handling for a write to a "cpus" or "mems" file.
1347 */
1348static int cpuset_write_resmask(struct cgroup *cgrp, struct cftype *cft,
1349 const char *buf)
1350{
1351 int retval = 0;
1352
1353 if (!cgroup_lock_live_group(cgrp))
1354 return -ENODEV;
1355
1356 switch (cft->private) {
1357 case FILE_CPULIST:
1358 retval = update_cpumask(cgroup_cs(cgrp), buf);
1359 break;
1360 case FILE_MEMLIST:
1361 retval = update_nodemask(cgroup_cs(cgrp), buf);
1362 break;
1363 default:
1364 retval = -EINVAL;
1365 break;
1366 }
1367 cgroup_unlock();
1368 return retval;
1369}
1370
1371/*
1375 * These ascii lists should be read in a single call, by using a user 1372 * These ascii lists should be read in a single call, by using a user
1376 * buffer large enough to hold the entire map. If read in smaller 1373 * buffer large enough to hold the entire map. If read in smaller
1377 * chunks, there is no guarantee of atomicity. Since the display format 1374 * chunks, there is no guarantee of atomicity. Since the display format
@@ -1490,14 +1487,16 @@ static struct cftype files[] = {
1490 { 1487 {
1491 .name = "cpus", 1488 .name = "cpus",
1492 .read = cpuset_common_file_read, 1489 .read = cpuset_common_file_read,
1493 .write = cpuset_common_file_write, 1490 .write_string = cpuset_write_resmask,
1491 .max_write_len = (100U + 6 * NR_CPUS),
1494 .private = FILE_CPULIST, 1492 .private = FILE_CPULIST,
1495 }, 1493 },
1496 1494
1497 { 1495 {
1498 .name = "mems", 1496 .name = "mems",
1499 .read = cpuset_common_file_read, 1497 .read = cpuset_common_file_read,
1500 .write = cpuset_common_file_write, 1498 .write_string = cpuset_write_resmask,
1499 .max_write_len = (100U + 6 * MAX_NUMNODES),
1501 .private = FILE_MEMLIST, 1500 .private = FILE_MEMLIST,
1502 }, 1501 },
1503 1502
@@ -1778,7 +1777,7 @@ static void move_member_tasks_to_cpuset(struct cpuset *from, struct cpuset *to)
1778 scan.scan.heap = NULL; 1777 scan.scan.heap = NULL;
1779 scan.to = to->css.cgroup; 1778 scan.to = to->css.cgroup;
1780 1779
1781 if (cgroup_scan_tasks((struct cgroup_scanner *)&scan)) 1780 if (cgroup_scan_tasks(&scan.scan))
1782 printk(KERN_ERR "move_member_tasks_to_cpuset: " 1781 printk(KERN_ERR "move_member_tasks_to_cpuset: "
1783 "cgroup_scan_tasks failed\n"); 1782 "cgroup_scan_tasks failed\n");
1784} 1783}
@@ -1838,6 +1837,7 @@ static void scan_for_empty_cpusets(const struct cpuset *root)
1838 struct cpuset *child; /* scans child cpusets of cp */ 1837 struct cpuset *child; /* scans child cpusets of cp */
1839 struct list_head queue; 1838 struct list_head queue;
1840 struct cgroup *cont; 1839 struct cgroup *cont;
1840 nodemask_t oldmems;
1841 1841
1842 INIT_LIST_HEAD(&queue); 1842 INIT_LIST_HEAD(&queue);
1843 1843
@@ -1857,6 +1857,8 @@ static void scan_for_empty_cpusets(const struct cpuset *root)
1857 nodes_subset(cp->mems_allowed, node_states[N_HIGH_MEMORY])) 1857 nodes_subset(cp->mems_allowed, node_states[N_HIGH_MEMORY]))
1858 continue; 1858 continue;
1859 1859
1860 oldmems = cp->mems_allowed;
1861
1860 /* Remove offline cpus and mems from this cpuset. */ 1862 /* Remove offline cpus and mems from this cpuset. */
1861 mutex_lock(&callback_mutex); 1863 mutex_lock(&callback_mutex);
1862 cpus_and(cp->cpus_allowed, cp->cpus_allowed, cpu_online_map); 1864 cpus_and(cp->cpus_allowed, cp->cpus_allowed, cpu_online_map);
@@ -1868,6 +1870,10 @@ static void scan_for_empty_cpusets(const struct cpuset *root)
1868 if (cpus_empty(cp->cpus_allowed) || 1870 if (cpus_empty(cp->cpus_allowed) ||
1869 nodes_empty(cp->mems_allowed)) 1871 nodes_empty(cp->mems_allowed))
1870 remove_tasks_in_empty_cpuset(cp); 1872 remove_tasks_in_empty_cpuset(cp);
1873 else {
1874 update_tasks_cpumask(cp);
1875 update_tasks_nodemask(cp, &oldmems);
1876 }
1871 } 1877 }
1872} 1878}
1873 1879
@@ -1960,7 +1966,6 @@ void __init cpuset_init_smp(void)
1960} 1966}
1961 1967
1962/** 1968/**
1963
1964 * cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset. 1969 * cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset.
1965 * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed. 1970 * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed.
1966 * @pmask: pointer to cpumask_t variable to receive cpus_allowed set. 1971 * @pmask: pointer to cpumask_t variable to receive cpus_allowed set.
diff --git a/kernel/delayacct.c b/kernel/delayacct.c
index 10e43fd8b721..b3179dad71be 100644
--- a/kernel/delayacct.c
+++ b/kernel/delayacct.c
@@ -145,8 +145,11 @@ int __delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk)
145 d->blkio_delay_total = (tmp < d->blkio_delay_total) ? 0 : tmp; 145 d->blkio_delay_total = (tmp < d->blkio_delay_total) ? 0 : tmp;
146 tmp = d->swapin_delay_total + tsk->delays->swapin_delay; 146 tmp = d->swapin_delay_total + tsk->delays->swapin_delay;
147 d->swapin_delay_total = (tmp < d->swapin_delay_total) ? 0 : tmp; 147 d->swapin_delay_total = (tmp < d->swapin_delay_total) ? 0 : tmp;
148 tmp = d->freepages_delay_total + tsk->delays->freepages_delay;
149 d->freepages_delay_total = (tmp < d->freepages_delay_total) ? 0 : tmp;
148 d->blkio_count += tsk->delays->blkio_count; 150 d->blkio_count += tsk->delays->blkio_count;
149 d->swapin_count += tsk->delays->swapin_count; 151 d->swapin_count += tsk->delays->swapin_count;
152 d->freepages_count += tsk->delays->freepages_count;
150 spin_unlock_irqrestore(&tsk->delays->lock, flags); 153 spin_unlock_irqrestore(&tsk->delays->lock, flags);
151 154
152done: 155done:
@@ -165,3 +168,16 @@ __u64 __delayacct_blkio_ticks(struct task_struct *tsk)
165 return ret; 168 return ret;
166} 169}
167 170
171void __delayacct_freepages_start(void)
172{
173 delayacct_start(&current->delays->freepages_start);
174}
175
176void __delayacct_freepages_end(void)
177{
178 delayacct_end(&current->delays->freepages_start,
179 &current->delays->freepages_end,
180 &current->delays->freepages_delay,
181 &current->delays->freepages_count);
182}
183
diff --git a/kernel/exec_domain.c b/kernel/exec_domain.c
index a9e6bad9f706..0d407e886735 100644
--- a/kernel/exec_domain.c
+++ b/kernel/exec_domain.c
@@ -65,7 +65,7 @@ lookup_exec_domain(u_long personality)
65 goto out; 65 goto out;
66 } 66 }
67 67
68#ifdef CONFIG_KMOD 68#ifdef CONFIG_MODULES
69 read_unlock(&exec_domains_lock); 69 read_unlock(&exec_domains_lock);
70 request_module("personality-%ld", pers); 70 request_module("personality-%ld", pers);
71 read_lock(&exec_domains_lock); 71 read_lock(&exec_domains_lock);
@@ -168,7 +168,6 @@ __set_personality(u_long personality)
168 current->personality = personality; 168 current->personality = personality;
169 oep = current_thread_info()->exec_domain; 169 oep = current_thread_info()->exec_domain;
170 current_thread_info()->exec_domain = ep; 170 current_thread_info()->exec_domain = ep;
171 set_fs_altroot();
172 171
173 module_put(oep->module); 172 module_put(oep->module);
174 return 0; 173 return 0;
diff --git a/kernel/exit.c b/kernel/exit.c
index 8f6185e69b69..eb4d6470d1d0 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -13,6 +13,7 @@
13#include <linux/personality.h> 13#include <linux/personality.h>
14#include <linux/tty.h> 14#include <linux/tty.h>
15#include <linux/mnt_namespace.h> 15#include <linux/mnt_namespace.h>
16#include <linux/iocontext.h>
16#include <linux/key.h> 17#include <linux/key.h>
17#include <linux/security.h> 18#include <linux/security.h>
18#include <linux/cpu.h> 19#include <linux/cpu.h>
@@ -45,6 +46,7 @@
45#include <linux/resource.h> 46#include <linux/resource.h>
46#include <linux/blkdev.h> 47#include <linux/blkdev.h>
47#include <linux/task_io_accounting_ops.h> 48#include <linux/task_io_accounting_ops.h>
49#include <linux/tracehook.h>
48 50
49#include <asm/uaccess.h> 51#include <asm/uaccess.h>
50#include <asm/unistd.h> 52#include <asm/unistd.h>
@@ -70,7 +72,7 @@ static void __unhash_process(struct task_struct *p)
70 __get_cpu_var(process_counts)--; 72 __get_cpu_var(process_counts)--;
71 } 73 }
72 list_del_rcu(&p->thread_group); 74 list_del_rcu(&p->thread_group);
73 remove_parent(p); 75 list_del_init(&p->sibling);
74} 76}
75 77
76/* 78/*
@@ -84,7 +86,6 @@ static void __exit_signal(struct task_struct *tsk)
84 BUG_ON(!sig); 86 BUG_ON(!sig);
85 BUG_ON(!atomic_read(&sig->count)); 87 BUG_ON(!atomic_read(&sig->count));
86 88
87 rcu_read_lock();
88 sighand = rcu_dereference(tsk->sighand); 89 sighand = rcu_dereference(tsk->sighand);
89 spin_lock(&sighand->siglock); 90 spin_lock(&sighand->siglock);
90 91
@@ -120,6 +121,7 @@ static void __exit_signal(struct task_struct *tsk)
120 sig->nivcsw += tsk->nivcsw; 121 sig->nivcsw += tsk->nivcsw;
121 sig->inblock += task_io_get_inblock(tsk); 122 sig->inblock += task_io_get_inblock(tsk);
122 sig->oublock += task_io_get_oublock(tsk); 123 sig->oublock += task_io_get_oublock(tsk);
124 task_io_accounting_add(&sig->ioac, &tsk->ioac);
123 sig->sum_sched_runtime += tsk->se.sum_exec_runtime; 125 sig->sum_sched_runtime += tsk->se.sum_exec_runtime;
124 sig = NULL; /* Marker for below. */ 126 sig = NULL; /* Marker for below. */
125 } 127 }
@@ -135,7 +137,6 @@ static void __exit_signal(struct task_struct *tsk)
135 tsk->signal = NULL; 137 tsk->signal = NULL;
136 tsk->sighand = NULL; 138 tsk->sighand = NULL;
137 spin_unlock(&sighand->siglock); 139 spin_unlock(&sighand->siglock);
138 rcu_read_unlock();
139 140
140 __cleanup_sighand(sighand); 141 __cleanup_sighand(sighand);
141 clear_tsk_thread_flag(tsk,TIF_SIGPENDING); 142 clear_tsk_thread_flag(tsk,TIF_SIGPENDING);
@@ -151,16 +152,17 @@ static void delayed_put_task_struct(struct rcu_head *rhp)
151 put_task_struct(container_of(rhp, struct task_struct, rcu)); 152 put_task_struct(container_of(rhp, struct task_struct, rcu));
152} 153}
153 154
155
154void release_task(struct task_struct * p) 156void release_task(struct task_struct * p)
155{ 157{
156 struct task_struct *leader; 158 struct task_struct *leader;
157 int zap_leader; 159 int zap_leader;
158repeat: 160repeat:
161 tracehook_prepare_release_task(p);
159 atomic_dec(&p->user->processes); 162 atomic_dec(&p->user->processes);
160 proc_flush_task(p); 163 proc_flush_task(p);
161 write_lock_irq(&tasklist_lock); 164 write_lock_irq(&tasklist_lock);
162 ptrace_unlink(p); 165 tracehook_finish_release_task(p);
163 BUG_ON(!list_empty(&p->ptrace_list) || !list_empty(&p->ptrace_children));
164 __exit_signal(p); 166 __exit_signal(p);
165 167
166 /* 168 /*
@@ -182,6 +184,13 @@ repeat:
182 * that case. 184 * that case.
183 */ 185 */
184 zap_leader = task_detached(leader); 186 zap_leader = task_detached(leader);
187
188 /*
189 * This maintains the invariant that release_task()
190 * only runs on a task in EXIT_DEAD, just for sanity.
191 */
192 if (zap_leader)
193 leader->exit_state = EXIT_DEAD;
185 } 194 }
186 195
187 write_unlock_irq(&tasklist_lock); 196 write_unlock_irq(&tasklist_lock);
@@ -314,9 +323,8 @@ static void reparent_to_kthreadd(void)
314 323
315 ptrace_unlink(current); 324 ptrace_unlink(current);
316 /* Reparent to init */ 325 /* Reparent to init */
317 remove_parent(current);
318 current->real_parent = current->parent = kthreadd_task; 326 current->real_parent = current->parent = kthreadd_task;
319 add_parent(current); 327 list_move_tail(&current->sibling, &current->real_parent->children);
320 328
321 /* Set the exit signal to SIGCHLD so we signal init on exit */ 329 /* Set the exit signal to SIGCHLD so we signal init on exit */
322 current->exit_signal = SIGCHLD; 330 current->exit_signal = SIGCHLD;
@@ -421,7 +429,7 @@ void daemonize(const char *name, ...)
421 * We don't want to have TIF_FREEZE set if the system-wide hibernation 429 * We don't want to have TIF_FREEZE set if the system-wide hibernation
422 * or suspend transition begins right now. 430 * or suspend transition begins right now.
423 */ 431 */
424 current->flags |= PF_NOFREEZE; 432 current->flags |= (PF_NOFREEZE | PF_KTHREAD);
425 433
426 if (current->nsproxy != &init_nsproxy) { 434 if (current->nsproxy != &init_nsproxy) {
427 get_nsproxy(&init_nsproxy); 435 get_nsproxy(&init_nsproxy);
@@ -546,8 +554,6 @@ void put_fs_struct(struct fs_struct *fs)
546 if (atomic_dec_and_test(&fs->count)) { 554 if (atomic_dec_and_test(&fs->count)) {
547 path_put(&fs->root); 555 path_put(&fs->root);
548 path_put(&fs->pwd); 556 path_put(&fs->pwd);
549 if (fs->altroot.dentry)
550 path_put(&fs->altroot);
551 kmem_cache_free(fs_cachep, fs); 557 kmem_cache_free(fs_cachep, fs);
552 } 558 }
553} 559}
@@ -655,26 +661,40 @@ assign_new_owner:
655static void exit_mm(struct task_struct * tsk) 661static void exit_mm(struct task_struct * tsk)
656{ 662{
657 struct mm_struct *mm = tsk->mm; 663 struct mm_struct *mm = tsk->mm;
664 struct core_state *core_state;
658 665
659 mm_release(tsk, mm); 666 mm_release(tsk, mm);
660 if (!mm) 667 if (!mm)
661 return; 668 return;
662 /* 669 /*
663 * Serialize with any possible pending coredump. 670 * Serialize with any possible pending coredump.
664 * We must hold mmap_sem around checking core_waiters 671 * We must hold mmap_sem around checking core_state
665 * and clearing tsk->mm. The core-inducing thread 672 * and clearing tsk->mm. The core-inducing thread
666 * will increment core_waiters for each thread in the 673 * will increment ->nr_threads for each thread in the
667 * group with ->mm != NULL. 674 * group with ->mm != NULL.
668 */ 675 */
669 down_read(&mm->mmap_sem); 676 down_read(&mm->mmap_sem);
670 if (mm->core_waiters) { 677 core_state = mm->core_state;
678 if (core_state) {
679 struct core_thread self;
671 up_read(&mm->mmap_sem); 680 up_read(&mm->mmap_sem);
672 down_write(&mm->mmap_sem);
673 if (!--mm->core_waiters)
674 complete(mm->core_startup_done);
675 up_write(&mm->mmap_sem);
676 681
677 wait_for_completion(&mm->core_done); 682 self.task = tsk;
683 self.next = xchg(&core_state->dumper.next, &self);
684 /*
685 * Implies mb(), the result of xchg() must be visible
686 * to core_state->dumper.
687 */
688 if (atomic_dec_and_test(&core_state->nr_threads))
689 complete(&core_state->startup);
690
691 for (;;) {
692 set_task_state(tsk, TASK_UNINTERRUPTIBLE);
693 if (!self.task) /* see coredump_finish() */
694 break;
695 schedule();
696 }
697 __set_task_state(tsk, TASK_RUNNING);
678 down_read(&mm->mmap_sem); 698 down_read(&mm->mmap_sem);
679 } 699 }
680 atomic_inc(&mm->mm_count); 700 atomic_inc(&mm->mm_count);
@@ -691,37 +711,97 @@ static void exit_mm(struct task_struct * tsk)
691 mmput(mm); 711 mmput(mm);
692} 712}
693 713
694static void 714/*
695reparent_thread(struct task_struct *p, struct task_struct *father, int traced) 715 * Return nonzero if @parent's children should reap themselves.
716 *
717 * Called with write_lock_irq(&tasklist_lock) held.
718 */
719static int ignoring_children(struct task_struct *parent)
696{ 720{
697 if (p->pdeath_signal) 721 int ret;
698 /* We already hold the tasklist_lock here. */ 722 struct sighand_struct *psig = parent->sighand;
699 group_send_sig_info(p->pdeath_signal, SEND_SIG_NOINFO, p); 723 unsigned long flags;
724 spin_lock_irqsave(&psig->siglock, flags);
725 ret = (psig->action[SIGCHLD-1].sa.sa_handler == SIG_IGN ||
726 (psig->action[SIGCHLD-1].sa.sa_flags & SA_NOCLDWAIT));
727 spin_unlock_irqrestore(&psig->siglock, flags);
728 return ret;
729}
700 730
701 /* Move the child from its dying parent to the new one. */ 731/*
702 if (unlikely(traced)) { 732 * Detach all tasks we were using ptrace on.
703 /* Preserve ptrace links if someone else is tracing this child. */ 733 * Any that need to be release_task'd are put on the @dead list.
704 list_del_init(&p->ptrace_list); 734 *
705 if (ptrace_reparented(p)) 735 * Called with write_lock(&tasklist_lock) held.
706 list_add(&p->ptrace_list, &p->real_parent->ptrace_children); 736 */
707 } else { 737static void ptrace_exit(struct task_struct *parent, struct list_head *dead)
708 /* If this child is being traced, then we're the one tracing it 738{
709 * anyway, so let go of it. 739 struct task_struct *p, *n;
740 int ign = -1;
741
742 list_for_each_entry_safe(p, n, &parent->ptraced, ptrace_entry) {
743 __ptrace_unlink(p);
744
745 if (p->exit_state != EXIT_ZOMBIE)
746 continue;
747
748 /*
749 * If it's a zombie, our attachedness prevented normal
750 * parent notification or self-reaping. Do notification
751 * now if it would have happened earlier. If it should
752 * reap itself, add it to the @dead list. We can't call
753 * release_task() here because we already hold tasklist_lock.
754 *
755 * If it's our own child, there is no notification to do.
756 * But if our normal children self-reap, then this child
757 * was prevented by ptrace and we must reap it now.
710 */ 758 */
711 p->ptrace = 0; 759 if (!task_detached(p) && thread_group_empty(p)) {
712 remove_parent(p); 760 if (!same_thread_group(p->real_parent, parent))
713 p->parent = p->real_parent; 761 do_notify_parent(p, p->exit_signal);
714 add_parent(p); 762 else {
763 if (ign < 0)
764 ign = ignoring_children(parent);
765 if (ign)
766 p->exit_signal = -1;
767 }
768 }
715 769
716 if (task_is_traced(p)) { 770 if (task_detached(p)) {
717 /* 771 /*
718 * If it was at a trace stop, turn it into 772 * Mark it as in the process of being reaped.
719 * a normal stop since it's no longer being
720 * traced.
721 */ 773 */
722 ptrace_untrace(p); 774 p->exit_state = EXIT_DEAD;
775 list_add(&p->ptrace_entry, dead);
723 } 776 }
724 } 777 }
778}
779
780/*
781 * Finish up exit-time ptrace cleanup.
782 *
783 * Called without locks.
784 */
785static void ptrace_exit_finish(struct task_struct *parent,
786 struct list_head *dead)
787{
788 struct task_struct *p, *n;
789
790 BUG_ON(!list_empty(&parent->ptraced));
791
792 list_for_each_entry_safe(p, n, dead, ptrace_entry) {
793 list_del_init(&p->ptrace_entry);
794 release_task(p);
795 }
796}
797
798static void reparent_thread(struct task_struct *p, struct task_struct *father)
799{
800 if (p->pdeath_signal)
801 /* We already hold the tasklist_lock here. */
802 group_send_sig_info(p->pdeath_signal, SEND_SIG_NOINFO, p);
803
804 list_move_tail(&p->sibling, &p->real_parent->children);
725 805
726 /* If this is a threaded reparent there is no need to 806 /* If this is a threaded reparent there is no need to
727 * notify anyone anything has happened. 807 * notify anyone anything has happened.
@@ -736,7 +816,8 @@ reparent_thread(struct task_struct *p, struct task_struct *father, int traced)
736 /* If we'd notified the old parent about this child's death, 816 /* If we'd notified the old parent about this child's death,
737 * also notify the new parent. 817 * also notify the new parent.
738 */ 818 */
739 if (!traced && p->exit_state == EXIT_ZOMBIE && 819 if (!ptrace_reparented(p) &&
820 p->exit_state == EXIT_ZOMBIE &&
740 !task_detached(p) && thread_group_empty(p)) 821 !task_detached(p) && thread_group_empty(p))
741 do_notify_parent(p, p->exit_signal); 822 do_notify_parent(p, p->exit_signal);
742 823
@@ -753,12 +834,15 @@ reparent_thread(struct task_struct *p, struct task_struct *father, int traced)
753static void forget_original_parent(struct task_struct *father) 834static void forget_original_parent(struct task_struct *father)
754{ 835{
755 struct task_struct *p, *n, *reaper = father; 836 struct task_struct *p, *n, *reaper = father;
756 struct list_head ptrace_dead; 837 LIST_HEAD(ptrace_dead);
757
758 INIT_LIST_HEAD(&ptrace_dead);
759 838
760 write_lock_irq(&tasklist_lock); 839 write_lock_irq(&tasklist_lock);
761 840
841 /*
842 * First clean up ptrace if we were using it.
843 */
844 ptrace_exit(father, &ptrace_dead);
845
762 do { 846 do {
763 reaper = next_thread(reaper); 847 reaper = next_thread(reaper);
764 if (reaper == father) { 848 if (reaper == father) {
@@ -767,58 +851,19 @@ static void forget_original_parent(struct task_struct *father)
767 } 851 }
768 } while (reaper->flags & PF_EXITING); 852 } while (reaper->flags & PF_EXITING);
769 853
770 /*
771 * There are only two places where our children can be:
772 *
773 * - in our child list
774 * - in our ptraced child list
775 *
776 * Search them and reparent children.
777 */
778 list_for_each_entry_safe(p, n, &father->children, sibling) { 854 list_for_each_entry_safe(p, n, &father->children, sibling) {
779 int ptrace;
780
781 ptrace = p->ptrace;
782
783 /* if father isn't the real parent, then ptrace must be enabled */
784 BUG_ON(father != p->real_parent && !ptrace);
785
786 if (father == p->real_parent) {
787 /* reparent with a reaper, real father it's us */
788 p->real_parent = reaper;
789 reparent_thread(p, father, 0);
790 } else {
791 /* reparent ptraced task to its real parent */
792 __ptrace_unlink (p);
793 if (p->exit_state == EXIT_ZOMBIE && !task_detached(p) &&
794 thread_group_empty(p))
795 do_notify_parent(p, p->exit_signal);
796 }
797
798 /*
799 * if the ptraced child is a detached zombie we must collect
800 * it before we exit, or it will remain zombie forever since
801 * we prevented it from self-reap itself while it was being
802 * traced by us, to be able to see it in wait4.
803 */
804 if (unlikely(ptrace && p->exit_state == EXIT_ZOMBIE && task_detached(p)))
805 list_add(&p->ptrace_list, &ptrace_dead);
806 }
807
808 list_for_each_entry_safe(p, n, &father->ptrace_children, ptrace_list) {
809 p->real_parent = reaper; 855 p->real_parent = reaper;
810 reparent_thread(p, father, 1); 856 if (p->parent == father) {
857 BUG_ON(p->ptrace);
858 p->parent = p->real_parent;
859 }
860 reparent_thread(p, father);
811 } 861 }
812 862
813 write_unlock_irq(&tasklist_lock); 863 write_unlock_irq(&tasklist_lock);
814 BUG_ON(!list_empty(&father->children)); 864 BUG_ON(!list_empty(&father->children));
815 BUG_ON(!list_empty(&father->ptrace_children));
816
817 list_for_each_entry_safe(p, n, &ptrace_dead, ptrace_list) {
818 list_del_init(&p->ptrace_list);
819 release_task(p);
820 }
821 865
866 ptrace_exit_finish(father, &ptrace_dead);
822} 867}
823 868
824/* 869/*
@@ -827,7 +872,8 @@ static void forget_original_parent(struct task_struct *father)
827 */ 872 */
828static void exit_notify(struct task_struct *tsk, int group_dead) 873static void exit_notify(struct task_struct *tsk, int group_dead)
829{ 874{
830 int state; 875 int signal;
876 void *cookie;
831 877
832 /* 878 /*
833 * This does two things: 879 * This does two things:
@@ -864,22 +910,11 @@ static void exit_notify(struct task_struct *tsk, int group_dead)
864 !capable(CAP_KILL)) 910 !capable(CAP_KILL))
865 tsk->exit_signal = SIGCHLD; 911 tsk->exit_signal = SIGCHLD;
866 912
867 /* If something other than our normal parent is ptracing us, then 913 signal = tracehook_notify_death(tsk, &cookie, group_dead);
868 * send it a SIGCHLD instead of honoring exit_signal. exit_signal 914 if (signal > 0)
869 * only has special meaning to our real parent. 915 signal = do_notify_parent(tsk, signal);
870 */
871 if (!task_detached(tsk) && thread_group_empty(tsk)) {
872 int signal = ptrace_reparented(tsk) ?
873 SIGCHLD : tsk->exit_signal;
874 do_notify_parent(tsk, signal);
875 } else if (tsk->ptrace) {
876 do_notify_parent(tsk, SIGCHLD);
877 }
878 916
879 state = EXIT_ZOMBIE; 917 tsk->exit_state = signal < 0 ? EXIT_DEAD : EXIT_ZOMBIE;
880 if (task_detached(tsk) && likely(!tsk->ptrace))
881 state = EXIT_DEAD;
882 tsk->exit_state = state;
883 918
884 /* mt-exec, de_thread() is waiting for us */ 919 /* mt-exec, de_thread() is waiting for us */
885 if (thread_group_leader(tsk) && 920 if (thread_group_leader(tsk) &&
@@ -889,8 +924,10 @@ static void exit_notify(struct task_struct *tsk, int group_dead)
889 924
890 write_unlock_irq(&tasklist_lock); 925 write_unlock_irq(&tasklist_lock);
891 926
927 tracehook_report_death(tsk, signal, cookie, group_dead);
928
892 /* If the process is dead, release it - nobody will wait for it */ 929 /* If the process is dead, release it - nobody will wait for it */
893 if (state == EXIT_DEAD) 930 if (signal < 0)
894 release_task(tsk); 931 release_task(tsk);
895} 932}
896 933
@@ -969,10 +1006,7 @@ NORET_TYPE void do_exit(long code)
969 if (unlikely(!tsk->pid)) 1006 if (unlikely(!tsk->pid))
970 panic("Attempted to kill the idle task!"); 1007 panic("Attempted to kill the idle task!");
971 1008
972 if (unlikely(current->ptrace & PT_TRACE_EXIT)) { 1009 tracehook_report_exit(&code);
973 current->ptrace_message = code;
974 ptrace_notify((PTRACE_EVENT_EXIT << 8) | SIGTRAP);
975 }
976 1010
977 /* 1011 /*
978 * We're taking recursive faults here in do_exit. Safest is to just 1012 * We're taking recursive faults here in do_exit. Safest is to just
@@ -1179,13 +1213,6 @@ static int eligible_child(enum pid_type type, struct pid *pid, int options,
1179 return 0; 1213 return 0;
1180 } 1214 }
1181 1215
1182 /*
1183 * Do not consider detached threads that are
1184 * not ptraced:
1185 */
1186 if (task_detached(p) && !p->ptrace)
1187 return 0;
1188
1189 /* Wait for all children (clone and not) if __WALL is set; 1216 /* Wait for all children (clone and not) if __WALL is set;
1190 * otherwise, wait for clone children *only* if __WCLONE is 1217 * otherwise, wait for clone children *only* if __WCLONE is
1191 * set; otherwise, wait for non-clone children *only*. (Note: 1218 * set; otherwise, wait for non-clone children *only*. (Note:
@@ -1196,14 +1223,10 @@ static int eligible_child(enum pid_type type, struct pid *pid, int options,
1196 return 0; 1223 return 0;
1197 1224
1198 err = security_task_wait(p); 1225 err = security_task_wait(p);
1199 if (likely(!err)) 1226 if (err)
1200 return 1; 1227 return err;
1201 1228
1202 if (type != PIDTYPE_PID) 1229 return 1;
1203 return 0;
1204 /* This child was explicitly requested, abort */
1205 read_unlock(&tasklist_lock);
1206 return err;
1207} 1230}
1208 1231
1209static int wait_noreap_copyout(struct task_struct *p, pid_t pid, uid_t uid, 1232static int wait_noreap_copyout(struct task_struct *p, pid_t pid, uid_t uid,
@@ -1237,7 +1260,7 @@ static int wait_noreap_copyout(struct task_struct *p, pid_t pid, uid_t uid,
1237 * the lock and this task is uninteresting. If we return nonzero, we have 1260 * the lock and this task is uninteresting. If we return nonzero, we have
1238 * released the lock and the system call should return. 1261 * released the lock and the system call should return.
1239 */ 1262 */
1240static int wait_task_zombie(struct task_struct *p, int noreap, 1263static int wait_task_zombie(struct task_struct *p, int options,
1241 struct siginfo __user *infop, 1264 struct siginfo __user *infop,
1242 int __user *stat_addr, struct rusage __user *ru) 1265 int __user *stat_addr, struct rusage __user *ru)
1243{ 1266{
@@ -1245,7 +1268,10 @@ static int wait_task_zombie(struct task_struct *p, int noreap,
1245 int retval, status, traced; 1268 int retval, status, traced;
1246 pid_t pid = task_pid_vnr(p); 1269 pid_t pid = task_pid_vnr(p);
1247 1270
1248 if (unlikely(noreap)) { 1271 if (!likely(options & WEXITED))
1272 return 0;
1273
1274 if (unlikely(options & WNOWAIT)) {
1249 uid_t uid = p->uid; 1275 uid_t uid = p->uid;
1250 int exit_code = p->exit_code; 1276 int exit_code = p->exit_code;
1251 int why, status; 1277 int why, status;
@@ -1326,6 +1352,8 @@ static int wait_task_zombie(struct task_struct *p, int noreap,
1326 psig->coublock += 1352 psig->coublock +=
1327 task_io_get_oublock(p) + 1353 task_io_get_oublock(p) +
1328 sig->oublock + sig->coublock; 1354 sig->oublock + sig->coublock;
1355 task_io_accounting_add(&psig->ioac, &p->ioac);
1356 task_io_accounting_add(&psig->ioac, &sig->ioac);
1329 spin_unlock_irq(&p->parent->sighand->siglock); 1357 spin_unlock_irq(&p->parent->sighand->siglock);
1330 } 1358 }
1331 1359
@@ -1395,21 +1423,24 @@ static int wait_task_zombie(struct task_struct *p, int noreap,
1395 * the lock and this task is uninteresting. If we return nonzero, we have 1423 * the lock and this task is uninteresting. If we return nonzero, we have
1396 * released the lock and the system call should return. 1424 * released the lock and the system call should return.
1397 */ 1425 */
1398static int wait_task_stopped(struct task_struct *p, 1426static int wait_task_stopped(int ptrace, struct task_struct *p,
1399 int noreap, struct siginfo __user *infop, 1427 int options, struct siginfo __user *infop,
1400 int __user *stat_addr, struct rusage __user *ru) 1428 int __user *stat_addr, struct rusage __user *ru)
1401{ 1429{
1402 int retval, exit_code, why; 1430 int retval, exit_code, why;
1403 uid_t uid = 0; /* unneeded, required by compiler */ 1431 uid_t uid = 0; /* unneeded, required by compiler */
1404 pid_t pid; 1432 pid_t pid;
1405 1433
1434 if (!(options & WUNTRACED))
1435 return 0;
1436
1406 exit_code = 0; 1437 exit_code = 0;
1407 spin_lock_irq(&p->sighand->siglock); 1438 spin_lock_irq(&p->sighand->siglock);
1408 1439
1409 if (unlikely(!task_is_stopped_or_traced(p))) 1440 if (unlikely(!task_is_stopped_or_traced(p)))
1410 goto unlock_sig; 1441 goto unlock_sig;
1411 1442
1412 if (!(p->ptrace & PT_PTRACED) && p->signal->group_stop_count > 0) 1443 if (!ptrace && p->signal->group_stop_count > 0)
1413 /* 1444 /*
1414 * A group stop is in progress and this is the group leader. 1445 * A group stop is in progress and this is the group leader.
1415 * We won't report until all threads have stopped. 1446 * We won't report until all threads have stopped.
@@ -1420,7 +1451,7 @@ static int wait_task_stopped(struct task_struct *p,
1420 if (!exit_code) 1451 if (!exit_code)
1421 goto unlock_sig; 1452 goto unlock_sig;
1422 1453
1423 if (!noreap) 1454 if (!unlikely(options & WNOWAIT))
1424 p->exit_code = 0; 1455 p->exit_code = 0;
1425 1456
1426 uid = p->uid; 1457 uid = p->uid;
@@ -1438,10 +1469,10 @@ unlock_sig:
1438 */ 1469 */
1439 get_task_struct(p); 1470 get_task_struct(p);
1440 pid = task_pid_vnr(p); 1471 pid = task_pid_vnr(p);
1441 why = (p->ptrace & PT_PTRACED) ? CLD_TRAPPED : CLD_STOPPED; 1472 why = ptrace ? CLD_TRAPPED : CLD_STOPPED;
1442 read_unlock(&tasklist_lock); 1473 read_unlock(&tasklist_lock);
1443 1474
1444 if (unlikely(noreap)) 1475 if (unlikely(options & WNOWAIT))
1445 return wait_noreap_copyout(p, pid, uid, 1476 return wait_noreap_copyout(p, pid, uid,
1446 why, exit_code, 1477 why, exit_code,
1447 infop, ru); 1478 infop, ru);
@@ -1475,7 +1506,7 @@ unlock_sig:
1475 * the lock and this task is uninteresting. If we return nonzero, we have 1506 * the lock and this task is uninteresting. If we return nonzero, we have
1476 * released the lock and the system call should return. 1507 * released the lock and the system call should return.
1477 */ 1508 */
1478static int wait_task_continued(struct task_struct *p, int noreap, 1509static int wait_task_continued(struct task_struct *p, int options,
1479 struct siginfo __user *infop, 1510 struct siginfo __user *infop,
1480 int __user *stat_addr, struct rusage __user *ru) 1511 int __user *stat_addr, struct rusage __user *ru)
1481{ 1512{
@@ -1483,6 +1514,9 @@ static int wait_task_continued(struct task_struct *p, int noreap,
1483 pid_t pid; 1514 pid_t pid;
1484 uid_t uid; 1515 uid_t uid;
1485 1516
1517 if (!unlikely(options & WCONTINUED))
1518 return 0;
1519
1486 if (!(p->signal->flags & SIGNAL_STOP_CONTINUED)) 1520 if (!(p->signal->flags & SIGNAL_STOP_CONTINUED))
1487 return 0; 1521 return 0;
1488 1522
@@ -1492,7 +1526,7 @@ static int wait_task_continued(struct task_struct *p, int noreap,
1492 spin_unlock_irq(&p->sighand->siglock); 1526 spin_unlock_irq(&p->sighand->siglock);
1493 return 0; 1527 return 0;
1494 } 1528 }
1495 if (!noreap) 1529 if (!unlikely(options & WNOWAIT))
1496 p->signal->flags &= ~SIGNAL_STOP_CONTINUED; 1530 p->signal->flags &= ~SIGNAL_STOP_CONTINUED;
1497 spin_unlock_irq(&p->sighand->siglock); 1531 spin_unlock_irq(&p->sighand->siglock);
1498 1532
@@ -1518,89 +1552,161 @@ static int wait_task_continued(struct task_struct *p, int noreap,
1518 return retval; 1552 return retval;
1519} 1553}
1520 1554
1555/*
1556 * Consider @p for a wait by @parent.
1557 *
1558 * -ECHILD should be in *@notask_error before the first call.
1559 * Returns nonzero for a final return, when we have unlocked tasklist_lock.
1560 * Returns zero if the search for a child should continue;
1561 * then *@notask_error is 0 if @p is an eligible child,
1562 * or another error from security_task_wait(), or still -ECHILD.
1563 */
1564static int wait_consider_task(struct task_struct *parent, int ptrace,
1565 struct task_struct *p, int *notask_error,
1566 enum pid_type type, struct pid *pid, int options,
1567 struct siginfo __user *infop,
1568 int __user *stat_addr, struct rusage __user *ru)
1569{
1570 int ret = eligible_child(type, pid, options, p);
1571 if (!ret)
1572 return ret;
1573
1574 if (unlikely(ret < 0)) {
1575 /*
1576 * If we have not yet seen any eligible child,
1577 * then let this error code replace -ECHILD.
1578 * A permission error will give the user a clue
1579 * to look for security policy problems, rather
1580 * than for mysterious wait bugs.
1581 */
1582 if (*notask_error)
1583 *notask_error = ret;
1584 }
1585
1586 if (likely(!ptrace) && unlikely(p->ptrace)) {
1587 /*
1588 * This child is hidden by ptrace.
1589 * We aren't allowed to see it now, but eventually we will.
1590 */
1591 *notask_error = 0;
1592 return 0;
1593 }
1594
1595 if (p->exit_state == EXIT_DEAD)
1596 return 0;
1597
1598 /*
1599 * We don't reap group leaders with subthreads.
1600 */
1601 if (p->exit_state == EXIT_ZOMBIE && !delay_group_leader(p))
1602 return wait_task_zombie(p, options, infop, stat_addr, ru);
1603
1604 /*
1605 * It's stopped or running now, so it might
1606 * later continue, exit, or stop again.
1607 */
1608 *notask_error = 0;
1609
1610 if (task_is_stopped_or_traced(p))
1611 return wait_task_stopped(ptrace, p, options,
1612 infop, stat_addr, ru);
1613
1614 return wait_task_continued(p, options, infop, stat_addr, ru);
1615}
1616
1617/*
1618 * Do the work of do_wait() for one thread in the group, @tsk.
1619 *
1620 * -ECHILD should be in *@notask_error before the first call.
1621 * Returns nonzero for a final return, when we have unlocked tasklist_lock.
1622 * Returns zero if the search for a child should continue; then
1623 * *@notask_error is 0 if there were any eligible children,
1624 * or another error from security_task_wait(), or still -ECHILD.
1625 */
1626static int do_wait_thread(struct task_struct *tsk, int *notask_error,
1627 enum pid_type type, struct pid *pid, int options,
1628 struct siginfo __user *infop, int __user *stat_addr,
1629 struct rusage __user *ru)
1630{
1631 struct task_struct *p;
1632
1633 list_for_each_entry(p, &tsk->children, sibling) {
1634 /*
1635 * Do not consider detached threads.
1636 */
1637 if (!task_detached(p)) {
1638 int ret = wait_consider_task(tsk, 0, p, notask_error,
1639 type, pid, options,
1640 infop, stat_addr, ru);
1641 if (ret)
1642 return ret;
1643 }
1644 }
1645
1646 return 0;
1647}
1648
1649static int ptrace_do_wait(struct task_struct *tsk, int *notask_error,
1650 enum pid_type type, struct pid *pid, int options,
1651 struct siginfo __user *infop, int __user *stat_addr,
1652 struct rusage __user *ru)
1653{
1654 struct task_struct *p;
1655
1656 /*
1657 * Traditionally we see ptrace'd stopped tasks regardless of options.
1658 */
1659 options |= WUNTRACED;
1660
1661 list_for_each_entry(p, &tsk->ptraced, ptrace_entry) {
1662 int ret = wait_consider_task(tsk, 1, p, notask_error,
1663 type, pid, options,
1664 infop, stat_addr, ru);
1665 if (ret)
1666 return ret;
1667 }
1668
1669 return 0;
1670}
1671
1521static long do_wait(enum pid_type type, struct pid *pid, int options, 1672static long do_wait(enum pid_type type, struct pid *pid, int options,
1522 struct siginfo __user *infop, int __user *stat_addr, 1673 struct siginfo __user *infop, int __user *stat_addr,
1523 struct rusage __user *ru) 1674 struct rusage __user *ru)
1524{ 1675{
1525 DECLARE_WAITQUEUE(wait, current); 1676 DECLARE_WAITQUEUE(wait, current);
1526 struct task_struct *tsk; 1677 struct task_struct *tsk;
1527 int flag, retval; 1678 int retval;
1528 1679
1529 add_wait_queue(&current->signal->wait_chldexit,&wait); 1680 add_wait_queue(&current->signal->wait_chldexit,&wait);
1530repeat: 1681repeat:
1531 /* If there is nothing that can match our critier just get out */ 1682 /*
1683 * If there is nothing that can match our critiera just get out.
1684 * We will clear @retval to zero if we see any child that might later
1685 * match our criteria, even if we are not able to reap it yet.
1686 */
1532 retval = -ECHILD; 1687 retval = -ECHILD;
1533 if ((type < PIDTYPE_MAX) && (!pid || hlist_empty(&pid->tasks[type]))) 1688 if ((type < PIDTYPE_MAX) && (!pid || hlist_empty(&pid->tasks[type])))
1534 goto end; 1689 goto end;
1535 1690
1536 /*
1537 * We will set this flag if we see any child that might later
1538 * match our criteria, even if we are not able to reap it yet.
1539 */
1540 flag = retval = 0;
1541 current->state = TASK_INTERRUPTIBLE; 1691 current->state = TASK_INTERRUPTIBLE;
1542 read_lock(&tasklist_lock); 1692 read_lock(&tasklist_lock);
1543 tsk = current; 1693 tsk = current;
1544 do { 1694 do {
1545 struct task_struct *p; 1695 int tsk_result = do_wait_thread(tsk, &retval,
1546 1696 type, pid, options,
1547 list_for_each_entry(p, &tsk->children, sibling) { 1697 infop, stat_addr, ru);
1548 int ret = eligible_child(type, pid, options, p); 1698 if (!tsk_result)
1549 if (!ret) 1699 tsk_result = ptrace_do_wait(tsk, &retval,
1550 continue; 1700 type, pid, options,
1551 1701 infop, stat_addr, ru);
1552 if (unlikely(ret < 0)) { 1702 if (tsk_result) {
1553 retval = ret; 1703 /*
1554 } else if (task_is_stopped_or_traced(p)) { 1704 * tasklist_lock is unlocked and we have a final result.
1555 /* 1705 */
1556 * It's stopped now, so it might later 1706 retval = tsk_result;
1557 * continue, exit, or stop again. 1707 goto end;
1558 */
1559 flag = 1;
1560 if (!(p->ptrace & PT_PTRACED) &&
1561 !(options & WUNTRACED))
1562 continue;
1563
1564 retval = wait_task_stopped(p,
1565 (options & WNOWAIT), infop,
1566 stat_addr, ru);
1567 } else if (p->exit_state == EXIT_ZOMBIE &&
1568 !delay_group_leader(p)) {
1569 /*
1570 * We don't reap group leaders with subthreads.
1571 */
1572 if (!likely(options & WEXITED))
1573 continue;
1574 retval = wait_task_zombie(p,
1575 (options & WNOWAIT), infop,
1576 stat_addr, ru);
1577 } else if (p->exit_state != EXIT_DEAD) {
1578 /*
1579 * It's running now, so it might later
1580 * exit, stop, or stop and then continue.
1581 */
1582 flag = 1;
1583 if (!unlikely(options & WCONTINUED))
1584 continue;
1585 retval = wait_task_continued(p,
1586 (options & WNOWAIT), infop,
1587 stat_addr, ru);
1588 }
1589 if (retval != 0) /* tasklist_lock released */
1590 goto end;
1591 }
1592 if (!flag) {
1593 list_for_each_entry(p, &tsk->ptrace_children,
1594 ptrace_list) {
1595 flag = eligible_child(type, pid, options, p);
1596 if (!flag)
1597 continue;
1598 if (likely(flag > 0))
1599 break;
1600 retval = flag;
1601 goto end;
1602 }
1603 } 1708 }
1709
1604 if (options & __WNOTHREAD) 1710 if (options & __WNOTHREAD)
1605 break; 1711 break;
1606 tsk = next_thread(tsk); 1712 tsk = next_thread(tsk);
@@ -1608,16 +1714,14 @@ repeat:
1608 } while (tsk != current); 1714 } while (tsk != current);
1609 read_unlock(&tasklist_lock); 1715 read_unlock(&tasklist_lock);
1610 1716
1611 if (flag) { 1717 if (!retval && !(options & WNOHANG)) {
1612 if (options & WNOHANG)
1613 goto end;
1614 retval = -ERESTARTSYS; 1718 retval = -ERESTARTSYS;
1615 if (signal_pending(current)) 1719 if (!signal_pending(current)) {
1616 goto end; 1720 schedule();
1617 schedule(); 1721 goto repeat;
1618 goto repeat; 1722 }
1619 } 1723 }
1620 retval = -ECHILD; 1724
1621end: 1725end:
1622 current->state = TASK_RUNNING; 1726 current->state = TASK_RUNNING;
1623 remove_wait_queue(&current->signal->wait_chldexit,&wait); 1727 remove_wait_queue(&current->signal->wait_chldexit,&wait);
diff --git a/kernel/fork.c b/kernel/fork.c
index 19908b26cf80..8214ba7c8bb1 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -23,6 +23,7 @@
23#include <linux/sem.h> 23#include <linux/sem.h>
24#include <linux/file.h> 24#include <linux/file.h>
25#include <linux/fdtable.h> 25#include <linux/fdtable.h>
26#include <linux/iocontext.h>
26#include <linux/key.h> 27#include <linux/key.h>
27#include <linux/binfmts.h> 28#include <linux/binfmts.h>
28#include <linux/mman.h> 29#include <linux/mman.h>
@@ -32,9 +33,11 @@
32#include <linux/cpu.h> 33#include <linux/cpu.h>
33#include <linux/cgroup.h> 34#include <linux/cgroup.h>
34#include <linux/security.h> 35#include <linux/security.h>
36#include <linux/hugetlb.h>
35#include <linux/swap.h> 37#include <linux/swap.h>
36#include <linux/syscalls.h> 38#include <linux/syscalls.h>
37#include <linux/jiffies.h> 39#include <linux/jiffies.h>
40#include <linux/tracehook.h>
38#include <linux/futex.h> 41#include <linux/futex.h>
39#include <linux/task_io_accounting_ops.h> 42#include <linux/task_io_accounting_ops.h>
40#include <linux/rcupdate.h> 43#include <linux/rcupdate.h>
@@ -91,6 +94,23 @@ int nr_processes(void)
91static struct kmem_cache *task_struct_cachep; 94static struct kmem_cache *task_struct_cachep;
92#endif 95#endif
93 96
97#ifndef __HAVE_ARCH_THREAD_INFO_ALLOCATOR
98static inline struct thread_info *alloc_thread_info(struct task_struct *tsk)
99{
100#ifdef CONFIG_DEBUG_STACK_USAGE
101 gfp_t mask = GFP_KERNEL | __GFP_ZERO;
102#else
103 gfp_t mask = GFP_KERNEL;
104#endif
105 return (struct thread_info *)__get_free_pages(mask, THREAD_SIZE_ORDER);
106}
107
108static inline void free_thread_info(struct thread_info *ti)
109{
110 free_pages((unsigned long)ti, THREAD_SIZE_ORDER);
111}
112#endif
113
94/* SLAB cache for signal_struct structures (tsk->signal) */ 114/* SLAB cache for signal_struct structures (tsk->signal) */
95static struct kmem_cache *signal_cachep; 115static struct kmem_cache *signal_cachep;
96 116
@@ -306,6 +326,14 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
306 } 326 }
307 327
308 /* 328 /*
329 * Clear hugetlb-related page reserves for children. This only
330 * affects MAP_PRIVATE mappings. Faults generated by the child
331 * are not guaranteed to succeed, even if read-only
332 */
333 if (is_vm_hugetlb_page(tmp))
334 reset_vma_resv_huge_pages(tmp);
335
336 /*
309 * Link in the new vma and copy the page table entries. 337 * Link in the new vma and copy the page table entries.
310 */ 338 */
311 *pprev = tmp; 339 *pprev = tmp;
@@ -373,7 +401,7 @@ static struct mm_struct * mm_init(struct mm_struct * mm, struct task_struct *p)
373 INIT_LIST_HEAD(&mm->mmlist); 401 INIT_LIST_HEAD(&mm->mmlist);
374 mm->flags = (current->mm) ? current->mm->flags 402 mm->flags = (current->mm) ? current->mm->flags
375 : MMF_DUMP_FILTER_DEFAULT; 403 : MMF_DUMP_FILTER_DEFAULT;
376 mm->core_waiters = 0; 404 mm->core_state = NULL;
377 mm->nr_ptes = 0; 405 mm->nr_ptes = 0;
378 set_mm_counter(mm, file_rss, 0); 406 set_mm_counter(mm, file_rss, 0);
379 set_mm_counter(mm, anon_rss, 0); 407 set_mm_counter(mm, anon_rss, 0);
@@ -447,7 +475,7 @@ EXPORT_SYMBOL_GPL(mmput);
447/** 475/**
448 * get_task_mm - acquire a reference to the task's mm 476 * get_task_mm - acquire a reference to the task's mm
449 * 477 *
450 * Returns %NULL if the task has no mm. Checks PF_BORROWED_MM (meaning 478 * Returns %NULL if the task has no mm. Checks PF_KTHREAD (meaning
451 * this kernel workthread has transiently adopted a user mm with use_mm, 479 * this kernel workthread has transiently adopted a user mm with use_mm,
452 * to do its AIO) is not set and if so returns a reference to it, after 480 * to do its AIO) is not set and if so returns a reference to it, after
453 * bumping up the use count. User must release the mm via mmput() 481 * bumping up the use count. User must release the mm via mmput()
@@ -460,7 +488,7 @@ struct mm_struct *get_task_mm(struct task_struct *task)
460 task_lock(task); 488 task_lock(task);
461 mm = task->mm; 489 mm = task->mm;
462 if (mm) { 490 if (mm) {
463 if (task->flags & PF_BORROWED_MM) 491 if (task->flags & PF_KTHREAD)
464 mm = NULL; 492 mm = NULL;
465 else 493 else
466 atomic_inc(&mm->mm_users); 494 atomic_inc(&mm->mm_users);
@@ -629,13 +657,6 @@ static struct fs_struct *__copy_fs_struct(struct fs_struct *old)
629 path_get(&old->root); 657 path_get(&old->root);
630 fs->pwd = old->pwd; 658 fs->pwd = old->pwd;
631 path_get(&old->pwd); 659 path_get(&old->pwd);
632 if (old->altroot.dentry) {
633 fs->altroot = old->altroot;
634 path_get(&old->altroot);
635 } else {
636 fs->altroot.mnt = NULL;
637 fs->altroot.dentry = NULL;
638 }
639 read_unlock(&old->lock); 660 read_unlock(&old->lock);
640 } 661 }
641 return fs; 662 return fs;
@@ -785,6 +806,7 @@ static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
785 sig->nvcsw = sig->nivcsw = sig->cnvcsw = sig->cnivcsw = 0; 806 sig->nvcsw = sig->nivcsw = sig->cnvcsw = sig->cnivcsw = 0;
786 sig->min_flt = sig->maj_flt = sig->cmin_flt = sig->cmaj_flt = 0; 807 sig->min_flt = sig->maj_flt = sig->cmin_flt = sig->cmaj_flt = 0;
787 sig->inblock = sig->oublock = sig->cinblock = sig->coublock = 0; 808 sig->inblock = sig->oublock = sig->cinblock = sig->coublock = 0;
809 task_io_accounting_init(&sig->ioac);
788 sig->sum_sched_runtime = 0; 810 sig->sum_sched_runtime = 0;
789 INIT_LIST_HEAD(&sig->cpu_timers[0]); 811 INIT_LIST_HEAD(&sig->cpu_timers[0]);
790 INIT_LIST_HEAD(&sig->cpu_timers[1]); 812 INIT_LIST_HEAD(&sig->cpu_timers[1]);
@@ -832,8 +854,7 @@ static void copy_flags(unsigned long clone_flags, struct task_struct *p)
832 854
833 new_flags &= ~PF_SUPERPRIV; 855 new_flags &= ~PF_SUPERPRIV;
834 new_flags |= PF_FORKNOEXEC; 856 new_flags |= PF_FORKNOEXEC;
835 if (!(clone_flags & CLONE_PTRACE)) 857 new_flags |= PF_STARTING;
836 p->ptrace = 0;
837 p->flags = new_flags; 858 p->flags = new_flags;
838 clear_freeze_flag(p); 859 clear_freeze_flag(p);
839} 860}
@@ -874,7 +895,8 @@ static struct task_struct *copy_process(unsigned long clone_flags,
874 struct pt_regs *regs, 895 struct pt_regs *regs,
875 unsigned long stack_size, 896 unsigned long stack_size,
876 int __user *child_tidptr, 897 int __user *child_tidptr,
877 struct pid *pid) 898 struct pid *pid,
899 int trace)
878{ 900{
879 int retval; 901 int retval;
880 struct task_struct *p; 902 struct task_struct *p;
@@ -909,7 +931,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
909 931
910 rt_mutex_init_task(p); 932 rt_mutex_init_task(p);
911 933
912#ifdef CONFIG_TRACE_IRQFLAGS 934#ifdef CONFIG_PROVE_LOCKING
913 DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled); 935 DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled);
914 DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled); 936 DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
915#endif 937#endif
@@ -967,13 +989,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
967 p->last_switch_timestamp = 0; 989 p->last_switch_timestamp = 0;
968#endif 990#endif
969 991
970#ifdef CONFIG_TASK_XACCT 992 task_io_accounting_init(&p->ioac);
971 p->rchar = 0; /* I/O counter: bytes read */
972 p->wchar = 0; /* I/O counter: bytes written */
973 p->syscr = 0; /* I/O counter: read syscalls */
974 p->syscw = 0; /* I/O counter: write syscalls */
975#endif
976 task_io_accounting_init(p);
977 acct_clear_integrals(p); 993 acct_clear_integrals(p);
978 994
979 p->it_virt_expires = cputime_zero; 995 p->it_virt_expires = cputime_zero;
@@ -1080,6 +1096,12 @@ static struct task_struct *copy_process(unsigned long clone_flags,
1080 if (clone_flags & CLONE_THREAD) 1096 if (clone_flags & CLONE_THREAD)
1081 p->tgid = current->tgid; 1097 p->tgid = current->tgid;
1082 1098
1099 if (current->nsproxy != p->nsproxy) {
1100 retval = ns_cgroup_clone(p, pid);
1101 if (retval)
1102 goto bad_fork_free_pid;
1103 }
1104
1083 p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL; 1105 p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL;
1084 /* 1106 /*
1085 * Clear TID on mm_release()? 1107 * Clear TID on mm_release()?
@@ -1124,8 +1146,6 @@ static struct task_struct *copy_process(unsigned long clone_flags,
1124 */ 1146 */
1125 p->group_leader = p; 1147 p->group_leader = p;
1126 INIT_LIST_HEAD(&p->thread_group); 1148 INIT_LIST_HEAD(&p->thread_group);
1127 INIT_LIST_HEAD(&p->ptrace_children);
1128 INIT_LIST_HEAD(&p->ptrace_list);
1129 1149
1130 /* Now that the task is set up, run cgroup callbacks if 1150 /* Now that the task is set up, run cgroup callbacks if
1131 * necessary. We need to run them before the task is visible 1151 * necessary. We need to run them before the task is visible
@@ -1156,7 +1176,6 @@ static struct task_struct *copy_process(unsigned long clone_flags,
1156 p->real_parent = current->real_parent; 1176 p->real_parent = current->real_parent;
1157 else 1177 else
1158 p->real_parent = current; 1178 p->real_parent = current;
1159 p->parent = p->real_parent;
1160 1179
1161 spin_lock(&current->sighand->siglock); 1180 spin_lock(&current->sighand->siglock);
1162 1181
@@ -1197,9 +1216,8 @@ static struct task_struct *copy_process(unsigned long clone_flags,
1197 } 1216 }
1198 1217
1199 if (likely(p->pid)) { 1218 if (likely(p->pid)) {
1200 add_parent(p); 1219 list_add_tail(&p->sibling, &p->real_parent->children);
1201 if (unlikely(p->ptrace & PT_PTRACED)) 1220 tracehook_finish_clone(p, clone_flags, trace);
1202 __ptrace_link(p, current->parent);
1203 1221
1204 if (thread_group_leader(p)) { 1222 if (thread_group_leader(p)) {
1205 if (clone_flags & CLONE_NEWPID) 1223 if (clone_flags & CLONE_NEWPID)
@@ -1284,29 +1302,13 @@ struct task_struct * __cpuinit fork_idle(int cpu)
1284 struct pt_regs regs; 1302 struct pt_regs regs;
1285 1303
1286 task = copy_process(CLONE_VM, 0, idle_regs(&regs), 0, NULL, 1304 task = copy_process(CLONE_VM, 0, idle_regs(&regs), 0, NULL,
1287 &init_struct_pid); 1305 &init_struct_pid, 0);
1288 if (!IS_ERR(task)) 1306 if (!IS_ERR(task))
1289 init_idle(task, cpu); 1307 init_idle(task, cpu);
1290 1308
1291 return task; 1309 return task;
1292} 1310}
1293 1311
1294static int fork_traceflag(unsigned clone_flags)
1295{
1296 if (clone_flags & CLONE_UNTRACED)
1297 return 0;
1298 else if (clone_flags & CLONE_VFORK) {
1299 if (current->ptrace & PT_TRACE_VFORK)
1300 return PTRACE_EVENT_VFORK;
1301 } else if ((clone_flags & CSIGNAL) != SIGCHLD) {
1302 if (current->ptrace & PT_TRACE_CLONE)
1303 return PTRACE_EVENT_CLONE;
1304 } else if (current->ptrace & PT_TRACE_FORK)
1305 return PTRACE_EVENT_FORK;
1306
1307 return 0;
1308}
1309
1310/* 1312/*
1311 * Ok, this is the main fork-routine. 1313 * Ok, this is the main fork-routine.
1312 * 1314 *
@@ -1341,14 +1343,14 @@ long do_fork(unsigned long clone_flags,
1341 } 1343 }
1342 } 1344 }
1343 1345
1344 if (unlikely(current->ptrace)) { 1346 /*
1345 trace = fork_traceflag (clone_flags); 1347 * When called from kernel_thread, don't do user tracing stuff.
1346 if (trace) 1348 */
1347 clone_flags |= CLONE_PTRACE; 1349 if (likely(user_mode(regs)))
1348 } 1350 trace = tracehook_prepare_clone(clone_flags);
1349 1351
1350 p = copy_process(clone_flags, stack_start, regs, stack_size, 1352 p = copy_process(clone_flags, stack_start, regs, stack_size,
1351 child_tidptr, NULL); 1353 child_tidptr, NULL, trace);
1352 /* 1354 /*
1353 * Do this prior waking up the new thread - the thread pointer 1355 * Do this prior waking up the new thread - the thread pointer
1354 * might get invalid after that point, if the thread exits quickly. 1356 * might get invalid after that point, if the thread exits quickly.
@@ -1366,32 +1368,35 @@ long do_fork(unsigned long clone_flags,
1366 init_completion(&vfork); 1368 init_completion(&vfork);
1367 } 1369 }
1368 1370
1369 if ((p->ptrace & PT_PTRACED) || (clone_flags & CLONE_STOPPED)) { 1371 tracehook_report_clone(trace, regs, clone_flags, nr, p);
1372
1373 /*
1374 * We set PF_STARTING at creation in case tracing wants to
1375 * use this to distinguish a fully live task from one that
1376 * hasn't gotten to tracehook_report_clone() yet. Now we
1377 * clear it and set the child going.
1378 */
1379 p->flags &= ~PF_STARTING;
1380
1381 if (unlikely(clone_flags & CLONE_STOPPED)) {
1370 /* 1382 /*
1371 * We'll start up with an immediate SIGSTOP. 1383 * We'll start up with an immediate SIGSTOP.
1372 */ 1384 */
1373 sigaddset(&p->pending.signal, SIGSTOP); 1385 sigaddset(&p->pending.signal, SIGSTOP);
1374 set_tsk_thread_flag(p, TIF_SIGPENDING); 1386 set_tsk_thread_flag(p, TIF_SIGPENDING);
1375 }
1376
1377 if (!(clone_flags & CLONE_STOPPED))
1378 wake_up_new_task(p, clone_flags);
1379 else
1380 __set_task_state(p, TASK_STOPPED); 1387 __set_task_state(p, TASK_STOPPED);
1381 1388 } else {
1382 if (unlikely (trace)) { 1389 wake_up_new_task(p, clone_flags);
1383 current->ptrace_message = nr;
1384 ptrace_notify ((trace << 8) | SIGTRAP);
1385 } 1390 }
1386 1391
1392 tracehook_report_clone_complete(trace, regs,
1393 clone_flags, nr, p);
1394
1387 if (clone_flags & CLONE_VFORK) { 1395 if (clone_flags & CLONE_VFORK) {
1388 freezer_do_not_count(); 1396 freezer_do_not_count();
1389 wait_for_completion(&vfork); 1397 wait_for_completion(&vfork);
1390 freezer_count(); 1398 freezer_count();
1391 if (unlikely (current->ptrace & PT_TRACE_VFORK_DONE)) { 1399 tracehook_report_vfork_done(p, nr);
1392 current->ptrace_message = nr;
1393 ptrace_notify ((PTRACE_EVENT_VFORK_DONE << 8) | SIGTRAP);
1394 }
1395 } 1400 }
1396 } else { 1401 } else {
1397 nr = PTR_ERR(p); 1402 nr = PTR_ERR(p);
@@ -1403,7 +1408,7 @@ long do_fork(unsigned long clone_flags,
1403#define ARCH_MIN_MMSTRUCT_ALIGN 0 1408#define ARCH_MIN_MMSTRUCT_ALIGN 0
1404#endif 1409#endif
1405 1410
1406static void sighand_ctor(struct kmem_cache *cachep, void *data) 1411static void sighand_ctor(void *data)
1407{ 1412{
1408 struct sighand_struct *sighand = data; 1413 struct sighand_struct *sighand = data;
1409 1414
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c
index ab80515008f4..b8e4dce80a74 100644
--- a/kernel/hrtimer.c
+++ b/kernel/hrtimer.c
@@ -300,11 +300,10 @@ EXPORT_SYMBOL_GPL(ktime_sub_ns);
300 */ 300 */
301u64 ktime_divns(const ktime_t kt, s64 div) 301u64 ktime_divns(const ktime_t kt, s64 div)
302{ 302{
303 u64 dclc, inc, dns; 303 u64 dclc;
304 int sft = 0; 304 int sft = 0;
305 305
306 dclc = dns = ktime_to_ns(kt); 306 dclc = ktime_to_ns(kt);
307 inc = div;
308 /* Make sure the divisor is less than 2^32: */ 307 /* Make sure the divisor is less than 2^32: */
309 while (div >> 32) { 308 while (div >> 32) {
310 sft++; 309 sft++;
@@ -623,7 +622,7 @@ static void retrigger_next_event(void *arg)
623void clock_was_set(void) 622void clock_was_set(void)
624{ 623{
625 /* Retrigger the CPU local events everywhere */ 624 /* Retrigger the CPU local events everywhere */
626 on_each_cpu(retrigger_next_event, NULL, 0, 1); 625 on_each_cpu(retrigger_next_event, NULL, 1);
627} 626}
628 627
629/* 628/*
@@ -632,8 +631,6 @@ void clock_was_set(void)
632 */ 631 */
633void hres_timers_resume(void) 632void hres_timers_resume(void)
634{ 633{
635 WARN_ON_ONCE(num_online_cpus() > 1);
636
637 /* Retrigger the CPU local events: */ 634 /* Retrigger the CPU local events: */
638 retrigger_next_event(NULL); 635 retrigger_next_event(NULL);
639} 636}
@@ -1086,7 +1083,7 @@ ktime_t hrtimer_get_remaining(const struct hrtimer *timer)
1086} 1083}
1087EXPORT_SYMBOL_GPL(hrtimer_get_remaining); 1084EXPORT_SYMBOL_GPL(hrtimer_get_remaining);
1088 1085
1089#if defined(CONFIG_NO_IDLE_HZ) || defined(CONFIG_NO_HZ) 1086#ifdef CONFIG_NO_HZ
1090/** 1087/**
1091 * hrtimer_get_next_event - get the time until next expiry event 1088 * hrtimer_get_next_event - get the time until next expiry event
1092 * 1089 *
@@ -1677,7 +1674,7 @@ void __init hrtimers_init(void)
1677 (void *)(long)smp_processor_id()); 1674 (void *)(long)smp_processor_id());
1678 register_cpu_notifier(&hrtimers_nb); 1675 register_cpu_notifier(&hrtimers_nb);
1679#ifdef CONFIG_HIGH_RES_TIMERS 1676#ifdef CONFIG_HIGH_RES_TIMERS
1680 open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq, NULL); 1677 open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq);
1681#endif 1678#endif
1682} 1679}
1683 1680
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c
index 964964baefa2..3cd441ebf5d2 100644
--- a/kernel/irq/chip.c
+++ b/kernel/irq/chip.c
@@ -28,8 +28,7 @@ void dynamic_irq_init(unsigned int irq)
28 unsigned long flags; 28 unsigned long flags;
29 29
30 if (irq >= NR_IRQS) { 30 if (irq >= NR_IRQS) {
31 printk(KERN_ERR "Trying to initialize invalid IRQ%d\n", irq); 31 WARN(1, KERN_ERR "Trying to initialize invalid IRQ%d\n", irq);
32 WARN_ON(1);
33 return; 32 return;
34 } 33 }
35 34
@@ -62,8 +61,7 @@ void dynamic_irq_cleanup(unsigned int irq)
62 unsigned long flags; 61 unsigned long flags;
63 62
64 if (irq >= NR_IRQS) { 63 if (irq >= NR_IRQS) {
65 printk(KERN_ERR "Trying to cleanup invalid IRQ%d\n", irq); 64 WARN(1, KERN_ERR "Trying to cleanup invalid IRQ%d\n", irq);
66 WARN_ON(1);
67 return; 65 return;
68 } 66 }
69 67
@@ -71,9 +69,8 @@ void dynamic_irq_cleanup(unsigned int irq)
71 spin_lock_irqsave(&desc->lock, flags); 69 spin_lock_irqsave(&desc->lock, flags);
72 if (desc->action) { 70 if (desc->action) {
73 spin_unlock_irqrestore(&desc->lock, flags); 71 spin_unlock_irqrestore(&desc->lock, flags);
74 printk(KERN_ERR "Destroying IRQ%d without calling free_irq\n", 72 WARN(1, KERN_ERR "Destroying IRQ%d without calling free_irq\n",
75 irq); 73 irq);
76 WARN_ON(1);
77 return; 74 return;
78 } 75 }
79 desc->msi_desc = NULL; 76 desc->msi_desc = NULL;
@@ -96,8 +93,7 @@ int set_irq_chip(unsigned int irq, struct irq_chip *chip)
96 unsigned long flags; 93 unsigned long flags;
97 94
98 if (irq >= NR_IRQS) { 95 if (irq >= NR_IRQS) {
99 printk(KERN_ERR "Trying to install chip for IRQ%d\n", irq); 96 WARN(1, KERN_ERR "Trying to install chip for IRQ%d\n", irq);
100 WARN_ON(1);
101 return -EINVAL; 97 return -EINVAL;
102 } 98 }
103 99
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index 46d6611a33bb..152abfd3589f 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -17,6 +17,8 @@
17 17
18#ifdef CONFIG_SMP 18#ifdef CONFIG_SMP
19 19
20cpumask_t irq_default_affinity = CPU_MASK_ALL;
21
20/** 22/**
21 * synchronize_irq - wait for pending IRQ handlers (on other CPUs) 23 * synchronize_irq - wait for pending IRQ handlers (on other CPUs)
22 * @irq: interrupt number to wait for 24 * @irq: interrupt number to wait for
@@ -95,6 +97,27 @@ int irq_set_affinity(unsigned int irq, cpumask_t cpumask)
95 return 0; 97 return 0;
96} 98}
97 99
100#ifndef CONFIG_AUTO_IRQ_AFFINITY
101/*
102 * Generic version of the affinity autoselector.
103 */
104int irq_select_affinity(unsigned int irq)
105{
106 cpumask_t mask;
107
108 if (!irq_can_set_affinity(irq))
109 return 0;
110
111 cpus_and(mask, cpu_online_map, irq_default_affinity);
112
113 irq_desc[irq].affinity = mask;
114 irq_desc[irq].chip->set_affinity(irq, mask);
115
116 set_balance_irq_affinity(irq, mask);
117 return 0;
118}
119#endif
120
98#endif 121#endif
99 122
100/** 123/**
@@ -154,8 +177,7 @@ static void __enable_irq(struct irq_desc *desc, unsigned int irq)
154{ 177{
155 switch (desc->depth) { 178 switch (desc->depth) {
156 case 0: 179 case 0:
157 printk(KERN_WARNING "Unbalanced enable for IRQ %d\n", irq); 180 WARN(1, KERN_WARNING "Unbalanced enable for IRQ %d\n", irq);
158 WARN_ON(1);
159 break; 181 break;
160 case 1: { 182 case 1: {
161 unsigned int status = desc->status & ~IRQ_DISABLED; 183 unsigned int status = desc->status & ~IRQ_DISABLED;
@@ -194,6 +216,17 @@ void enable_irq(unsigned int irq)
194} 216}
195EXPORT_SYMBOL(enable_irq); 217EXPORT_SYMBOL(enable_irq);
196 218
219int set_irq_wake_real(unsigned int irq, unsigned int on)
220{
221 struct irq_desc *desc = irq_desc + irq;
222 int ret = -ENXIO;
223
224 if (desc->chip->set_wake)
225 ret = desc->chip->set_wake(irq, on);
226
227 return ret;
228}
229
197/** 230/**
198 * set_irq_wake - control irq power management wakeup 231 * set_irq_wake - control irq power management wakeup
199 * @irq: interrupt to control 232 * @irq: interrupt to control
@@ -210,30 +243,32 @@ int set_irq_wake(unsigned int irq, unsigned int on)
210{ 243{
211 struct irq_desc *desc = irq_desc + irq; 244 struct irq_desc *desc = irq_desc + irq;
212 unsigned long flags; 245 unsigned long flags;
213 int ret = -ENXIO; 246 int ret = 0;
214 int (*set_wake)(unsigned, unsigned) = desc->chip->set_wake;
215 247
216 /* wakeup-capable irqs can be shared between drivers that 248 /* wakeup-capable irqs can be shared between drivers that
217 * don't need to have the same sleep mode behaviors. 249 * don't need to have the same sleep mode behaviors.
218 */ 250 */
219 spin_lock_irqsave(&desc->lock, flags); 251 spin_lock_irqsave(&desc->lock, flags);
220 if (on) { 252 if (on) {
221 if (desc->wake_depth++ == 0) 253 if (desc->wake_depth++ == 0) {
222 desc->status |= IRQ_WAKEUP; 254 ret = set_irq_wake_real(irq, on);
223 else 255 if (ret)
224 set_wake = NULL; 256 desc->wake_depth = 0;
257 else
258 desc->status |= IRQ_WAKEUP;
259 }
225 } else { 260 } else {
226 if (desc->wake_depth == 0) { 261 if (desc->wake_depth == 0) {
227 printk(KERN_WARNING "Unbalanced IRQ %d " 262 WARN(1, "Unbalanced IRQ %d wake disable\n", irq);
228 "wake disable\n", irq); 263 } else if (--desc->wake_depth == 0) {
229 WARN_ON(1); 264 ret = set_irq_wake_real(irq, on);
230 } else if (--desc->wake_depth == 0) 265 if (ret)
231 desc->status &= ~IRQ_WAKEUP; 266 desc->wake_depth = 1;
232 else 267 else
233 set_wake = NULL; 268 desc->status &= ~IRQ_WAKEUP;
269 }
234 } 270 }
235 if (set_wake) 271
236 ret = desc->chip->set_wake(irq, on);
237 spin_unlock_irqrestore(&desc->lock, flags); 272 spin_unlock_irqrestore(&desc->lock, flags);
238 return ret; 273 return ret;
239} 274}
@@ -270,6 +305,30 @@ void compat_irq_chip_set_default_handler(struct irq_desc *desc)
270 desc->handle_irq = NULL; 305 desc->handle_irq = NULL;
271} 306}
272 307
308static int __irq_set_trigger(struct irq_chip *chip, unsigned int irq,
309 unsigned long flags)
310{
311 int ret;
312
313 if (!chip || !chip->set_type) {
314 /*
315 * IRQF_TRIGGER_* but the PIC does not support multiple
316 * flow-types?
317 */
318 pr_warning("No set_type function for IRQ %d (%s)\n", irq,
319 chip ? (chip->name ? : "unknown") : "unknown");
320 return 0;
321 }
322
323 ret = chip->set_type(irq, flags & IRQF_TRIGGER_MASK);
324
325 if (ret)
326 pr_err("setting flow type for irq %u failed (%pF)\n",
327 irq, chip->set_type);
328
329 return ret;
330}
331
273/* 332/*
274 * Internal function to register an irqaction - typically used to 333 * Internal function to register an irqaction - typically used to
275 * allocate special interrupts that are part of the architecture. 334 * allocate special interrupts that are part of the architecture.
@@ -281,6 +340,7 @@ int setup_irq(unsigned int irq, struct irqaction *new)
281 const char *old_name = NULL; 340 const char *old_name = NULL;
282 unsigned long flags; 341 unsigned long flags;
283 int shared = 0; 342 int shared = 0;
343 int ret;
284 344
285 if (irq >= NR_IRQS) 345 if (irq >= NR_IRQS)
286 return -EINVAL; 346 return -EINVAL;
@@ -338,36 +398,23 @@ int setup_irq(unsigned int irq, struct irqaction *new)
338 shared = 1; 398 shared = 1;
339 } 399 }
340 400
341 *p = new;
342
343 /* Exclude IRQ from balancing */
344 if (new->flags & IRQF_NOBALANCING)
345 desc->status |= IRQ_NO_BALANCING;
346
347 if (!shared) { 401 if (!shared) {
348 irq_chip_set_defaults(desc->chip); 402 irq_chip_set_defaults(desc->chip);
349 403
350#if defined(CONFIG_IRQ_PER_CPU)
351 if (new->flags & IRQF_PERCPU)
352 desc->status |= IRQ_PER_CPU;
353#endif
354
355 /* Setup the type (level, edge polarity) if configured: */ 404 /* Setup the type (level, edge polarity) if configured: */
356 if (new->flags & IRQF_TRIGGER_MASK) { 405 if (new->flags & IRQF_TRIGGER_MASK) {
357 if (desc->chip && desc->chip->set_type) 406 ret = __irq_set_trigger(desc->chip, irq, new->flags);
358 desc->chip->set_type(irq, 407
359 new->flags & IRQF_TRIGGER_MASK); 408 if (ret) {
360 else 409 spin_unlock_irqrestore(&desc->lock, flags);
361 /* 410 return ret;
362 * IRQF_TRIGGER_* but the PIC does not support 411 }
363 * multiple flow-types?
364 */
365 printk(KERN_WARNING "No IRQF_TRIGGER set_type "
366 "function for IRQ %d (%s)\n", irq,
367 desc->chip ? desc->chip->name :
368 "unknown");
369 } else 412 } else
370 compat_irq_chip_set_default_handler(desc); 413 compat_irq_chip_set_default_handler(desc);
414#if defined(CONFIG_IRQ_PER_CPU)
415 if (new->flags & IRQF_PERCPU)
416 desc->status |= IRQ_PER_CPU;
417#endif
371 418
372 desc->status &= ~(IRQ_AUTODETECT | IRQ_WAITING | 419 desc->status &= ~(IRQ_AUTODETECT | IRQ_WAITING |
373 IRQ_INPROGRESS | IRQ_SPURIOUS_DISABLED); 420 IRQ_INPROGRESS | IRQ_SPURIOUS_DISABLED);
@@ -382,7 +429,17 @@ int setup_irq(unsigned int irq, struct irqaction *new)
382 } else 429 } else
383 /* Undo nested disables: */ 430 /* Undo nested disables: */
384 desc->depth = 1; 431 desc->depth = 1;
432
433 /* Set default affinity mask once everything is setup */
434 irq_select_affinity(irq);
385 } 435 }
436
437 *p = new;
438
439 /* Exclude IRQ from balancing */
440 if (new->flags & IRQF_NOBALANCING)
441 desc->status |= IRQ_NO_BALANCING;
442
386 /* Reset broken irq detection when installing new handler */ 443 /* Reset broken irq detection when installing new handler */
387 desc->irq_count = 0; 444 desc->irq_count = 0;
388 desc->irqs_unhandled = 0; 445 desc->irqs_unhandled = 0;
@@ -571,8 +628,6 @@ int request_irq(unsigned int irq, irq_handler_t handler,
571 action->next = NULL; 628 action->next = NULL;
572 action->dev_id = dev_id; 629 action->dev_id = dev_id;
573 630
574 select_smp_affinity(irq);
575
576#ifdef CONFIG_DEBUG_SHIRQ 631#ifdef CONFIG_DEBUG_SHIRQ
577 if (irqflags & IRQF_SHARED) { 632 if (irqflags & IRQF_SHARED) {
578 /* 633 /*
diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c
index c2f2ccb0549a..6c6d35d68ee9 100644
--- a/kernel/irq/proc.c
+++ b/kernel/irq/proc.c
@@ -44,7 +44,7 @@ static int irq_affinity_write_proc(struct file *file, const char __user *buffer,
44 unsigned long count, void *data) 44 unsigned long count, void *data)
45{ 45{
46 unsigned int irq = (int)(long)data, full_count = count, err; 46 unsigned int irq = (int)(long)data, full_count = count, err;
47 cpumask_t new_value, tmp; 47 cpumask_t new_value;
48 48
49 if (!irq_desc[irq].chip->set_affinity || no_irq_affinity || 49 if (!irq_desc[irq].chip->set_affinity || no_irq_affinity ||
50 irq_balancing_disabled(irq)) 50 irq_balancing_disabled(irq))
@@ -62,17 +62,51 @@ static int irq_affinity_write_proc(struct file *file, const char __user *buffer,
62 * way to make the system unusable accidentally :-) At least 62 * way to make the system unusable accidentally :-) At least
63 * one online CPU still has to be targeted. 63 * one online CPU still has to be targeted.
64 */ 64 */
65 cpus_and(tmp, new_value, cpu_online_map); 65 if (!cpus_intersects(new_value, cpu_online_map))
66 if (cpus_empty(tmp))
67 /* Special case for empty set - allow the architecture 66 /* Special case for empty set - allow the architecture
68 code to set default SMP affinity. */ 67 code to set default SMP affinity. */
69 return select_smp_affinity(irq) ? -EINVAL : full_count; 68 return irq_select_affinity(irq) ? -EINVAL : full_count;
70 69
71 irq_set_affinity(irq, new_value); 70 irq_set_affinity(irq, new_value);
72 71
73 return full_count; 72 return full_count;
74} 73}
75 74
75static int default_affinity_read(char *page, char **start, off_t off,
76 int count, int *eof, void *data)
77{
78 int len = cpumask_scnprintf(page, count, irq_default_affinity);
79 if (count - len < 2)
80 return -EINVAL;
81 len += sprintf(page + len, "\n");
82 return len;
83}
84
85static int default_affinity_write(struct file *file, const char __user *buffer,
86 unsigned long count, void *data)
87{
88 unsigned int full_count = count, err;
89 cpumask_t new_value;
90
91 err = cpumask_parse_user(buffer, count, new_value);
92 if (err)
93 return err;
94
95 if (!is_affinity_mask_valid(new_value))
96 return -EINVAL;
97
98 /*
99 * Do not allow disabling IRQs completely - it's a too easy
100 * way to make the system unusable accidentally :-) At least
101 * one online CPU still has to be targeted.
102 */
103 if (!cpus_intersects(new_value, cpu_online_map))
104 return -EINVAL;
105
106 irq_default_affinity = new_value;
107
108 return full_count;
109}
76#endif 110#endif
77 111
78static int irq_spurious_read(char *page, char **start, off_t off, 112static int irq_spurious_read(char *page, char **start, off_t off,
@@ -171,6 +205,21 @@ void unregister_handler_proc(unsigned int irq, struct irqaction *action)
171 remove_proc_entry(action->dir->name, irq_desc[irq].dir); 205 remove_proc_entry(action->dir->name, irq_desc[irq].dir);
172} 206}
173 207
208void register_default_affinity_proc(void)
209{
210#ifdef CONFIG_SMP
211 struct proc_dir_entry *entry;
212
213 /* create /proc/irq/default_smp_affinity */
214 entry = create_proc_entry("default_smp_affinity", 0600, root_irq_dir);
215 if (entry) {
216 entry->data = NULL;
217 entry->read_proc = default_affinity_read;
218 entry->write_proc = default_affinity_write;
219 }
220#endif
221}
222
174void init_irq_proc(void) 223void init_irq_proc(void)
175{ 224{
176 int i; 225 int i;
@@ -180,6 +229,8 @@ void init_irq_proc(void)
180 if (!root_irq_dir) 229 if (!root_irq_dir)
181 return; 230 return;
182 231
232 register_default_affinity_proc();
233
183 /* 234 /*
184 * Create entries for all existing IRQs. 235 * Create entries for all existing IRQs.
185 */ 236 */
diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c
index 6fc0040f3e3a..38fc10ac7541 100644
--- a/kernel/kallsyms.c
+++ b/kernel/kallsyms.c
@@ -176,7 +176,7 @@ static unsigned long get_symbol_pos(unsigned long addr,
176 high = kallsyms_num_syms; 176 high = kallsyms_num_syms;
177 177
178 while (high - low > 1) { 178 while (high - low > 1) {
179 mid = (low + high) / 2; 179 mid = low + (high - low) / 2;
180 if (kallsyms_addresses[mid] <= addr) 180 if (kallsyms_addresses[mid] <= addr)
181 low = mid; 181 low = mid;
182 else 182 else
diff --git a/kernel/kexec.c b/kernel/kexec.c
index 1c5fcacbcf33..c8a4370e2a34 100644
--- a/kernel/kexec.c
+++ b/kernel/kexec.c
@@ -24,6 +24,12 @@
24#include <linux/utsrelease.h> 24#include <linux/utsrelease.h>
25#include <linux/utsname.h> 25#include <linux/utsname.h>
26#include <linux/numa.h> 26#include <linux/numa.h>
27#include <linux/suspend.h>
28#include <linux/device.h>
29#include <linux/freezer.h>
30#include <linux/pm.h>
31#include <linux/cpu.h>
32#include <linux/console.h>
27 33
28#include <asm/page.h> 34#include <asm/page.h>
29#include <asm/uaccess.h> 35#include <asm/uaccess.h>
@@ -242,6 +248,12 @@ static int kimage_normal_alloc(struct kimage **rimage, unsigned long entry,
242 goto out; 248 goto out;
243 } 249 }
244 250
251 image->swap_page = kimage_alloc_control_pages(image, 0);
252 if (!image->swap_page) {
253 printk(KERN_ERR "Could not allocate swap buffer\n");
254 goto out;
255 }
256
245 result = 0; 257 result = 0;
246 out: 258 out:
247 if (result == 0) 259 if (result == 0)
@@ -589,14 +601,12 @@ static void kimage_free_extra_pages(struct kimage *image)
589 kimage_free_page_list(&image->unuseable_pages); 601 kimage_free_page_list(&image->unuseable_pages);
590 602
591} 603}
592static int kimage_terminate(struct kimage *image) 604static void kimage_terminate(struct kimage *image)
593{ 605{
594 if (*image->entry != 0) 606 if (*image->entry != 0)
595 image->entry++; 607 image->entry++;
596 608
597 *image->entry = IND_DONE; 609 *image->entry = IND_DONE;
598
599 return 0;
600} 610}
601 611
602#define for_each_kimage_entry(image, ptr, entry) \ 612#define for_each_kimage_entry(image, ptr, entry) \
@@ -988,6 +998,8 @@ asmlinkage long sys_kexec_load(unsigned long entry, unsigned long nr_segments,
988 if (result) 998 if (result)
989 goto out; 999 goto out;
990 1000
1001 if (flags & KEXEC_PRESERVE_CONTEXT)
1002 image->preserve_context = 1;
991 result = machine_kexec_prepare(image); 1003 result = machine_kexec_prepare(image);
992 if (result) 1004 if (result)
993 goto out; 1005 goto out;
@@ -997,9 +1009,7 @@ asmlinkage long sys_kexec_load(unsigned long entry, unsigned long nr_segments,
997 if (result) 1009 if (result)
998 goto out; 1010 goto out;
999 } 1011 }
1000 result = kimage_terminate(image); 1012 kimage_terminate(image);
1001 if (result)
1002 goto out;
1003 } 1013 }
1004 /* Install the new kernel, and Uninstall the old */ 1014 /* Install the new kernel, and Uninstall the old */
1005 image = xchg(dest_image, image); 1015 image = xchg(dest_image, image);
@@ -1415,3 +1425,85 @@ static int __init crash_save_vmcoreinfo_init(void)
1415} 1425}
1416 1426
1417module_init(crash_save_vmcoreinfo_init) 1427module_init(crash_save_vmcoreinfo_init)
1428
1429/**
1430 * kernel_kexec - reboot the system
1431 *
1432 * Move into place and start executing a preloaded standalone
1433 * executable. If nothing was preloaded return an error.
1434 */
1435int kernel_kexec(void)
1436{
1437 int error = 0;
1438
1439 if (xchg(&kexec_lock, 1))
1440 return -EBUSY;
1441 if (!kexec_image) {
1442 error = -EINVAL;
1443 goto Unlock;
1444 }
1445
1446 if (kexec_image->preserve_context) {
1447#ifdef CONFIG_KEXEC_JUMP
1448 mutex_lock(&pm_mutex);
1449 pm_prepare_console();
1450 error = freeze_processes();
1451 if (error) {
1452 error = -EBUSY;
1453 goto Restore_console;
1454 }
1455 suspend_console();
1456 error = device_suspend(PMSG_FREEZE);
1457 if (error)
1458 goto Resume_console;
1459 error = disable_nonboot_cpus();
1460 if (error)
1461 goto Resume_devices;
1462 local_irq_disable();
1463 /* At this point, device_suspend() has been called,
1464 * but *not* device_power_down(). We *must*
1465 * device_power_down() now. Otherwise, drivers for
1466 * some devices (e.g. interrupt controllers) become
1467 * desynchronized with the actual state of the
1468 * hardware at resume time, and evil weirdness ensues.
1469 */
1470 error = device_power_down(PMSG_FREEZE);
1471 if (error)
1472 goto Enable_irqs;
1473 save_processor_state();
1474#endif
1475 } else {
1476 blocking_notifier_call_chain(&reboot_notifier_list,
1477 SYS_RESTART, NULL);
1478 system_state = SYSTEM_RESTART;
1479 device_shutdown();
1480 sysdev_shutdown();
1481 printk(KERN_EMERG "Starting new kernel\n");
1482 machine_shutdown();
1483 }
1484
1485 machine_kexec(kexec_image);
1486
1487 if (kexec_image->preserve_context) {
1488#ifdef CONFIG_KEXEC_JUMP
1489 restore_processor_state();
1490 device_power_up(PMSG_RESTORE);
1491 Enable_irqs:
1492 local_irq_enable();
1493 enable_nonboot_cpus();
1494 Resume_devices:
1495 device_resume(PMSG_RESTORE);
1496 Resume_console:
1497 resume_console();
1498 thaw_processes();
1499 Restore_console:
1500 pm_restore_console();
1501 mutex_unlock(&pm_mutex);
1502#endif
1503 }
1504
1505 Unlock:
1506 xchg(&kexec_lock, 0);
1507
1508 return error;
1509}
diff --git a/kernel/kmod.c b/kernel/kmod.c
index 8df97d3dfda8..2456d1a0befb 100644
--- a/kernel/kmod.c
+++ b/kernel/kmod.c
@@ -42,7 +42,7 @@ extern int max_threads;
42 42
43static struct workqueue_struct *khelper_wq; 43static struct workqueue_struct *khelper_wq;
44 44
45#ifdef CONFIG_KMOD 45#ifdef CONFIG_MODULES
46 46
47/* 47/*
48 modprobe_path is set via /proc/sys. 48 modprobe_path is set via /proc/sys.
@@ -352,16 +352,17 @@ static inline void register_pm_notifier_callback(void) {}
352 * @path: path to usermode executable 352 * @path: path to usermode executable
353 * @argv: arg vector for process 353 * @argv: arg vector for process
354 * @envp: environment for process 354 * @envp: environment for process
355 * @gfp_mask: gfp mask for memory allocation
355 * 356 *
356 * Returns either %NULL on allocation failure, or a subprocess_info 357 * Returns either %NULL on allocation failure, or a subprocess_info
357 * structure. This should be passed to call_usermodehelper_exec to 358 * structure. This should be passed to call_usermodehelper_exec to
358 * exec the process and free the structure. 359 * exec the process and free the structure.
359 */ 360 */
360struct subprocess_info *call_usermodehelper_setup(char *path, 361struct subprocess_info *call_usermodehelper_setup(char *path, char **argv,
361 char **argv, char **envp) 362 char **envp, gfp_t gfp_mask)
362{ 363{
363 struct subprocess_info *sub_info; 364 struct subprocess_info *sub_info;
364 sub_info = kzalloc(sizeof(struct subprocess_info), GFP_ATOMIC); 365 sub_info = kzalloc(sizeof(struct subprocess_info), gfp_mask);
365 if (!sub_info) 366 if (!sub_info)
366 goto out; 367 goto out;
367 368
@@ -417,12 +418,12 @@ int call_usermodehelper_stdinpipe(struct subprocess_info *sub_info,
417{ 418{
418 struct file *f; 419 struct file *f;
419 420
420 f = create_write_pipe(); 421 f = create_write_pipe(0);
421 if (IS_ERR(f)) 422 if (IS_ERR(f))
422 return PTR_ERR(f); 423 return PTR_ERR(f);
423 *filp = f; 424 *filp = f;
424 425
425 f = create_read_pipe(f); 426 f = create_read_pipe(f, 0);
426 if (IS_ERR(f)) { 427 if (IS_ERR(f)) {
427 free_write_pipe(*filp); 428 free_write_pipe(*filp);
428 return PTR_ERR(f); 429 return PTR_ERR(f);
@@ -494,7 +495,7 @@ int call_usermodehelper_pipe(char *path, char **argv, char **envp,
494 struct subprocess_info *sub_info; 495 struct subprocess_info *sub_info;
495 int ret; 496 int ret;
496 497
497 sub_info = call_usermodehelper_setup(path, argv, envp); 498 sub_info = call_usermodehelper_setup(path, argv, envp, GFP_KERNEL);
498 if (sub_info == NULL) 499 if (sub_info == NULL)
499 return -ENOMEM; 500 return -ENOMEM;
500 501
diff --git a/kernel/kprobes.c b/kernel/kprobes.c
index 1485ca8d0e00..75bc2cd9ebc6 100644
--- a/kernel/kprobes.c
+++ b/kernel/kprobes.c
@@ -62,6 +62,7 @@
62 addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name))) 62 addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name)))
63#endif 63#endif
64 64
65static int kprobes_initialized;
65static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE]; 66static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
66static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE]; 67static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
67 68
@@ -69,8 +70,15 @@ static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
69static bool kprobe_enabled; 70static bool kprobe_enabled;
70 71
71DEFINE_MUTEX(kprobe_mutex); /* Protects kprobe_table */ 72DEFINE_MUTEX(kprobe_mutex); /* Protects kprobe_table */
72DEFINE_SPINLOCK(kretprobe_lock); /* Protects kretprobe_inst_table */
73static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL; 73static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
74static struct {
75 spinlock_t lock ____cacheline_aligned;
76} kretprobe_table_locks[KPROBE_TABLE_SIZE];
77
78static spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
79{
80 return &(kretprobe_table_locks[hash].lock);
81}
74 82
75/* 83/*
76 * Normally, functions that we'd want to prohibit kprobes in, are marked 84 * Normally, functions that we'd want to prohibit kprobes in, are marked
@@ -368,26 +376,53 @@ void __kprobes kprobes_inc_nmissed_count(struct kprobe *p)
368 return; 376 return;
369} 377}
370 378
371/* Called with kretprobe_lock held */
372void __kprobes recycle_rp_inst(struct kretprobe_instance *ri, 379void __kprobes recycle_rp_inst(struct kretprobe_instance *ri,
373 struct hlist_head *head) 380 struct hlist_head *head)
374{ 381{
382 struct kretprobe *rp = ri->rp;
383
375 /* remove rp inst off the rprobe_inst_table */ 384 /* remove rp inst off the rprobe_inst_table */
376 hlist_del(&ri->hlist); 385 hlist_del(&ri->hlist);
377 if (ri->rp) { 386 INIT_HLIST_NODE(&ri->hlist);
378 /* remove rp inst off the used list */ 387 if (likely(rp)) {
379 hlist_del(&ri->uflist); 388 spin_lock(&rp->lock);
380 /* put rp inst back onto the free list */ 389 hlist_add_head(&ri->hlist, &rp->free_instances);
381 INIT_HLIST_NODE(&ri->uflist); 390 spin_unlock(&rp->lock);
382 hlist_add_head(&ri->uflist, &ri->rp->free_instances);
383 } else 391 } else
384 /* Unregistering */ 392 /* Unregistering */
385 hlist_add_head(&ri->hlist, head); 393 hlist_add_head(&ri->hlist, head);
386} 394}
387 395
388struct hlist_head __kprobes *kretprobe_inst_table_head(struct task_struct *tsk) 396void kretprobe_hash_lock(struct task_struct *tsk,
397 struct hlist_head **head, unsigned long *flags)
389{ 398{
390 return &kretprobe_inst_table[hash_ptr(tsk, KPROBE_HASH_BITS)]; 399 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
400 spinlock_t *hlist_lock;
401
402 *head = &kretprobe_inst_table[hash];
403 hlist_lock = kretprobe_table_lock_ptr(hash);
404 spin_lock_irqsave(hlist_lock, *flags);
405}
406
407void kretprobe_table_lock(unsigned long hash, unsigned long *flags)
408{
409 spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
410 spin_lock_irqsave(hlist_lock, *flags);
411}
412
413void kretprobe_hash_unlock(struct task_struct *tsk, unsigned long *flags)
414{
415 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
416 spinlock_t *hlist_lock;
417
418 hlist_lock = kretprobe_table_lock_ptr(hash);
419 spin_unlock_irqrestore(hlist_lock, *flags);
420}
421
422void kretprobe_table_unlock(unsigned long hash, unsigned long *flags)
423{
424 spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
425 spin_unlock_irqrestore(hlist_lock, *flags);
391} 426}
392 427
393/* 428/*
@@ -401,17 +436,21 @@ void __kprobes kprobe_flush_task(struct task_struct *tk)
401 struct kretprobe_instance *ri; 436 struct kretprobe_instance *ri;
402 struct hlist_head *head, empty_rp; 437 struct hlist_head *head, empty_rp;
403 struct hlist_node *node, *tmp; 438 struct hlist_node *node, *tmp;
404 unsigned long flags = 0; 439 unsigned long hash, flags = 0;
405 440
406 INIT_HLIST_HEAD(&empty_rp); 441 if (unlikely(!kprobes_initialized))
407 spin_lock_irqsave(&kretprobe_lock, flags); 442 /* Early boot. kretprobe_table_locks not yet initialized. */
408 head = kretprobe_inst_table_head(tk); 443 return;
444
445 hash = hash_ptr(tk, KPROBE_HASH_BITS);
446 head = &kretprobe_inst_table[hash];
447 kretprobe_table_lock(hash, &flags);
409 hlist_for_each_entry_safe(ri, node, tmp, head, hlist) { 448 hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
410 if (ri->task == tk) 449 if (ri->task == tk)
411 recycle_rp_inst(ri, &empty_rp); 450 recycle_rp_inst(ri, &empty_rp);
412 } 451 }
413 spin_unlock_irqrestore(&kretprobe_lock, flags); 452 kretprobe_table_unlock(hash, &flags);
414 453 INIT_HLIST_HEAD(&empty_rp);
415 hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) { 454 hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
416 hlist_del(&ri->hlist); 455 hlist_del(&ri->hlist);
417 kfree(ri); 456 kfree(ri);
@@ -423,24 +462,29 @@ static inline void free_rp_inst(struct kretprobe *rp)
423 struct kretprobe_instance *ri; 462 struct kretprobe_instance *ri;
424 struct hlist_node *pos, *next; 463 struct hlist_node *pos, *next;
425 464
426 hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, uflist) { 465 hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, hlist) {
427 hlist_del(&ri->uflist); 466 hlist_del(&ri->hlist);
428 kfree(ri); 467 kfree(ri);
429 } 468 }
430} 469}
431 470
432static void __kprobes cleanup_rp_inst(struct kretprobe *rp) 471static void __kprobes cleanup_rp_inst(struct kretprobe *rp)
433{ 472{
434 unsigned long flags; 473 unsigned long flags, hash;
435 struct kretprobe_instance *ri; 474 struct kretprobe_instance *ri;
436 struct hlist_node *pos, *next; 475 struct hlist_node *pos, *next;
476 struct hlist_head *head;
477
437 /* No race here */ 478 /* No race here */
438 spin_lock_irqsave(&kretprobe_lock, flags); 479 for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
439 hlist_for_each_entry_safe(ri, pos, next, &rp->used_instances, uflist) { 480 kretprobe_table_lock(hash, &flags);
440 ri->rp = NULL; 481 head = &kretprobe_inst_table[hash];
441 hlist_del(&ri->uflist); 482 hlist_for_each_entry_safe(ri, pos, next, head, hlist) {
483 if (ri->rp == rp)
484 ri->rp = NULL;
485 }
486 kretprobe_table_unlock(hash, &flags);
442 } 487 }
443 spin_unlock_irqrestore(&kretprobe_lock, flags);
444 free_rp_inst(rp); 488 free_rp_inst(rp);
445} 489}
446 490
@@ -831,32 +875,37 @@ static int __kprobes pre_handler_kretprobe(struct kprobe *p,
831 struct pt_regs *regs) 875 struct pt_regs *regs)
832{ 876{
833 struct kretprobe *rp = container_of(p, struct kretprobe, kp); 877 struct kretprobe *rp = container_of(p, struct kretprobe, kp);
834 unsigned long flags = 0; 878 unsigned long hash, flags = 0;
879 struct kretprobe_instance *ri;
835 880
836 /*TODO: consider to only swap the RA after the last pre_handler fired */ 881 /*TODO: consider to only swap the RA after the last pre_handler fired */
837 spin_lock_irqsave(&kretprobe_lock, flags); 882 hash = hash_ptr(current, KPROBE_HASH_BITS);
883 spin_lock_irqsave(&rp->lock, flags);
838 if (!hlist_empty(&rp->free_instances)) { 884 if (!hlist_empty(&rp->free_instances)) {
839 struct kretprobe_instance *ri;
840
841 ri = hlist_entry(rp->free_instances.first, 885 ri = hlist_entry(rp->free_instances.first,
842 struct kretprobe_instance, uflist); 886 struct kretprobe_instance, hlist);
887 hlist_del(&ri->hlist);
888 spin_unlock_irqrestore(&rp->lock, flags);
889
843 ri->rp = rp; 890 ri->rp = rp;
844 ri->task = current; 891 ri->task = current;
845 892
846 if (rp->entry_handler && rp->entry_handler(ri, regs)) { 893 if (rp->entry_handler && rp->entry_handler(ri, regs)) {
847 spin_unlock_irqrestore(&kretprobe_lock, flags); 894 spin_unlock_irqrestore(&rp->lock, flags);
848 return 0; 895 return 0;
849 } 896 }
850 897
851 arch_prepare_kretprobe(ri, regs); 898 arch_prepare_kretprobe(ri, regs);
852 899
853 /* XXX(hch): why is there no hlist_move_head? */ 900 /* XXX(hch): why is there no hlist_move_head? */
854 hlist_del(&ri->uflist); 901 INIT_HLIST_NODE(&ri->hlist);
855 hlist_add_head(&ri->uflist, &ri->rp->used_instances); 902 kretprobe_table_lock(hash, &flags);
856 hlist_add_head(&ri->hlist, kretprobe_inst_table_head(ri->task)); 903 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]);
857 } else 904 kretprobe_table_unlock(hash, &flags);
905 } else {
858 rp->nmissed++; 906 rp->nmissed++;
859 spin_unlock_irqrestore(&kretprobe_lock, flags); 907 spin_unlock_irqrestore(&rp->lock, flags);
908 }
860 return 0; 909 return 0;
861} 910}
862 911
@@ -892,7 +941,7 @@ static int __kprobes __register_kretprobe(struct kretprobe *rp,
892 rp->maxactive = NR_CPUS; 941 rp->maxactive = NR_CPUS;
893#endif 942#endif
894 } 943 }
895 INIT_HLIST_HEAD(&rp->used_instances); 944 spin_lock_init(&rp->lock);
896 INIT_HLIST_HEAD(&rp->free_instances); 945 INIT_HLIST_HEAD(&rp->free_instances);
897 for (i = 0; i < rp->maxactive; i++) { 946 for (i = 0; i < rp->maxactive; i++) {
898 inst = kmalloc(sizeof(struct kretprobe_instance) + 947 inst = kmalloc(sizeof(struct kretprobe_instance) +
@@ -901,8 +950,8 @@ static int __kprobes __register_kretprobe(struct kretprobe *rp,
901 free_rp_inst(rp); 950 free_rp_inst(rp);
902 return -ENOMEM; 951 return -ENOMEM;
903 } 952 }
904 INIT_HLIST_NODE(&inst->uflist); 953 INIT_HLIST_NODE(&inst->hlist);
905 hlist_add_head(&inst->uflist, &rp->free_instances); 954 hlist_add_head(&inst->hlist, &rp->free_instances);
906 } 955 }
907 956
908 rp->nmissed = 0; 957 rp->nmissed = 0;
@@ -1009,6 +1058,7 @@ static int __init init_kprobes(void)
1009 for (i = 0; i < KPROBE_TABLE_SIZE; i++) { 1058 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1010 INIT_HLIST_HEAD(&kprobe_table[i]); 1059 INIT_HLIST_HEAD(&kprobe_table[i]);
1011 INIT_HLIST_HEAD(&kretprobe_inst_table[i]); 1060 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
1061 spin_lock_init(&(kretprobe_table_locks[i].lock));
1012 } 1062 }
1013 1063
1014 /* 1064 /*
@@ -1050,6 +1100,7 @@ static int __init init_kprobes(void)
1050 err = arch_init_kprobes(); 1100 err = arch_init_kprobes();
1051 if (!err) 1101 if (!err)
1052 err = register_die_notifier(&kprobe_exceptions_nb); 1102 err = register_die_notifier(&kprobe_exceptions_nb);
1103 kprobes_initialized = (err == 0);
1053 1104
1054 if (!err) 1105 if (!err)
1055 init_test_probes(); 1106 init_test_probes();
@@ -1286,13 +1337,8 @@ EXPORT_SYMBOL_GPL(register_jprobe);
1286EXPORT_SYMBOL_GPL(unregister_jprobe); 1337EXPORT_SYMBOL_GPL(unregister_jprobe);
1287EXPORT_SYMBOL_GPL(register_jprobes); 1338EXPORT_SYMBOL_GPL(register_jprobes);
1288EXPORT_SYMBOL_GPL(unregister_jprobes); 1339EXPORT_SYMBOL_GPL(unregister_jprobes);
1289#ifdef CONFIG_KPROBES
1290EXPORT_SYMBOL_GPL(jprobe_return); 1340EXPORT_SYMBOL_GPL(jprobe_return);
1291#endif
1292
1293#ifdef CONFIG_KPROBES
1294EXPORT_SYMBOL_GPL(register_kretprobe); 1341EXPORT_SYMBOL_GPL(register_kretprobe);
1295EXPORT_SYMBOL_GPL(unregister_kretprobe); 1342EXPORT_SYMBOL_GPL(unregister_kretprobe);
1296EXPORT_SYMBOL_GPL(register_kretprobes); 1343EXPORT_SYMBOL_GPL(register_kretprobes);
1297EXPORT_SYMBOL_GPL(unregister_kretprobes); 1344EXPORT_SYMBOL_GPL(unregister_kretprobes);
1298#endif
diff --git a/kernel/kthread.c b/kernel/kthread.c
index bd1b9ea024e1..96cff2f8710b 100644
--- a/kernel/kthread.c
+++ b/kernel/kthread.c
@@ -106,7 +106,7 @@ static void create_kthread(struct kthread_create_info *create)
106 */ 106 */
107 sched_setscheduler(create->result, SCHED_NORMAL, &param); 107 sched_setscheduler(create->result, SCHED_NORMAL, &param);
108 set_user_nice(create->result, KTHREAD_NICE_LEVEL); 108 set_user_nice(create->result, KTHREAD_NICE_LEVEL);
109 set_cpus_allowed(create->result, CPU_MASK_ALL); 109 set_cpus_allowed_ptr(create->result, CPU_MASK_ALL_PTR);
110 } 110 }
111 complete(&create->done); 111 complete(&create->done);
112} 112}
@@ -176,10 +176,11 @@ void kthread_bind(struct task_struct *k, unsigned int cpu)
176 return; 176 return;
177 } 177 }
178 /* Must have done schedule() in kthread() before we set_task_cpu */ 178 /* Must have done schedule() in kthread() before we set_task_cpu */
179 wait_task_inactive(k); 179 wait_task_inactive(k, 0);
180 set_task_cpu(k, cpu); 180 set_task_cpu(k, cpu);
181 k->cpus_allowed = cpumask_of_cpu(cpu); 181 k->cpus_allowed = cpumask_of_cpu(cpu);
182 k->rt.nr_cpus_allowed = 1; 182 k->rt.nr_cpus_allowed = 1;
183 k->flags |= PF_THREAD_BOUND;
183} 184}
184EXPORT_SYMBOL(kthread_bind); 185EXPORT_SYMBOL(kthread_bind);
185 186
@@ -232,9 +233,9 @@ int kthreadd(void *unused)
232 set_task_comm(tsk, "kthreadd"); 233 set_task_comm(tsk, "kthreadd");
233 ignore_signals(tsk); 234 ignore_signals(tsk);
234 set_user_nice(tsk, KTHREAD_NICE_LEVEL); 235 set_user_nice(tsk, KTHREAD_NICE_LEVEL);
235 set_cpus_allowed(tsk, CPU_MASK_ALL); 236 set_cpus_allowed_ptr(tsk, CPU_MASK_ALL_PTR);
236 237
237 current->flags |= PF_NOFREEZE; 238 current->flags |= PF_NOFREEZE | PF_FREEZER_NOSIG;
238 239
239 for (;;) { 240 for (;;) {
240 set_current_state(TASK_INTERRUPTIBLE); 241 set_current_state(TASK_INTERRUPTIBLE);
diff --git a/kernel/lockdep.c b/kernel/lockdep.c
index 81a4e4a3f087..d38a64362973 100644
--- a/kernel/lockdep.c
+++ b/kernel/lockdep.c
@@ -39,6 +39,7 @@
39#include <linux/irqflags.h> 39#include <linux/irqflags.h>
40#include <linux/utsname.h> 40#include <linux/utsname.h>
41#include <linux/hash.h> 41#include <linux/hash.h>
42#include <linux/ftrace.h>
42 43
43#include <asm/sections.h> 44#include <asm/sections.h>
44 45
@@ -81,6 +82,8 @@ static int graph_lock(void)
81 __raw_spin_unlock(&lockdep_lock); 82 __raw_spin_unlock(&lockdep_lock);
82 return 0; 83 return 0;
83 } 84 }
85 /* prevent any recursions within lockdep from causing deadlocks */
86 current->lockdep_recursion++;
84 return 1; 87 return 1;
85} 88}
86 89
@@ -89,6 +92,7 @@ static inline int graph_unlock(void)
89 if (debug_locks && !__raw_spin_is_locked(&lockdep_lock)) 92 if (debug_locks && !__raw_spin_is_locked(&lockdep_lock))
90 return DEBUG_LOCKS_WARN_ON(1); 93 return DEBUG_LOCKS_WARN_ON(1);
91 94
95 current->lockdep_recursion--;
92 __raw_spin_unlock(&lockdep_lock); 96 __raw_spin_unlock(&lockdep_lock);
93 return 0; 97 return 0;
94} 98}
@@ -982,7 +986,7 @@ check_noncircular(struct lock_class *source, unsigned int depth)
982 return 1; 986 return 1;
983} 987}
984 988
985#ifdef CONFIG_TRACE_IRQFLAGS 989#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING)
986/* 990/*
987 * Forwards and backwards subgraph searching, for the purposes of 991 * Forwards and backwards subgraph searching, for the purposes of
988 * proving that two subgraphs can be connected by a new dependency 992 * proving that two subgraphs can be connected by a new dependency
@@ -1458,7 +1462,14 @@ out_bug:
1458} 1462}
1459 1463
1460unsigned long nr_lock_chains; 1464unsigned long nr_lock_chains;
1461static struct lock_chain lock_chains[MAX_LOCKDEP_CHAINS]; 1465struct lock_chain lock_chains[MAX_LOCKDEP_CHAINS];
1466int nr_chain_hlocks;
1467static u16 chain_hlocks[MAX_LOCKDEP_CHAIN_HLOCKS];
1468
1469struct lock_class *lock_chain_get_class(struct lock_chain *chain, int i)
1470{
1471 return lock_classes + chain_hlocks[chain->base + i];
1472}
1462 1473
1463/* 1474/*
1464 * Look up a dependency chain. If the key is not present yet then 1475 * Look up a dependency chain. If the key is not present yet then
@@ -1466,10 +1477,15 @@ static struct lock_chain lock_chains[MAX_LOCKDEP_CHAINS];
1466 * validated. If the key is already hashed, return 0. 1477 * validated. If the key is already hashed, return 0.
1467 * (On return with 1 graph_lock is held.) 1478 * (On return with 1 graph_lock is held.)
1468 */ 1479 */
1469static inline int lookup_chain_cache(u64 chain_key, struct lock_class *class) 1480static inline int lookup_chain_cache(struct task_struct *curr,
1481 struct held_lock *hlock,
1482 u64 chain_key)
1470{ 1483{
1484 struct lock_class *class = hlock->class;
1471 struct list_head *hash_head = chainhashentry(chain_key); 1485 struct list_head *hash_head = chainhashentry(chain_key);
1472 struct lock_chain *chain; 1486 struct lock_chain *chain;
1487 struct held_lock *hlock_curr, *hlock_next;
1488 int i, j, n, cn;
1473 1489
1474 if (DEBUG_LOCKS_WARN_ON(!irqs_disabled())) 1490 if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
1475 return 0; 1491 return 0;
@@ -1517,6 +1533,32 @@ cache_hit:
1517 } 1533 }
1518 chain = lock_chains + nr_lock_chains++; 1534 chain = lock_chains + nr_lock_chains++;
1519 chain->chain_key = chain_key; 1535 chain->chain_key = chain_key;
1536 chain->irq_context = hlock->irq_context;
1537 /* Find the first held_lock of current chain */
1538 hlock_next = hlock;
1539 for (i = curr->lockdep_depth - 1; i >= 0; i--) {
1540 hlock_curr = curr->held_locks + i;
1541 if (hlock_curr->irq_context != hlock_next->irq_context)
1542 break;
1543 hlock_next = hlock;
1544 }
1545 i++;
1546 chain->depth = curr->lockdep_depth + 1 - i;
1547 cn = nr_chain_hlocks;
1548 while (cn + chain->depth <= MAX_LOCKDEP_CHAIN_HLOCKS) {
1549 n = cmpxchg(&nr_chain_hlocks, cn, cn + chain->depth);
1550 if (n == cn)
1551 break;
1552 cn = n;
1553 }
1554 if (likely(cn + chain->depth <= MAX_LOCKDEP_CHAIN_HLOCKS)) {
1555 chain->base = cn;
1556 for (j = 0; j < chain->depth - 1; j++, i++) {
1557 int lock_id = curr->held_locks[i].class - lock_classes;
1558 chain_hlocks[chain->base + j] = lock_id;
1559 }
1560 chain_hlocks[chain->base + j] = class - lock_classes;
1561 }
1520 list_add_tail_rcu(&chain->entry, hash_head); 1562 list_add_tail_rcu(&chain->entry, hash_head);
1521 debug_atomic_inc(&chain_lookup_misses); 1563 debug_atomic_inc(&chain_lookup_misses);
1522 inc_chains(); 1564 inc_chains();
@@ -1538,7 +1580,7 @@ static int validate_chain(struct task_struct *curr, struct lockdep_map *lock,
1538 * graph_lock for us) 1580 * graph_lock for us)
1539 */ 1581 */
1540 if (!hlock->trylock && (hlock->check == 2) && 1582 if (!hlock->trylock && (hlock->check == 2) &&
1541 lookup_chain_cache(chain_key, hlock->class)) { 1583 lookup_chain_cache(curr, hlock, chain_key)) {
1542 /* 1584 /*
1543 * Check whether last held lock: 1585 * Check whether last held lock:
1544 * 1586 *
@@ -1680,7 +1722,7 @@ valid_state(struct task_struct *curr, struct held_lock *this,
1680static int mark_lock(struct task_struct *curr, struct held_lock *this, 1722static int mark_lock(struct task_struct *curr, struct held_lock *this,
1681 enum lock_usage_bit new_bit); 1723 enum lock_usage_bit new_bit);
1682 1724
1683#ifdef CONFIG_TRACE_IRQFLAGS 1725#if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_PROVE_LOCKING)
1684 1726
1685/* 1727/*
1686 * print irq inversion bug: 1728 * print irq inversion bug:
@@ -2013,11 +2055,13 @@ void early_boot_irqs_on(void)
2013/* 2055/*
2014 * Hardirqs will be enabled: 2056 * Hardirqs will be enabled:
2015 */ 2057 */
2016void trace_hardirqs_on(void) 2058void trace_hardirqs_on_caller(unsigned long a0)
2017{ 2059{
2018 struct task_struct *curr = current; 2060 struct task_struct *curr = current;
2019 unsigned long ip; 2061 unsigned long ip;
2020 2062
2063 time_hardirqs_on(CALLER_ADDR0, a0);
2064
2021 if (unlikely(!debug_locks || current->lockdep_recursion)) 2065 if (unlikely(!debug_locks || current->lockdep_recursion))
2022 return; 2066 return;
2023 2067
@@ -2055,16 +2099,23 @@ void trace_hardirqs_on(void)
2055 curr->hardirq_enable_event = ++curr->irq_events; 2099 curr->hardirq_enable_event = ++curr->irq_events;
2056 debug_atomic_inc(&hardirqs_on_events); 2100 debug_atomic_inc(&hardirqs_on_events);
2057} 2101}
2102EXPORT_SYMBOL(trace_hardirqs_on_caller);
2058 2103
2104void trace_hardirqs_on(void)
2105{
2106 trace_hardirqs_on_caller(CALLER_ADDR0);
2107}
2059EXPORT_SYMBOL(trace_hardirqs_on); 2108EXPORT_SYMBOL(trace_hardirqs_on);
2060 2109
2061/* 2110/*
2062 * Hardirqs were disabled: 2111 * Hardirqs were disabled:
2063 */ 2112 */
2064void trace_hardirqs_off(void) 2113void trace_hardirqs_off_caller(unsigned long a0)
2065{ 2114{
2066 struct task_struct *curr = current; 2115 struct task_struct *curr = current;
2067 2116
2117 time_hardirqs_off(CALLER_ADDR0, a0);
2118
2068 if (unlikely(!debug_locks || current->lockdep_recursion)) 2119 if (unlikely(!debug_locks || current->lockdep_recursion))
2069 return; 2120 return;
2070 2121
@@ -2082,7 +2133,12 @@ void trace_hardirqs_off(void)
2082 } else 2133 } else
2083 debug_atomic_inc(&redundant_hardirqs_off); 2134 debug_atomic_inc(&redundant_hardirqs_off);
2084} 2135}
2136EXPORT_SYMBOL(trace_hardirqs_off_caller);
2085 2137
2138void trace_hardirqs_off(void)
2139{
2140 trace_hardirqs_off_caller(CALLER_ADDR0);
2141}
2086EXPORT_SYMBOL(trace_hardirqs_off); 2142EXPORT_SYMBOL(trace_hardirqs_off);
2087 2143
2088/* 2144/*
@@ -2246,7 +2302,7 @@ static inline int separate_irq_context(struct task_struct *curr,
2246 * Mark a lock with a usage bit, and validate the state transition: 2302 * Mark a lock with a usage bit, and validate the state transition:
2247 */ 2303 */
2248static int mark_lock(struct task_struct *curr, struct held_lock *this, 2304static int mark_lock(struct task_struct *curr, struct held_lock *this,
2249 enum lock_usage_bit new_bit) 2305 enum lock_usage_bit new_bit)
2250{ 2306{
2251 unsigned int new_mask = 1 << new_bit, ret = 1; 2307 unsigned int new_mask = 1 << new_bit, ret = 1;
2252 2308
@@ -2650,7 +2706,8 @@ __lock_release(struct lockdep_map *lock, int nested, unsigned long ip)
2650 */ 2706 */
2651static void check_flags(unsigned long flags) 2707static void check_flags(unsigned long flags)
2652{ 2708{
2653#if defined(CONFIG_DEBUG_LOCKDEP) && defined(CONFIG_TRACE_IRQFLAGS) 2709#if defined(CONFIG_PROVE_LOCKING) && defined(CONFIG_DEBUG_LOCKDEP) && \
2710 defined(CONFIG_TRACE_IRQFLAGS)
2654 if (!debug_locks) 2711 if (!debug_locks)
2655 return; 2712 return;
2656 2713
@@ -2686,7 +2743,7 @@ static void check_flags(unsigned long flags)
2686 * and also avoid lockdep recursion: 2743 * and also avoid lockdep recursion:
2687 */ 2744 */
2688void lock_acquire(struct lockdep_map *lock, unsigned int subclass, 2745void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
2689 int trylock, int read, int check, unsigned long ip) 2746 int trylock, int read, int check, unsigned long ip)
2690{ 2747{
2691 unsigned long flags; 2748 unsigned long flags;
2692 2749
@@ -2708,7 +2765,8 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
2708 2765
2709EXPORT_SYMBOL_GPL(lock_acquire); 2766EXPORT_SYMBOL_GPL(lock_acquire);
2710 2767
2711void lock_release(struct lockdep_map *lock, int nested, unsigned long ip) 2768void lock_release(struct lockdep_map *lock, int nested,
2769 unsigned long ip)
2712{ 2770{
2713 unsigned long flags; 2771 unsigned long flags;
2714 2772
diff --git a/kernel/lockdep_internals.h b/kernel/lockdep_internals.h
index 8ce09bc4613d..c3600a091a28 100644
--- a/kernel/lockdep_internals.h
+++ b/kernel/lockdep_internals.h
@@ -23,6 +23,8 @@
23#define MAX_LOCKDEP_CHAINS_BITS 14 23#define MAX_LOCKDEP_CHAINS_BITS 14
24#define MAX_LOCKDEP_CHAINS (1UL << MAX_LOCKDEP_CHAINS_BITS) 24#define MAX_LOCKDEP_CHAINS (1UL << MAX_LOCKDEP_CHAINS_BITS)
25 25
26#define MAX_LOCKDEP_CHAIN_HLOCKS (MAX_LOCKDEP_CHAINS*5)
27
26/* 28/*
27 * Stack-trace: tightly packed array of stack backtrace 29 * Stack-trace: tightly packed array of stack backtrace
28 * addresses. Protected by the hash_lock. 30 * addresses. Protected by the hash_lock.
@@ -30,15 +32,19 @@
30#define MAX_STACK_TRACE_ENTRIES 262144UL 32#define MAX_STACK_TRACE_ENTRIES 262144UL
31 33
32extern struct list_head all_lock_classes; 34extern struct list_head all_lock_classes;
35extern struct lock_chain lock_chains[];
33 36
34extern void 37extern void
35get_usage_chars(struct lock_class *class, char *c1, char *c2, char *c3, char *c4); 38get_usage_chars(struct lock_class *class, char *c1, char *c2, char *c3, char *c4);
36 39
37extern const char * __get_key_name(struct lockdep_subclass_key *key, char *str); 40extern const char * __get_key_name(struct lockdep_subclass_key *key, char *str);
38 41
42struct lock_class *lock_chain_get_class(struct lock_chain *chain, int i);
43
39extern unsigned long nr_lock_classes; 44extern unsigned long nr_lock_classes;
40extern unsigned long nr_list_entries; 45extern unsigned long nr_list_entries;
41extern unsigned long nr_lock_chains; 46extern unsigned long nr_lock_chains;
47extern int nr_chain_hlocks;
42extern unsigned long nr_stack_trace_entries; 48extern unsigned long nr_stack_trace_entries;
43 49
44extern unsigned int nr_hardirq_chains; 50extern unsigned int nr_hardirq_chains;
diff --git a/kernel/lockdep_proc.c b/kernel/lockdep_proc.c
index dc5d29648d85..9b0e940e2545 100644
--- a/kernel/lockdep_proc.c
+++ b/kernel/lockdep_proc.c
@@ -139,7 +139,7 @@ static int l_show(struct seq_file *m, void *v)
139 139
140 list_for_each_entry(entry, &class->locks_after, entry) { 140 list_for_each_entry(entry, &class->locks_after, entry) {
141 if (entry->distance == 1) { 141 if (entry->distance == 1) {
142 seq_printf(m, " -> [%p] ", entry->class); 142 seq_printf(m, " -> [%p] ", entry->class->key);
143 print_name(m, entry->class); 143 print_name(m, entry->class);
144 seq_puts(m, "\n"); 144 seq_puts(m, "\n");
145 } 145 }
@@ -178,6 +178,95 @@ static const struct file_operations proc_lockdep_operations = {
178 .release = seq_release, 178 .release = seq_release,
179}; 179};
180 180
181#ifdef CONFIG_PROVE_LOCKING
182static void *lc_next(struct seq_file *m, void *v, loff_t *pos)
183{
184 struct lock_chain *chain;
185
186 (*pos)++;
187
188 if (v == SEQ_START_TOKEN)
189 chain = m->private;
190 else {
191 chain = v;
192
193 if (*pos < nr_lock_chains)
194 chain = lock_chains + *pos;
195 else
196 chain = NULL;
197 }
198
199 return chain;
200}
201
202static void *lc_start(struct seq_file *m, loff_t *pos)
203{
204 if (*pos == 0)
205 return SEQ_START_TOKEN;
206
207 if (*pos < nr_lock_chains)
208 return lock_chains + *pos;
209
210 return NULL;
211}
212
213static void lc_stop(struct seq_file *m, void *v)
214{
215}
216
217static int lc_show(struct seq_file *m, void *v)
218{
219 struct lock_chain *chain = v;
220 struct lock_class *class;
221 int i;
222
223 if (v == SEQ_START_TOKEN) {
224 seq_printf(m, "all lock chains:\n");
225 return 0;
226 }
227
228 seq_printf(m, "irq_context: %d\n", chain->irq_context);
229
230 for (i = 0; i < chain->depth; i++) {
231 class = lock_chain_get_class(chain, i);
232 seq_printf(m, "[%p] ", class->key);
233 print_name(m, class);
234 seq_puts(m, "\n");
235 }
236 seq_puts(m, "\n");
237
238 return 0;
239}
240
241static const struct seq_operations lockdep_chains_ops = {
242 .start = lc_start,
243 .next = lc_next,
244 .stop = lc_stop,
245 .show = lc_show,
246};
247
248static int lockdep_chains_open(struct inode *inode, struct file *file)
249{
250 int res = seq_open(file, &lockdep_chains_ops);
251 if (!res) {
252 struct seq_file *m = file->private_data;
253
254 if (nr_lock_chains)
255 m->private = lock_chains;
256 else
257 m->private = NULL;
258 }
259 return res;
260}
261
262static const struct file_operations proc_lockdep_chains_operations = {
263 .open = lockdep_chains_open,
264 .read = seq_read,
265 .llseek = seq_lseek,
266 .release = seq_release,
267};
268#endif /* CONFIG_PROVE_LOCKING */
269
181static void lockdep_stats_debug_show(struct seq_file *m) 270static void lockdep_stats_debug_show(struct seq_file *m)
182{ 271{
183#ifdef CONFIG_DEBUG_LOCKDEP 272#ifdef CONFIG_DEBUG_LOCKDEP
@@ -294,6 +383,8 @@ static int lockdep_stats_show(struct seq_file *m, void *v)
294#ifdef CONFIG_PROVE_LOCKING 383#ifdef CONFIG_PROVE_LOCKING
295 seq_printf(m, " dependency chains: %11lu [max: %lu]\n", 384 seq_printf(m, " dependency chains: %11lu [max: %lu]\n",
296 nr_lock_chains, MAX_LOCKDEP_CHAINS); 385 nr_lock_chains, MAX_LOCKDEP_CHAINS);
386 seq_printf(m, " dependency chain hlocks: %11d [max: %lu]\n",
387 nr_chain_hlocks, MAX_LOCKDEP_CHAIN_HLOCKS);
297#endif 388#endif
298 389
299#ifdef CONFIG_TRACE_IRQFLAGS 390#ifdef CONFIG_TRACE_IRQFLAGS
@@ -661,6 +752,10 @@ static const struct file_operations proc_lock_stat_operations = {
661static int __init lockdep_proc_init(void) 752static int __init lockdep_proc_init(void)
662{ 753{
663 proc_create("lockdep", S_IRUSR, NULL, &proc_lockdep_operations); 754 proc_create("lockdep", S_IRUSR, NULL, &proc_lockdep_operations);
755#ifdef CONFIG_PROVE_LOCKING
756 proc_create("lockdep_chains", S_IRUSR, NULL,
757 &proc_lockdep_chains_operations);
758#endif
664 proc_create("lockdep_stats", S_IRUSR, NULL, 759 proc_create("lockdep_stats", S_IRUSR, NULL,
665 &proc_lockdep_stats_operations); 760 &proc_lockdep_stats_operations);
666 761
diff --git a/kernel/marker.c b/kernel/marker.c
index b5a9fe1d50d5..971da5317903 100644
--- a/kernel/marker.c
+++ b/kernel/marker.c
@@ -55,8 +55,8 @@ static DEFINE_MUTEX(markers_mutex);
55struct marker_entry { 55struct marker_entry {
56 struct hlist_node hlist; 56 struct hlist_node hlist;
57 char *format; 57 char *format;
58 void (*call)(const struct marker *mdata, /* Probe wrapper */ 58 /* Probe wrapper */
59 void *call_private, const char *fmt, ...); 59 void (*call)(const struct marker *mdata, void *call_private, ...);
60 struct marker_probe_closure single; 60 struct marker_probe_closure single;
61 struct marker_probe_closure *multi; 61 struct marker_probe_closure *multi;
62 int refcount; /* Number of times armed. 0 if disarmed. */ 62 int refcount; /* Number of times armed. 0 if disarmed. */
@@ -91,15 +91,13 @@ EXPORT_SYMBOL_GPL(__mark_empty_function);
91 * marker_probe_cb Callback that prepares the variable argument list for probes. 91 * marker_probe_cb Callback that prepares the variable argument list for probes.
92 * @mdata: pointer of type struct marker 92 * @mdata: pointer of type struct marker
93 * @call_private: caller site private data 93 * @call_private: caller site private data
94 * @fmt: format string
95 * @...: Variable argument list. 94 * @...: Variable argument list.
96 * 95 *
97 * Since we do not use "typical" pointer based RCU in the 1 argument case, we 96 * Since we do not use "typical" pointer based RCU in the 1 argument case, we
98 * need to put a full smp_rmb() in this branch. This is why we do not use 97 * need to put a full smp_rmb() in this branch. This is why we do not use
99 * rcu_dereference() for the pointer read. 98 * rcu_dereference() for the pointer read.
100 */ 99 */
101void marker_probe_cb(const struct marker *mdata, void *call_private, 100void marker_probe_cb(const struct marker *mdata, void *call_private, ...)
102 const char *fmt, ...)
103{ 101{
104 va_list args; 102 va_list args;
105 char ptype; 103 char ptype;
@@ -120,8 +118,9 @@ void marker_probe_cb(const struct marker *mdata, void *call_private,
120 /* Must read the ptr before private data. They are not data 118 /* Must read the ptr before private data. They are not data
121 * dependant, so we put an explicit smp_rmb() here. */ 119 * dependant, so we put an explicit smp_rmb() here. */
122 smp_rmb(); 120 smp_rmb();
123 va_start(args, fmt); 121 va_start(args, call_private);
124 func(mdata->single.probe_private, call_private, fmt, &args); 122 func(mdata->single.probe_private, call_private, mdata->format,
123 &args);
125 va_end(args); 124 va_end(args);
126 } else { 125 } else {
127 struct marker_probe_closure *multi; 126 struct marker_probe_closure *multi;
@@ -136,9 +135,9 @@ void marker_probe_cb(const struct marker *mdata, void *call_private,
136 smp_read_barrier_depends(); 135 smp_read_barrier_depends();
137 multi = mdata->multi; 136 multi = mdata->multi;
138 for (i = 0; multi[i].func; i++) { 137 for (i = 0; multi[i].func; i++) {
139 va_start(args, fmt); 138 va_start(args, call_private);
140 multi[i].func(multi[i].probe_private, call_private, fmt, 139 multi[i].func(multi[i].probe_private, call_private,
141 &args); 140 mdata->format, &args);
142 va_end(args); 141 va_end(args);
143 } 142 }
144 } 143 }
@@ -150,13 +149,11 @@ EXPORT_SYMBOL_GPL(marker_probe_cb);
150 * marker_probe_cb Callback that does not prepare the variable argument list. 149 * marker_probe_cb Callback that does not prepare the variable argument list.
151 * @mdata: pointer of type struct marker 150 * @mdata: pointer of type struct marker
152 * @call_private: caller site private data 151 * @call_private: caller site private data
153 * @fmt: format string
154 * @...: Variable argument list. 152 * @...: Variable argument list.
155 * 153 *
156 * Should be connected to markers "MARK_NOARGS". 154 * Should be connected to markers "MARK_NOARGS".
157 */ 155 */
158void marker_probe_cb_noarg(const struct marker *mdata, 156void marker_probe_cb_noarg(const struct marker *mdata, void *call_private, ...)
159 void *call_private, const char *fmt, ...)
160{ 157{
161 va_list args; /* not initialized */ 158 va_list args; /* not initialized */
162 char ptype; 159 char ptype;
@@ -172,7 +169,8 @@ void marker_probe_cb_noarg(const struct marker *mdata,
172 /* Must read the ptr before private data. They are not data 169 /* Must read the ptr before private data. They are not data
173 * dependant, so we put an explicit smp_rmb() here. */ 170 * dependant, so we put an explicit smp_rmb() here. */
174 smp_rmb(); 171 smp_rmb();
175 func(mdata->single.probe_private, call_private, fmt, &args); 172 func(mdata->single.probe_private, call_private, mdata->format,
173 &args);
176 } else { 174 } else {
177 struct marker_probe_closure *multi; 175 struct marker_probe_closure *multi;
178 int i; 176 int i;
@@ -186,8 +184,8 @@ void marker_probe_cb_noarg(const struct marker *mdata,
186 smp_read_barrier_depends(); 184 smp_read_barrier_depends();
187 multi = mdata->multi; 185 multi = mdata->multi;
188 for (i = 0; multi[i].func; i++) 186 for (i = 0; multi[i].func; i++)
189 multi[i].func(multi[i].probe_private, call_private, fmt, 187 multi[i].func(multi[i].probe_private, call_private,
190 &args); 188 mdata->format, &args);
191 } 189 }
192 preempt_enable(); 190 preempt_enable();
193} 191}
@@ -443,7 +441,7 @@ static int remove_marker(const char *name)
443 hlist_del(&e->hlist); 441 hlist_del(&e->hlist);
444 /* Make sure the call_rcu has been executed */ 442 /* Make sure the call_rcu has been executed */
445 if (e->rcu_pending) 443 if (e->rcu_pending)
446 rcu_barrier(); 444 rcu_barrier_sched();
447 kfree(e); 445 kfree(e);
448 return 0; 446 return 0;
449} 447}
@@ -478,7 +476,7 @@ static int marker_set_format(struct marker_entry **entry, const char *format)
478 hlist_del(&(*entry)->hlist); 476 hlist_del(&(*entry)->hlist);
479 /* Make sure the call_rcu has been executed */ 477 /* Make sure the call_rcu has been executed */
480 if ((*entry)->rcu_pending) 478 if ((*entry)->rcu_pending)
481 rcu_barrier(); 479 rcu_barrier_sched();
482 kfree(*entry); 480 kfree(*entry);
483 *entry = e; 481 *entry = e;
484 trace_mark(core_marker_format, "name %s format %s", 482 trace_mark(core_marker_format, "name %s format %s",
@@ -657,7 +655,7 @@ int marker_probe_register(const char *name, const char *format,
657 * make sure it's executed now. 655 * make sure it's executed now.
658 */ 656 */
659 if (entry->rcu_pending) 657 if (entry->rcu_pending)
660 rcu_barrier(); 658 rcu_barrier_sched();
661 old = marker_entry_add_probe(entry, probe, probe_private); 659 old = marker_entry_add_probe(entry, probe, probe_private);
662 if (IS_ERR(old)) { 660 if (IS_ERR(old)) {
663 ret = PTR_ERR(old); 661 ret = PTR_ERR(old);
@@ -672,10 +670,7 @@ int marker_probe_register(const char *name, const char *format,
672 entry->rcu_pending = 1; 670 entry->rcu_pending = 1;
673 /* write rcu_pending before calling the RCU callback */ 671 /* write rcu_pending before calling the RCU callback */
674 smp_wmb(); 672 smp_wmb();
675#ifdef CONFIG_PREEMPT_RCU 673 call_rcu_sched(&entry->rcu, free_old_closure);
676 synchronize_sched(); /* Until we have the call_rcu_sched() */
677#endif
678 call_rcu(&entry->rcu, free_old_closure);
679end: 674end:
680 mutex_unlock(&markers_mutex); 675 mutex_unlock(&markers_mutex);
681 return ret; 676 return ret;
@@ -706,7 +701,7 @@ int marker_probe_unregister(const char *name,
706 if (!entry) 701 if (!entry)
707 goto end; 702 goto end;
708 if (entry->rcu_pending) 703 if (entry->rcu_pending)
709 rcu_barrier(); 704 rcu_barrier_sched();
710 old = marker_entry_remove_probe(entry, probe, probe_private); 705 old = marker_entry_remove_probe(entry, probe, probe_private);
711 mutex_unlock(&markers_mutex); 706 mutex_unlock(&markers_mutex);
712 marker_update_probes(); /* may update entry */ 707 marker_update_probes(); /* may update entry */
@@ -718,10 +713,7 @@ int marker_probe_unregister(const char *name,
718 entry->rcu_pending = 1; 713 entry->rcu_pending = 1;
719 /* write rcu_pending before calling the RCU callback */ 714 /* write rcu_pending before calling the RCU callback */
720 smp_wmb(); 715 smp_wmb();
721#ifdef CONFIG_PREEMPT_RCU 716 call_rcu_sched(&entry->rcu, free_old_closure);
722 synchronize_sched(); /* Until we have the call_rcu_sched() */
723#endif
724 call_rcu(&entry->rcu, free_old_closure);
725 remove_marker(name); /* Ignore busy error message */ 717 remove_marker(name); /* Ignore busy error message */
726 ret = 0; 718 ret = 0;
727end: 719end:
@@ -788,7 +780,7 @@ int marker_probe_unregister_private_data(marker_probe_func *probe,
788 goto end; 780 goto end;
789 } 781 }
790 if (entry->rcu_pending) 782 if (entry->rcu_pending)
791 rcu_barrier(); 783 rcu_barrier_sched();
792 old = marker_entry_remove_probe(entry, NULL, probe_private); 784 old = marker_entry_remove_probe(entry, NULL, probe_private);
793 mutex_unlock(&markers_mutex); 785 mutex_unlock(&markers_mutex);
794 marker_update_probes(); /* may update entry */ 786 marker_update_probes(); /* may update entry */
@@ -799,10 +791,7 @@ int marker_probe_unregister_private_data(marker_probe_func *probe,
799 entry->rcu_pending = 1; 791 entry->rcu_pending = 1;
800 /* write rcu_pending before calling the RCU callback */ 792 /* write rcu_pending before calling the RCU callback */
801 smp_wmb(); 793 smp_wmb();
802#ifdef CONFIG_PREEMPT_RCU 794 call_rcu_sched(&entry->rcu, free_old_closure);
803 synchronize_sched(); /* Until we have the call_rcu_sched() */
804#endif
805 call_rcu(&entry->rcu, free_old_closure);
806 remove_marker(entry->name); /* Ignore busy error message */ 795 remove_marker(entry->name); /* Ignore busy error message */
807end: 796end:
808 mutex_unlock(&markers_mutex); 797 mutex_unlock(&markers_mutex);
diff --git a/kernel/module.c b/kernel/module.c
index 5f80478b746d..d8b5605132a0 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -70,6 +70,9 @@ static DECLARE_WAIT_QUEUE_HEAD(module_wq);
70 70
71static BLOCKING_NOTIFIER_HEAD(module_notify_list); 71static BLOCKING_NOTIFIER_HEAD(module_notify_list);
72 72
73/* Bounds of module allocation, for speeding __module_text_address */
74static unsigned long module_addr_min = -1UL, module_addr_max = 0;
75
73int register_module_notifier(struct notifier_block * nb) 76int register_module_notifier(struct notifier_block * nb)
74{ 77{
75 return blocking_notifier_chain_register(&module_notify_list, nb); 78 return blocking_notifier_chain_register(&module_notify_list, nb);
@@ -134,17 +137,19 @@ extern const struct kernel_symbol __start___ksymtab_gpl[];
134extern const struct kernel_symbol __stop___ksymtab_gpl[]; 137extern const struct kernel_symbol __stop___ksymtab_gpl[];
135extern const struct kernel_symbol __start___ksymtab_gpl_future[]; 138extern const struct kernel_symbol __start___ksymtab_gpl_future[];
136extern const struct kernel_symbol __stop___ksymtab_gpl_future[]; 139extern const struct kernel_symbol __stop___ksymtab_gpl_future[];
137extern const struct kernel_symbol __start___ksymtab_unused[];
138extern const struct kernel_symbol __stop___ksymtab_unused[];
139extern const struct kernel_symbol __start___ksymtab_unused_gpl[];
140extern const struct kernel_symbol __stop___ksymtab_unused_gpl[];
141extern const struct kernel_symbol __start___ksymtab_gpl_future[]; 140extern const struct kernel_symbol __start___ksymtab_gpl_future[];
142extern const struct kernel_symbol __stop___ksymtab_gpl_future[]; 141extern const struct kernel_symbol __stop___ksymtab_gpl_future[];
143extern const unsigned long __start___kcrctab[]; 142extern const unsigned long __start___kcrctab[];
144extern const unsigned long __start___kcrctab_gpl[]; 143extern const unsigned long __start___kcrctab_gpl[];
145extern const unsigned long __start___kcrctab_gpl_future[]; 144extern const unsigned long __start___kcrctab_gpl_future[];
145#ifdef CONFIG_UNUSED_SYMBOLS
146extern const struct kernel_symbol __start___ksymtab_unused[];
147extern const struct kernel_symbol __stop___ksymtab_unused[];
148extern const struct kernel_symbol __start___ksymtab_unused_gpl[];
149extern const struct kernel_symbol __stop___ksymtab_unused_gpl[];
146extern const unsigned long __start___kcrctab_unused[]; 150extern const unsigned long __start___kcrctab_unused[];
147extern const unsigned long __start___kcrctab_unused_gpl[]; 151extern const unsigned long __start___kcrctab_unused_gpl[];
152#endif
148 153
149#ifndef CONFIG_MODVERSIONS 154#ifndef CONFIG_MODVERSIONS
150#define symversion(base, idx) NULL 155#define symversion(base, idx) NULL
@@ -152,156 +157,186 @@ extern const unsigned long __start___kcrctab_unused_gpl[];
152#define symversion(base, idx) ((base != NULL) ? ((base) + (idx)) : NULL) 157#define symversion(base, idx) ((base != NULL) ? ((base) + (idx)) : NULL)
153#endif 158#endif
154 159
155/* lookup symbol in given range of kernel_symbols */
156static const struct kernel_symbol *lookup_symbol(const char *name,
157 const struct kernel_symbol *start,
158 const struct kernel_symbol *stop)
159{
160 const struct kernel_symbol *ks = start;
161 for (; ks < stop; ks++)
162 if (strcmp(ks->name, name) == 0)
163 return ks;
164 return NULL;
165}
166
167static bool always_ok(bool gplok, bool warn, const char *name)
168{
169 return true;
170}
171
172static bool printk_unused_warning(bool gplok, bool warn, const char *name)
173{
174 if (warn) {
175 printk(KERN_WARNING "Symbol %s is marked as UNUSED, "
176 "however this module is using it.\n", name);
177 printk(KERN_WARNING
178 "This symbol will go away in the future.\n");
179 printk(KERN_WARNING
180 "Please evalute if this is the right api to use and if "
181 "it really is, submit a report the linux kernel "
182 "mailinglist together with submitting your code for "
183 "inclusion.\n");
184 }
185 return true;
186}
187
188static bool gpl_only_unused_warning(bool gplok, bool warn, const char *name)
189{
190 if (!gplok)
191 return false;
192 return printk_unused_warning(gplok, warn, name);
193}
194
195static bool gpl_only(bool gplok, bool warn, const char *name)
196{
197 return gplok;
198}
199
200static bool warn_if_not_gpl(bool gplok, bool warn, const char *name)
201{
202 if (!gplok && warn) {
203 printk(KERN_WARNING "Symbol %s is being used "
204 "by a non-GPL module, which will not "
205 "be allowed in the future\n", name);
206 printk(KERN_WARNING "Please see the file "
207 "Documentation/feature-removal-schedule.txt "
208 "in the kernel source tree for more details.\n");
209 }
210 return true;
211}
212
213struct symsearch { 160struct symsearch {
214 const struct kernel_symbol *start, *stop; 161 const struct kernel_symbol *start, *stop;
215 const unsigned long *crcs; 162 const unsigned long *crcs;
216 bool (*check)(bool gplok, bool warn, const char *name); 163 enum {
164 NOT_GPL_ONLY,
165 GPL_ONLY,
166 WILL_BE_GPL_ONLY,
167 } licence;
168 bool unused;
217}; 169};
218 170
219/* Look through this array of symbol tables for a symbol match which 171static bool each_symbol_in_section(const struct symsearch *arr,
220 * passes the check function. */ 172 unsigned int arrsize,
221static const struct kernel_symbol *search_symarrays(const struct symsearch *arr, 173 struct module *owner,
222 unsigned int num, 174 bool (*fn)(const struct symsearch *syms,
223 const char *name, 175 struct module *owner,
224 bool gplok, 176 unsigned int symnum, void *data),
225 bool warn, 177 void *data)
226 const unsigned long **crc)
227{ 178{
228 unsigned int i; 179 unsigned int i, j;
229 const struct kernel_symbol *ks;
230 180
231 for (i = 0; i < num; i++) { 181 for (j = 0; j < arrsize; j++) {
232 ks = lookup_symbol(name, arr[i].start, arr[i].stop); 182 for (i = 0; i < arr[j].stop - arr[j].start; i++)
233 if (!ks || !arr[i].check(gplok, warn, name)) 183 if (fn(&arr[j], owner, i, data))
234 continue; 184 return true;
235
236 if (crc)
237 *crc = symversion(arr[i].crcs, ks - arr[i].start);
238 return ks;
239 } 185 }
240 return NULL; 186
187 return false;
241} 188}
242 189
243/* Find a symbol, return value, (optional) crc and (optional) module 190/* Returns true as soon as fn returns true, otherwise false. */
244 * which owns it */ 191static bool each_symbol(bool (*fn)(const struct symsearch *arr,
245static unsigned long find_symbol(const char *name, 192 struct module *owner,
246 struct module **owner, 193 unsigned int symnum, void *data),
247 const unsigned long **crc, 194 void *data)
248 bool gplok,
249 bool warn)
250{ 195{
251 struct module *mod; 196 struct module *mod;
252 const struct kernel_symbol *ks;
253 const struct symsearch arr[] = { 197 const struct symsearch arr[] = {
254 { __start___ksymtab, __stop___ksymtab, __start___kcrctab, 198 { __start___ksymtab, __stop___ksymtab, __start___kcrctab,
255 always_ok }, 199 NOT_GPL_ONLY, false },
256 { __start___ksymtab_gpl, __stop___ksymtab_gpl, 200 { __start___ksymtab_gpl, __stop___ksymtab_gpl,
257 __start___kcrctab_gpl, gpl_only }, 201 __start___kcrctab_gpl,
202 GPL_ONLY, false },
258 { __start___ksymtab_gpl_future, __stop___ksymtab_gpl_future, 203 { __start___ksymtab_gpl_future, __stop___ksymtab_gpl_future,
259 __start___kcrctab_gpl_future, warn_if_not_gpl }, 204 __start___kcrctab_gpl_future,
205 WILL_BE_GPL_ONLY, false },
206#ifdef CONFIG_UNUSED_SYMBOLS
260 { __start___ksymtab_unused, __stop___ksymtab_unused, 207 { __start___ksymtab_unused, __stop___ksymtab_unused,
261 __start___kcrctab_unused, printk_unused_warning }, 208 __start___kcrctab_unused,
209 NOT_GPL_ONLY, true },
262 { __start___ksymtab_unused_gpl, __stop___ksymtab_unused_gpl, 210 { __start___ksymtab_unused_gpl, __stop___ksymtab_unused_gpl,
263 __start___kcrctab_unused_gpl, gpl_only_unused_warning }, 211 __start___kcrctab_unused_gpl,
212 GPL_ONLY, true },
213#endif
264 }; 214 };
265 215
266 /* Core kernel first. */ 216 if (each_symbol_in_section(arr, ARRAY_SIZE(arr), NULL, fn, data))
267 ks = search_symarrays(arr, ARRAY_SIZE(arr), name, gplok, warn, crc); 217 return true;
268 if (ks) {
269 if (owner)
270 *owner = NULL;
271 return ks->value;
272 }
273 218
274 /* Now try modules. */
275 list_for_each_entry(mod, &modules, list) { 219 list_for_each_entry(mod, &modules, list) {
276 struct symsearch arr[] = { 220 struct symsearch arr[] = {
277 { mod->syms, mod->syms + mod->num_syms, mod->crcs, 221 { mod->syms, mod->syms + mod->num_syms, mod->crcs,
278 always_ok }, 222 NOT_GPL_ONLY, false },
279 { mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms, 223 { mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms,
280 mod->gpl_crcs, gpl_only }, 224 mod->gpl_crcs,
225 GPL_ONLY, false },
281 { mod->gpl_future_syms, 226 { mod->gpl_future_syms,
282 mod->gpl_future_syms + mod->num_gpl_future_syms, 227 mod->gpl_future_syms + mod->num_gpl_future_syms,
283 mod->gpl_future_crcs, warn_if_not_gpl }, 228 mod->gpl_future_crcs,
229 WILL_BE_GPL_ONLY, false },
230#ifdef CONFIG_UNUSED_SYMBOLS
284 { mod->unused_syms, 231 { mod->unused_syms,
285 mod->unused_syms + mod->num_unused_syms, 232 mod->unused_syms + mod->num_unused_syms,
286 mod->unused_crcs, printk_unused_warning }, 233 mod->unused_crcs,
234 NOT_GPL_ONLY, true },
287 { mod->unused_gpl_syms, 235 { mod->unused_gpl_syms,
288 mod->unused_gpl_syms + mod->num_unused_gpl_syms, 236 mod->unused_gpl_syms + mod->num_unused_gpl_syms,
289 mod->unused_gpl_crcs, gpl_only_unused_warning }, 237 mod->unused_gpl_crcs,
238 GPL_ONLY, true },
239#endif
290 }; 240 };
291 241
292 ks = search_symarrays(arr, ARRAY_SIZE(arr), 242 if (each_symbol_in_section(arr, ARRAY_SIZE(arr), mod, fn, data))
293 name, gplok, warn, crc); 243 return true;
294 if (ks) { 244 }
295 if (owner) 245 return false;
296 *owner = mod; 246}
297 return ks->value; 247
248struct find_symbol_arg {
249 /* Input */
250 const char *name;
251 bool gplok;
252 bool warn;
253
254 /* Output */
255 struct module *owner;
256 const unsigned long *crc;
257 unsigned long value;
258};
259
260static bool find_symbol_in_section(const struct symsearch *syms,
261 struct module *owner,
262 unsigned int symnum, void *data)
263{
264 struct find_symbol_arg *fsa = data;
265
266 if (strcmp(syms->start[symnum].name, fsa->name) != 0)
267 return false;
268
269 if (!fsa->gplok) {
270 if (syms->licence == GPL_ONLY)
271 return false;
272 if (syms->licence == WILL_BE_GPL_ONLY && fsa->warn) {
273 printk(KERN_WARNING "Symbol %s is being used "
274 "by a non-GPL module, which will not "
275 "be allowed in the future\n", fsa->name);
276 printk(KERN_WARNING "Please see the file "
277 "Documentation/feature-removal-schedule.txt "
278 "in the kernel source tree for more details.\n");
298 } 279 }
299 } 280 }
300 281
282#ifdef CONFIG_UNUSED_SYMBOLS
283 if (syms->unused && fsa->warn) {
284 printk(KERN_WARNING "Symbol %s is marked as UNUSED, "
285 "however this module is using it.\n", fsa->name);
286 printk(KERN_WARNING
287 "This symbol will go away in the future.\n");
288 printk(KERN_WARNING
289 "Please evalute if this is the right api to use and if "
290 "it really is, submit a report the linux kernel "
291 "mailinglist together with submitting your code for "
292 "inclusion.\n");
293 }
294#endif
295
296 fsa->owner = owner;
297 fsa->crc = symversion(syms->crcs, symnum);
298 fsa->value = syms->start[symnum].value;
299 return true;
300}
301
302/* Find a symbol, return value, (optional) crc and (optional) module
303 * which owns it */
304static unsigned long find_symbol(const char *name,
305 struct module **owner,
306 const unsigned long **crc,
307 bool gplok,
308 bool warn)
309{
310 struct find_symbol_arg fsa;
311
312 fsa.name = name;
313 fsa.gplok = gplok;
314 fsa.warn = warn;
315
316 if (each_symbol(find_symbol_in_section, &fsa)) {
317 if (owner)
318 *owner = fsa.owner;
319 if (crc)
320 *crc = fsa.crc;
321 return fsa.value;
322 }
323
301 DEBUGP("Failed to find symbol %s\n", name); 324 DEBUGP("Failed to find symbol %s\n", name);
302 return -ENOENT; 325 return -ENOENT;
303} 326}
304 327
328/* lookup symbol in given range of kernel_symbols */
329static const struct kernel_symbol *lookup_symbol(const char *name,
330 const struct kernel_symbol *start,
331 const struct kernel_symbol *stop)
332{
333 const struct kernel_symbol *ks = start;
334 for (; ks < stop; ks++)
335 if (strcmp(ks->name, name) == 0)
336 return ks;
337 return NULL;
338}
339
305/* Search for module by name: must hold module_mutex. */ 340/* Search for module by name: must hold module_mutex. */
306static struct module *find_module(const char *name) 341static struct module *find_module(const char *name)
307{ 342{
@@ -639,8 +674,8 @@ static int __try_stop_module(void *_sref)
639{ 674{
640 struct stopref *sref = _sref; 675 struct stopref *sref = _sref;
641 676
642 /* If it's not unused, quit unless we are told to block. */ 677 /* If it's not unused, quit unless we're forcing. */
643 if ((sref->flags & O_NONBLOCK) && module_refcount(sref->mod) != 0) { 678 if (module_refcount(sref->mod) != 0) {
644 if (!(*sref->forced = try_force_unload(sref->flags))) 679 if (!(*sref->forced = try_force_unload(sref->flags)))
645 return -EWOULDBLOCK; 680 return -EWOULDBLOCK;
646 } 681 }
@@ -652,9 +687,16 @@ static int __try_stop_module(void *_sref)
652 687
653static int try_stop_module(struct module *mod, int flags, int *forced) 688static int try_stop_module(struct module *mod, int flags, int *forced)
654{ 689{
655 struct stopref sref = { mod, flags, forced }; 690 if (flags & O_NONBLOCK) {
691 struct stopref sref = { mod, flags, forced };
656 692
657 return stop_machine_run(__try_stop_module, &sref, NR_CPUS); 693 return stop_machine_run(__try_stop_module, &sref, NR_CPUS);
694 } else {
695 /* We don't need to stop the machine for this. */
696 mod->state = MODULE_STATE_GOING;
697 synchronize_sched();
698 return 0;
699 }
658} 700}
659 701
660unsigned int module_refcount(struct module *mod) 702unsigned int module_refcount(struct module *mod)
@@ -1445,8 +1487,10 @@ static int verify_export_symbols(struct module *mod)
1445 { mod->syms, mod->num_syms }, 1487 { mod->syms, mod->num_syms },
1446 { mod->gpl_syms, mod->num_gpl_syms }, 1488 { mod->gpl_syms, mod->num_gpl_syms },
1447 { mod->gpl_future_syms, mod->num_gpl_future_syms }, 1489 { mod->gpl_future_syms, mod->num_gpl_future_syms },
1490#ifdef CONFIG_UNUSED_SYMBOLS
1448 { mod->unused_syms, mod->num_unused_syms }, 1491 { mod->unused_syms, mod->num_unused_syms },
1449 { mod->unused_gpl_syms, mod->num_unused_gpl_syms }, 1492 { mod->unused_gpl_syms, mod->num_unused_gpl_syms },
1493#endif
1450 }; 1494 };
1451 1495
1452 for (i = 0; i < ARRAY_SIZE(arr); i++) { 1496 for (i = 0; i < ARRAY_SIZE(arr); i++) {
@@ -1526,7 +1570,7 @@ static int simplify_symbols(Elf_Shdr *sechdrs,
1526} 1570}
1527 1571
1528/* Update size with this section: return offset. */ 1572/* Update size with this section: return offset. */
1529static long get_offset(unsigned long *size, Elf_Shdr *sechdr) 1573static long get_offset(unsigned int *size, Elf_Shdr *sechdr)
1530{ 1574{
1531 long ret; 1575 long ret;
1532 1576
@@ -1738,6 +1782,20 @@ static inline void add_kallsyms(struct module *mod,
1738} 1782}
1739#endif /* CONFIG_KALLSYMS */ 1783#endif /* CONFIG_KALLSYMS */
1740 1784
1785static void *module_alloc_update_bounds(unsigned long size)
1786{
1787 void *ret = module_alloc(size);
1788
1789 if (ret) {
1790 /* Update module bounds. */
1791 if ((unsigned long)ret < module_addr_min)
1792 module_addr_min = (unsigned long)ret;
1793 if ((unsigned long)ret + size > module_addr_max)
1794 module_addr_max = (unsigned long)ret + size;
1795 }
1796 return ret;
1797}
1798
1741/* Allocate and load the module: note that size of section 0 is always 1799/* Allocate and load the module: note that size of section 0 is always
1742 zero, and we rely on this for optional sections. */ 1800 zero, and we rely on this for optional sections. */
1743static struct module *load_module(void __user *umod, 1801static struct module *load_module(void __user *umod,
@@ -1764,10 +1822,12 @@ static struct module *load_module(void __user *umod,
1764 unsigned int gplfutureindex; 1822 unsigned int gplfutureindex;
1765 unsigned int gplfuturecrcindex; 1823 unsigned int gplfuturecrcindex;
1766 unsigned int unwindex = 0; 1824 unsigned int unwindex = 0;
1825#ifdef CONFIG_UNUSED_SYMBOLS
1767 unsigned int unusedindex; 1826 unsigned int unusedindex;
1768 unsigned int unusedcrcindex; 1827 unsigned int unusedcrcindex;
1769 unsigned int unusedgplindex; 1828 unsigned int unusedgplindex;
1770 unsigned int unusedgplcrcindex; 1829 unsigned int unusedgplcrcindex;
1830#endif
1771 unsigned int markersindex; 1831 unsigned int markersindex;
1772 unsigned int markersstringsindex; 1832 unsigned int markersstringsindex;
1773 struct module *mod; 1833 struct module *mod;
@@ -1850,13 +1910,15 @@ static struct module *load_module(void __user *umod,
1850 exportindex = find_sec(hdr, sechdrs, secstrings, "__ksymtab"); 1910 exportindex = find_sec(hdr, sechdrs, secstrings, "__ksymtab");
1851 gplindex = find_sec(hdr, sechdrs, secstrings, "__ksymtab_gpl"); 1911 gplindex = find_sec(hdr, sechdrs, secstrings, "__ksymtab_gpl");
1852 gplfutureindex = find_sec(hdr, sechdrs, secstrings, "__ksymtab_gpl_future"); 1912 gplfutureindex = find_sec(hdr, sechdrs, secstrings, "__ksymtab_gpl_future");
1853 unusedindex = find_sec(hdr, sechdrs, secstrings, "__ksymtab_unused");
1854 unusedgplindex = find_sec(hdr, sechdrs, secstrings, "__ksymtab_unused_gpl");
1855 crcindex = find_sec(hdr, sechdrs, secstrings, "__kcrctab"); 1913 crcindex = find_sec(hdr, sechdrs, secstrings, "__kcrctab");
1856 gplcrcindex = find_sec(hdr, sechdrs, secstrings, "__kcrctab_gpl"); 1914 gplcrcindex = find_sec(hdr, sechdrs, secstrings, "__kcrctab_gpl");
1857 gplfuturecrcindex = find_sec(hdr, sechdrs, secstrings, "__kcrctab_gpl_future"); 1915 gplfuturecrcindex = find_sec(hdr, sechdrs, secstrings, "__kcrctab_gpl_future");
1916#ifdef CONFIG_UNUSED_SYMBOLS
1917 unusedindex = find_sec(hdr, sechdrs, secstrings, "__ksymtab_unused");
1918 unusedgplindex = find_sec(hdr, sechdrs, secstrings, "__ksymtab_unused_gpl");
1858 unusedcrcindex = find_sec(hdr, sechdrs, secstrings, "__kcrctab_unused"); 1919 unusedcrcindex = find_sec(hdr, sechdrs, secstrings, "__kcrctab_unused");
1859 unusedgplcrcindex = find_sec(hdr, sechdrs, secstrings, "__kcrctab_unused_gpl"); 1920 unusedgplcrcindex = find_sec(hdr, sechdrs, secstrings, "__kcrctab_unused_gpl");
1921#endif
1860 setupindex = find_sec(hdr, sechdrs, secstrings, "__param"); 1922 setupindex = find_sec(hdr, sechdrs, secstrings, "__param");
1861 exindex = find_sec(hdr, sechdrs, secstrings, "__ex_table"); 1923 exindex = find_sec(hdr, sechdrs, secstrings, "__ex_table");
1862 obsparmindex = find_sec(hdr, sechdrs, secstrings, "__obsparm"); 1924 obsparmindex = find_sec(hdr, sechdrs, secstrings, "__obsparm");
@@ -1935,7 +1997,7 @@ static struct module *load_module(void __user *umod,
1935 layout_sections(mod, hdr, sechdrs, secstrings); 1997 layout_sections(mod, hdr, sechdrs, secstrings);
1936 1998
1937 /* Do the allocs. */ 1999 /* Do the allocs. */
1938 ptr = module_alloc(mod->core_size); 2000 ptr = module_alloc_update_bounds(mod->core_size);
1939 if (!ptr) { 2001 if (!ptr) {
1940 err = -ENOMEM; 2002 err = -ENOMEM;
1941 goto free_percpu; 2003 goto free_percpu;
@@ -1943,7 +2005,7 @@ static struct module *load_module(void __user *umod,
1943 memset(ptr, 0, mod->core_size); 2005 memset(ptr, 0, mod->core_size);
1944 mod->module_core = ptr; 2006 mod->module_core = ptr;
1945 2007
1946 ptr = module_alloc(mod->init_size); 2008 ptr = module_alloc_update_bounds(mod->init_size);
1947 if (!ptr && mod->init_size) { 2009 if (!ptr && mod->init_size) {
1948 err = -ENOMEM; 2010 err = -ENOMEM;
1949 goto free_core; 2011 goto free_core;
@@ -2018,14 +2080,15 @@ static struct module *load_module(void __user *umod,
2018 mod->gpl_crcs = (void *)sechdrs[gplcrcindex].sh_addr; 2080 mod->gpl_crcs = (void *)sechdrs[gplcrcindex].sh_addr;
2019 mod->num_gpl_future_syms = sechdrs[gplfutureindex].sh_size / 2081 mod->num_gpl_future_syms = sechdrs[gplfutureindex].sh_size /
2020 sizeof(*mod->gpl_future_syms); 2082 sizeof(*mod->gpl_future_syms);
2021 mod->num_unused_syms = sechdrs[unusedindex].sh_size /
2022 sizeof(*mod->unused_syms);
2023 mod->num_unused_gpl_syms = sechdrs[unusedgplindex].sh_size /
2024 sizeof(*mod->unused_gpl_syms);
2025 mod->gpl_future_syms = (void *)sechdrs[gplfutureindex].sh_addr; 2083 mod->gpl_future_syms = (void *)sechdrs[gplfutureindex].sh_addr;
2026 if (gplfuturecrcindex) 2084 if (gplfuturecrcindex)
2027 mod->gpl_future_crcs = (void *)sechdrs[gplfuturecrcindex].sh_addr; 2085 mod->gpl_future_crcs = (void *)sechdrs[gplfuturecrcindex].sh_addr;
2028 2086
2087#ifdef CONFIG_UNUSED_SYMBOLS
2088 mod->num_unused_syms = sechdrs[unusedindex].sh_size /
2089 sizeof(*mod->unused_syms);
2090 mod->num_unused_gpl_syms = sechdrs[unusedgplindex].sh_size /
2091 sizeof(*mod->unused_gpl_syms);
2029 mod->unused_syms = (void *)sechdrs[unusedindex].sh_addr; 2092 mod->unused_syms = (void *)sechdrs[unusedindex].sh_addr;
2030 if (unusedcrcindex) 2093 if (unusedcrcindex)
2031 mod->unused_crcs = (void *)sechdrs[unusedcrcindex].sh_addr; 2094 mod->unused_crcs = (void *)sechdrs[unusedcrcindex].sh_addr;
@@ -2033,13 +2096,17 @@ static struct module *load_module(void __user *umod,
2033 if (unusedgplcrcindex) 2096 if (unusedgplcrcindex)
2034 mod->unused_gpl_crcs 2097 mod->unused_gpl_crcs
2035 = (void *)sechdrs[unusedgplcrcindex].sh_addr; 2098 = (void *)sechdrs[unusedgplcrcindex].sh_addr;
2099#endif
2036 2100
2037#ifdef CONFIG_MODVERSIONS 2101#ifdef CONFIG_MODVERSIONS
2038 if ((mod->num_syms && !crcindex) || 2102 if ((mod->num_syms && !crcindex)
2039 (mod->num_gpl_syms && !gplcrcindex) || 2103 || (mod->num_gpl_syms && !gplcrcindex)
2040 (mod->num_gpl_future_syms && !gplfuturecrcindex) || 2104 || (mod->num_gpl_future_syms && !gplfuturecrcindex)
2041 (mod->num_unused_syms && !unusedcrcindex) || 2105#ifdef CONFIG_UNUSED_SYMBOLS
2042 (mod->num_unused_gpl_syms && !unusedgplcrcindex)) { 2106 || (mod->num_unused_syms && !unusedcrcindex)
2107 || (mod->num_unused_gpl_syms && !unusedgplcrcindex)
2108#endif
2109 ) {
2043 printk(KERN_WARNING "%s: No versions for exported symbols.\n", mod->name); 2110 printk(KERN_WARNING "%s: No versions for exported symbols.\n", mod->name);
2044 err = try_to_force_load(mod, "nocrc"); 2111 err = try_to_force_load(mod, "nocrc");
2045 if (err) 2112 if (err)
@@ -2512,7 +2579,7 @@ static int m_show(struct seq_file *m, void *p)
2512 struct module *mod = list_entry(p, struct module, list); 2579 struct module *mod = list_entry(p, struct module, list);
2513 char buf[8]; 2580 char buf[8];
2514 2581
2515 seq_printf(m, "%s %lu", 2582 seq_printf(m, "%s %u",
2516 mod->name, mod->init_size + mod->core_size); 2583 mod->name, mod->init_size + mod->core_size);
2517 print_unload_info(m, mod); 2584 print_unload_info(m, mod);
2518 2585
@@ -2595,6 +2662,9 @@ struct module *__module_text_address(unsigned long addr)
2595{ 2662{
2596 struct module *mod; 2663 struct module *mod;
2597 2664
2665 if (addr < module_addr_min || addr > module_addr_max)
2666 return NULL;
2667
2598 list_for_each_entry(mod, &modules, list) 2668 list_for_each_entry(mod, &modules, list)
2599 if (within(addr, mod->module_init, mod->init_text_size) 2669 if (within(addr, mod->module_init, mod->init_text_size)
2600 || within(addr, mod->module_core, mod->core_text_size)) 2670 || within(addr, mod->module_core, mod->core_text_size))
diff --git a/kernel/mutex-debug.c b/kernel/mutex-debug.c
index 3aaa06c561de..1d94160eb532 100644
--- a/kernel/mutex-debug.c
+++ b/kernel/mutex-debug.c
@@ -79,8 +79,8 @@ void debug_mutex_unlock(struct mutex *lock)
79 if (unlikely(!debug_locks)) 79 if (unlikely(!debug_locks))
80 return; 80 return;
81 81
82 DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info());
83 DEBUG_LOCKS_WARN_ON(lock->magic != lock); 82 DEBUG_LOCKS_WARN_ON(lock->magic != lock);
83 DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info());
84 DEBUG_LOCKS_WARN_ON(!lock->wait_list.prev && !lock->wait_list.next); 84 DEBUG_LOCKS_WARN_ON(!lock->wait_list.prev && !lock->wait_list.next);
85 DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info()); 85 DEBUG_LOCKS_WARN_ON(lock->owner != current_thread_info());
86} 86}
diff --git a/kernel/mutex.c b/kernel/mutex.c
index d046a345d365..bcdc9ac8ef60 100644
--- a/kernel/mutex.c
+++ b/kernel/mutex.c
@@ -165,10 +165,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
165 * got a signal? (This code gets eliminated in the 165 * got a signal? (This code gets eliminated in the
166 * TASK_UNINTERRUPTIBLE case.) 166 * TASK_UNINTERRUPTIBLE case.)
167 */ 167 */
168 if (unlikely((state == TASK_INTERRUPTIBLE && 168 if (unlikely(signal_pending_state(state, task))) {
169 signal_pending(task)) ||
170 (state == TASK_KILLABLE &&
171 fatal_signal_pending(task)))) {
172 mutex_remove_waiter(lock, &waiter, 169 mutex_remove_waiter(lock, &waiter,
173 task_thread_info(task)); 170 task_thread_info(task));
174 mutex_release(&lock->dep_map, 1, ip); 171 mutex_release(&lock->dep_map, 1, ip);
diff --git a/kernel/ns_cgroup.c b/kernel/ns_cgroup.c
index 48d7ed6fc3a4..43c2111cd54d 100644
--- a/kernel/ns_cgroup.c
+++ b/kernel/ns_cgroup.c
@@ -7,6 +7,7 @@
7#include <linux/module.h> 7#include <linux/module.h>
8#include <linux/cgroup.h> 8#include <linux/cgroup.h>
9#include <linux/fs.h> 9#include <linux/fs.h>
10#include <linux/proc_fs.h>
10#include <linux/slab.h> 11#include <linux/slab.h>
11#include <linux/nsproxy.h> 12#include <linux/nsproxy.h>
12 13
@@ -24,9 +25,12 @@ static inline struct ns_cgroup *cgroup_to_ns(
24 struct ns_cgroup, css); 25 struct ns_cgroup, css);
25} 26}
26 27
27int ns_cgroup_clone(struct task_struct *task) 28int ns_cgroup_clone(struct task_struct *task, struct pid *pid)
28{ 29{
29 return cgroup_clone(task, &ns_subsys); 30 char name[PROC_NUMBUF];
31
32 snprintf(name, PROC_NUMBUF, "%d", pid_vnr(pid));
33 return cgroup_clone(task, &ns_subsys, name);
30} 34}
31 35
32/* 36/*
diff --git a/kernel/nsproxy.c b/kernel/nsproxy.c
index adc785146a1c..21575fc46d05 100644
--- a/kernel/nsproxy.c
+++ b/kernel/nsproxy.c
@@ -157,12 +157,6 @@ int copy_namespaces(unsigned long flags, struct task_struct *tsk)
157 goto out; 157 goto out;
158 } 158 }
159 159
160 err = ns_cgroup_clone(tsk);
161 if (err) {
162 put_nsproxy(new_ns);
163 goto out;
164 }
165
166 tsk->nsproxy = new_ns; 160 tsk->nsproxy = new_ns;
167 161
168out: 162out:
@@ -209,7 +203,7 @@ int unshare_nsproxy_namespaces(unsigned long unshare_flags,
209 goto out; 203 goto out;
210 } 204 }
211 205
212 err = ns_cgroup_clone(current); 206 err = ns_cgroup_clone(current, task_pid(current));
213 if (err) 207 if (err)
214 put_nsproxy(*new_nsp); 208 put_nsproxy(*new_nsp);
215 209
diff --git a/kernel/panic.c b/kernel/panic.c
index 425567f45b9f..12c5a0a6c89b 100644
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -318,6 +318,28 @@ void warn_on_slowpath(const char *file, int line)
318 add_taint(TAINT_WARN); 318 add_taint(TAINT_WARN);
319} 319}
320EXPORT_SYMBOL(warn_on_slowpath); 320EXPORT_SYMBOL(warn_on_slowpath);
321
322
323void warn_slowpath(const char *file, int line, const char *fmt, ...)
324{
325 va_list args;
326 char function[KSYM_SYMBOL_LEN];
327 unsigned long caller = (unsigned long)__builtin_return_address(0);
328 sprint_symbol(function, caller);
329
330 printk(KERN_WARNING "------------[ cut here ]------------\n");
331 printk(KERN_WARNING "WARNING: at %s:%d %s()\n", file,
332 line, function);
333 va_start(args, fmt);
334 vprintk(fmt, args);
335 va_end(args);
336
337 print_modules();
338 dump_stack();
339 print_oops_end_marker();
340 add_taint(TAINT_WARN);
341}
342EXPORT_SYMBOL(warn_slowpath);
321#endif 343#endif
322 344
323#ifdef CONFIG_CC_STACKPROTECTOR 345#ifdef CONFIG_CC_STACKPROTECTOR
diff --git a/kernel/pid.c b/kernel/pid.c
index 20d59fa2d493..064e76afa507 100644
--- a/kernel/pid.c
+++ b/kernel/pid.c
@@ -30,6 +30,7 @@
30#include <linux/module.h> 30#include <linux/module.h>
31#include <linux/slab.h> 31#include <linux/slab.h>
32#include <linux/init.h> 32#include <linux/init.h>
33#include <linux/rculist.h>
33#include <linux/bootmem.h> 34#include <linux/bootmem.h>
34#include <linux/hash.h> 35#include <linux/hash.h>
35#include <linux/pid_namespace.h> 36#include <linux/pid_namespace.h>
@@ -308,12 +309,6 @@ struct pid *find_vpid(int nr)
308} 309}
309EXPORT_SYMBOL_GPL(find_vpid); 310EXPORT_SYMBOL_GPL(find_vpid);
310 311
311struct pid *find_pid(int nr)
312{
313 return find_pid_ns(nr, &init_pid_ns);
314}
315EXPORT_SYMBOL_GPL(find_pid);
316
317/* 312/*
318 * attach_pid() must be called with the tasklist_lock write-held. 313 * attach_pid() must be called with the tasklist_lock write-held.
319 */ 314 */
@@ -434,6 +429,7 @@ struct pid *find_get_pid(pid_t nr)
434 429
435 return pid; 430 return pid;
436} 431}
432EXPORT_SYMBOL_GPL(find_get_pid);
437 433
438pid_t pid_nr_ns(struct pid *pid, struct pid_namespace *ns) 434pid_t pid_nr_ns(struct pid *pid, struct pid_namespace *ns)
439{ 435{
@@ -481,7 +477,7 @@ EXPORT_SYMBOL(task_session_nr_ns);
481/* 477/*
482 * Used by proc to find the first pid that is greater then or equal to nr. 478 * Used by proc to find the first pid that is greater then or equal to nr.
483 * 479 *
484 * If there is a pid at nr this function is exactly the same as find_pid. 480 * If there is a pid at nr this function is exactly the same as find_pid_ns.
485 */ 481 */
486struct pid *find_ge_pid(int nr, struct pid_namespace *ns) 482struct pid *find_ge_pid(int nr, struct pid_namespace *ns)
487{ 483{
@@ -496,7 +492,6 @@ struct pid *find_ge_pid(int nr, struct pid_namespace *ns)
496 492
497 return pid; 493 return pid;
498} 494}
499EXPORT_SYMBOL_GPL(find_get_pid);
500 495
501/* 496/*
502 * The pid hash table is scaled according to the amount of memory in the 497 * The pid hash table is scaled according to the amount of memory in the
diff --git a/kernel/pid_namespace.c b/kernel/pid_namespace.c
index 98702b4b8851..ea567b78d1aa 100644
--- a/kernel/pid_namespace.c
+++ b/kernel/pid_namespace.c
@@ -12,6 +12,7 @@
12#include <linux/pid_namespace.h> 12#include <linux/pid_namespace.h>
13#include <linux/syscalls.h> 13#include <linux/syscalls.h>
14#include <linux/err.h> 14#include <linux/err.h>
15#include <linux/acct.h>
15 16
16#define BITS_PER_PAGE (PAGE_SIZE*8) 17#define BITS_PER_PAGE (PAGE_SIZE*8)
17 18
@@ -71,7 +72,7 @@ static struct pid_namespace *create_pid_namespace(unsigned int level)
71 struct pid_namespace *ns; 72 struct pid_namespace *ns;
72 int i; 73 int i;
73 74
74 ns = kmem_cache_alloc(pid_ns_cachep, GFP_KERNEL); 75 ns = kmem_cache_zalloc(pid_ns_cachep, GFP_KERNEL);
75 if (ns == NULL) 76 if (ns == NULL)
76 goto out; 77 goto out;
77 78
@@ -84,17 +85,13 @@ static struct pid_namespace *create_pid_namespace(unsigned int level)
84 goto out_free_map; 85 goto out_free_map;
85 86
86 kref_init(&ns->kref); 87 kref_init(&ns->kref);
87 ns->last_pid = 0;
88 ns->child_reaper = NULL;
89 ns->level = level; 88 ns->level = level;
90 89
91 set_bit(0, ns->pidmap[0].page); 90 set_bit(0, ns->pidmap[0].page);
92 atomic_set(&ns->pidmap[0].nr_free, BITS_PER_PAGE - 1); 91 atomic_set(&ns->pidmap[0].nr_free, BITS_PER_PAGE - 1);
93 92
94 for (i = 1; i < PIDMAP_ENTRIES; i++) { 93 for (i = 1; i < PIDMAP_ENTRIES; i++)
95 ns->pidmap[i].page = NULL;
96 atomic_set(&ns->pidmap[i].nr_free, BITS_PER_PAGE); 94 atomic_set(&ns->pidmap[i].nr_free, BITS_PER_PAGE);
97 }
98 95
99 return ns; 96 return ns;
100 97
@@ -185,6 +182,7 @@ void zap_pid_ns_processes(struct pid_namespace *pid_ns)
185 182
186 /* Child reaper for the pid namespace is going away */ 183 /* Child reaper for the pid namespace is going away */
187 pid_ns->child_reaper = NULL; 184 pid_ns->child_reaper = NULL;
185 acct_exit_ns(pid_ns);
188 return; 186 return;
189} 187}
190 188
diff --git a/kernel/pm_qos_params.c b/kernel/pm_qos_params.c
index 0afe32be4c85..8cb757026386 100644
--- a/kernel/pm_qos_params.c
+++ b/kernel/pm_qos_params.c
@@ -29,6 +29,7 @@
29 29
30#include <linux/pm_qos_params.h> 30#include <linux/pm_qos_params.h>
31#include <linux/sched.h> 31#include <linux/sched.h>
32#include <linux/smp_lock.h>
32#include <linux/spinlock.h> 33#include <linux/spinlock.h>
33#include <linux/slab.h> 34#include <linux/slab.h>
34#include <linux/time.h> 35#include <linux/time.h>
@@ -358,15 +359,19 @@ static int pm_qos_power_open(struct inode *inode, struct file *filp)
358 int ret; 359 int ret;
359 long pm_qos_class; 360 long pm_qos_class;
360 361
362 lock_kernel();
361 pm_qos_class = find_pm_qos_object_by_minor(iminor(inode)); 363 pm_qos_class = find_pm_qos_object_by_minor(iminor(inode));
362 if (pm_qos_class >= 0) { 364 if (pm_qos_class >= 0) {
363 filp->private_data = (void *)pm_qos_class; 365 filp->private_data = (void *)pm_qos_class;
364 sprintf(name, "process_%d", current->pid); 366 sprintf(name, "process_%d", current->pid);
365 ret = pm_qos_add_requirement(pm_qos_class, name, 367 ret = pm_qos_add_requirement(pm_qos_class, name,
366 PM_QOS_DEFAULT_VALUE); 368 PM_QOS_DEFAULT_VALUE);
367 if (ret >= 0) 369 if (ret >= 0) {
370 unlock_kernel();
368 return 0; 371 return 0;
372 }
369 } 373 }
374 unlock_kernel();
370 375
371 return -EPERM; 376 return -EPERM;
372} 377}
diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c
index f1525ad06cb3..c42a03aef36f 100644
--- a/kernel/posix-cpu-timers.c
+++ b/kernel/posix-cpu-timers.c
@@ -1037,6 +1037,9 @@ static void check_thread_timers(struct task_struct *tsk,
1037 sig->rlim[RLIMIT_RTTIME].rlim_cur += 1037 sig->rlim[RLIMIT_RTTIME].rlim_cur +=
1038 USEC_PER_SEC; 1038 USEC_PER_SEC;
1039 } 1039 }
1040 printk(KERN_INFO
1041 "RT Watchdog Timeout: %s[%d]\n",
1042 tsk->comm, task_pid_nr(tsk));
1040 __group_send_sig_info(SIGXCPU, SEND_SIG_PRIV, tsk); 1043 __group_send_sig_info(SIGXCPU, SEND_SIG_PRIV, tsk);
1041 } 1044 }
1042 } 1045 }
diff --git a/kernel/posix-timers.c b/kernel/posix-timers.c
index dbd8398ddb0b..9a21681aa80f 100644
--- a/kernel/posix-timers.c
+++ b/kernel/posix-timers.c
@@ -449,9 +449,6 @@ static void release_posix_timer(struct k_itimer *tmr, int it_id_set)
449 spin_unlock_irqrestore(&idr_lock, flags); 449 spin_unlock_irqrestore(&idr_lock, flags);
450 } 450 }
451 sigqueue_free(tmr->sigq); 451 sigqueue_free(tmr->sigq);
452 if (unlikely(tmr->it_process) &&
453 tmr->it_sigev_notify == (SIGEV_SIGNAL|SIGEV_THREAD_ID))
454 put_task_struct(tmr->it_process);
455 kmem_cache_free(posix_timers_cache, tmr); 452 kmem_cache_free(posix_timers_cache, tmr);
456} 453}
457 454
@@ -856,11 +853,10 @@ retry_delete:
856 * This keeps any tasks waiting on the spin lock from thinking 853 * This keeps any tasks waiting on the spin lock from thinking
857 * they got something (see the lock code above). 854 * they got something (see the lock code above).
858 */ 855 */
859 if (timer->it_process) { 856 if (timer->it_sigev_notify == (SIGEV_SIGNAL|SIGEV_THREAD_ID))
860 if (timer->it_sigev_notify == (SIGEV_SIGNAL|SIGEV_THREAD_ID)) 857 put_task_struct(timer->it_process);
861 put_task_struct(timer->it_process); 858 timer->it_process = NULL;
862 timer->it_process = NULL; 859
863 }
864 unlock_timer(timer, flags); 860 unlock_timer(timer, flags);
865 release_posix_timer(timer, IT_ID_SET); 861 release_posix_timer(timer, IT_ID_SET);
866 return 0; 862 return 0;
@@ -885,11 +881,10 @@ retry_delete:
885 * This keeps any tasks waiting on the spin lock from thinking 881 * This keeps any tasks waiting on the spin lock from thinking
886 * they got something (see the lock code above). 882 * they got something (see the lock code above).
887 */ 883 */
888 if (timer->it_process) { 884 if (timer->it_sigev_notify == (SIGEV_SIGNAL|SIGEV_THREAD_ID))
889 if (timer->it_sigev_notify == (SIGEV_SIGNAL|SIGEV_THREAD_ID)) 885 put_task_struct(timer->it_process);
890 put_task_struct(timer->it_process); 886 timer->it_process = NULL;
891 timer->it_process = NULL; 887
892 }
893 unlock_timer(timer, flags); 888 unlock_timer(timer, flags);
894 release_posix_timer(timer, IT_ID_SET); 889 release_posix_timer(timer, IT_ID_SET);
895} 890}
diff --git a/kernel/power/Kconfig b/kernel/power/Kconfig
index b45da40e8d25..dcd165f92a88 100644
--- a/kernel/power/Kconfig
+++ b/kernel/power/Kconfig
@@ -82,7 +82,7 @@ config PM_SLEEP_SMP
82 82
83config PM_SLEEP 83config PM_SLEEP
84 bool 84 bool
85 depends on SUSPEND || HIBERNATION 85 depends on SUSPEND || HIBERNATION || XEN_SAVE_RESTORE
86 default y 86 default y
87 87
88config SUSPEND 88config SUSPEND
@@ -94,6 +94,17 @@ config SUSPEND
94 powered and thus its contents are preserved, such as the 94 powered and thus its contents are preserved, such as the
95 suspend-to-RAM state (e.g. the ACPI S3 state). 95 suspend-to-RAM state (e.g. the ACPI S3 state).
96 96
97config PM_TEST_SUSPEND
98 bool "Test suspend/resume and wakealarm during bootup"
99 depends on SUSPEND && PM_DEBUG && RTC_LIB=y
100 ---help---
101 This option will let you suspend your machine during bootup, and
102 make it wake up a few seconds later using an RTC wakeup alarm.
103 Enable this with a kernel parameter like "test_suspend=mem".
104
105 You probably want to have your system's RTC driver statically
106 linked, ensuring that it's available when this test runs.
107
97config SUSPEND_FREEZER 108config SUSPEND_FREEZER
98 bool "Enable freezer for suspend to RAM/standby" \ 109 bool "Enable freezer for suspend to RAM/standby" \
99 if ARCH_WANTS_FREEZER_CONTROL || BROKEN 110 if ARCH_WANTS_FREEZER_CONTROL || BROKEN
diff --git a/kernel/power/disk.c b/kernel/power/disk.c
index 14a656cdc652..f011e0870b52 100644
--- a/kernel/power/disk.c
+++ b/kernel/power/disk.c
@@ -180,6 +180,17 @@ static void platform_restore_cleanup(int platform_mode)
180} 180}
181 181
182/** 182/**
183 * platform_recover - recover the platform from a failure to suspend
184 * devices.
185 */
186
187static void platform_recover(int platform_mode)
188{
189 if (platform_mode && hibernation_ops && hibernation_ops->recover)
190 hibernation_ops->recover();
191}
192
193/**
183 * create_image - freeze devices that need to be frozen with interrupts 194 * create_image - freeze devices that need to be frozen with interrupts
184 * off, create the hibernation image and thaw those devices. Control 195 * off, create the hibernation image and thaw those devices. Control
185 * reappears in this routine after a restore. 196 * reappears in this routine after a restore.
@@ -193,6 +204,7 @@ static int create_image(int platform_mode)
193 if (error) 204 if (error)
194 return error; 205 return error;
195 206
207 device_pm_lock();
196 local_irq_disable(); 208 local_irq_disable();
197 /* At this point, device_suspend() has been called, but *not* 209 /* At this point, device_suspend() has been called, but *not*
198 * device_power_down(). We *must* call device_power_down() now. 210 * device_power_down(). We *must* call device_power_down() now.
@@ -224,9 +236,11 @@ static int create_image(int platform_mode)
224 /* NOTE: device_power_up() is just a resume() for devices 236 /* NOTE: device_power_up() is just a resume() for devices
225 * that suspended with irqs off ... no overall powerup. 237 * that suspended with irqs off ... no overall powerup.
226 */ 238 */
227 device_power_up(); 239 device_power_up(in_suspend ?
240 (error ? PMSG_RECOVER : PMSG_THAW) : PMSG_RESTORE);
228 Enable_irqs: 241 Enable_irqs:
229 local_irq_enable(); 242 local_irq_enable();
243 device_pm_unlock();
230 return error; 244 return error;
231} 245}
232 246
@@ -255,10 +269,10 @@ int hibernation_snapshot(int platform_mode)
255 suspend_console(); 269 suspend_console();
256 error = device_suspend(PMSG_FREEZE); 270 error = device_suspend(PMSG_FREEZE);
257 if (error) 271 if (error)
258 goto Resume_console; 272 goto Recover_platform;
259 273
260 if (hibernation_test(TEST_DEVICES)) 274 if (hibernation_test(TEST_DEVICES))
261 goto Resume_devices; 275 goto Recover_platform;
262 276
263 error = platform_pre_snapshot(platform_mode); 277 error = platform_pre_snapshot(platform_mode);
264 if (error || hibernation_test(TEST_PLATFORM)) 278 if (error || hibernation_test(TEST_PLATFORM))
@@ -280,12 +294,16 @@ int hibernation_snapshot(int platform_mode)
280 Finish: 294 Finish:
281 platform_finish(platform_mode); 295 platform_finish(platform_mode);
282 Resume_devices: 296 Resume_devices:
283 device_resume(); 297 device_resume(in_suspend ?
284 Resume_console: 298 (error ? PMSG_RECOVER : PMSG_THAW) : PMSG_RESTORE);
285 resume_console(); 299 resume_console();
286 Close: 300 Close:
287 platform_end(platform_mode); 301 platform_end(platform_mode);
288 return error; 302 return error;
303
304 Recover_platform:
305 platform_recover(platform_mode);
306 goto Resume_devices;
289} 307}
290 308
291/** 309/**
@@ -300,8 +318,9 @@ static int resume_target_kernel(void)
300{ 318{
301 int error; 319 int error;
302 320
321 device_pm_lock();
303 local_irq_disable(); 322 local_irq_disable();
304 error = device_power_down(PMSG_PRETHAW); 323 error = device_power_down(PMSG_QUIESCE);
305 if (error) { 324 if (error) {
306 printk(KERN_ERR "PM: Some devices failed to power down, " 325 printk(KERN_ERR "PM: Some devices failed to power down, "
307 "aborting resume\n"); 326 "aborting resume\n");
@@ -329,9 +348,10 @@ static int resume_target_kernel(void)
329 swsusp_free(); 348 swsusp_free();
330 restore_processor_state(); 349 restore_processor_state();
331 touch_softlockup_watchdog(); 350 touch_softlockup_watchdog();
332 device_power_up(); 351 device_power_up(PMSG_RECOVER);
333 Enable_irqs: 352 Enable_irqs:
334 local_irq_enable(); 353 local_irq_enable();
354 device_pm_unlock();
335 return error; 355 return error;
336} 356}
337 357
@@ -350,7 +370,7 @@ int hibernation_restore(int platform_mode)
350 370
351 pm_prepare_console(); 371 pm_prepare_console();
352 suspend_console(); 372 suspend_console();
353 error = device_suspend(PMSG_PRETHAW); 373 error = device_suspend(PMSG_QUIESCE);
354 if (error) 374 if (error)
355 goto Finish; 375 goto Finish;
356 376
@@ -362,7 +382,7 @@ int hibernation_restore(int platform_mode)
362 enable_nonboot_cpus(); 382 enable_nonboot_cpus();
363 } 383 }
364 platform_restore_cleanup(platform_mode); 384 platform_restore_cleanup(platform_mode);
365 device_resume(); 385 device_resume(PMSG_RECOVER);
366 Finish: 386 Finish:
367 resume_console(); 387 resume_console();
368 pm_restore_console(); 388 pm_restore_console();
@@ -392,8 +412,11 @@ int hibernation_platform_enter(void)
392 412
393 suspend_console(); 413 suspend_console();
394 error = device_suspend(PMSG_HIBERNATE); 414 error = device_suspend(PMSG_HIBERNATE);
395 if (error) 415 if (error) {
396 goto Resume_console; 416 if (hibernation_ops->recover)
417 hibernation_ops->recover();
418 goto Resume_devices;
419 }
397 420
398 error = hibernation_ops->prepare(); 421 error = hibernation_ops->prepare();
399 if (error) 422 if (error)
@@ -403,6 +426,7 @@ int hibernation_platform_enter(void)
403 if (error) 426 if (error)
404 goto Finish; 427 goto Finish;
405 428
429 device_pm_lock();
406 local_irq_disable(); 430 local_irq_disable();
407 error = device_power_down(PMSG_HIBERNATE); 431 error = device_power_down(PMSG_HIBERNATE);
408 if (!error) { 432 if (!error) {
@@ -411,6 +435,7 @@ int hibernation_platform_enter(void)
411 while (1); 435 while (1);
412 } 436 }
413 local_irq_enable(); 437 local_irq_enable();
438 device_pm_unlock();
414 439
415 /* 440 /*
416 * We don't need to reenable the nonboot CPUs or resume consoles, since 441 * We don't need to reenable the nonboot CPUs or resume consoles, since
@@ -419,8 +444,7 @@ int hibernation_platform_enter(void)
419 Finish: 444 Finish:
420 hibernation_ops->finish(); 445 hibernation_ops->finish();
421 Resume_devices: 446 Resume_devices:
422 device_resume(); 447 device_resume(PMSG_RESTORE);
423 Resume_console:
424 resume_console(); 448 resume_console();
425 Close: 449 Close:
426 hibernation_ops->end(); 450 hibernation_ops->end();
diff --git a/kernel/power/main.c b/kernel/power/main.c
index 6a6d5eb3524e..0b7476f5d2a6 100644
--- a/kernel/power/main.c
+++ b/kernel/power/main.c
@@ -132,6 +132,61 @@ static inline int suspend_test(int level) { return 0; }
132 132
133#ifdef CONFIG_SUSPEND 133#ifdef CONFIG_SUSPEND
134 134
135#ifdef CONFIG_PM_TEST_SUSPEND
136
137/*
138 * We test the system suspend code by setting an RTC wakealarm a short
139 * time in the future, then suspending. Suspending the devices won't
140 * normally take long ... some systems only need a few milliseconds.
141 *
142 * The time it takes is system-specific though, so when we test this
143 * during system bootup we allow a LOT of time.
144 */
145#define TEST_SUSPEND_SECONDS 5
146
147static unsigned long suspend_test_start_time;
148
149static void suspend_test_start(void)
150{
151 /* FIXME Use better timebase than "jiffies", ideally a clocksource.
152 * What we want is a hardware counter that will work correctly even
153 * during the irqs-are-off stages of the suspend/resume cycle...
154 */
155 suspend_test_start_time = jiffies;
156}
157
158static void suspend_test_finish(const char *label)
159{
160 long nj = jiffies - suspend_test_start_time;
161 unsigned msec;
162
163 msec = jiffies_to_msecs(abs(nj));
164 pr_info("PM: %s took %d.%03d seconds\n", label,
165 msec / 1000, msec % 1000);
166
167 /* Warning on suspend means the RTC alarm period needs to be
168 * larger -- the system was sooo slooowwww to suspend that the
169 * alarm (should have) fired before the system went to sleep!
170 *
171 * Warning on either suspend or resume also means the system
172 * has some performance issues. The stack dump of a WARN_ON
173 * is more likely to get the right attention than a printk...
174 */
175 WARN_ON(msec > (TEST_SUSPEND_SECONDS * 1000));
176}
177
178#else
179
180static void suspend_test_start(void)
181{
182}
183
184static void suspend_test_finish(const char *label)
185{
186}
187
188#endif
189
135/* This is just an arbitrary number */ 190/* This is just an arbitrary number */
136#define FREE_PAGE_NUMBER (100) 191#define FREE_PAGE_NUMBER (100)
137 192
@@ -228,6 +283,7 @@ static int suspend_enter(suspend_state_t state)
228{ 283{
229 int error = 0; 284 int error = 0;
230 285
286 device_pm_lock();
231 arch_suspend_disable_irqs(); 287 arch_suspend_disable_irqs();
232 BUG_ON(!irqs_disabled()); 288 BUG_ON(!irqs_disabled());
233 289
@@ -239,10 +295,11 @@ static int suspend_enter(suspend_state_t state)
239 if (!suspend_test(TEST_CORE)) 295 if (!suspend_test(TEST_CORE))
240 error = suspend_ops->enter(state); 296 error = suspend_ops->enter(state);
241 297
242 device_power_up(); 298 device_power_up(PMSG_RESUME);
243 Done: 299 Done:
244 arch_suspend_enable_irqs(); 300 arch_suspend_enable_irqs();
245 BUG_ON(irqs_disabled()); 301 BUG_ON(irqs_disabled());
302 device_pm_unlock();
246 return error; 303 return error;
247} 304}
248 305
@@ -264,14 +321,15 @@ int suspend_devices_and_enter(suspend_state_t state)
264 goto Close; 321 goto Close;
265 } 322 }
266 suspend_console(); 323 suspend_console();
324 suspend_test_start();
267 error = device_suspend(PMSG_SUSPEND); 325 error = device_suspend(PMSG_SUSPEND);
268 if (error) { 326 if (error) {
269 printk(KERN_ERR "PM: Some devices failed to suspend\n"); 327 printk(KERN_ERR "PM: Some devices failed to suspend\n");
270 goto Resume_console; 328 goto Recover_platform;
271 } 329 }
272 330 suspend_test_finish("suspend devices");
273 if (suspend_test(TEST_DEVICES)) 331 if (suspend_test(TEST_DEVICES))
274 goto Resume_devices; 332 goto Recover_platform;
275 333
276 if (suspend_ops->prepare) { 334 if (suspend_ops->prepare) {
277 error = suspend_ops->prepare(); 335 error = suspend_ops->prepare();
@@ -291,13 +349,19 @@ int suspend_devices_and_enter(suspend_state_t state)
291 if (suspend_ops->finish) 349 if (suspend_ops->finish)
292 suspend_ops->finish(); 350 suspend_ops->finish();
293 Resume_devices: 351 Resume_devices:
294 device_resume(); 352 suspend_test_start();
295 Resume_console: 353 device_resume(PMSG_RESUME);
354 suspend_test_finish("resume devices");
296 resume_console(); 355 resume_console();
297 Close: 356 Close:
298 if (suspend_ops->end) 357 if (suspend_ops->end)
299 suspend_ops->end(); 358 suspend_ops->end();
300 return error; 359 return error;
360
361 Recover_platform:
362 if (suspend_ops->recover)
363 suspend_ops->recover();
364 goto Resume_devices;
301} 365}
302 366
303/** 367/**
@@ -515,3 +579,144 @@ static int __init pm_init(void)
515} 579}
516 580
517core_initcall(pm_init); 581core_initcall(pm_init);
582
583
584#ifdef CONFIG_PM_TEST_SUSPEND
585
586#include <linux/rtc.h>
587
588/*
589 * To test system suspend, we need a hands-off mechanism to resume the
590 * system. RTCs wake alarms are a common self-contained mechanism.
591 */
592
593static void __init test_wakealarm(struct rtc_device *rtc, suspend_state_t state)
594{
595 static char err_readtime[] __initdata =
596 KERN_ERR "PM: can't read %s time, err %d\n";
597 static char err_wakealarm [] __initdata =
598 KERN_ERR "PM: can't set %s wakealarm, err %d\n";
599 static char err_suspend[] __initdata =
600 KERN_ERR "PM: suspend test failed, error %d\n";
601 static char info_test[] __initdata =
602 KERN_INFO "PM: test RTC wakeup from '%s' suspend\n";
603
604 unsigned long now;
605 struct rtc_wkalrm alm;
606 int status;
607
608 /* this may fail if the RTC hasn't been initialized */
609 status = rtc_read_time(rtc, &alm.time);
610 if (status < 0) {
611 printk(err_readtime, rtc->dev.bus_id, status);
612 return;
613 }
614 rtc_tm_to_time(&alm.time, &now);
615
616 memset(&alm, 0, sizeof alm);
617 rtc_time_to_tm(now + TEST_SUSPEND_SECONDS, &alm.time);
618 alm.enabled = true;
619
620 status = rtc_set_alarm(rtc, &alm);
621 if (status < 0) {
622 printk(err_wakealarm, rtc->dev.bus_id, status);
623 return;
624 }
625
626 if (state == PM_SUSPEND_MEM) {
627 printk(info_test, pm_states[state]);
628 status = pm_suspend(state);
629 if (status == -ENODEV)
630 state = PM_SUSPEND_STANDBY;
631 }
632 if (state == PM_SUSPEND_STANDBY) {
633 printk(info_test, pm_states[state]);
634 status = pm_suspend(state);
635 }
636 if (status < 0)
637 printk(err_suspend, status);
638
639 /* Some platforms can't detect that the alarm triggered the
640 * wakeup, or (accordingly) disable it after it afterwards.
641 * It's supposed to give oneshot behavior; cope.
642 */
643 alm.enabled = false;
644 rtc_set_alarm(rtc, &alm);
645}
646
647static int __init has_wakealarm(struct device *dev, void *name_ptr)
648{
649 struct rtc_device *candidate = to_rtc_device(dev);
650
651 if (!candidate->ops->set_alarm)
652 return 0;
653 if (!device_may_wakeup(candidate->dev.parent))
654 return 0;
655
656 *(char **)name_ptr = dev->bus_id;
657 return 1;
658}
659
660/*
661 * Kernel options like "test_suspend=mem" force suspend/resume sanity tests
662 * at startup time. They're normally disabled, for faster boot and because
663 * we can't know which states really work on this particular system.
664 */
665static suspend_state_t test_state __initdata = PM_SUSPEND_ON;
666
667static char warn_bad_state[] __initdata =
668 KERN_WARNING "PM: can't test '%s' suspend state\n";
669
670static int __init setup_test_suspend(char *value)
671{
672 unsigned i;
673
674 /* "=mem" ==> "mem" */
675 value++;
676 for (i = 0; i < PM_SUSPEND_MAX; i++) {
677 if (!pm_states[i])
678 continue;
679 if (strcmp(pm_states[i], value) != 0)
680 continue;
681 test_state = (__force suspend_state_t) i;
682 return 0;
683 }
684 printk(warn_bad_state, value);
685 return 0;
686}
687__setup("test_suspend", setup_test_suspend);
688
689static int __init test_suspend(void)
690{
691 static char warn_no_rtc[] __initdata =
692 KERN_WARNING "PM: no wakealarm-capable RTC driver is ready\n";
693
694 char *pony = NULL;
695 struct rtc_device *rtc = NULL;
696
697 /* PM is initialized by now; is that state testable? */
698 if (test_state == PM_SUSPEND_ON)
699 goto done;
700 if (!valid_state(test_state)) {
701 printk(warn_bad_state, pm_states[test_state]);
702 goto done;
703 }
704
705 /* RTCs have initialized by now too ... can we use one? */
706 class_find_device(rtc_class, NULL, &pony, has_wakealarm);
707 if (pony)
708 rtc = rtc_class_open(pony);
709 if (!rtc) {
710 printk(warn_no_rtc);
711 goto done;
712 }
713
714 /* go for it */
715 test_wakealarm(rtc, test_state);
716 rtc_class_close(rtc);
717done:
718 return 0;
719}
720late_initcall(test_suspend);
721
722#endif /* CONFIG_PM_TEST_SUSPEND */
diff --git a/kernel/power/power.h b/kernel/power/power.h
index 700f44ec8406..acc0c101dbd5 100644
--- a/kernel/power/power.h
+++ b/kernel/power/power.h
@@ -53,8 +53,6 @@ extern int hibernation_platform_enter(void);
53 53
54extern int pfn_is_nosave(unsigned long); 54extern int pfn_is_nosave(unsigned long);
55 55
56extern struct mutex pm_mutex;
57
58#define power_attr(_name) \ 56#define power_attr(_name) \
59static struct kobj_attribute _name##_attr = { \ 57static struct kobj_attribute _name##_attr = { \
60 .attr = { \ 58 .attr = { \
diff --git a/kernel/power/poweroff.c b/kernel/power/poweroff.c
index 678ec736076b..72016f051477 100644
--- a/kernel/power/poweroff.c
+++ b/kernel/power/poweroff.c
@@ -10,6 +10,7 @@
10#include <linux/pm.h> 10#include <linux/pm.h>
11#include <linux/workqueue.h> 11#include <linux/workqueue.h>
12#include <linux/reboot.h> 12#include <linux/reboot.h>
13#include <linux/cpumask.h>
13 14
14/* 15/*
15 * When the user hits Sys-Rq o to power down the machine this is the 16 * When the user hits Sys-Rq o to power down the machine this is the
@@ -25,7 +26,8 @@ static DECLARE_WORK(poweroff_work, do_poweroff);
25 26
26static void handle_poweroff(int key, struct tty_struct *tty) 27static void handle_poweroff(int key, struct tty_struct *tty)
27{ 28{
28 schedule_work(&poweroff_work); 29 /* run sysrq poweroff on boot cpu */
30 schedule_work_on(first_cpu(cpu_online_map), &poweroff_work);
29} 31}
30 32
31static struct sysrq_key_op sysrq_poweroff_op = { 33static struct sysrq_key_op sysrq_poweroff_op = {
diff --git a/kernel/power/process.c b/kernel/power/process.c
index f1d0b345c9ba..278946aecaf0 100644
--- a/kernel/power/process.c
+++ b/kernel/power/process.c
@@ -19,9 +19,6 @@
19 */ 19 */
20#define TIMEOUT (20 * HZ) 20#define TIMEOUT (20 * HZ)
21 21
22#define FREEZER_KERNEL_THREADS 0
23#define FREEZER_USER_SPACE 1
24
25static inline int freezeable(struct task_struct * p) 22static inline int freezeable(struct task_struct * p)
26{ 23{
27 if ((p == current) || 24 if ((p == current) ||
@@ -84,63 +81,53 @@ static void fake_signal_wake_up(struct task_struct *p)
84 spin_unlock_irqrestore(&p->sighand->siglock, flags); 81 spin_unlock_irqrestore(&p->sighand->siglock, flags);
85} 82}
86 83
87static int has_mm(struct task_struct *p) 84static inline bool should_send_signal(struct task_struct *p)
88{ 85{
89 return (p->mm && !(p->flags & PF_BORROWED_MM)); 86 return !(p->flags & PF_FREEZER_NOSIG);
90} 87}
91 88
92/** 89/**
93 * freeze_task - send a freeze request to given task 90 * freeze_task - send a freeze request to given task
94 * @p: task to send the request to 91 * @p: task to send the request to
95 * @with_mm_only: if set, the request will only be sent if the task has its 92 * @sig_only: if set, the request will only be sent if the task has the
96 * own mm 93 * PF_FREEZER_NOSIG flag unset
97 * Return value: 0, if @with_mm_only is set and the task has no mm of its 94 * Return value: 'false', if @sig_only is set and the task has
98 * own or the task is frozen, 1, otherwise 95 * PF_FREEZER_NOSIG set or the task is frozen, 'true', otherwise
99 * 96 *
100 * The freeze request is sent by seting the tasks's TIF_FREEZE flag and 97 * The freeze request is sent by setting the tasks's TIF_FREEZE flag and
101 * either sending a fake signal to it or waking it up, depending on whether 98 * either sending a fake signal to it or waking it up, depending on whether
102 * or not it has its own mm (ie. it is a user land task). If @with_mm_only 99 * or not it has PF_FREEZER_NOSIG set. If @sig_only is set and the task
103 * is set and the task has no mm of its own (ie. it is a kernel thread), 100 * has PF_FREEZER_NOSIG set (ie. it is a typical kernel thread), its
104 * its TIF_FREEZE flag should not be set. 101 * TIF_FREEZE flag will not be set.
105 *
106 * The task_lock() is necessary to prevent races with exit_mm() or
107 * use_mm()/unuse_mm() from occuring.
108 */ 102 */
109static int freeze_task(struct task_struct *p, int with_mm_only) 103static bool freeze_task(struct task_struct *p, bool sig_only)
110{ 104{
111 int ret = 1; 105 /*
106 * We first check if the task is freezing and next if it has already
107 * been frozen to avoid the race with frozen_process() which first marks
108 * the task as frozen and next clears its TIF_FREEZE.
109 */
110 if (!freezing(p)) {
111 rmb();
112 if (frozen(p))
113 return false;
112 114
113 task_lock(p); 115 if (!sig_only || should_send_signal(p))
114 if (freezing(p)) { 116 set_freeze_flag(p);
115 if (has_mm(p)) { 117 else
116 if (!signal_pending(p)) 118 return false;
117 fake_signal_wake_up(p); 119 }
118 } else { 120
119 if (with_mm_only) 121 if (should_send_signal(p)) {
120 ret = 0; 122 if (!signal_pending(p))
121 else 123 fake_signal_wake_up(p);
122 wake_up_state(p, TASK_INTERRUPTIBLE); 124 } else if (sig_only) {
123 } 125 return false;
124 } else { 126 } else {
125 rmb(); 127 wake_up_state(p, TASK_INTERRUPTIBLE);
126 if (frozen(p)) {
127 ret = 0;
128 } else {
129 if (has_mm(p)) {
130 set_freeze_flag(p);
131 fake_signal_wake_up(p);
132 } else {
133 if (with_mm_only) {
134 ret = 0;
135 } else {
136 set_freeze_flag(p);
137 wake_up_state(p, TASK_INTERRUPTIBLE);
138 }
139 }
140 }
141 } 128 }
142 task_unlock(p); 129
143 return ret; 130 return true;
144} 131}
145 132
146static void cancel_freezing(struct task_struct *p) 133static void cancel_freezing(struct task_struct *p)
@@ -156,13 +143,13 @@ static void cancel_freezing(struct task_struct *p)
156 } 143 }
157} 144}
158 145
159static int try_to_freeze_tasks(int freeze_user_space) 146static int try_to_freeze_tasks(bool sig_only)
160{ 147{
161 struct task_struct *g, *p; 148 struct task_struct *g, *p;
162 unsigned long end_time; 149 unsigned long end_time;
163 unsigned int todo; 150 unsigned int todo;
164 struct timeval start, end; 151 struct timeval start, end;
165 s64 elapsed_csecs64; 152 u64 elapsed_csecs64;
166 unsigned int elapsed_csecs; 153 unsigned int elapsed_csecs;
167 154
168 do_gettimeofday(&start); 155 do_gettimeofday(&start);
@@ -175,7 +162,7 @@ static int try_to_freeze_tasks(int freeze_user_space)
175 if (frozen(p) || !freezeable(p)) 162 if (frozen(p) || !freezeable(p))
176 continue; 163 continue;
177 164
178 if (!freeze_task(p, freeze_user_space)) 165 if (!freeze_task(p, sig_only))
179 continue; 166 continue;
180 167
181 /* 168 /*
@@ -235,13 +222,13 @@ int freeze_processes(void)
235 int error; 222 int error;
236 223
237 printk("Freezing user space processes ... "); 224 printk("Freezing user space processes ... ");
238 error = try_to_freeze_tasks(FREEZER_USER_SPACE); 225 error = try_to_freeze_tasks(true);
239 if (error) 226 if (error)
240 goto Exit; 227 goto Exit;
241 printk("done.\n"); 228 printk("done.\n");
242 229
243 printk("Freezing remaining freezable tasks ... "); 230 printk("Freezing remaining freezable tasks ... ");
244 error = try_to_freeze_tasks(FREEZER_KERNEL_THREADS); 231 error = try_to_freeze_tasks(false);
245 if (error) 232 if (error)
246 goto Exit; 233 goto Exit;
247 printk("done."); 234 printk("done.");
@@ -251,7 +238,7 @@ int freeze_processes(void)
251 return error; 238 return error;
252} 239}
253 240
254static void thaw_tasks(int thaw_user_space) 241static void thaw_tasks(bool nosig_only)
255{ 242{
256 struct task_struct *g, *p; 243 struct task_struct *g, *p;
257 244
@@ -260,7 +247,7 @@ static void thaw_tasks(int thaw_user_space)
260 if (!freezeable(p)) 247 if (!freezeable(p))
261 continue; 248 continue;
262 249
263 if (!p->mm == thaw_user_space) 250 if (nosig_only && should_send_signal(p))
264 continue; 251 continue;
265 252
266 thaw_process(p); 253 thaw_process(p);
@@ -271,8 +258,8 @@ static void thaw_tasks(int thaw_user_space)
271void thaw_processes(void) 258void thaw_processes(void)
272{ 259{
273 printk("Restarting tasks ... "); 260 printk("Restarting tasks ... ");
274 thaw_tasks(FREEZER_KERNEL_THREADS); 261 thaw_tasks(true);
275 thaw_tasks(FREEZER_USER_SPACE); 262 thaw_tasks(false);
276 schedule(); 263 schedule();
277 printk("done.\n"); 264 printk("done.\n");
278} 265}
diff --git a/kernel/power/snapshot.c b/kernel/power/snapshot.c
index 5f91a07c4eac..5d2ab836e998 100644
--- a/kernel/power/snapshot.c
+++ b/kernel/power/snapshot.c
@@ -205,8 +205,7 @@ static void chain_free(struct chain_allocator *ca, int clear_page_nosave)
205 * objects. The main list's elements are of type struct zone_bitmap 205 * objects. The main list's elements are of type struct zone_bitmap
206 * and each of them corresonds to one zone. For each zone bitmap 206 * and each of them corresonds to one zone. For each zone bitmap
207 * object there is a list of objects of type struct bm_block that 207 * object there is a list of objects of type struct bm_block that
208 * represent each blocks of bit chunks in which information is 208 * represent each blocks of bitmap in which information is stored.
209 * stored.
210 * 209 *
211 * struct memory_bitmap contains a pointer to the main list of zone 210 * struct memory_bitmap contains a pointer to the main list of zone
212 * bitmap objects, a struct bm_position used for browsing the bitmap, 211 * bitmap objects, a struct bm_position used for browsing the bitmap,
@@ -224,26 +223,27 @@ static void chain_free(struct chain_allocator *ca, int clear_page_nosave)
224 * pfns that correspond to the start and end of the represented zone. 223 * pfns that correspond to the start and end of the represented zone.
225 * 224 *
226 * struct bm_block contains a pointer to the memory page in which 225 * struct bm_block contains a pointer to the memory page in which
227 * information is stored (in the form of a block of bit chunks 226 * information is stored (in the form of a block of bitmap)
228 * of type unsigned long each). It also contains the pfns that 227 * It also contains the pfns that correspond to the start and end of
229 * correspond to the start and end of the represented memory area and 228 * the represented memory area.
230 * the number of bit chunks in the block.
231 */ 229 */
232 230
233#define BM_END_OF_MAP (~0UL) 231#define BM_END_OF_MAP (~0UL)
234 232
235#define BM_CHUNKS_PER_BLOCK (PAGE_SIZE / sizeof(long))
236#define BM_BITS_PER_CHUNK (sizeof(long) << 3)
237#define BM_BITS_PER_BLOCK (PAGE_SIZE << 3) 233#define BM_BITS_PER_BLOCK (PAGE_SIZE << 3)
238 234
239struct bm_block { 235struct bm_block {
240 struct bm_block *next; /* next element of the list */ 236 struct bm_block *next; /* next element of the list */
241 unsigned long start_pfn; /* pfn represented by the first bit */ 237 unsigned long start_pfn; /* pfn represented by the first bit */
242 unsigned long end_pfn; /* pfn represented by the last bit plus 1 */ 238 unsigned long end_pfn; /* pfn represented by the last bit plus 1 */
243 unsigned int size; /* number of bit chunks */ 239 unsigned long *data; /* bitmap representing pages */
244 unsigned long *data; /* chunks of bits representing pages */
245}; 240};
246 241
242static inline unsigned long bm_block_bits(struct bm_block *bb)
243{
244 return bb->end_pfn - bb->start_pfn;
245}
246
247struct zone_bitmap { 247struct zone_bitmap {
248 struct zone_bitmap *next; /* next element of the list */ 248 struct zone_bitmap *next; /* next element of the list */
249 unsigned long start_pfn; /* minimal pfn in this zone */ 249 unsigned long start_pfn; /* minimal pfn in this zone */
@@ -257,7 +257,6 @@ struct zone_bitmap {
257struct bm_position { 257struct bm_position {
258 struct zone_bitmap *zone_bm; 258 struct zone_bitmap *zone_bm;
259 struct bm_block *block; 259 struct bm_block *block;
260 int chunk;
261 int bit; 260 int bit;
262}; 261};
263 262
@@ -272,12 +271,6 @@ struct memory_bitmap {
272 271
273/* Functions that operate on memory bitmaps */ 272/* Functions that operate on memory bitmaps */
274 273
275static inline void memory_bm_reset_chunk(struct memory_bitmap *bm)
276{
277 bm->cur.chunk = 0;
278 bm->cur.bit = -1;
279}
280
281static void memory_bm_position_reset(struct memory_bitmap *bm) 274static void memory_bm_position_reset(struct memory_bitmap *bm)
282{ 275{
283 struct zone_bitmap *zone_bm; 276 struct zone_bitmap *zone_bm;
@@ -285,7 +278,7 @@ static void memory_bm_position_reset(struct memory_bitmap *bm)
285 zone_bm = bm->zone_bm_list; 278 zone_bm = bm->zone_bm_list;
286 bm->cur.zone_bm = zone_bm; 279 bm->cur.zone_bm = zone_bm;
287 bm->cur.block = zone_bm->bm_blocks; 280 bm->cur.block = zone_bm->bm_blocks;
288 memory_bm_reset_chunk(bm); 281 bm->cur.bit = 0;
289} 282}
290 283
291static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free); 284static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free);
@@ -394,12 +387,10 @@ memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, int safe_needed)
394 bb->start_pfn = pfn; 387 bb->start_pfn = pfn;
395 if (nr >= BM_BITS_PER_BLOCK) { 388 if (nr >= BM_BITS_PER_BLOCK) {
396 pfn += BM_BITS_PER_BLOCK; 389 pfn += BM_BITS_PER_BLOCK;
397 bb->size = BM_CHUNKS_PER_BLOCK;
398 nr -= BM_BITS_PER_BLOCK; 390 nr -= BM_BITS_PER_BLOCK;
399 } else { 391 } else {
400 /* This is executed only once in the loop */ 392 /* This is executed only once in the loop */
401 pfn += nr; 393 pfn += nr;
402 bb->size = DIV_ROUND_UP(nr, BM_BITS_PER_CHUNK);
403 } 394 }
404 bb->end_pfn = pfn; 395 bb->end_pfn = pfn;
405 bb = bb->next; 396 bb = bb->next;
@@ -478,8 +469,8 @@ static int memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn,
478 } 469 }
479 zone_bm->cur_block = bb; 470 zone_bm->cur_block = bb;
480 pfn -= bb->start_pfn; 471 pfn -= bb->start_pfn;
481 *bit_nr = pfn % BM_BITS_PER_CHUNK; 472 *bit_nr = pfn;
482 *addr = bb->data + pfn / BM_BITS_PER_CHUNK; 473 *addr = bb->data;
483 return 0; 474 return 0;
484} 475}
485 476
@@ -528,36 +519,6 @@ static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn)
528 return test_bit(bit, addr); 519 return test_bit(bit, addr);
529} 520}
530 521
531/* Two auxiliary functions for memory_bm_next_pfn */
532
533/* Find the first set bit in the given chunk, if there is one */
534
535static inline int next_bit_in_chunk(int bit, unsigned long *chunk_p)
536{
537 bit++;
538 while (bit < BM_BITS_PER_CHUNK) {
539 if (test_bit(bit, chunk_p))
540 return bit;
541
542 bit++;
543 }
544 return -1;
545}
546
547/* Find a chunk containing some bits set in given block of bits */
548
549static inline int next_chunk_in_block(int n, struct bm_block *bb)
550{
551 n++;
552 while (n < bb->size) {
553 if (bb->data[n])
554 return n;
555
556 n++;
557 }
558 return -1;
559}
560
561/** 522/**
562 * memory_bm_next_pfn - find the pfn that corresponds to the next set bit 523 * memory_bm_next_pfn - find the pfn that corresponds to the next set bit
563 * in the bitmap @bm. If the pfn cannot be found, BM_END_OF_MAP is 524 * in the bitmap @bm. If the pfn cannot be found, BM_END_OF_MAP is
@@ -571,40 +532,33 @@ static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm)
571{ 532{
572 struct zone_bitmap *zone_bm; 533 struct zone_bitmap *zone_bm;
573 struct bm_block *bb; 534 struct bm_block *bb;
574 int chunk;
575 int bit; 535 int bit;
576 536
577 do { 537 do {
578 bb = bm->cur.block; 538 bb = bm->cur.block;
579 do { 539 do {
580 chunk = bm->cur.chunk;
581 bit = bm->cur.bit; 540 bit = bm->cur.bit;
582 do { 541 bit = find_next_bit(bb->data, bm_block_bits(bb), bit);
583 bit = next_bit_in_chunk(bit, bb->data + chunk); 542 if (bit < bm_block_bits(bb))
584 if (bit >= 0) 543 goto Return_pfn;
585 goto Return_pfn; 544
586
587 chunk = next_chunk_in_block(chunk, bb);
588 bit = -1;
589 } while (chunk >= 0);
590 bb = bb->next; 545 bb = bb->next;
591 bm->cur.block = bb; 546 bm->cur.block = bb;
592 memory_bm_reset_chunk(bm); 547 bm->cur.bit = 0;
593 } while (bb); 548 } while (bb);
594 zone_bm = bm->cur.zone_bm->next; 549 zone_bm = bm->cur.zone_bm->next;
595 if (zone_bm) { 550 if (zone_bm) {
596 bm->cur.zone_bm = zone_bm; 551 bm->cur.zone_bm = zone_bm;
597 bm->cur.block = zone_bm->bm_blocks; 552 bm->cur.block = zone_bm->bm_blocks;
598 memory_bm_reset_chunk(bm); 553 bm->cur.bit = 0;
599 } 554 }
600 } while (zone_bm); 555 } while (zone_bm);
601 memory_bm_position_reset(bm); 556 memory_bm_position_reset(bm);
602 return BM_END_OF_MAP; 557 return BM_END_OF_MAP;
603 558
604 Return_pfn: 559 Return_pfn:
605 bm->cur.chunk = chunk; 560 bm->cur.bit = bit + 1;
606 bm->cur.bit = bit; 561 return bb->start_pfn + bit;
607 return bb->start_pfn + chunk * BM_BITS_PER_CHUNK + bit;
608} 562}
609 563
610/** 564/**
diff --git a/kernel/power/user.c b/kernel/power/user.c
index f5512cb3aa86..a6332a313262 100644
--- a/kernel/power/user.c
+++ b/kernel/power/user.c
@@ -23,6 +23,7 @@
23#include <linux/console.h> 23#include <linux/console.h>
24#include <linux/cpu.h> 24#include <linux/cpu.h>
25#include <linux/freezer.h> 25#include <linux/freezer.h>
26#include <linux/smp_lock.h>
26 27
27#include <asm/uaccess.h> 28#include <asm/uaccess.h>
28 29
@@ -69,16 +70,22 @@ static int snapshot_open(struct inode *inode, struct file *filp)
69 struct snapshot_data *data; 70 struct snapshot_data *data;
70 int error; 71 int error;
71 72
72 if (!atomic_add_unless(&snapshot_device_available, -1, 0)) 73 mutex_lock(&pm_mutex);
73 return -EBUSY; 74
75 if (!atomic_add_unless(&snapshot_device_available, -1, 0)) {
76 error = -EBUSY;
77 goto Unlock;
78 }
74 79
75 if ((filp->f_flags & O_ACCMODE) == O_RDWR) { 80 if ((filp->f_flags & O_ACCMODE) == O_RDWR) {
76 atomic_inc(&snapshot_device_available); 81 atomic_inc(&snapshot_device_available);
77 return -ENOSYS; 82 error = -ENOSYS;
83 goto Unlock;
78 } 84 }
79 if(create_basic_memory_bitmaps()) { 85 if(create_basic_memory_bitmaps()) {
80 atomic_inc(&snapshot_device_available); 86 atomic_inc(&snapshot_device_available);
81 return -ENOMEM; 87 error = -ENOMEM;
88 goto Unlock;
82 } 89 }
83 nonseekable_open(inode, filp); 90 nonseekable_open(inode, filp);
84 data = &snapshot_state; 91 data = &snapshot_state;
@@ -98,33 +105,36 @@ static int snapshot_open(struct inode *inode, struct file *filp)
98 if (error) 105 if (error)
99 pm_notifier_call_chain(PM_POST_HIBERNATION); 106 pm_notifier_call_chain(PM_POST_HIBERNATION);
100 } 107 }
101 if (error) { 108 if (error)
102 atomic_inc(&snapshot_device_available); 109 atomic_inc(&snapshot_device_available);
103 return error;
104 }
105 data->frozen = 0; 110 data->frozen = 0;
106 data->ready = 0; 111 data->ready = 0;
107 data->platform_support = 0; 112 data->platform_support = 0;
108 113
109 return 0; 114 Unlock:
115 mutex_unlock(&pm_mutex);
116
117 return error;
110} 118}
111 119
112static int snapshot_release(struct inode *inode, struct file *filp) 120static int snapshot_release(struct inode *inode, struct file *filp)
113{ 121{
114 struct snapshot_data *data; 122 struct snapshot_data *data;
115 123
124 mutex_lock(&pm_mutex);
125
116 swsusp_free(); 126 swsusp_free();
117 free_basic_memory_bitmaps(); 127 free_basic_memory_bitmaps();
118 data = filp->private_data; 128 data = filp->private_data;
119 free_all_swap_pages(data->swap); 129 free_all_swap_pages(data->swap);
120 if (data->frozen) { 130 if (data->frozen)
121 mutex_lock(&pm_mutex);
122 thaw_processes(); 131 thaw_processes();
123 mutex_unlock(&pm_mutex);
124 }
125 pm_notifier_call_chain(data->mode == O_WRONLY ? 132 pm_notifier_call_chain(data->mode == O_WRONLY ?
126 PM_POST_HIBERNATION : PM_POST_RESTORE); 133 PM_POST_HIBERNATION : PM_POST_RESTORE);
127 atomic_inc(&snapshot_device_available); 134 atomic_inc(&snapshot_device_available);
135
136 mutex_unlock(&pm_mutex);
137
128 return 0; 138 return 0;
129} 139}
130 140
@@ -134,9 +144,13 @@ static ssize_t snapshot_read(struct file *filp, char __user *buf,
134 struct snapshot_data *data; 144 struct snapshot_data *data;
135 ssize_t res; 145 ssize_t res;
136 146
147 mutex_lock(&pm_mutex);
148
137 data = filp->private_data; 149 data = filp->private_data;
138 if (!data->ready) 150 if (!data->ready) {
139 return -ENODATA; 151 res = -ENODATA;
152 goto Unlock;
153 }
140 res = snapshot_read_next(&data->handle, count); 154 res = snapshot_read_next(&data->handle, count);
141 if (res > 0) { 155 if (res > 0) {
142 if (copy_to_user(buf, data_of(data->handle), res)) 156 if (copy_to_user(buf, data_of(data->handle), res))
@@ -144,6 +158,10 @@ static ssize_t snapshot_read(struct file *filp, char __user *buf,
144 else 158 else
145 *offp = data->handle.offset; 159 *offp = data->handle.offset;
146 } 160 }
161
162 Unlock:
163 mutex_unlock(&pm_mutex);
164
147 return res; 165 return res;
148} 166}
149 167
@@ -153,6 +171,8 @@ static ssize_t snapshot_write(struct file *filp, const char __user *buf,
153 struct snapshot_data *data; 171 struct snapshot_data *data;
154 ssize_t res; 172 ssize_t res;
155 173
174 mutex_lock(&pm_mutex);
175
156 data = filp->private_data; 176 data = filp->private_data;
157 res = snapshot_write_next(&data->handle, count); 177 res = snapshot_write_next(&data->handle, count);
158 if (res > 0) { 178 if (res > 0) {
@@ -161,11 +181,14 @@ static ssize_t snapshot_write(struct file *filp, const char __user *buf,
161 else 181 else
162 *offp = data->handle.offset; 182 *offp = data->handle.offset;
163 } 183 }
184
185 mutex_unlock(&pm_mutex);
186
164 return res; 187 return res;
165} 188}
166 189
167static int snapshot_ioctl(struct inode *inode, struct file *filp, 190static long snapshot_ioctl(struct file *filp, unsigned int cmd,
168 unsigned int cmd, unsigned long arg) 191 unsigned long arg)
169{ 192{
170 int error = 0; 193 int error = 0;
171 struct snapshot_data *data; 194 struct snapshot_data *data;
@@ -179,6 +202,9 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp,
179 if (!capable(CAP_SYS_ADMIN)) 202 if (!capable(CAP_SYS_ADMIN))
180 return -EPERM; 203 return -EPERM;
181 204
205 if (!mutex_trylock(&pm_mutex))
206 return -EBUSY;
207
182 data = filp->private_data; 208 data = filp->private_data;
183 209
184 switch (cmd) { 210 switch (cmd) {
@@ -186,7 +212,6 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp,
186 case SNAPSHOT_FREEZE: 212 case SNAPSHOT_FREEZE:
187 if (data->frozen) 213 if (data->frozen)
188 break; 214 break;
189 mutex_lock(&pm_mutex);
190 printk("Syncing filesystems ... "); 215 printk("Syncing filesystems ... ");
191 sys_sync(); 216 sys_sync();
192 printk("done.\n"); 217 printk("done.\n");
@@ -194,7 +219,6 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp,
194 error = freeze_processes(); 219 error = freeze_processes();
195 if (error) 220 if (error)
196 thaw_processes(); 221 thaw_processes();
197 mutex_unlock(&pm_mutex);
198 if (!error) 222 if (!error)
199 data->frozen = 1; 223 data->frozen = 1;
200 break; 224 break;
@@ -202,9 +226,7 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp,
202 case SNAPSHOT_UNFREEZE: 226 case SNAPSHOT_UNFREEZE:
203 if (!data->frozen || data->ready) 227 if (!data->frozen || data->ready)
204 break; 228 break;
205 mutex_lock(&pm_mutex);
206 thaw_processes(); 229 thaw_processes();
207 mutex_unlock(&pm_mutex);
208 data->frozen = 0; 230 data->frozen = 0;
209 break; 231 break;
210 232
@@ -307,16 +329,11 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp,
307 error = -EPERM; 329 error = -EPERM;
308 break; 330 break;
309 } 331 }
310 if (!mutex_trylock(&pm_mutex)) {
311 error = -EBUSY;
312 break;
313 }
314 /* 332 /*
315 * Tasks are frozen and the notifiers have been called with 333 * Tasks are frozen and the notifiers have been called with
316 * PM_HIBERNATION_PREPARE 334 * PM_HIBERNATION_PREPARE
317 */ 335 */
318 error = suspend_devices_and_enter(PM_SUSPEND_MEM); 336 error = suspend_devices_and_enter(PM_SUSPEND_MEM);
319 mutex_unlock(&pm_mutex);
320 break; 337 break;
321 338
322 case SNAPSHOT_PLATFORM_SUPPORT: 339 case SNAPSHOT_PLATFORM_SUPPORT:
@@ -390,6 +407,8 @@ static int snapshot_ioctl(struct inode *inode, struct file *filp,
390 407
391 } 408 }
392 409
410 mutex_unlock(&pm_mutex);
411
393 return error; 412 return error;
394} 413}
395 414
@@ -399,7 +418,7 @@ static const struct file_operations snapshot_fops = {
399 .read = snapshot_read, 418 .read = snapshot_read,
400 .write = snapshot_write, 419 .write = snapshot_write,
401 .llseek = no_llseek, 420 .llseek = no_llseek,
402 .ioctl = snapshot_ioctl, 421 .unlocked_ioctl = snapshot_ioctl,
403}; 422};
404 423
405static struct miscdevice snapshot_device = { 424static struct miscdevice snapshot_device = {
diff --git a/kernel/printk.c b/kernel/printk.c
index e2129e83fd75..a7f7559c5f6c 100644
--- a/kernel/printk.c
+++ b/kernel/printk.c
@@ -38,7 +38,7 @@
38/* 38/*
39 * Architectures can override it: 39 * Architectures can override it:
40 */ 40 */
41void __attribute__((weak)) early_printk(const char *fmt, ...) 41void asmlinkage __attribute__((weak)) early_printk(const char *fmt, ...)
42{ 42{
43} 43}
44 44
@@ -75,6 +75,8 @@ EXPORT_SYMBOL(oops_in_progress);
75static DECLARE_MUTEX(console_sem); 75static DECLARE_MUTEX(console_sem);
76static DECLARE_MUTEX(secondary_console_sem); 76static DECLARE_MUTEX(secondary_console_sem);
77struct console *console_drivers; 77struct console *console_drivers;
78EXPORT_SYMBOL_GPL(console_drivers);
79
78/* 80/*
79 * This is used for debugging the mess that is the VT code by 81 * This is used for debugging the mess that is the VT code by
80 * keeping track if we have the console semaphore held. It's 82 * keeping track if we have the console semaphore held. It's
@@ -121,6 +123,8 @@ struct console_cmdline
121static struct console_cmdline console_cmdline[MAX_CMDLINECONSOLES]; 123static struct console_cmdline console_cmdline[MAX_CMDLINECONSOLES];
122static int selected_console = -1; 124static int selected_console = -1;
123static int preferred_console = -1; 125static int preferred_console = -1;
126int console_set_on_cmdline;
127EXPORT_SYMBOL(console_set_on_cmdline);
124 128
125/* Flag: console code may call schedule() */ 129/* Flag: console code may call schedule() */
126static int console_may_schedule; 130static int console_may_schedule;
@@ -231,7 +235,7 @@ static inline void boot_delay_msec(void)
231/* 235/*
232 * Return the number of unread characters in the log buffer. 236 * Return the number of unread characters in the log buffer.
233 */ 237 */
234int log_buf_get_len(void) 238static int log_buf_get_len(void)
235{ 239{
236 return logged_chars; 240 return logged_chars;
237} 241}
@@ -268,19 +272,6 @@ int log_buf_copy(char *dest, int idx, int len)
268} 272}
269 273
270/* 274/*
271 * Extract a single character from the log buffer.
272 */
273int log_buf_read(int idx)
274{
275 char ret;
276
277 if (log_buf_copy(&ret, idx, 1) == 1)
278 return ret;
279 else
280 return -1;
281}
282
283/*
284 * Commands to do_syslog: 275 * Commands to do_syslog:
285 * 276 *
286 * 0 -- Close the log. Currently a NOP. 277 * 0 -- Close the log. Currently a NOP.
@@ -665,18 +656,17 @@ static int acquire_console_semaphore_for_printk(unsigned int cpu)
665 spin_unlock(&logbuf_lock); 656 spin_unlock(&logbuf_lock);
666 return retval; 657 return retval;
667} 658}
668 659static const char recursion_bug_msg [] =
669static const char printk_recursion_bug_msg [] = 660 KERN_CRIT "BUG: recent printk recursion!\n";
670 KERN_CRIT "BUG: recent printk recursion!\n"; 661static int recursion_bug;
671static int printk_recursion_bug; 662 static int new_text_line = 1;
663static char printk_buf[1024];
672 664
673asmlinkage int vprintk(const char *fmt, va_list args) 665asmlinkage int vprintk(const char *fmt, va_list args)
674{ 666{
675 static int log_level_unknown = 1;
676 static char printk_buf[1024];
677
678 unsigned long flags;
679 int printed_len = 0; 667 int printed_len = 0;
668 int current_log_level = default_message_loglevel;
669 unsigned long flags;
680 int this_cpu; 670 int this_cpu;
681 char *p; 671 char *p;
682 672
@@ -699,7 +689,7 @@ asmlinkage int vprintk(const char *fmt, va_list args)
699 * it can be printed at the next appropriate moment: 689 * it can be printed at the next appropriate moment:
700 */ 690 */
701 if (!oops_in_progress) { 691 if (!oops_in_progress) {
702 printk_recursion_bug = 1; 692 recursion_bug = 1;
703 goto out_restore_irqs; 693 goto out_restore_irqs;
704 } 694 }
705 zap_locks(); 695 zap_locks();
@@ -709,70 +699,62 @@ asmlinkage int vprintk(const char *fmt, va_list args)
709 spin_lock(&logbuf_lock); 699 spin_lock(&logbuf_lock);
710 printk_cpu = this_cpu; 700 printk_cpu = this_cpu;
711 701
712 if (printk_recursion_bug) { 702 if (recursion_bug) {
713 printk_recursion_bug = 0; 703 recursion_bug = 0;
714 strcpy(printk_buf, printk_recursion_bug_msg); 704 strcpy(printk_buf, recursion_bug_msg);
715 printed_len = sizeof(printk_recursion_bug_msg); 705 printed_len = sizeof(recursion_bug_msg);
716 } 706 }
717 /* Emit the output into the temporary buffer */ 707 /* Emit the output into the temporary buffer */
718 printed_len += vscnprintf(printk_buf + printed_len, 708 printed_len += vscnprintf(printk_buf + printed_len,
719 sizeof(printk_buf) - printed_len, fmt, args); 709 sizeof(printk_buf) - printed_len, fmt, args);
720 710
711
721 /* 712 /*
722 * Copy the output into log_buf. If the caller didn't provide 713 * Copy the output into log_buf. If the caller didn't provide
723 * appropriate log level tags, we insert them here 714 * appropriate log level tags, we insert them here
724 */ 715 */
725 for (p = printk_buf; *p; p++) { 716 for (p = printk_buf; *p; p++) {
726 if (log_level_unknown) { 717 if (new_text_line) {
727 /* log_level_unknown signals the start of a new line */ 718 /* If a token, set current_log_level and skip over */
719 if (p[0] == '<' && p[1] >= '0' && p[1] <= '7' &&
720 p[2] == '>') {
721 current_log_level = p[1] - '0';
722 p += 3;
723 printed_len -= 3;
724 }
725
726 /* Always output the token */
727 emit_log_char('<');
728 emit_log_char(current_log_level + '0');
729 emit_log_char('>');
730 printed_len += 3;
731 new_text_line = 0;
732
728 if (printk_time) { 733 if (printk_time) {
729 int loglev_char; 734 /* Follow the token with the time */
730 char tbuf[50], *tp; 735 char tbuf[50], *tp;
731 unsigned tlen; 736 unsigned tlen;
732 unsigned long long t; 737 unsigned long long t;
733 unsigned long nanosec_rem; 738 unsigned long nanosec_rem;
734 739
735 /*
736 * force the log level token to be
737 * before the time output.
738 */
739 if (p[0] == '<' && p[1] >='0' &&
740 p[1] <= '7' && p[2] == '>') {
741 loglev_char = p[1];
742 p += 3;
743 printed_len -= 3;
744 } else {
745 loglev_char = default_message_loglevel
746 + '0';
747 }
748 t = cpu_clock(printk_cpu); 740 t = cpu_clock(printk_cpu);
749 nanosec_rem = do_div(t, 1000000000); 741 nanosec_rem = do_div(t, 1000000000);
750 tlen = sprintf(tbuf, 742 tlen = sprintf(tbuf, "[%5lu.%06lu] ",
751 "<%c>[%5lu.%06lu] ", 743 (unsigned long) t,
752 loglev_char, 744 nanosec_rem / 1000);
753 (unsigned long)t,
754 nanosec_rem/1000);
755 745
756 for (tp = tbuf; tp < tbuf + tlen; tp++) 746 for (tp = tbuf; tp < tbuf + tlen; tp++)
757 emit_log_char(*tp); 747 emit_log_char(*tp);
758 printed_len += tlen; 748 printed_len += tlen;
759 } else {
760 if (p[0] != '<' || p[1] < '0' ||
761 p[1] > '7' || p[2] != '>') {
762 emit_log_char('<');
763 emit_log_char(default_message_loglevel
764 + '0');
765 emit_log_char('>');
766 printed_len += 3;
767 }
768 } 749 }
769 log_level_unknown = 0; 750
770 if (!*p) 751 if (!*p)
771 break; 752 break;
772 } 753 }
754
773 emit_log_char(*p); 755 emit_log_char(*p);
774 if (*p == '\n') 756 if (*p == '\n')
775 log_level_unknown = 1; 757 new_text_line = 1;
776 } 758 }
777 759
778 /* 760 /*
@@ -890,6 +872,7 @@ static int __init console_setup(char *str)
890 *s = 0; 872 *s = 0;
891 873
892 __add_preferred_console(buf, idx, options, brl_options); 874 __add_preferred_console(buf, idx, options, brl_options);
875 console_set_on_cmdline = 1;
893 return 1; 876 return 1;
894} 877}
895__setup("console=", console_setup); 878__setup("console=", console_setup);
@@ -950,7 +933,7 @@ void suspend_console(void)
950{ 933{
951 if (!console_suspend_enabled) 934 if (!console_suspend_enabled)
952 return; 935 return;
953 printk("Suspending console(s)\n"); 936 printk("Suspending console(s) (use no_console_suspend to debug)\n");
954 acquire_console_sem(); 937 acquire_console_sem();
955 console_suspended = 1; 938 console_suspended = 1;
956} 939}
@@ -1041,7 +1024,9 @@ void release_console_sem(void)
1041 _log_end = log_end; 1024 _log_end = log_end;
1042 con_start = log_end; /* Flush */ 1025 con_start = log_end; /* Flush */
1043 spin_unlock(&logbuf_lock); 1026 spin_unlock(&logbuf_lock);
1027 stop_critical_timings(); /* don't trace print latency */
1044 call_console_drivers(_con_start, _log_end); 1028 call_console_drivers(_con_start, _log_end);
1029 start_critical_timings();
1045 local_irq_restore(flags); 1030 local_irq_restore(flags);
1046 } 1031 }
1047 console_locked = 0; 1032 console_locked = 0;
@@ -1172,8 +1157,11 @@ void register_console(struct console *console)
1172 console->index = 0; 1157 console->index = 0;
1173 if (console->setup == NULL || 1158 if (console->setup == NULL ||
1174 console->setup(console, NULL) == 0) { 1159 console->setup(console, NULL) == 0) {
1175 console->flags |= CON_ENABLED | CON_CONSDEV; 1160 console->flags |= CON_ENABLED;
1176 preferred_console = 0; 1161 if (console->device) {
1162 console->flags |= CON_CONSDEV;
1163 preferred_console = 0;
1164 }
1177 } 1165 }
1178 } 1166 }
1179 1167
@@ -1320,6 +1308,8 @@ void tty_write_message(struct tty_struct *tty, char *msg)
1320} 1308}
1321 1309
1322#if defined CONFIG_PRINTK 1310#if defined CONFIG_PRINTK
1311
1312DEFINE_RATELIMIT_STATE(printk_ratelimit_state, 5 * HZ, 10);
1323/* 1313/*
1324 * printk rate limiting, lifted from the networking subsystem. 1314 * printk rate limiting, lifted from the networking subsystem.
1325 * 1315 *
@@ -1327,22 +1317,9 @@ void tty_write_message(struct tty_struct *tty, char *msg)
1327 * every printk_ratelimit_jiffies to make a denial-of-service 1317 * every printk_ratelimit_jiffies to make a denial-of-service
1328 * attack impossible. 1318 * attack impossible.
1329 */ 1319 */
1330int __printk_ratelimit(int ratelimit_jiffies, int ratelimit_burst)
1331{
1332 return __ratelimit(ratelimit_jiffies, ratelimit_burst);
1333}
1334EXPORT_SYMBOL(__printk_ratelimit);
1335
1336/* minimum time in jiffies between messages */
1337int printk_ratelimit_jiffies = 5 * HZ;
1338
1339/* number of messages we send before ratelimiting */
1340int printk_ratelimit_burst = 10;
1341
1342int printk_ratelimit(void) 1320int printk_ratelimit(void)
1343{ 1321{
1344 return __printk_ratelimit(printk_ratelimit_jiffies, 1322 return __ratelimit(&printk_ratelimit_state);
1345 printk_ratelimit_burst);
1346} 1323}
1347EXPORT_SYMBOL(printk_ratelimit); 1324EXPORT_SYMBOL(printk_ratelimit);
1348 1325
diff --git a/kernel/profile.c b/kernel/profile.c
index ae7ead82cbc9..cd26bed4cc26 100644
--- a/kernel/profile.c
+++ b/kernel/profile.c
@@ -112,8 +112,6 @@ void __init profile_init(void)
112 112
113/* Profile event notifications */ 113/* Profile event notifications */
114 114
115#ifdef CONFIG_PROFILING
116
117static BLOCKING_NOTIFIER_HEAD(task_exit_notifier); 115static BLOCKING_NOTIFIER_HEAD(task_exit_notifier);
118static ATOMIC_NOTIFIER_HEAD(task_free_notifier); 116static ATOMIC_NOTIFIER_HEAD(task_free_notifier);
119static BLOCKING_NOTIFIER_HEAD(munmap_notifier); 117static BLOCKING_NOTIFIER_HEAD(munmap_notifier);
@@ -203,8 +201,6 @@ void unregister_timer_hook(int (*hook)(struct pt_regs *))
203} 201}
204EXPORT_SYMBOL_GPL(unregister_timer_hook); 202EXPORT_SYMBOL_GPL(unregister_timer_hook);
205 203
206#endif /* CONFIG_PROFILING */
207
208 204
209#ifdef CONFIG_SMP 205#ifdef CONFIG_SMP
210/* 206/*
@@ -252,7 +248,7 @@ static void profile_flip_buffers(void)
252 mutex_lock(&profile_flip_mutex); 248 mutex_lock(&profile_flip_mutex);
253 j = per_cpu(cpu_profile_flip, get_cpu()); 249 j = per_cpu(cpu_profile_flip, get_cpu());
254 put_cpu(); 250 put_cpu();
255 on_each_cpu(__profile_flip_buffers, NULL, 0, 1); 251 on_each_cpu(__profile_flip_buffers, NULL, 1);
256 for_each_online_cpu(cpu) { 252 for_each_online_cpu(cpu) {
257 struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[j]; 253 struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[j];
258 for (i = 0; i < NR_PROFILE_HIT; ++i) { 254 for (i = 0; i < NR_PROFILE_HIT; ++i) {
@@ -275,7 +271,7 @@ static void profile_discard_flip_buffers(void)
275 mutex_lock(&profile_flip_mutex); 271 mutex_lock(&profile_flip_mutex);
276 i = per_cpu(cpu_profile_flip, get_cpu()); 272 i = per_cpu(cpu_profile_flip, get_cpu());
277 put_cpu(); 273 put_cpu();
278 on_each_cpu(__profile_flip_buffers, NULL, 0, 1); 274 on_each_cpu(__profile_flip_buffers, NULL, 1);
279 for_each_online_cpu(cpu) { 275 for_each_online_cpu(cpu) {
280 struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[i]; 276 struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[i];
281 memset(hits, 0, NR_PROFILE_HIT*sizeof(struct profile_hit)); 277 memset(hits, 0, NR_PROFILE_HIT*sizeof(struct profile_hit));
@@ -558,7 +554,7 @@ static int __init create_hash_tables(void)
558out_cleanup: 554out_cleanup:
559 prof_on = 0; 555 prof_on = 0;
560 smp_mb(); 556 smp_mb();
561 on_each_cpu(profile_nop, NULL, 0, 1); 557 on_each_cpu(profile_nop, NULL, 1);
562 for_each_online_cpu(cpu) { 558 for_each_online_cpu(cpu) {
563 struct page *page; 559 struct page *page;
564 560
diff --git a/kernel/ptrace.c b/kernel/ptrace.c
index 6c19e94fd0a5..082b3fcb32a0 100644
--- a/kernel/ptrace.c
+++ b/kernel/ptrace.c
@@ -33,13 +33,9 @@
33 */ 33 */
34void __ptrace_link(struct task_struct *child, struct task_struct *new_parent) 34void __ptrace_link(struct task_struct *child, struct task_struct *new_parent)
35{ 35{
36 BUG_ON(!list_empty(&child->ptrace_list)); 36 BUG_ON(!list_empty(&child->ptrace_entry));
37 if (child->parent == new_parent) 37 list_add(&child->ptrace_entry, &new_parent->ptraced);
38 return;
39 list_add(&child->ptrace_list, &child->parent->ptrace_children);
40 remove_parent(child);
41 child->parent = new_parent; 38 child->parent = new_parent;
42 add_parent(child);
43} 39}
44 40
45/* 41/*
@@ -73,12 +69,8 @@ void __ptrace_unlink(struct task_struct *child)
73 BUG_ON(!child->ptrace); 69 BUG_ON(!child->ptrace);
74 70
75 child->ptrace = 0; 71 child->ptrace = 0;
76 if (ptrace_reparented(child)) { 72 child->parent = child->real_parent;
77 list_del_init(&child->ptrace_list); 73 list_del_init(&child->ptrace_entry);
78 remove_parent(child);
79 child->parent = child->real_parent;
80 add_parent(child);
81 }
82 74
83 if (task_is_traced(child)) 75 if (task_is_traced(child))
84 ptrace_untrace(child); 76 ptrace_untrace(child);
@@ -115,13 +107,13 @@ int ptrace_check_attach(struct task_struct *child, int kill)
115 read_unlock(&tasklist_lock); 107 read_unlock(&tasklist_lock);
116 108
117 if (!ret && !kill) 109 if (!ret && !kill)
118 wait_task_inactive(child); 110 ret = wait_task_inactive(child, TASK_TRACED) ? 0 : -ESRCH;
119 111
120 /* All systems go.. */ 112 /* All systems go.. */
121 return ret; 113 return ret;
122} 114}
123 115
124int __ptrace_may_attach(struct task_struct *task) 116int __ptrace_may_access(struct task_struct *task, unsigned int mode)
125{ 117{
126 /* May we inspect the given task? 118 /* May we inspect the given task?
127 * This check is used both for attaching with ptrace 119 * This check is used both for attaching with ptrace
@@ -148,16 +140,16 @@ int __ptrace_may_attach(struct task_struct *task)
148 if (!dumpable && !capable(CAP_SYS_PTRACE)) 140 if (!dumpable && !capable(CAP_SYS_PTRACE))
149 return -EPERM; 141 return -EPERM;
150 142
151 return security_ptrace(current, task); 143 return security_ptrace(current, task, mode);
152} 144}
153 145
154int ptrace_may_attach(struct task_struct *task) 146bool ptrace_may_access(struct task_struct *task, unsigned int mode)
155{ 147{
156 int err; 148 int err;
157 task_lock(task); 149 task_lock(task);
158 err = __ptrace_may_attach(task); 150 err = __ptrace_may_access(task, mode);
159 task_unlock(task); 151 task_unlock(task);
160 return !err; 152 return (!err ? true : false);
161} 153}
162 154
163int ptrace_attach(struct task_struct *task) 155int ptrace_attach(struct task_struct *task)
@@ -195,7 +187,7 @@ repeat:
195 /* the same process cannot be attached many times */ 187 /* the same process cannot be attached many times */
196 if (task->ptrace & PT_PTRACED) 188 if (task->ptrace & PT_PTRACED)
197 goto bad; 189 goto bad;
198 retval = __ptrace_may_attach(task); 190 retval = __ptrace_may_access(task, PTRACE_MODE_ATTACH);
199 if (retval) 191 if (retval)
200 goto bad; 192 goto bad;
201 193
@@ -492,14 +484,34 @@ int ptrace_traceme(void)
492 /* 484 /*
493 * Are we already being traced? 485 * Are we already being traced?
494 */ 486 */
487repeat:
495 task_lock(current); 488 task_lock(current);
496 if (!(current->ptrace & PT_PTRACED)) { 489 if (!(current->ptrace & PT_PTRACED)) {
497 ret = security_ptrace(current->parent, current); 490 /*
491 * See ptrace_attach() comments about the locking here.
492 */
493 unsigned long flags;
494 if (!write_trylock_irqsave(&tasklist_lock, flags)) {
495 task_unlock(current);
496 do {
497 cpu_relax();
498 } while (!write_can_lock(&tasklist_lock));
499 goto repeat;
500 }
501
502 ret = security_ptrace(current->parent, current,
503 PTRACE_MODE_ATTACH);
504
498 /* 505 /*
499 * Set the ptrace bit in the process ptrace flags. 506 * Set the ptrace bit in the process ptrace flags.
507 * Then link us on our parent's ptraced list.
500 */ 508 */
501 if (!ret) 509 if (!ret) {
502 current->ptrace |= PT_PTRACED; 510 current->ptrace |= PT_PTRACED;
511 __ptrace_link(current, current->real_parent);
512 }
513
514 write_unlock_irqrestore(&tasklist_lock, flags);
503 } 515 }
504 task_unlock(current); 516 task_unlock(current);
505 return ret; 517 return ret;
diff --git a/kernel/rcuclassic.c b/kernel/rcuclassic.c
index a38895a5b8e2..6f8696c502f4 100644
--- a/kernel/rcuclassic.c
+++ b/kernel/rcuclassic.c
@@ -106,7 +106,7 @@ static void force_quiescent_state(struct rcu_data *rdp,
106 */ 106 */
107 cpus_and(cpumask, rcp->cpumask, cpu_online_map); 107 cpus_and(cpumask, rcp->cpumask, cpu_online_map);
108 cpu_clear(rdp->cpu, cpumask); 108 cpu_clear(rdp->cpu, cpumask);
109 for_each_cpu_mask(cpu, cpumask) 109 for_each_cpu_mask_nr(cpu, cpumask)
110 smp_send_reschedule(cpu); 110 smp_send_reschedule(cpu);
111 } 111 }
112} 112}
@@ -387,6 +387,10 @@ static void __rcu_offline_cpu(struct rcu_data *this_rdp,
387 rcu_move_batch(this_rdp, rdp->donelist, rdp->donetail); 387 rcu_move_batch(this_rdp, rdp->donelist, rdp->donetail);
388 rcu_move_batch(this_rdp, rdp->curlist, rdp->curtail); 388 rcu_move_batch(this_rdp, rdp->curlist, rdp->curtail);
389 rcu_move_batch(this_rdp, rdp->nxtlist, rdp->nxttail); 389 rcu_move_batch(this_rdp, rdp->nxtlist, rdp->nxttail);
390
391 local_irq_disable();
392 this_rdp->qlen += rdp->qlen;
393 local_irq_enable();
390} 394}
391 395
392static void rcu_offline_cpu(int cpu) 396static void rcu_offline_cpu(int cpu)
@@ -516,10 +520,38 @@ void rcu_check_callbacks(int cpu, int user)
516 if (user || 520 if (user ||
517 (idle_cpu(cpu) && !in_softirq() && 521 (idle_cpu(cpu) && !in_softirq() &&
518 hardirq_count() <= (1 << HARDIRQ_SHIFT))) { 522 hardirq_count() <= (1 << HARDIRQ_SHIFT))) {
523
524 /*
525 * Get here if this CPU took its interrupt from user
526 * mode or from the idle loop, and if this is not a
527 * nested interrupt. In this case, the CPU is in
528 * a quiescent state, so count it.
529 *
530 * Also do a memory barrier. This is needed to handle
531 * the case where writes from a preempt-disable section
532 * of code get reordered into schedule() by this CPU's
533 * write buffer. The memory barrier makes sure that
534 * the rcu_qsctr_inc() and rcu_bh_qsctr_inc() are see
535 * by other CPUs to happen after any such write.
536 */
537
538 smp_mb(); /* See above block comment. */
519 rcu_qsctr_inc(cpu); 539 rcu_qsctr_inc(cpu);
520 rcu_bh_qsctr_inc(cpu); 540 rcu_bh_qsctr_inc(cpu);
521 } else if (!in_softirq()) 541
542 } else if (!in_softirq()) {
543
544 /*
545 * Get here if this CPU did not take its interrupt from
546 * softirq, in other words, if it is not interrupting
547 * a rcu_bh read-side critical section. This is an _bh
548 * critical section, so count it. The memory barrier
549 * is needed for the same reason as is the above one.
550 */
551
552 smp_mb(); /* See above block comment. */
522 rcu_bh_qsctr_inc(cpu); 553 rcu_bh_qsctr_inc(cpu);
554 }
523 raise_rcu_softirq(); 555 raise_rcu_softirq();
524} 556}
525 557
@@ -543,7 +575,7 @@ static void __cpuinit rcu_online_cpu(int cpu)
543 575
544 rcu_init_percpu_data(cpu, &rcu_ctrlblk, rdp); 576 rcu_init_percpu_data(cpu, &rcu_ctrlblk, rdp);
545 rcu_init_percpu_data(cpu, &rcu_bh_ctrlblk, bh_rdp); 577 rcu_init_percpu_data(cpu, &rcu_bh_ctrlblk, bh_rdp);
546 open_softirq(RCU_SOFTIRQ, rcu_process_callbacks, NULL); 578 open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
547} 579}
548 580
549static int __cpuinit rcu_cpu_notify(struct notifier_block *self, 581static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index c09605f8d16c..f14f372cf6f5 100644
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -39,16 +39,16 @@
39#include <linux/sched.h> 39#include <linux/sched.h>
40#include <asm/atomic.h> 40#include <asm/atomic.h>
41#include <linux/bitops.h> 41#include <linux/bitops.h>
42#include <linux/completion.h>
43#include <linux/percpu.h> 42#include <linux/percpu.h>
44#include <linux/notifier.h> 43#include <linux/notifier.h>
45#include <linux/cpu.h> 44#include <linux/cpu.h>
46#include <linux/mutex.h> 45#include <linux/mutex.h>
47#include <linux/module.h> 46#include <linux/module.h>
48 47
49struct rcu_synchronize { 48enum rcu_barrier {
50 struct rcu_head head; 49 RCU_BARRIER_STD,
51 struct completion completion; 50 RCU_BARRIER_BH,
51 RCU_BARRIER_SCHED,
52}; 52};
53 53
54static DEFINE_PER_CPU(struct rcu_head, rcu_barrier_head) = {NULL}; 54static DEFINE_PER_CPU(struct rcu_head, rcu_barrier_head) = {NULL};
@@ -60,7 +60,7 @@ static struct completion rcu_barrier_completion;
60 * Awaken the corresponding synchronize_rcu() instance now that a 60 * Awaken the corresponding synchronize_rcu() instance now that a
61 * grace period has elapsed. 61 * grace period has elapsed.
62 */ 62 */
63static void wakeme_after_rcu(struct rcu_head *head) 63void wakeme_after_rcu(struct rcu_head *head)
64{ 64{
65 struct rcu_synchronize *rcu; 65 struct rcu_synchronize *rcu;
66 66
@@ -77,17 +77,7 @@ static void wakeme_after_rcu(struct rcu_head *head)
77 * sections are delimited by rcu_read_lock() and rcu_read_unlock(), 77 * sections are delimited by rcu_read_lock() and rcu_read_unlock(),
78 * and may be nested. 78 * and may be nested.
79 */ 79 */
80void synchronize_rcu(void) 80synchronize_rcu_xxx(synchronize_rcu, call_rcu)
81{
82 struct rcu_synchronize rcu;
83
84 init_completion(&rcu.completion);
85 /* Will wake me after RCU finished */
86 call_rcu(&rcu.head, wakeme_after_rcu);
87
88 /* Wait for it */
89 wait_for_completion(&rcu.completion);
90}
91EXPORT_SYMBOL_GPL(synchronize_rcu); 81EXPORT_SYMBOL_GPL(synchronize_rcu);
92 82
93static void rcu_barrier_callback(struct rcu_head *notused) 83static void rcu_barrier_callback(struct rcu_head *notused)
@@ -99,19 +89,30 @@ static void rcu_barrier_callback(struct rcu_head *notused)
99/* 89/*
100 * Called with preemption disabled, and from cross-cpu IRQ context. 90 * Called with preemption disabled, and from cross-cpu IRQ context.
101 */ 91 */
102static void rcu_barrier_func(void *notused) 92static void rcu_barrier_func(void *type)
103{ 93{
104 int cpu = smp_processor_id(); 94 int cpu = smp_processor_id();
105 struct rcu_head *head = &per_cpu(rcu_barrier_head, cpu); 95 struct rcu_head *head = &per_cpu(rcu_barrier_head, cpu);
106 96
107 atomic_inc(&rcu_barrier_cpu_count); 97 atomic_inc(&rcu_barrier_cpu_count);
108 call_rcu(head, rcu_barrier_callback); 98 switch ((enum rcu_barrier)type) {
99 case RCU_BARRIER_STD:
100 call_rcu(head, rcu_barrier_callback);
101 break;
102 case RCU_BARRIER_BH:
103 call_rcu_bh(head, rcu_barrier_callback);
104 break;
105 case RCU_BARRIER_SCHED:
106 call_rcu_sched(head, rcu_barrier_callback);
107 break;
108 }
109} 109}
110 110
111/** 111/*
112 * rcu_barrier - Wait until all the in-flight RCUs are complete. 112 * Orchestrate the specified type of RCU barrier, waiting for all
113 * RCU callbacks of the specified type to complete.
113 */ 114 */
114void rcu_barrier(void) 115static void _rcu_barrier(enum rcu_barrier type)
115{ 116{
116 BUG_ON(in_interrupt()); 117 BUG_ON(in_interrupt());
117 /* Take cpucontrol mutex to protect against CPU hotplug */ 118 /* Take cpucontrol mutex to protect against CPU hotplug */
@@ -127,13 +128,39 @@ void rcu_barrier(void)
127 * until all the callbacks are queued. 128 * until all the callbacks are queued.
128 */ 129 */
129 rcu_read_lock(); 130 rcu_read_lock();
130 on_each_cpu(rcu_barrier_func, NULL, 0, 1); 131 on_each_cpu(rcu_barrier_func, (void *)type, 1);
131 rcu_read_unlock(); 132 rcu_read_unlock();
132 wait_for_completion(&rcu_barrier_completion); 133 wait_for_completion(&rcu_barrier_completion);
133 mutex_unlock(&rcu_barrier_mutex); 134 mutex_unlock(&rcu_barrier_mutex);
134} 135}
136
137/**
138 * rcu_barrier - Wait until all in-flight call_rcu() callbacks complete.
139 */
140void rcu_barrier(void)
141{
142 _rcu_barrier(RCU_BARRIER_STD);
143}
135EXPORT_SYMBOL_GPL(rcu_barrier); 144EXPORT_SYMBOL_GPL(rcu_barrier);
136 145
146/**
147 * rcu_barrier_bh - Wait until all in-flight call_rcu_bh() callbacks complete.
148 */
149void rcu_barrier_bh(void)
150{
151 _rcu_barrier(RCU_BARRIER_BH);
152}
153EXPORT_SYMBOL_GPL(rcu_barrier_bh);
154
155/**
156 * rcu_barrier_sched - Wait for in-flight call_rcu_sched() callbacks.
157 */
158void rcu_barrier_sched(void)
159{
160 _rcu_barrier(RCU_BARRIER_SCHED);
161}
162EXPORT_SYMBOL_GPL(rcu_barrier_sched);
163
137void __init rcu_init(void) 164void __init rcu_init(void)
138{ 165{
139 __rcu_init(); 166 __rcu_init();
diff --git a/kernel/rcupreempt.c b/kernel/rcupreempt.c
index 41d275a81df5..27827931ca0d 100644
--- a/kernel/rcupreempt.c
+++ b/kernel/rcupreempt.c
@@ -46,11 +46,11 @@
46#include <asm/atomic.h> 46#include <asm/atomic.h>
47#include <linux/bitops.h> 47#include <linux/bitops.h>
48#include <linux/module.h> 48#include <linux/module.h>
49#include <linux/kthread.h>
49#include <linux/completion.h> 50#include <linux/completion.h>
50#include <linux/moduleparam.h> 51#include <linux/moduleparam.h>
51#include <linux/percpu.h> 52#include <linux/percpu.h>
52#include <linux/notifier.h> 53#include <linux/notifier.h>
53#include <linux/rcupdate.h>
54#include <linux/cpu.h> 54#include <linux/cpu.h>
55#include <linux/random.h> 55#include <linux/random.h>
56#include <linux/delay.h> 56#include <linux/delay.h>
@@ -82,14 +82,18 @@ struct rcu_data {
82 spinlock_t lock; /* Protect rcu_data fields. */ 82 spinlock_t lock; /* Protect rcu_data fields. */
83 long completed; /* Number of last completed batch. */ 83 long completed; /* Number of last completed batch. */
84 int waitlistcount; 84 int waitlistcount;
85 struct tasklet_struct rcu_tasklet;
86 struct rcu_head *nextlist; 85 struct rcu_head *nextlist;
87 struct rcu_head **nexttail; 86 struct rcu_head **nexttail;
88 struct rcu_head *waitlist[GP_STAGES]; 87 struct rcu_head *waitlist[GP_STAGES];
89 struct rcu_head **waittail[GP_STAGES]; 88 struct rcu_head **waittail[GP_STAGES];
90 struct rcu_head *donelist; 89 struct rcu_head *donelist; /* from waitlist & waitschedlist */
91 struct rcu_head **donetail; 90 struct rcu_head **donetail;
92 long rcu_flipctr[2]; 91 long rcu_flipctr[2];
92 struct rcu_head *nextschedlist;
93 struct rcu_head **nextschedtail;
94 struct rcu_head *waitschedlist;
95 struct rcu_head **waitschedtail;
96 int rcu_sched_sleeping;
93#ifdef CONFIG_RCU_TRACE 97#ifdef CONFIG_RCU_TRACE
94 struct rcupreempt_trace trace; 98 struct rcupreempt_trace trace;
95#endif /* #ifdef CONFIG_RCU_TRACE */ 99#endif /* #ifdef CONFIG_RCU_TRACE */
@@ -131,11 +135,24 @@ enum rcu_try_flip_states {
131 rcu_try_flip_waitmb_state, 135 rcu_try_flip_waitmb_state,
132}; 136};
133 137
138/*
139 * States for rcu_ctrlblk.rcu_sched_sleep.
140 */
141
142enum rcu_sched_sleep_states {
143 rcu_sched_not_sleeping, /* Not sleeping, callbacks need GP. */
144 rcu_sched_sleep_prep, /* Thinking of sleeping, rechecking. */
145 rcu_sched_sleeping, /* Sleeping, awaken if GP needed. */
146};
147
134struct rcu_ctrlblk { 148struct rcu_ctrlblk {
135 spinlock_t fliplock; /* Protect state-machine transitions. */ 149 spinlock_t fliplock; /* Protect state-machine transitions. */
136 long completed; /* Number of last completed batch. */ 150 long completed; /* Number of last completed batch. */
137 enum rcu_try_flip_states rcu_try_flip_state; /* The current state of 151 enum rcu_try_flip_states rcu_try_flip_state; /* The current state of
138 the rcu state machine */ 152 the rcu state machine */
153 spinlock_t schedlock; /* Protect rcu_sched sleep state. */
154 enum rcu_sched_sleep_states sched_sleep; /* rcu_sched state. */
155 wait_queue_head_t sched_wq; /* Place for rcu_sched to sleep. */
139}; 156};
140 157
141static DEFINE_PER_CPU(struct rcu_data, rcu_data); 158static DEFINE_PER_CPU(struct rcu_data, rcu_data);
@@ -143,8 +160,12 @@ static struct rcu_ctrlblk rcu_ctrlblk = {
143 .fliplock = __SPIN_LOCK_UNLOCKED(rcu_ctrlblk.fliplock), 160 .fliplock = __SPIN_LOCK_UNLOCKED(rcu_ctrlblk.fliplock),
144 .completed = 0, 161 .completed = 0,
145 .rcu_try_flip_state = rcu_try_flip_idle_state, 162 .rcu_try_flip_state = rcu_try_flip_idle_state,
163 .schedlock = __SPIN_LOCK_UNLOCKED(rcu_ctrlblk.schedlock),
164 .sched_sleep = rcu_sched_not_sleeping,
165 .sched_wq = __WAIT_QUEUE_HEAD_INITIALIZER(rcu_ctrlblk.sched_wq),
146}; 166};
147 167
168static struct task_struct *rcu_sched_grace_period_task;
148 169
149#ifdef CONFIG_RCU_TRACE 170#ifdef CONFIG_RCU_TRACE
150static char *rcu_try_flip_state_names[] = 171static char *rcu_try_flip_state_names[] =
@@ -207,6 +228,8 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(enum rcu_mb_flag_values, rcu_mb_flag)
207 */ 228 */
208#define RCU_TRACE_RDP(f, rdp) RCU_TRACE(f, &((rdp)->trace)); 229#define RCU_TRACE_RDP(f, rdp) RCU_TRACE(f, &((rdp)->trace));
209 230
231#define RCU_SCHED_BATCH_TIME (HZ / 50)
232
210/* 233/*
211 * Return the number of RCU batches processed thus far. Useful 234 * Return the number of RCU batches processed thus far. Useful
212 * for debug and statistics. 235 * for debug and statistics.
@@ -411,32 +434,34 @@ static void __rcu_advance_callbacks(struct rcu_data *rdp)
411 } 434 }
412} 435}
413 436
414#ifdef CONFIG_NO_HZ 437DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_dyntick_sched, rcu_dyntick_sched) = {
438 .dynticks = 1,
439};
415 440
416DEFINE_PER_CPU(long, dynticks_progress_counter) = 1; 441#ifdef CONFIG_NO_HZ
417static DEFINE_PER_CPU(long, rcu_dyntick_snapshot);
418static DEFINE_PER_CPU(int, rcu_update_flag); 442static DEFINE_PER_CPU(int, rcu_update_flag);
419 443
420/** 444/**
421 * rcu_irq_enter - Called from Hard irq handlers and NMI/SMI. 445 * rcu_irq_enter - Called from Hard irq handlers and NMI/SMI.
422 * 446 *
423 * If the CPU was idle with dynamic ticks active, this updates the 447 * If the CPU was idle with dynamic ticks active, this updates the
424 * dynticks_progress_counter to let the RCU handling know that the 448 * rcu_dyntick_sched.dynticks to let the RCU handling know that the
425 * CPU is active. 449 * CPU is active.
426 */ 450 */
427void rcu_irq_enter(void) 451void rcu_irq_enter(void)
428{ 452{
429 int cpu = smp_processor_id(); 453 int cpu = smp_processor_id();
454 struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
430 455
431 if (per_cpu(rcu_update_flag, cpu)) 456 if (per_cpu(rcu_update_flag, cpu))
432 per_cpu(rcu_update_flag, cpu)++; 457 per_cpu(rcu_update_flag, cpu)++;
433 458
434 /* 459 /*
435 * Only update if we are coming from a stopped ticks mode 460 * Only update if we are coming from a stopped ticks mode
436 * (dynticks_progress_counter is even). 461 * (rcu_dyntick_sched.dynticks is even).
437 */ 462 */
438 if (!in_interrupt() && 463 if (!in_interrupt() &&
439 (per_cpu(dynticks_progress_counter, cpu) & 0x1) == 0) { 464 (rdssp->dynticks & 0x1) == 0) {
440 /* 465 /*
441 * The following might seem like we could have a race 466 * The following might seem like we could have a race
442 * with NMI/SMIs. But this really isn't a problem. 467 * with NMI/SMIs. But this really isn't a problem.
@@ -459,12 +484,12 @@ void rcu_irq_enter(void)
459 * RCU read-side critical sections on this CPU would 484 * RCU read-side critical sections on this CPU would
460 * have already completed. 485 * have already completed.
461 */ 486 */
462 per_cpu(dynticks_progress_counter, cpu)++; 487 rdssp->dynticks++;
463 /* 488 /*
464 * The following memory barrier ensures that any 489 * The following memory barrier ensures that any
465 * rcu_read_lock() primitives in the irq handler 490 * rcu_read_lock() primitives in the irq handler
466 * are seen by other CPUs to follow the above 491 * are seen by other CPUs to follow the above
467 * increment to dynticks_progress_counter. This is 492 * increment to rcu_dyntick_sched.dynticks. This is
468 * required in order for other CPUs to correctly 493 * required in order for other CPUs to correctly
469 * determine when it is safe to advance the RCU 494 * determine when it is safe to advance the RCU
470 * grace-period state machine. 495 * grace-period state machine.
@@ -472,7 +497,7 @@ void rcu_irq_enter(void)
472 smp_mb(); /* see above block comment. */ 497 smp_mb(); /* see above block comment. */
473 /* 498 /*
474 * Since we can't determine the dynamic tick mode from 499 * Since we can't determine the dynamic tick mode from
475 * the dynticks_progress_counter after this routine, 500 * the rcu_dyntick_sched.dynticks after this routine,
476 * we use a second flag to acknowledge that we came 501 * we use a second flag to acknowledge that we came
477 * from an idle state with ticks stopped. 502 * from an idle state with ticks stopped.
478 */ 503 */
@@ -480,7 +505,7 @@ void rcu_irq_enter(void)
480 /* 505 /*
481 * If we take an NMI/SMI now, they will also increment 506 * If we take an NMI/SMI now, they will also increment
482 * the rcu_update_flag, and will not update the 507 * the rcu_update_flag, and will not update the
483 * dynticks_progress_counter on exit. That is for 508 * rcu_dyntick_sched.dynticks on exit. That is for
484 * this IRQ to do. 509 * this IRQ to do.
485 */ 510 */
486 } 511 }
@@ -490,12 +515,13 @@ void rcu_irq_enter(void)
490 * rcu_irq_exit - Called from exiting Hard irq context. 515 * rcu_irq_exit - Called from exiting Hard irq context.
491 * 516 *
492 * If the CPU was idle with dynamic ticks active, update the 517 * If the CPU was idle with dynamic ticks active, update the
493 * dynticks_progress_counter to put let the RCU handling be 518 * rcu_dyntick_sched.dynticks to put let the RCU handling be
494 * aware that the CPU is going back to idle with no ticks. 519 * aware that the CPU is going back to idle with no ticks.
495 */ 520 */
496void rcu_irq_exit(void) 521void rcu_irq_exit(void)
497{ 522{
498 int cpu = smp_processor_id(); 523 int cpu = smp_processor_id();
524 struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
499 525
500 /* 526 /*
501 * rcu_update_flag is set if we interrupted the CPU 527 * rcu_update_flag is set if we interrupted the CPU
@@ -503,7 +529,7 @@ void rcu_irq_exit(void)
503 * Once this occurs, we keep track of interrupt nesting 529 * Once this occurs, we keep track of interrupt nesting
504 * because a NMI/SMI could also come in, and we still 530 * because a NMI/SMI could also come in, and we still
505 * only want the IRQ that started the increment of the 531 * only want the IRQ that started the increment of the
506 * dynticks_progress_counter to be the one that modifies 532 * rcu_dyntick_sched.dynticks to be the one that modifies
507 * it on exit. 533 * it on exit.
508 */ 534 */
509 if (per_cpu(rcu_update_flag, cpu)) { 535 if (per_cpu(rcu_update_flag, cpu)) {
@@ -515,28 +541,29 @@ void rcu_irq_exit(void)
515 541
516 /* 542 /*
517 * If an NMI/SMI happens now we are still 543 * If an NMI/SMI happens now we are still
518 * protected by the dynticks_progress_counter being odd. 544 * protected by the rcu_dyntick_sched.dynticks being odd.
519 */ 545 */
520 546
521 /* 547 /*
522 * The following memory barrier ensures that any 548 * The following memory barrier ensures that any
523 * rcu_read_unlock() primitives in the irq handler 549 * rcu_read_unlock() primitives in the irq handler
524 * are seen by other CPUs to preceed the following 550 * are seen by other CPUs to preceed the following
525 * increment to dynticks_progress_counter. This 551 * increment to rcu_dyntick_sched.dynticks. This
526 * is required in order for other CPUs to determine 552 * is required in order for other CPUs to determine
527 * when it is safe to advance the RCU grace-period 553 * when it is safe to advance the RCU grace-period
528 * state machine. 554 * state machine.
529 */ 555 */
530 smp_mb(); /* see above block comment. */ 556 smp_mb(); /* see above block comment. */
531 per_cpu(dynticks_progress_counter, cpu)++; 557 rdssp->dynticks++;
532 WARN_ON(per_cpu(dynticks_progress_counter, cpu) & 0x1); 558 WARN_ON(rdssp->dynticks & 0x1);
533 } 559 }
534} 560}
535 561
536static void dyntick_save_progress_counter(int cpu) 562static void dyntick_save_progress_counter(int cpu)
537{ 563{
538 per_cpu(rcu_dyntick_snapshot, cpu) = 564 struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
539 per_cpu(dynticks_progress_counter, cpu); 565
566 rdssp->dynticks_snap = rdssp->dynticks;
540} 567}
541 568
542static inline int 569static inline int
@@ -544,9 +571,10 @@ rcu_try_flip_waitack_needed(int cpu)
544{ 571{
545 long curr; 572 long curr;
546 long snap; 573 long snap;
574 struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
547 575
548 curr = per_cpu(dynticks_progress_counter, cpu); 576 curr = rdssp->dynticks;
549 snap = per_cpu(rcu_dyntick_snapshot, cpu); 577 snap = rdssp->dynticks_snap;
550 smp_mb(); /* force ordering with cpu entering/leaving dynticks. */ 578 smp_mb(); /* force ordering with cpu entering/leaving dynticks. */
551 579
552 /* 580 /*
@@ -567,7 +595,7 @@ rcu_try_flip_waitack_needed(int cpu)
567 * that this CPU already acknowledged the counter. 595 * that this CPU already acknowledged the counter.
568 */ 596 */
569 597
570 if ((curr - snap) > 2 || (snap & 0x1) == 0) 598 if ((curr - snap) > 2 || (curr & 0x1) == 0)
571 return 0; 599 return 0;
572 600
573 /* We need this CPU to explicitly acknowledge the counter flip. */ 601 /* We need this CPU to explicitly acknowledge the counter flip. */
@@ -580,9 +608,10 @@ rcu_try_flip_waitmb_needed(int cpu)
580{ 608{
581 long curr; 609 long curr;
582 long snap; 610 long snap;
611 struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
583 612
584 curr = per_cpu(dynticks_progress_counter, cpu); 613 curr = rdssp->dynticks;
585 snap = per_cpu(rcu_dyntick_snapshot, cpu); 614 snap = rdssp->dynticks_snap;
586 smp_mb(); /* force ordering with cpu entering/leaving dynticks. */ 615 smp_mb(); /* force ordering with cpu entering/leaving dynticks. */
587 616
588 /* 617 /*
@@ -609,14 +638,86 @@ rcu_try_flip_waitmb_needed(int cpu)
609 return 1; 638 return 1;
610} 639}
611 640
641static void dyntick_save_progress_counter_sched(int cpu)
642{
643 struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
644
645 rdssp->sched_dynticks_snap = rdssp->dynticks;
646}
647
648static int rcu_qsctr_inc_needed_dyntick(int cpu)
649{
650 long curr;
651 long snap;
652 struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
653
654 curr = rdssp->dynticks;
655 snap = rdssp->sched_dynticks_snap;
656 smp_mb(); /* force ordering with cpu entering/leaving dynticks. */
657
658 /*
659 * If the CPU remained in dynticks mode for the entire time
660 * and didn't take any interrupts, NMIs, SMIs, or whatever,
661 * then it cannot be in the middle of an rcu_read_lock(), so
662 * the next rcu_read_lock() it executes must use the new value
663 * of the counter. Therefore, this CPU has been in a quiescent
664 * state the entire time, and we don't need to wait for it.
665 */
666
667 if ((curr == snap) && ((curr & 0x1) == 0))
668 return 0;
669
670 /*
671 * If the CPU passed through or entered a dynticks idle phase with
672 * no active irq handlers, then, as above, this CPU has already
673 * passed through a quiescent state.
674 */
675
676 if ((curr - snap) > 2 || (snap & 0x1) == 0)
677 return 0;
678
679 /* We need this CPU to go through a quiescent state. */
680
681 return 1;
682}
683
612#else /* !CONFIG_NO_HZ */ 684#else /* !CONFIG_NO_HZ */
613 685
614# define dyntick_save_progress_counter(cpu) do { } while (0) 686# define dyntick_save_progress_counter(cpu) do { } while (0)
615# define rcu_try_flip_waitack_needed(cpu) (1) 687# define rcu_try_flip_waitack_needed(cpu) (1)
616# define rcu_try_flip_waitmb_needed(cpu) (1) 688# define rcu_try_flip_waitmb_needed(cpu) (1)
689
690# define dyntick_save_progress_counter_sched(cpu) do { } while (0)
691# define rcu_qsctr_inc_needed_dyntick(cpu) (1)
617 692
618#endif /* CONFIG_NO_HZ */ 693#endif /* CONFIG_NO_HZ */
619 694
695static void save_qsctr_sched(int cpu)
696{
697 struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
698
699 rdssp->sched_qs_snap = rdssp->sched_qs;
700}
701
702static inline int rcu_qsctr_inc_needed(int cpu)
703{
704 struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
705
706 /*
707 * If there has been a quiescent state, no more need to wait
708 * on this CPU.
709 */
710
711 if (rdssp->sched_qs != rdssp->sched_qs_snap) {
712 smp_mb(); /* force ordering with cpu entering schedule(). */
713 return 0;
714 }
715
716 /* We need this CPU to go through a quiescent state. */
717
718 return 1;
719}
720
620/* 721/*
621 * Get here when RCU is idle. Decide whether we need to 722 * Get here when RCU is idle. Decide whether we need to
622 * move out of idle state, and return non-zero if so. 723 * move out of idle state, and return non-zero if so.
@@ -655,7 +756,7 @@ rcu_try_flip_idle(void)
655 756
656 /* Now ask each CPU for acknowledgement of the flip. */ 757 /* Now ask each CPU for acknowledgement of the flip. */
657 758
658 for_each_cpu_mask(cpu, rcu_cpu_online_map) { 759 for_each_cpu_mask_nr(cpu, rcu_cpu_online_map) {
659 per_cpu(rcu_flip_flag, cpu) = rcu_flipped; 760 per_cpu(rcu_flip_flag, cpu) = rcu_flipped;
660 dyntick_save_progress_counter(cpu); 761 dyntick_save_progress_counter(cpu);
661 } 762 }
@@ -673,7 +774,7 @@ rcu_try_flip_waitack(void)
673 int cpu; 774 int cpu;
674 775
675 RCU_TRACE_ME(rcupreempt_trace_try_flip_a1); 776 RCU_TRACE_ME(rcupreempt_trace_try_flip_a1);
676 for_each_cpu_mask(cpu, rcu_cpu_online_map) 777 for_each_cpu_mask_nr(cpu, rcu_cpu_online_map)
677 if (rcu_try_flip_waitack_needed(cpu) && 778 if (rcu_try_flip_waitack_needed(cpu) &&
678 per_cpu(rcu_flip_flag, cpu) != rcu_flip_seen) { 779 per_cpu(rcu_flip_flag, cpu) != rcu_flip_seen) {
679 RCU_TRACE_ME(rcupreempt_trace_try_flip_ae1); 780 RCU_TRACE_ME(rcupreempt_trace_try_flip_ae1);
@@ -705,7 +806,7 @@ rcu_try_flip_waitzero(void)
705 /* Check to see if the sum of the "last" counters is zero. */ 806 /* Check to see if the sum of the "last" counters is zero. */
706 807
707 RCU_TRACE_ME(rcupreempt_trace_try_flip_z1); 808 RCU_TRACE_ME(rcupreempt_trace_try_flip_z1);
708 for_each_cpu_mask(cpu, rcu_cpu_online_map) 809 for_each_cpu_mask_nr(cpu, rcu_cpu_online_map)
709 sum += RCU_DATA_CPU(cpu)->rcu_flipctr[lastidx]; 810 sum += RCU_DATA_CPU(cpu)->rcu_flipctr[lastidx];
710 if (sum != 0) { 811 if (sum != 0) {
711 RCU_TRACE_ME(rcupreempt_trace_try_flip_ze1); 812 RCU_TRACE_ME(rcupreempt_trace_try_flip_ze1);
@@ -720,7 +821,7 @@ rcu_try_flip_waitzero(void)
720 smp_mb(); /* ^^^^^^^^^^^^ */ 821 smp_mb(); /* ^^^^^^^^^^^^ */
721 822
722 /* Call for a memory barrier from each CPU. */ 823 /* Call for a memory barrier from each CPU. */
723 for_each_cpu_mask(cpu, rcu_cpu_online_map) { 824 for_each_cpu_mask_nr(cpu, rcu_cpu_online_map) {
724 per_cpu(rcu_mb_flag, cpu) = rcu_mb_needed; 825 per_cpu(rcu_mb_flag, cpu) = rcu_mb_needed;
725 dyntick_save_progress_counter(cpu); 826 dyntick_save_progress_counter(cpu);
726 } 827 }
@@ -740,7 +841,7 @@ rcu_try_flip_waitmb(void)
740 int cpu; 841 int cpu;
741 842
742 RCU_TRACE_ME(rcupreempt_trace_try_flip_m1); 843 RCU_TRACE_ME(rcupreempt_trace_try_flip_m1);
743 for_each_cpu_mask(cpu, rcu_cpu_online_map) 844 for_each_cpu_mask_nr(cpu, rcu_cpu_online_map)
744 if (rcu_try_flip_waitmb_needed(cpu) && 845 if (rcu_try_flip_waitmb_needed(cpu) &&
745 per_cpu(rcu_mb_flag, cpu) != rcu_mb_done) { 846 per_cpu(rcu_mb_flag, cpu) != rcu_mb_done) {
746 RCU_TRACE_ME(rcupreempt_trace_try_flip_me1); 847 RCU_TRACE_ME(rcupreempt_trace_try_flip_me1);
@@ -819,6 +920,26 @@ void rcu_check_callbacks(int cpu, int user)
819 unsigned long flags; 920 unsigned long flags;
820 struct rcu_data *rdp = RCU_DATA_CPU(cpu); 921 struct rcu_data *rdp = RCU_DATA_CPU(cpu);
821 922
923 /*
924 * If this CPU took its interrupt from user mode or from the
925 * idle loop, and this is not a nested interrupt, then
926 * this CPU has to have exited all prior preept-disable
927 * sections of code. So increment the counter to note this.
928 *
929 * The memory barrier is needed to handle the case where
930 * writes from a preempt-disable section of code get reordered
931 * into schedule() by this CPU's write buffer. So the memory
932 * barrier makes sure that the rcu_qsctr_inc() is seen by other
933 * CPUs to happen after any such write.
934 */
935
936 if (user ||
937 (idle_cpu(cpu) && !in_softirq() &&
938 hardirq_count() <= (1 << HARDIRQ_SHIFT))) {
939 smp_mb(); /* Guard against aggressive schedule(). */
940 rcu_qsctr_inc(cpu);
941 }
942
822 rcu_check_mb(cpu); 943 rcu_check_mb(cpu);
823 if (rcu_ctrlblk.completed == rdp->completed) 944 if (rcu_ctrlblk.completed == rdp->completed)
824 rcu_try_flip(); 945 rcu_try_flip();
@@ -869,6 +990,8 @@ void rcu_offline_cpu(int cpu)
869 struct rcu_head *list = NULL; 990 struct rcu_head *list = NULL;
870 unsigned long flags; 991 unsigned long flags;
871 struct rcu_data *rdp = RCU_DATA_CPU(cpu); 992 struct rcu_data *rdp = RCU_DATA_CPU(cpu);
993 struct rcu_head *schedlist = NULL;
994 struct rcu_head **schedtail = &schedlist;
872 struct rcu_head **tail = &list; 995 struct rcu_head **tail = &list;
873 996
874 /* 997 /*
@@ -882,6 +1005,11 @@ void rcu_offline_cpu(int cpu)
882 rcu_offline_cpu_enqueue(rdp->waitlist[i], rdp->waittail[i], 1005 rcu_offline_cpu_enqueue(rdp->waitlist[i], rdp->waittail[i],
883 list, tail); 1006 list, tail);
884 rcu_offline_cpu_enqueue(rdp->nextlist, rdp->nexttail, list, tail); 1007 rcu_offline_cpu_enqueue(rdp->nextlist, rdp->nexttail, list, tail);
1008 rcu_offline_cpu_enqueue(rdp->waitschedlist, rdp->waitschedtail,
1009 schedlist, schedtail);
1010 rcu_offline_cpu_enqueue(rdp->nextschedlist, rdp->nextschedtail,
1011 schedlist, schedtail);
1012 rdp->rcu_sched_sleeping = 0;
885 spin_unlock_irqrestore(&rdp->lock, flags); 1013 spin_unlock_irqrestore(&rdp->lock, flags);
886 rdp->waitlistcount = 0; 1014 rdp->waitlistcount = 0;
887 1015
@@ -916,12 +1044,15 @@ void rcu_offline_cpu(int cpu)
916 * fix. 1044 * fix.
917 */ 1045 */
918 1046
919 local_irq_save(flags); 1047 local_irq_save(flags); /* disable preempt till we know what lock. */
920 rdp = RCU_DATA_ME(); 1048 rdp = RCU_DATA_ME();
921 spin_lock(&rdp->lock); 1049 spin_lock(&rdp->lock);
922 *rdp->nexttail = list; 1050 *rdp->nexttail = list;
923 if (list) 1051 if (list)
924 rdp->nexttail = tail; 1052 rdp->nexttail = tail;
1053 *rdp->nextschedtail = schedlist;
1054 if (schedlist)
1055 rdp->nextschedtail = schedtail;
925 spin_unlock_irqrestore(&rdp->lock, flags); 1056 spin_unlock_irqrestore(&rdp->lock, flags);
926} 1057}
927 1058
@@ -936,10 +1067,25 @@ void rcu_offline_cpu(int cpu)
936void __cpuinit rcu_online_cpu(int cpu) 1067void __cpuinit rcu_online_cpu(int cpu)
937{ 1068{
938 unsigned long flags; 1069 unsigned long flags;
1070 struct rcu_data *rdp;
939 1071
940 spin_lock_irqsave(&rcu_ctrlblk.fliplock, flags); 1072 spin_lock_irqsave(&rcu_ctrlblk.fliplock, flags);
941 cpu_set(cpu, rcu_cpu_online_map); 1073 cpu_set(cpu, rcu_cpu_online_map);
942 spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags); 1074 spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags);
1075
1076 /*
1077 * The rcu_sched grace-period processing might have bypassed
1078 * this CPU, given that it was not in the rcu_cpu_online_map
1079 * when the grace-period scan started. This means that the
1080 * grace-period task might sleep. So make sure that if this
1081 * should happen, the first callback posted to this CPU will
1082 * wake up the grace-period task if need be.
1083 */
1084
1085 rdp = RCU_DATA_CPU(cpu);
1086 spin_lock_irqsave(&rdp->lock, flags);
1087 rdp->rcu_sched_sleeping = 1;
1088 spin_unlock_irqrestore(&rdp->lock, flags);
943} 1089}
944 1090
945static void rcu_process_callbacks(struct softirq_action *unused) 1091static void rcu_process_callbacks(struct softirq_action *unused)
@@ -982,31 +1128,196 @@ void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
982 *rdp->nexttail = head; 1128 *rdp->nexttail = head;
983 rdp->nexttail = &head->next; 1129 rdp->nexttail = &head->next;
984 RCU_TRACE_RDP(rcupreempt_trace_next_add, rdp); 1130 RCU_TRACE_RDP(rcupreempt_trace_next_add, rdp);
985 spin_unlock(&rdp->lock); 1131 spin_unlock_irqrestore(&rdp->lock, flags);
986 local_irq_restore(flags);
987} 1132}
988EXPORT_SYMBOL_GPL(call_rcu); 1133EXPORT_SYMBOL_GPL(call_rcu);
989 1134
1135void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
1136{
1137 unsigned long flags;
1138 struct rcu_data *rdp;
1139 int wake_gp = 0;
1140
1141 head->func = func;
1142 head->next = NULL;
1143 local_irq_save(flags);
1144 rdp = RCU_DATA_ME();
1145 spin_lock(&rdp->lock);
1146 *rdp->nextschedtail = head;
1147 rdp->nextschedtail = &head->next;
1148 if (rdp->rcu_sched_sleeping) {
1149
1150 /* Grace-period processing might be sleeping... */
1151
1152 rdp->rcu_sched_sleeping = 0;
1153 wake_gp = 1;
1154 }
1155 spin_unlock_irqrestore(&rdp->lock, flags);
1156 if (wake_gp) {
1157
1158 /* Wake up grace-period processing, unless someone beat us. */
1159
1160 spin_lock_irqsave(&rcu_ctrlblk.schedlock, flags);
1161 if (rcu_ctrlblk.sched_sleep != rcu_sched_sleeping)
1162 wake_gp = 0;
1163 rcu_ctrlblk.sched_sleep = rcu_sched_not_sleeping;
1164 spin_unlock_irqrestore(&rcu_ctrlblk.schedlock, flags);
1165 if (wake_gp)
1166 wake_up_interruptible(&rcu_ctrlblk.sched_wq);
1167 }
1168}
1169EXPORT_SYMBOL_GPL(call_rcu_sched);
1170
990/* 1171/*
991 * Wait until all currently running preempt_disable() code segments 1172 * Wait until all currently running preempt_disable() code segments
992 * (including hardware-irq-disable segments) complete. Note that 1173 * (including hardware-irq-disable segments) complete. Note that
993 * in -rt this does -not- necessarily result in all currently executing 1174 * in -rt this does -not- necessarily result in all currently executing
994 * interrupt -handlers- having completed. 1175 * interrupt -handlers- having completed.
995 */ 1176 */
996void __synchronize_sched(void) 1177synchronize_rcu_xxx(__synchronize_sched, call_rcu_sched)
1178EXPORT_SYMBOL_GPL(__synchronize_sched);
1179
1180/*
1181 * kthread function that manages call_rcu_sched grace periods.
1182 */
1183static int rcu_sched_grace_period(void *arg)
997{ 1184{
998 cpumask_t oldmask; 1185 int couldsleep; /* might sleep after current pass. */
1186 int couldsleepnext = 0; /* might sleep after next pass. */
999 int cpu; 1187 int cpu;
1188 unsigned long flags;
1189 struct rcu_data *rdp;
1190 int ret;
1000 1191
1001 if (sched_getaffinity(0, &oldmask) < 0) 1192 /*
1002 oldmask = cpu_possible_map; 1193 * Each pass through the following loop handles one
1003 for_each_online_cpu(cpu) { 1194 * rcu_sched grace period cycle.
1004 sched_setaffinity(0, &cpumask_of_cpu(cpu)); 1195 */
1005 schedule(); 1196 do {
1006 } 1197 /* Save each CPU's current state. */
1007 sched_setaffinity(0, &oldmask); 1198
1199 for_each_online_cpu(cpu) {
1200 dyntick_save_progress_counter_sched(cpu);
1201 save_qsctr_sched(cpu);
1202 }
1203
1204 /*
1205 * Sleep for about an RCU grace-period's worth to
1206 * allow better batching and to consume less CPU.
1207 */
1208 schedule_timeout_interruptible(RCU_SCHED_BATCH_TIME);
1209
1210 /*
1211 * If there was nothing to do last time, prepare to
1212 * sleep at the end of the current grace period cycle.
1213 */
1214 couldsleep = couldsleepnext;
1215 couldsleepnext = 1;
1216 if (couldsleep) {
1217 spin_lock_irqsave(&rcu_ctrlblk.schedlock, flags);
1218 rcu_ctrlblk.sched_sleep = rcu_sched_sleep_prep;
1219 spin_unlock_irqrestore(&rcu_ctrlblk.schedlock, flags);
1220 }
1221
1222 /*
1223 * Wait on each CPU in turn to have either visited
1224 * a quiescent state or been in dynticks-idle mode.
1225 */
1226 for_each_online_cpu(cpu) {
1227 while (rcu_qsctr_inc_needed(cpu) &&
1228 rcu_qsctr_inc_needed_dyntick(cpu)) {
1229 /* resched_cpu(cpu); @@@ */
1230 schedule_timeout_interruptible(1);
1231 }
1232 }
1233
1234 /* Advance callbacks for each CPU. */
1235
1236 for_each_online_cpu(cpu) {
1237
1238 rdp = RCU_DATA_CPU(cpu);
1239 spin_lock_irqsave(&rdp->lock, flags);
1240
1241 /*
1242 * We are running on this CPU irq-disabled, so no
1243 * CPU can go offline until we re-enable irqs.
1244 * The current CPU might have already gone
1245 * offline (between the for_each_offline_cpu and
1246 * the spin_lock_irqsave), but in that case all its
1247 * callback lists will be empty, so no harm done.
1248 *
1249 * Advance the callbacks! We share normal RCU's
1250 * donelist, since callbacks are invoked the
1251 * same way in either case.
1252 */
1253 if (rdp->waitschedlist != NULL) {
1254 *rdp->donetail = rdp->waitschedlist;
1255 rdp->donetail = rdp->waitschedtail;
1256
1257 /*
1258 * Next rcu_check_callbacks() will
1259 * do the required raise_softirq().
1260 */
1261 }
1262 if (rdp->nextschedlist != NULL) {
1263 rdp->waitschedlist = rdp->nextschedlist;
1264 rdp->waitschedtail = rdp->nextschedtail;
1265 couldsleep = 0;
1266 couldsleepnext = 0;
1267 } else {
1268 rdp->waitschedlist = NULL;
1269 rdp->waitschedtail = &rdp->waitschedlist;
1270 }
1271 rdp->nextschedlist = NULL;
1272 rdp->nextschedtail = &rdp->nextschedlist;
1273
1274 /* Mark sleep intention. */
1275
1276 rdp->rcu_sched_sleeping = couldsleep;
1277
1278 spin_unlock_irqrestore(&rdp->lock, flags);
1279 }
1280
1281 /* If we saw callbacks on the last scan, go deal with them. */
1282
1283 if (!couldsleep)
1284 continue;
1285
1286 /* Attempt to block... */
1287
1288 spin_lock_irqsave(&rcu_ctrlblk.schedlock, flags);
1289 if (rcu_ctrlblk.sched_sleep != rcu_sched_sleep_prep) {
1290
1291 /*
1292 * Someone posted a callback after we scanned.
1293 * Go take care of it.
1294 */
1295 spin_unlock_irqrestore(&rcu_ctrlblk.schedlock, flags);
1296 couldsleepnext = 0;
1297 continue;
1298 }
1299
1300 /* Block until the next person posts a callback. */
1301
1302 rcu_ctrlblk.sched_sleep = rcu_sched_sleeping;
1303 spin_unlock_irqrestore(&rcu_ctrlblk.schedlock, flags);
1304 ret = 0;
1305 __wait_event_interruptible(rcu_ctrlblk.sched_wq,
1306 rcu_ctrlblk.sched_sleep != rcu_sched_sleeping,
1307 ret);
1308
1309 /*
1310 * Signals would prevent us from sleeping, and we cannot
1311 * do much with them in any case. So flush them.
1312 */
1313 if (ret)
1314 flush_signals(current);
1315 couldsleepnext = 0;
1316
1317 } while (!kthread_should_stop());
1318
1319 return (0);
1008} 1320}
1009EXPORT_SYMBOL_GPL(__synchronize_sched);
1010 1321
1011/* 1322/*
1012 * Check to see if any future RCU-related work will need to be done 1323 * Check to see if any future RCU-related work will need to be done
@@ -1023,7 +1334,9 @@ int rcu_needs_cpu(int cpu)
1023 1334
1024 return (rdp->donelist != NULL || 1335 return (rdp->donelist != NULL ||
1025 !!rdp->waitlistcount || 1336 !!rdp->waitlistcount ||
1026 rdp->nextlist != NULL); 1337 rdp->nextlist != NULL ||
1338 rdp->nextschedlist != NULL ||
1339 rdp->waitschedlist != NULL);
1027} 1340}
1028 1341
1029int rcu_pending(int cpu) 1342int rcu_pending(int cpu)
@@ -1034,7 +1347,9 @@ int rcu_pending(int cpu)
1034 1347
1035 if (rdp->donelist != NULL || 1348 if (rdp->donelist != NULL ||
1036 !!rdp->waitlistcount || 1349 !!rdp->waitlistcount ||
1037 rdp->nextlist != NULL) 1350 rdp->nextlist != NULL ||
1351 rdp->nextschedlist != NULL ||
1352 rdp->waitschedlist != NULL)
1038 return 1; 1353 return 1;
1039 1354
1040 /* The RCU core needs an acknowledgement from this CPU. */ 1355 /* The RCU core needs an acknowledgement from this CPU. */
@@ -1101,6 +1416,11 @@ void __init __rcu_init(void)
1101 rdp->donetail = &rdp->donelist; 1416 rdp->donetail = &rdp->donelist;
1102 rdp->rcu_flipctr[0] = 0; 1417 rdp->rcu_flipctr[0] = 0;
1103 rdp->rcu_flipctr[1] = 0; 1418 rdp->rcu_flipctr[1] = 0;
1419 rdp->nextschedlist = NULL;
1420 rdp->nextschedtail = &rdp->nextschedlist;
1421 rdp->waitschedlist = NULL;
1422 rdp->waitschedtail = &rdp->waitschedlist;
1423 rdp->rcu_sched_sleeping = 0;
1104 } 1424 }
1105 register_cpu_notifier(&rcu_nb); 1425 register_cpu_notifier(&rcu_nb);
1106 1426
@@ -1119,15 +1439,19 @@ void __init __rcu_init(void)
1119 for_each_online_cpu(cpu) 1439 for_each_online_cpu(cpu)
1120 rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, (void *)(long) cpu); 1440 rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, (void *)(long) cpu);
1121 1441
1122 open_softirq(RCU_SOFTIRQ, rcu_process_callbacks, NULL); 1442 open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
1123} 1443}
1124 1444
1125/* 1445/*
1126 * Deprecated, use synchronize_rcu() or synchronize_sched() instead. 1446 * Late-boot-time RCU initialization that must wait until after scheduler
1447 * has been initialized.
1127 */ 1448 */
1128void synchronize_kernel(void) 1449void __init rcu_init_sched(void)
1129{ 1450{
1130 synchronize_rcu(); 1451 rcu_sched_grace_period_task = kthread_run(rcu_sched_grace_period,
1452 NULL,
1453 "rcu_sched_grace_period");
1454 WARN_ON(IS_ERR(rcu_sched_grace_period_task));
1131} 1455}
1132 1456
1133#ifdef CONFIG_RCU_TRACE 1457#ifdef CONFIG_RCU_TRACE
diff --git a/kernel/rcupreempt_trace.c b/kernel/rcupreempt_trace.c
index 49ac4947af24..5edf82c34bbc 100644
--- a/kernel/rcupreempt_trace.c
+++ b/kernel/rcupreempt_trace.c
@@ -38,7 +38,6 @@
38#include <linux/moduleparam.h> 38#include <linux/moduleparam.h>
39#include <linux/percpu.h> 39#include <linux/percpu.h>
40#include <linux/notifier.h> 40#include <linux/notifier.h>
41#include <linux/rcupdate.h>
42#include <linux/cpu.h> 41#include <linux/cpu.h>
43#include <linux/mutex.h> 42#include <linux/mutex.h>
44#include <linux/rcupreempt_trace.h> 43#include <linux/rcupreempt_trace.h>
diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c
index 33acc424667e..90b5b123f7a1 100644
--- a/kernel/rcutorture.c
+++ b/kernel/rcutorture.c
@@ -57,7 +57,9 @@ static int stat_interval; /* Interval between stats, in seconds. */
57 /* Defaults to "only at end of test". */ 57 /* Defaults to "only at end of test". */
58static int verbose; /* Print more debug info. */ 58static int verbose; /* Print more debug info. */
59static int test_no_idle_hz; /* Test RCU's support for tickless idle CPUs. */ 59static int test_no_idle_hz; /* Test RCU's support for tickless idle CPUs. */
60static int shuffle_interval = 5; /* Interval between shuffles (in sec)*/ 60static int shuffle_interval = 3; /* Interval between shuffles (in sec)*/
61static int stutter = 5; /* Start/stop testing interval (in sec) */
62static int irqreader = 1; /* RCU readers from irq (timers). */
61static char *torture_type = "rcu"; /* What RCU implementation to torture. */ 63static char *torture_type = "rcu"; /* What RCU implementation to torture. */
62 64
63module_param(nreaders, int, 0444); 65module_param(nreaders, int, 0444);
@@ -72,6 +74,10 @@ module_param(test_no_idle_hz, bool, 0444);
72MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs"); 74MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs");
73module_param(shuffle_interval, int, 0444); 75module_param(shuffle_interval, int, 0444);
74MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles"); 76MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles");
77module_param(stutter, int, 0444);
78MODULE_PARM_DESC(stutter, "Number of seconds to run/halt test");
79module_param(irqreader, int, 0444);
80MODULE_PARM_DESC(irqreader, "Allow RCU readers from irq handlers");
75module_param(torture_type, charp, 0444); 81module_param(torture_type, charp, 0444);
76MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, srcu)"); 82MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, srcu)");
77 83
@@ -91,6 +97,7 @@ static struct task_struct **fakewriter_tasks;
91static struct task_struct **reader_tasks; 97static struct task_struct **reader_tasks;
92static struct task_struct *stats_task; 98static struct task_struct *stats_task;
93static struct task_struct *shuffler_task; 99static struct task_struct *shuffler_task;
100static struct task_struct *stutter_task;
94 101
95#define RCU_TORTURE_PIPE_LEN 10 102#define RCU_TORTURE_PIPE_LEN 10
96 103
@@ -117,8 +124,18 @@ static atomic_t n_rcu_torture_alloc_fail;
117static atomic_t n_rcu_torture_free; 124static atomic_t n_rcu_torture_free;
118static atomic_t n_rcu_torture_mberror; 125static atomic_t n_rcu_torture_mberror;
119static atomic_t n_rcu_torture_error; 126static atomic_t n_rcu_torture_error;
127static long n_rcu_torture_timers = 0;
120static struct list_head rcu_torture_removed; 128static struct list_head rcu_torture_removed;
121 129
130static int stutter_pause_test = 0;
131
132#if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE)
133#define RCUTORTURE_RUNNABLE_INIT 1
134#else
135#define RCUTORTURE_RUNNABLE_INIT 0
136#endif
137int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT;
138
122/* 139/*
123 * Allocate an element from the rcu_tortures pool. 140 * Allocate an element from the rcu_tortures pool.
124 */ 141 */
@@ -179,6 +196,16 @@ rcu_random(struct rcu_random_state *rrsp)
179 return swahw32(rrsp->rrs_state); 196 return swahw32(rrsp->rrs_state);
180} 197}
181 198
199static void
200rcu_stutter_wait(void)
201{
202 while (stutter_pause_test || !rcutorture_runnable)
203 if (rcutorture_runnable)
204 schedule_timeout_interruptible(1);
205 else
206 schedule_timeout_interruptible(round_jiffies_relative(HZ));
207}
208
182/* 209/*
183 * Operations vector for selecting different types of tests. 210 * Operations vector for selecting different types of tests.
184 */ 211 */
@@ -192,7 +219,9 @@ struct rcu_torture_ops {
192 int (*completed)(void); 219 int (*completed)(void);
193 void (*deferredfree)(struct rcu_torture *p); 220 void (*deferredfree)(struct rcu_torture *p);
194 void (*sync)(void); 221 void (*sync)(void);
222 void (*cb_barrier)(void);
195 int (*stats)(char *page); 223 int (*stats)(char *page);
224 int irqcapable;
196 char *name; 225 char *name;
197}; 226};
198static struct rcu_torture_ops *cur_ops = NULL; 227static struct rcu_torture_ops *cur_ops = NULL;
@@ -265,7 +294,9 @@ static struct rcu_torture_ops rcu_ops = {
265 .completed = rcu_torture_completed, 294 .completed = rcu_torture_completed,
266 .deferredfree = rcu_torture_deferred_free, 295 .deferredfree = rcu_torture_deferred_free,
267 .sync = synchronize_rcu, 296 .sync = synchronize_rcu,
297 .cb_barrier = rcu_barrier,
268 .stats = NULL, 298 .stats = NULL,
299 .irqcapable = 1,
269 .name = "rcu" 300 .name = "rcu"
270}; 301};
271 302
@@ -304,7 +335,9 @@ static struct rcu_torture_ops rcu_sync_ops = {
304 .completed = rcu_torture_completed, 335 .completed = rcu_torture_completed,
305 .deferredfree = rcu_sync_torture_deferred_free, 336 .deferredfree = rcu_sync_torture_deferred_free,
306 .sync = synchronize_rcu, 337 .sync = synchronize_rcu,
338 .cb_barrier = NULL,
307 .stats = NULL, 339 .stats = NULL,
340 .irqcapable = 1,
308 .name = "rcu_sync" 341 .name = "rcu_sync"
309}; 342};
310 343
@@ -364,7 +397,9 @@ static struct rcu_torture_ops rcu_bh_ops = {
364 .completed = rcu_bh_torture_completed, 397 .completed = rcu_bh_torture_completed,
365 .deferredfree = rcu_bh_torture_deferred_free, 398 .deferredfree = rcu_bh_torture_deferred_free,
366 .sync = rcu_bh_torture_synchronize, 399 .sync = rcu_bh_torture_synchronize,
400 .cb_barrier = rcu_barrier_bh,
367 .stats = NULL, 401 .stats = NULL,
402 .irqcapable = 1,
368 .name = "rcu_bh" 403 .name = "rcu_bh"
369}; 404};
370 405
@@ -377,7 +412,9 @@ static struct rcu_torture_ops rcu_bh_sync_ops = {
377 .completed = rcu_bh_torture_completed, 412 .completed = rcu_bh_torture_completed,
378 .deferredfree = rcu_sync_torture_deferred_free, 413 .deferredfree = rcu_sync_torture_deferred_free,
379 .sync = rcu_bh_torture_synchronize, 414 .sync = rcu_bh_torture_synchronize,
415 .cb_barrier = NULL,
380 .stats = NULL, 416 .stats = NULL,
417 .irqcapable = 1,
381 .name = "rcu_bh_sync" 418 .name = "rcu_bh_sync"
382}; 419};
383 420
@@ -458,6 +495,7 @@ static struct rcu_torture_ops srcu_ops = {
458 .completed = srcu_torture_completed, 495 .completed = srcu_torture_completed,
459 .deferredfree = rcu_sync_torture_deferred_free, 496 .deferredfree = rcu_sync_torture_deferred_free,
460 .sync = srcu_torture_synchronize, 497 .sync = srcu_torture_synchronize,
498 .cb_barrier = NULL,
461 .stats = srcu_torture_stats, 499 .stats = srcu_torture_stats,
462 .name = "srcu" 500 .name = "srcu"
463}; 501};
@@ -482,6 +520,11 @@ static int sched_torture_completed(void)
482 return 0; 520 return 0;
483} 521}
484 522
523static void rcu_sched_torture_deferred_free(struct rcu_torture *p)
524{
525 call_rcu_sched(&p->rtort_rcu, rcu_torture_cb);
526}
527
485static void sched_torture_synchronize(void) 528static void sched_torture_synchronize(void)
486{ 529{
487 synchronize_sched(); 530 synchronize_sched();
@@ -494,12 +537,28 @@ static struct rcu_torture_ops sched_ops = {
494 .readdelay = rcu_read_delay, /* just reuse rcu's version. */ 537 .readdelay = rcu_read_delay, /* just reuse rcu's version. */
495 .readunlock = sched_torture_read_unlock, 538 .readunlock = sched_torture_read_unlock,
496 .completed = sched_torture_completed, 539 .completed = sched_torture_completed,
497 .deferredfree = rcu_sync_torture_deferred_free, 540 .deferredfree = rcu_sched_torture_deferred_free,
498 .sync = sched_torture_synchronize, 541 .sync = sched_torture_synchronize,
542 .cb_barrier = rcu_barrier_sched,
499 .stats = NULL, 543 .stats = NULL,
544 .irqcapable = 1,
500 .name = "sched" 545 .name = "sched"
501}; 546};
502 547
548static struct rcu_torture_ops sched_ops_sync = {
549 .init = rcu_sync_torture_init,
550 .cleanup = NULL,
551 .readlock = sched_torture_read_lock,
552 .readdelay = rcu_read_delay, /* just reuse rcu's version. */
553 .readunlock = sched_torture_read_unlock,
554 .completed = sched_torture_completed,
555 .deferredfree = rcu_sync_torture_deferred_free,
556 .sync = sched_torture_synchronize,
557 .cb_barrier = NULL,
558 .stats = NULL,
559 .name = "sched_sync"
560};
561
503/* 562/*
504 * RCU torture writer kthread. Repeatedly substitutes a new structure 563 * RCU torture writer kthread. Repeatedly substitutes a new structure
505 * for that pointed to by rcu_torture_current, freeing the old structure 564 * for that pointed to by rcu_torture_current, freeing the old structure
@@ -537,6 +596,7 @@ rcu_torture_writer(void *arg)
537 } 596 }
538 rcu_torture_current_version++; 597 rcu_torture_current_version++;
539 oldbatch = cur_ops->completed(); 598 oldbatch = cur_ops->completed();
599 rcu_stutter_wait();
540 } while (!kthread_should_stop() && !fullstop); 600 } while (!kthread_should_stop() && !fullstop);
541 VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping"); 601 VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping");
542 while (!kthread_should_stop()) 602 while (!kthread_should_stop())
@@ -560,6 +620,7 @@ rcu_torture_fakewriter(void *arg)
560 schedule_timeout_uninterruptible(1 + rcu_random(&rand)%10); 620 schedule_timeout_uninterruptible(1 + rcu_random(&rand)%10);
561 udelay(rcu_random(&rand) & 0x3ff); 621 udelay(rcu_random(&rand) & 0x3ff);
562 cur_ops->sync(); 622 cur_ops->sync();
623 rcu_stutter_wait();
563 } while (!kthread_should_stop() && !fullstop); 624 } while (!kthread_should_stop() && !fullstop);
564 625
565 VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task stopping"); 626 VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task stopping");
@@ -569,6 +630,52 @@ rcu_torture_fakewriter(void *arg)
569} 630}
570 631
571/* 632/*
633 * RCU torture reader from timer handler. Dereferences rcu_torture_current,
634 * incrementing the corresponding element of the pipeline array. The
635 * counter in the element should never be greater than 1, otherwise, the
636 * RCU implementation is broken.
637 */
638static void rcu_torture_timer(unsigned long unused)
639{
640 int idx;
641 int completed;
642 static DEFINE_RCU_RANDOM(rand);
643 static DEFINE_SPINLOCK(rand_lock);
644 struct rcu_torture *p;
645 int pipe_count;
646
647 idx = cur_ops->readlock();
648 completed = cur_ops->completed();
649 p = rcu_dereference(rcu_torture_current);
650 if (p == NULL) {
651 /* Leave because rcu_torture_writer is not yet underway */
652 cur_ops->readunlock(idx);
653 return;
654 }
655 if (p->rtort_mbtest == 0)
656 atomic_inc(&n_rcu_torture_mberror);
657 spin_lock(&rand_lock);
658 cur_ops->readdelay(&rand);
659 n_rcu_torture_timers++;
660 spin_unlock(&rand_lock);
661 preempt_disable();
662 pipe_count = p->rtort_pipe_count;
663 if (pipe_count > RCU_TORTURE_PIPE_LEN) {
664 /* Should not happen, but... */
665 pipe_count = RCU_TORTURE_PIPE_LEN;
666 }
667 ++__get_cpu_var(rcu_torture_count)[pipe_count];
668 completed = cur_ops->completed() - completed;
669 if (completed > RCU_TORTURE_PIPE_LEN) {
670 /* Should not happen, but... */
671 completed = RCU_TORTURE_PIPE_LEN;
672 }
673 ++__get_cpu_var(rcu_torture_batch)[completed];
674 preempt_enable();
675 cur_ops->readunlock(idx);
676}
677
678/*
572 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current, 679 * RCU torture reader kthread. Repeatedly dereferences rcu_torture_current,
573 * incrementing the corresponding element of the pipeline array. The 680 * incrementing the corresponding element of the pipeline array. The
574 * counter in the element should never be greater than 1, otherwise, the 681 * counter in the element should never be greater than 1, otherwise, the
@@ -582,11 +689,18 @@ rcu_torture_reader(void *arg)
582 DEFINE_RCU_RANDOM(rand); 689 DEFINE_RCU_RANDOM(rand);
583 struct rcu_torture *p; 690 struct rcu_torture *p;
584 int pipe_count; 691 int pipe_count;
692 struct timer_list t;
585 693
586 VERBOSE_PRINTK_STRING("rcu_torture_reader task started"); 694 VERBOSE_PRINTK_STRING("rcu_torture_reader task started");
587 set_user_nice(current, 19); 695 set_user_nice(current, 19);
696 if (irqreader && cur_ops->irqcapable)
697 setup_timer_on_stack(&t, rcu_torture_timer, 0);
588 698
589 do { 699 do {
700 if (irqreader && cur_ops->irqcapable) {
701 if (!timer_pending(&t))
702 mod_timer(&t, 1);
703 }
590 idx = cur_ops->readlock(); 704 idx = cur_ops->readlock();
591 completed = cur_ops->completed(); 705 completed = cur_ops->completed();
592 p = rcu_dereference(rcu_torture_current); 706 p = rcu_dereference(rcu_torture_current);
@@ -615,8 +729,11 @@ rcu_torture_reader(void *arg)
615 preempt_enable(); 729 preempt_enable();
616 cur_ops->readunlock(idx); 730 cur_ops->readunlock(idx);
617 schedule(); 731 schedule();
732 rcu_stutter_wait();
618 } while (!kthread_should_stop() && !fullstop); 733 } while (!kthread_should_stop() && !fullstop);
619 VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping"); 734 VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping");
735 if (irqreader && cur_ops->irqcapable)
736 del_timer_sync(&t);
620 while (!kthread_should_stop()) 737 while (!kthread_should_stop())
621 schedule_timeout_uninterruptible(1); 738 schedule_timeout_uninterruptible(1);
622 return 0; 739 return 0;
@@ -647,20 +764,22 @@ rcu_torture_printk(char *page)
647 cnt += sprintf(&page[cnt], "%s%s ", torture_type, TORTURE_FLAG); 764 cnt += sprintf(&page[cnt], "%s%s ", torture_type, TORTURE_FLAG);
648 cnt += sprintf(&page[cnt], 765 cnt += sprintf(&page[cnt],
649 "rtc: %p ver: %ld tfle: %d rta: %d rtaf: %d rtf: %d " 766 "rtc: %p ver: %ld tfle: %d rta: %d rtaf: %d rtf: %d "
650 "rtmbe: %d", 767 "rtmbe: %d nt: %ld",
651 rcu_torture_current, 768 rcu_torture_current,
652 rcu_torture_current_version, 769 rcu_torture_current_version,
653 list_empty(&rcu_torture_freelist), 770 list_empty(&rcu_torture_freelist),
654 atomic_read(&n_rcu_torture_alloc), 771 atomic_read(&n_rcu_torture_alloc),
655 atomic_read(&n_rcu_torture_alloc_fail), 772 atomic_read(&n_rcu_torture_alloc_fail),
656 atomic_read(&n_rcu_torture_free), 773 atomic_read(&n_rcu_torture_free),
657 atomic_read(&n_rcu_torture_mberror)); 774 atomic_read(&n_rcu_torture_mberror),
775 n_rcu_torture_timers);
658 if (atomic_read(&n_rcu_torture_mberror) != 0) 776 if (atomic_read(&n_rcu_torture_mberror) != 0)
659 cnt += sprintf(&page[cnt], " !!!"); 777 cnt += sprintf(&page[cnt], " !!!");
660 cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG); 778 cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
661 if (i > 1) { 779 if (i > 1) {
662 cnt += sprintf(&page[cnt], "!!! "); 780 cnt += sprintf(&page[cnt], "!!! ");
663 atomic_inc(&n_rcu_torture_error); 781 atomic_inc(&n_rcu_torture_error);
782 WARN_ON_ONCE(1);
664 } 783 }
665 cnt += sprintf(&page[cnt], "Reader Pipe: "); 784 cnt += sprintf(&page[cnt], "Reader Pipe: ");
666 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) 785 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
@@ -785,15 +904,34 @@ rcu_torture_shuffle(void *arg)
785 return 0; 904 return 0;
786} 905}
787 906
907/* Cause the rcutorture test to "stutter", starting and stopping all
908 * threads periodically.
909 */
910static int
911rcu_torture_stutter(void *arg)
912{
913 VERBOSE_PRINTK_STRING("rcu_torture_stutter task started");
914 do {
915 schedule_timeout_interruptible(stutter * HZ);
916 stutter_pause_test = 1;
917 if (!kthread_should_stop())
918 schedule_timeout_interruptible(stutter * HZ);
919 stutter_pause_test = 0;
920 } while (!kthread_should_stop());
921 VERBOSE_PRINTK_STRING("rcu_torture_stutter task stopping");
922 return 0;
923}
924
788static inline void 925static inline void
789rcu_torture_print_module_parms(char *tag) 926rcu_torture_print_module_parms(char *tag)
790{ 927{
791 printk(KERN_ALERT "%s" TORTURE_FLAG 928 printk(KERN_ALERT "%s" TORTURE_FLAG
792 "--- %s: nreaders=%d nfakewriters=%d " 929 "--- %s: nreaders=%d nfakewriters=%d "
793 "stat_interval=%d verbose=%d test_no_idle_hz=%d " 930 "stat_interval=%d verbose=%d test_no_idle_hz=%d "
794 "shuffle_interval = %d\n", 931 "shuffle_interval=%d stutter=%d irqreader=%d\n",
795 torture_type, tag, nrealreaders, nfakewriters, 932 torture_type, tag, nrealreaders, nfakewriters,
796 stat_interval, verbose, test_no_idle_hz, shuffle_interval); 933 stat_interval, verbose, test_no_idle_hz, shuffle_interval,
934 stutter, irqreader);
797} 935}
798 936
799static void 937static void
@@ -802,6 +940,11 @@ rcu_torture_cleanup(void)
802 int i; 940 int i;
803 941
804 fullstop = 1; 942 fullstop = 1;
943 if (stutter_task) {
944 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task");
945 kthread_stop(stutter_task);
946 }
947 stutter_task = NULL;
805 if (shuffler_task) { 948 if (shuffler_task) {
806 VERBOSE_PRINTK_STRING("Stopping rcu_torture_shuffle task"); 949 VERBOSE_PRINTK_STRING("Stopping rcu_torture_shuffle task");
807 kthread_stop(shuffler_task); 950 kthread_stop(shuffler_task);
@@ -848,7 +991,9 @@ rcu_torture_cleanup(void)
848 stats_task = NULL; 991 stats_task = NULL;
849 992
850 /* Wait for all RCU callbacks to fire. */ 993 /* Wait for all RCU callbacks to fire. */
851 rcu_barrier(); 994
995 if (cur_ops->cb_barrier != NULL)
996 cur_ops->cb_barrier();
852 997
853 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ 998 rcu_torture_stats_print(); /* -After- the stats thread is stopped! */
854 999
@@ -868,7 +1013,7 @@ rcu_torture_init(void)
868 int firsterr = 0; 1013 int firsterr = 0;
869 static struct rcu_torture_ops *torture_ops[] = 1014 static struct rcu_torture_ops *torture_ops[] =
870 { &rcu_ops, &rcu_sync_ops, &rcu_bh_ops, &rcu_bh_sync_ops, 1015 { &rcu_ops, &rcu_sync_ops, &rcu_bh_ops, &rcu_bh_sync_ops,
871 &srcu_ops, &sched_ops, }; 1016 &srcu_ops, &sched_ops, &sched_ops_sync, };
872 1017
873 /* Process args and tell the world that the torturer is on the job. */ 1018 /* Process args and tell the world that the torturer is on the job. */
874 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 1019 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
@@ -988,6 +1133,19 @@ rcu_torture_init(void)
988 goto unwind; 1133 goto unwind;
989 } 1134 }
990 } 1135 }
1136 if (stutter < 0)
1137 stutter = 0;
1138 if (stutter) {
1139 /* Create the stutter thread */
1140 stutter_task = kthread_run(rcu_torture_stutter, NULL,
1141 "rcu_torture_stutter");
1142 if (IS_ERR(stutter_task)) {
1143 firsterr = PTR_ERR(stutter_task);
1144 VERBOSE_PRINTK_ERRSTRING("Failed to create stutter");
1145 stutter_task = NULL;
1146 goto unwind;
1147 }
1148 }
991 return 0; 1149 return 0;
992 1150
993unwind: 1151unwind:
diff --git a/kernel/relay.c b/kernel/relay.c
index 7de644cdec43..04006ef970b8 100644
--- a/kernel/relay.c
+++ b/kernel/relay.c
@@ -407,6 +407,35 @@ void relay_reset(struct rchan *chan)
407} 407}
408EXPORT_SYMBOL_GPL(relay_reset); 408EXPORT_SYMBOL_GPL(relay_reset);
409 409
410static inline void relay_set_buf_dentry(struct rchan_buf *buf,
411 struct dentry *dentry)
412{
413 buf->dentry = dentry;
414 buf->dentry->d_inode->i_size = buf->early_bytes;
415}
416
417static struct dentry *relay_create_buf_file(struct rchan *chan,
418 struct rchan_buf *buf,
419 unsigned int cpu)
420{
421 struct dentry *dentry;
422 char *tmpname;
423
424 tmpname = kzalloc(NAME_MAX + 1, GFP_KERNEL);
425 if (!tmpname)
426 return NULL;
427 snprintf(tmpname, NAME_MAX, "%s%d", chan->base_filename, cpu);
428
429 /* Create file in fs */
430 dentry = chan->cb->create_buf_file(tmpname, chan->parent,
431 S_IRUSR, buf,
432 &chan->is_global);
433
434 kfree(tmpname);
435
436 return dentry;
437}
438
410/* 439/*
411 * relay_open_buf - create a new relay channel buffer 440 * relay_open_buf - create a new relay channel buffer
412 * 441 *
@@ -416,45 +445,34 @@ static struct rchan_buf *relay_open_buf(struct rchan *chan, unsigned int cpu)
416{ 445{
417 struct rchan_buf *buf = NULL; 446 struct rchan_buf *buf = NULL;
418 struct dentry *dentry; 447 struct dentry *dentry;
419 char *tmpname;
420 448
421 if (chan->is_global) 449 if (chan->is_global)
422 return chan->buf[0]; 450 return chan->buf[0];
423 451
424 tmpname = kzalloc(NAME_MAX + 1, GFP_KERNEL);
425 if (!tmpname)
426 goto end;
427 snprintf(tmpname, NAME_MAX, "%s%d", chan->base_filename, cpu);
428
429 buf = relay_create_buf(chan); 452 buf = relay_create_buf(chan);
430 if (!buf) 453 if (!buf)
431 goto free_name; 454 return NULL;
455
456 if (chan->has_base_filename) {
457 dentry = relay_create_buf_file(chan, buf, cpu);
458 if (!dentry)
459 goto free_buf;
460 relay_set_buf_dentry(buf, dentry);
461 }
432 462
433 buf->cpu = cpu; 463 buf->cpu = cpu;
434 __relay_reset(buf, 1); 464 __relay_reset(buf, 1);
435 465
436 /* Create file in fs */
437 dentry = chan->cb->create_buf_file(tmpname, chan->parent, S_IRUSR,
438 buf, &chan->is_global);
439 if (!dentry)
440 goto free_buf;
441
442 buf->dentry = dentry;
443
444 if(chan->is_global) { 466 if(chan->is_global) {
445 chan->buf[0] = buf; 467 chan->buf[0] = buf;
446 buf->cpu = 0; 468 buf->cpu = 0;
447 } 469 }
448 470
449 goto free_name; 471 return buf;
450 472
451free_buf: 473free_buf:
452 relay_destroy_buf(buf); 474 relay_destroy_buf(buf);
453 buf = NULL; 475 return NULL;
454free_name:
455 kfree(tmpname);
456end:
457 return buf;
458} 476}
459 477
460/** 478/**
@@ -537,8 +555,8 @@ static int __cpuinit relay_hotcpu_callback(struct notifier_block *nb,
537 555
538/** 556/**
539 * relay_open - create a new relay channel 557 * relay_open - create a new relay channel
540 * @base_filename: base name of files to create 558 * @base_filename: base name of files to create, %NULL for buffering only
541 * @parent: dentry of parent directory, %NULL for root directory 559 * @parent: dentry of parent directory, %NULL for root directory or buffer
542 * @subbuf_size: size of sub-buffers 560 * @subbuf_size: size of sub-buffers
543 * @n_subbufs: number of sub-buffers 561 * @n_subbufs: number of sub-buffers
544 * @cb: client callback functions 562 * @cb: client callback functions
@@ -560,8 +578,6 @@ struct rchan *relay_open(const char *base_filename,
560{ 578{
561 unsigned int i; 579 unsigned int i;
562 struct rchan *chan; 580 struct rchan *chan;
563 if (!base_filename)
564 return NULL;
565 581
566 if (!(subbuf_size && n_subbufs)) 582 if (!(subbuf_size && n_subbufs))
567 return NULL; 583 return NULL;
@@ -576,7 +592,10 @@ struct rchan *relay_open(const char *base_filename,
576 chan->alloc_size = FIX_SIZE(subbuf_size * n_subbufs); 592 chan->alloc_size = FIX_SIZE(subbuf_size * n_subbufs);
577 chan->parent = parent; 593 chan->parent = parent;
578 chan->private_data = private_data; 594 chan->private_data = private_data;
579 strlcpy(chan->base_filename, base_filename, NAME_MAX); 595 if (base_filename) {
596 chan->has_base_filename = 1;
597 strlcpy(chan->base_filename, base_filename, NAME_MAX);
598 }
580 setup_callbacks(chan, cb); 599 setup_callbacks(chan, cb);
581 kref_init(&chan->kref); 600 kref_init(&chan->kref);
582 601
@@ -604,6 +623,94 @@ free_bufs:
604} 623}
605EXPORT_SYMBOL_GPL(relay_open); 624EXPORT_SYMBOL_GPL(relay_open);
606 625
626struct rchan_percpu_buf_dispatcher {
627 struct rchan_buf *buf;
628 struct dentry *dentry;
629};
630
631/* Called in atomic context. */
632static void __relay_set_buf_dentry(void *info)
633{
634 struct rchan_percpu_buf_dispatcher *p = info;
635
636 relay_set_buf_dentry(p->buf, p->dentry);
637}
638
639/**
640 * relay_late_setup_files - triggers file creation
641 * @chan: channel to operate on
642 * @base_filename: base name of files to create
643 * @parent: dentry of parent directory, %NULL for root directory
644 *
645 * Returns 0 if successful, non-zero otherwise.
646 *
647 * Use to setup files for a previously buffer-only channel.
648 * Useful to do early tracing in kernel, before VFS is up, for example.
649 */
650int relay_late_setup_files(struct rchan *chan,
651 const char *base_filename,
652 struct dentry *parent)
653{
654 int err = 0;
655 unsigned int i, curr_cpu;
656 unsigned long flags;
657 struct dentry *dentry;
658 struct rchan_percpu_buf_dispatcher disp;
659
660 if (!chan || !base_filename)
661 return -EINVAL;
662
663 strlcpy(chan->base_filename, base_filename, NAME_MAX);
664
665 mutex_lock(&relay_channels_mutex);
666 /* Is chan already set up? */
667 if (unlikely(chan->has_base_filename))
668 return -EEXIST;
669 chan->has_base_filename = 1;
670 chan->parent = parent;
671 curr_cpu = get_cpu();
672 /*
673 * The CPU hotplug notifier ran before us and created buffers with
674 * no files associated. So it's safe to call relay_setup_buf_file()
675 * on all currently online CPUs.
676 */
677 for_each_online_cpu(i) {
678 if (unlikely(!chan->buf[i])) {
679 printk(KERN_ERR "relay_late_setup_files: CPU %u "
680 "has no buffer, it must have!\n", i);
681 BUG();
682 err = -EINVAL;
683 break;
684 }
685
686 dentry = relay_create_buf_file(chan, chan->buf[i], i);
687 if (unlikely(!dentry)) {
688 err = -EINVAL;
689 break;
690 }
691
692 if (curr_cpu == i) {
693 local_irq_save(flags);
694 relay_set_buf_dentry(chan->buf[i], dentry);
695 local_irq_restore(flags);
696 } else {
697 disp.buf = chan->buf[i];
698 disp.dentry = dentry;
699 smp_mb();
700 /* relay_channels_mutex must be held, so wait. */
701 err = smp_call_function_single(i,
702 __relay_set_buf_dentry,
703 &disp, 1);
704 }
705 if (unlikely(err))
706 break;
707 }
708 put_cpu();
709 mutex_unlock(&relay_channels_mutex);
710
711 return err;
712}
713
607/** 714/**
608 * relay_switch_subbuf - switch to a new sub-buffer 715 * relay_switch_subbuf - switch to a new sub-buffer
609 * @buf: channel buffer 716 * @buf: channel buffer
@@ -627,8 +734,13 @@ size_t relay_switch_subbuf(struct rchan_buf *buf, size_t length)
627 old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs; 734 old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs;
628 buf->padding[old_subbuf] = buf->prev_padding; 735 buf->padding[old_subbuf] = buf->prev_padding;
629 buf->subbufs_produced++; 736 buf->subbufs_produced++;
630 buf->dentry->d_inode->i_size += buf->chan->subbuf_size - 737 if (buf->dentry)
631 buf->padding[old_subbuf]; 738 buf->dentry->d_inode->i_size +=
739 buf->chan->subbuf_size -
740 buf->padding[old_subbuf];
741 else
742 buf->early_bytes += buf->chan->subbuf_size -
743 buf->padding[old_subbuf];
632 smp_mb(); 744 smp_mb();
633 if (waitqueue_active(&buf->read_wait)) 745 if (waitqueue_active(&buf->read_wait))
634 /* 746 /*
@@ -1237,4 +1349,4 @@ static __init int relay_init(void)
1237 return 0; 1349 return 0;
1238} 1350}
1239 1351
1240module_init(relay_init); 1352early_initcall(relay_init);
diff --git a/kernel/res_counter.c b/kernel/res_counter.c
index d3c61b4ebef2..f275c8eca772 100644
--- a/kernel/res_counter.c
+++ b/kernel/res_counter.c
@@ -13,6 +13,7 @@
13#include <linux/slab.h> 13#include <linux/slab.h>
14#include <linux/res_counter.h> 14#include <linux/res_counter.h>
15#include <linux/uaccess.h> 15#include <linux/uaccess.h>
16#include <linux/mm.h>
16 17
17void res_counter_init(struct res_counter *counter) 18void res_counter_init(struct res_counter *counter)
18{ 19{
@@ -102,44 +103,37 @@ u64 res_counter_read_u64(struct res_counter *counter, int member)
102 return *res_counter_member(counter, member); 103 return *res_counter_member(counter, member);
103} 104}
104 105
105ssize_t res_counter_write(struct res_counter *counter, int member, 106int res_counter_memparse_write_strategy(const char *buf,
106 const char __user *userbuf, size_t nbytes, loff_t *pos, 107 unsigned long long *res)
107 int (*write_strategy)(char *st_buf, unsigned long long *val))
108{ 108{
109 int ret; 109 char *end;
110 char *buf, *end; 110 /* FIXME - make memparse() take const char* args */
111 unsigned long flags; 111 *res = memparse((char *)buf, &end);
112 unsigned long long tmp, *val; 112 if (*end != '\0')
113 113 return -EINVAL;
114 buf = kmalloc(nbytes + 1, GFP_KERNEL);
115 ret = -ENOMEM;
116 if (buf == NULL)
117 goto out;
118 114
119 buf[nbytes] = '\0'; 115 *res = PAGE_ALIGN(*res);
120 ret = -EFAULT; 116 return 0;
121 if (copy_from_user(buf, userbuf, nbytes)) 117}
122 goto out_free;
123 118
124 ret = -EINVAL; 119int res_counter_write(struct res_counter *counter, int member,
120 const char *buf, write_strategy_fn write_strategy)
121{
122 char *end;
123 unsigned long flags;
124 unsigned long long tmp, *val;
125 125
126 strstrip(buf);
127 if (write_strategy) { 126 if (write_strategy) {
128 if (write_strategy(buf, &tmp)) { 127 if (write_strategy(buf, &tmp))
129 goto out_free; 128 return -EINVAL;
130 }
131 } else { 129 } else {
132 tmp = simple_strtoull(buf, &end, 10); 130 tmp = simple_strtoull(buf, &end, 10);
133 if (*end != '\0') 131 if (*end != '\0')
134 goto out_free; 132 return -EINVAL;
135 } 133 }
136 spin_lock_irqsave(&counter->lock, flags); 134 spin_lock_irqsave(&counter->lock, flags);
137 val = res_counter_member(counter, member); 135 val = res_counter_member(counter, member);
138 *val = tmp; 136 *val = tmp;
139 spin_unlock_irqrestore(&counter->lock, flags); 137 spin_unlock_irqrestore(&counter->lock, flags);
140 ret = nbytes; 138 return 0;
141out_free:
142 kfree(buf);
143out:
144 return ret;
145} 139}
diff --git a/kernel/rtmutex-tester.c b/kernel/rtmutex-tester.c
index 092e4c620af9..a56f629b057a 100644
--- a/kernel/rtmutex-tester.c
+++ b/kernel/rtmutex-tester.c
@@ -297,8 +297,8 @@ static int test_func(void *data)
297 * 297 *
298 * opcode:data 298 * opcode:data
299 */ 299 */
300static ssize_t sysfs_test_command(struct sys_device *dev, const char *buf, 300static ssize_t sysfs_test_command(struct sys_device *dev, struct sysdev_attribute *attr,
301 size_t count) 301 const char *buf, size_t count)
302{ 302{
303 struct sched_param schedpar; 303 struct sched_param schedpar;
304 struct test_thread_data *td; 304 struct test_thread_data *td;
@@ -360,7 +360,8 @@ static ssize_t sysfs_test_command(struct sys_device *dev, const char *buf,
360 * @dev: thread to query 360 * @dev: thread to query
361 * @buf: char buffer to be filled with thread status info 361 * @buf: char buffer to be filled with thread status info
362 */ 362 */
363static ssize_t sysfs_test_status(struct sys_device *dev, char *buf) 363static ssize_t sysfs_test_status(struct sys_device *dev, struct sysdev_attribute *attr,
364 char *buf)
364{ 365{
365 struct test_thread_data *td; 366 struct test_thread_data *td;
366 struct task_struct *tsk; 367 struct task_struct *tsk;
diff --git a/kernel/sched.c b/kernel/sched.c
index 4e2f60335656..0236958addcb 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -70,10 +70,13 @@
70#include <linux/bootmem.h> 70#include <linux/bootmem.h>
71#include <linux/debugfs.h> 71#include <linux/debugfs.h>
72#include <linux/ctype.h> 72#include <linux/ctype.h>
73#include <linux/ftrace.h>
73 74
74#include <asm/tlb.h> 75#include <asm/tlb.h>
75#include <asm/irq_regs.h> 76#include <asm/irq_regs.h>
76 77
78#include "sched_cpupri.h"
79
77/* 80/*
78 * Convert user-nice values [ -20 ... 0 ... 19 ] 81 * Convert user-nice values [ -20 ... 0 ... 19 ]
79 * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ], 82 * to static priority [ MAX_RT_PRIO..MAX_PRIO-1 ],
@@ -289,15 +292,15 @@ struct task_group root_task_group;
289static DEFINE_PER_CPU(struct sched_entity, init_sched_entity); 292static DEFINE_PER_CPU(struct sched_entity, init_sched_entity);
290/* Default task group's cfs_rq on each cpu */ 293/* Default task group's cfs_rq on each cpu */
291static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp; 294static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp;
292#endif 295#endif /* CONFIG_FAIR_GROUP_SCHED */
293 296
294#ifdef CONFIG_RT_GROUP_SCHED 297#ifdef CONFIG_RT_GROUP_SCHED
295static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity); 298static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
296static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp; 299static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp;
297#endif 300#endif /* CONFIG_RT_GROUP_SCHED */
298#else 301#else /* !CONFIG_FAIR_GROUP_SCHED */
299#define root_task_group init_task_group 302#define root_task_group init_task_group
300#endif 303#endif /* CONFIG_FAIR_GROUP_SCHED */
301 304
302/* task_group_lock serializes add/remove of task groups and also changes to 305/* task_group_lock serializes add/remove of task groups and also changes to
303 * a task group's cpu shares. 306 * a task group's cpu shares.
@@ -307,9 +310,9 @@ static DEFINE_SPINLOCK(task_group_lock);
307#ifdef CONFIG_FAIR_GROUP_SCHED 310#ifdef CONFIG_FAIR_GROUP_SCHED
308#ifdef CONFIG_USER_SCHED 311#ifdef CONFIG_USER_SCHED
309# define INIT_TASK_GROUP_LOAD (2*NICE_0_LOAD) 312# define INIT_TASK_GROUP_LOAD (2*NICE_0_LOAD)
310#else 313#else /* !CONFIG_USER_SCHED */
311# define INIT_TASK_GROUP_LOAD NICE_0_LOAD 314# define INIT_TASK_GROUP_LOAD NICE_0_LOAD
312#endif 315#endif /* CONFIG_USER_SCHED */
313 316
314/* 317/*
315 * A weight of 0 or 1 can cause arithmetics problems. 318 * A weight of 0 or 1 can cause arithmetics problems.
@@ -363,6 +366,10 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu)
363#else 366#else
364 367
365static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { } 368static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { }
369static inline struct task_group *task_group(struct task_struct *p)
370{
371 return NULL;
372}
366 373
367#endif /* CONFIG_GROUP_SCHED */ 374#endif /* CONFIG_GROUP_SCHED */
368 375
@@ -373,6 +380,7 @@ struct cfs_rq {
373 380
374 u64 exec_clock; 381 u64 exec_clock;
375 u64 min_vruntime; 382 u64 min_vruntime;
383 u64 pair_start;
376 384
377 struct rb_root tasks_timeline; 385 struct rb_root tasks_timeline;
378 struct rb_node *rb_leftmost; 386 struct rb_node *rb_leftmost;
@@ -401,6 +409,31 @@ struct cfs_rq {
401 */ 409 */
402 struct list_head leaf_cfs_rq_list; 410 struct list_head leaf_cfs_rq_list;
403 struct task_group *tg; /* group that "owns" this runqueue */ 411 struct task_group *tg; /* group that "owns" this runqueue */
412
413#ifdef CONFIG_SMP
414 /*
415 * the part of load.weight contributed by tasks
416 */
417 unsigned long task_weight;
418
419 /*
420 * h_load = weight * f(tg)
421 *
422 * Where f(tg) is the recursive weight fraction assigned to
423 * this group.
424 */
425 unsigned long h_load;
426
427 /*
428 * this cpu's part of tg->shares
429 */
430 unsigned long shares;
431
432 /*
433 * load.weight at the time we set shares
434 */
435 unsigned long rq_weight;
436#endif
404#endif 437#endif
405}; 438};
406 439
@@ -452,6 +485,9 @@ struct root_domain {
452 */ 485 */
453 cpumask_t rto_mask; 486 cpumask_t rto_mask;
454 atomic_t rto_count; 487 atomic_t rto_count;
488#ifdef CONFIG_SMP
489 struct cpupri cpupri;
490#endif
455}; 491};
456 492
457/* 493/*
@@ -526,14 +562,19 @@ struct rq {
526 int push_cpu; 562 int push_cpu;
527 /* cpu of this runqueue: */ 563 /* cpu of this runqueue: */
528 int cpu; 564 int cpu;
565 int online;
566
567 unsigned long avg_load_per_task;
529 568
530 struct task_struct *migration_thread; 569 struct task_struct *migration_thread;
531 struct list_head migration_queue; 570 struct list_head migration_queue;
532#endif 571#endif
533 572
534#ifdef CONFIG_SCHED_HRTICK 573#ifdef CONFIG_SCHED_HRTICK
535 unsigned long hrtick_flags; 574#ifdef CONFIG_SMP
536 ktime_t hrtick_expire; 575 int hrtick_csd_pending;
576 struct call_single_data hrtick_csd;
577#endif
537 struct hrtimer hrtick_timer; 578 struct hrtimer hrtick_timer;
538#endif 579#endif
539 580
@@ -607,6 +648,24 @@ static inline void update_rq_clock(struct rq *rq)
607# define const_debug static const 648# define const_debug static const
608#endif 649#endif
609 650
651/**
652 * runqueue_is_locked
653 *
654 * Returns true if the current cpu runqueue is locked.
655 * This interface allows printk to be called with the runqueue lock
656 * held and know whether or not it is OK to wake up the klogd.
657 */
658int runqueue_is_locked(void)
659{
660 int cpu = get_cpu();
661 struct rq *rq = cpu_rq(cpu);
662 int ret;
663
664 ret = spin_is_locked(&rq->lock);
665 put_cpu();
666 return ret;
667}
668
610/* 669/*
611 * Debugging: various feature bits 670 * Debugging: various feature bits
612 */ 671 */
@@ -749,6 +808,12 @@ late_initcall(sched_init_debug);
749const_debug unsigned int sysctl_sched_nr_migrate = 32; 808const_debug unsigned int sysctl_sched_nr_migrate = 32;
750 809
751/* 810/*
811 * ratelimit for updating the group shares.
812 * default: 0.5ms
813 */
814const_debug unsigned int sysctl_sched_shares_ratelimit = 500000;
815
816/*
752 * period over which we measure -rt task cpu usage in us. 817 * period over which we measure -rt task cpu usage in us.
753 * default: 1s 818 * default: 1s
754 */ 819 */
@@ -775,82 +840,6 @@ static inline u64 global_rt_runtime(void)
775 return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC; 840 return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC;
776} 841}
777 842
778unsigned long long time_sync_thresh = 100000;
779
780static DEFINE_PER_CPU(unsigned long long, time_offset);
781static DEFINE_PER_CPU(unsigned long long, prev_cpu_time);
782
783/*
784 * Global lock which we take every now and then to synchronize
785 * the CPUs time. This method is not warp-safe, but it's good
786 * enough to synchronize slowly diverging time sources and thus
787 * it's good enough for tracing:
788 */
789static DEFINE_SPINLOCK(time_sync_lock);
790static unsigned long long prev_global_time;
791
792static unsigned long long __sync_cpu_clock(unsigned long long time, int cpu)
793{
794 /*
795 * We want this inlined, to not get tracer function calls
796 * in this critical section:
797 */
798 spin_acquire(&time_sync_lock.dep_map, 0, 0, _THIS_IP_);
799 __raw_spin_lock(&time_sync_lock.raw_lock);
800
801 if (time < prev_global_time) {
802 per_cpu(time_offset, cpu) += prev_global_time - time;
803 time = prev_global_time;
804 } else {
805 prev_global_time = time;
806 }
807
808 __raw_spin_unlock(&time_sync_lock.raw_lock);
809 spin_release(&time_sync_lock.dep_map, 1, _THIS_IP_);
810
811 return time;
812}
813
814static unsigned long long __cpu_clock(int cpu)
815{
816 unsigned long long now;
817
818 /*
819 * Only call sched_clock() if the scheduler has already been
820 * initialized (some code might call cpu_clock() very early):
821 */
822 if (unlikely(!scheduler_running))
823 return 0;
824
825 now = sched_clock_cpu(cpu);
826
827 return now;
828}
829
830/*
831 * For kernel-internal use: high-speed (but slightly incorrect) per-cpu
832 * clock constructed from sched_clock():
833 */
834unsigned long long cpu_clock(int cpu)
835{
836 unsigned long long prev_cpu_time, time, delta_time;
837 unsigned long flags;
838
839 local_irq_save(flags);
840 prev_cpu_time = per_cpu(prev_cpu_time, cpu);
841 time = __cpu_clock(cpu) + per_cpu(time_offset, cpu);
842 delta_time = time-prev_cpu_time;
843
844 if (unlikely(delta_time > time_sync_thresh)) {
845 time = __sync_cpu_clock(time, cpu);
846 per_cpu(prev_cpu_time, cpu) = time;
847 }
848 local_irq_restore(flags);
849
850 return time;
851}
852EXPORT_SYMBOL_GPL(cpu_clock);
853
854#ifndef prepare_arch_switch 843#ifndef prepare_arch_switch
855# define prepare_arch_switch(next) do { } while (0) 844# define prepare_arch_switch(next) do { } while (0)
856#endif 845#endif
@@ -996,13 +985,6 @@ static struct rq *this_rq_lock(void)
996 return rq; 985 return rq;
997} 986}
998 987
999static void __resched_task(struct task_struct *p, int tif_bit);
1000
1001static inline void resched_task(struct task_struct *p)
1002{
1003 __resched_task(p, TIF_NEED_RESCHED);
1004}
1005
1006#ifdef CONFIG_SCHED_HRTICK 988#ifdef CONFIG_SCHED_HRTICK
1007/* 989/*
1008 * Use HR-timers to deliver accurate preemption points. 990 * Use HR-timers to deliver accurate preemption points.
@@ -1014,25 +996,6 @@ static inline void resched_task(struct task_struct *p)
1014 * When we get rescheduled we reprogram the hrtick_timer outside of the 996 * When we get rescheduled we reprogram the hrtick_timer outside of the
1015 * rq->lock. 997 * rq->lock.
1016 */ 998 */
1017static inline void resched_hrt(struct task_struct *p)
1018{
1019 __resched_task(p, TIF_HRTICK_RESCHED);
1020}
1021
1022static inline void resched_rq(struct rq *rq)
1023{
1024 unsigned long flags;
1025
1026 spin_lock_irqsave(&rq->lock, flags);
1027 resched_task(rq->curr);
1028 spin_unlock_irqrestore(&rq->lock, flags);
1029}
1030
1031enum {
1032 HRTICK_SET, /* re-programm hrtick_timer */
1033 HRTICK_RESET, /* not a new slice */
1034 HRTICK_BLOCK, /* stop hrtick operations */
1035};
1036 999
1037/* 1000/*
1038 * Use hrtick when: 1001 * Use hrtick when:
@@ -1043,40 +1006,11 @@ static inline int hrtick_enabled(struct rq *rq)
1043{ 1006{
1044 if (!sched_feat(HRTICK)) 1007 if (!sched_feat(HRTICK))
1045 return 0; 1008 return 0;
1046 if (unlikely(test_bit(HRTICK_BLOCK, &rq->hrtick_flags))) 1009 if (!cpu_active(cpu_of(rq)))
1047 return 0; 1010 return 0;
1048 return hrtimer_is_hres_active(&rq->hrtick_timer); 1011 return hrtimer_is_hres_active(&rq->hrtick_timer);
1049} 1012}
1050 1013
1051/*
1052 * Called to set the hrtick timer state.
1053 *
1054 * called with rq->lock held and irqs disabled
1055 */
1056static void hrtick_start(struct rq *rq, u64 delay, int reset)
1057{
1058 assert_spin_locked(&rq->lock);
1059
1060 /*
1061 * preempt at: now + delay
1062 */
1063 rq->hrtick_expire =
1064 ktime_add_ns(rq->hrtick_timer.base->get_time(), delay);
1065 /*
1066 * indicate we need to program the timer
1067 */
1068 __set_bit(HRTICK_SET, &rq->hrtick_flags);
1069 if (reset)
1070 __set_bit(HRTICK_RESET, &rq->hrtick_flags);
1071
1072 /*
1073 * New slices are called from the schedule path and don't need a
1074 * forced reschedule.
1075 */
1076 if (reset)
1077 resched_hrt(rq->curr);
1078}
1079
1080static void hrtick_clear(struct rq *rq) 1014static void hrtick_clear(struct rq *rq)
1081{ 1015{
1082 if (hrtimer_active(&rq->hrtick_timer)) 1016 if (hrtimer_active(&rq->hrtick_timer))
@@ -1084,32 +1018,6 @@ static void hrtick_clear(struct rq *rq)
1084} 1018}
1085 1019
1086/* 1020/*
1087 * Update the timer from the possible pending state.
1088 */
1089static void hrtick_set(struct rq *rq)
1090{
1091 ktime_t time;
1092 int set, reset;
1093 unsigned long flags;
1094
1095 WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
1096
1097 spin_lock_irqsave(&rq->lock, flags);
1098 set = __test_and_clear_bit(HRTICK_SET, &rq->hrtick_flags);
1099 reset = __test_and_clear_bit(HRTICK_RESET, &rq->hrtick_flags);
1100 time = rq->hrtick_expire;
1101 clear_thread_flag(TIF_HRTICK_RESCHED);
1102 spin_unlock_irqrestore(&rq->lock, flags);
1103
1104 if (set) {
1105 hrtimer_start(&rq->hrtick_timer, time, HRTIMER_MODE_ABS);
1106 if (reset && !hrtimer_active(&rq->hrtick_timer))
1107 resched_rq(rq);
1108 } else
1109 hrtick_clear(rq);
1110}
1111
1112/*
1113 * High-resolution timer tick. 1021 * High-resolution timer tick.
1114 * Runs from hardirq context with interrupts disabled. 1022 * Runs from hardirq context with interrupts disabled.
1115 */ 1023 */
@@ -1128,27 +1036,37 @@ static enum hrtimer_restart hrtick(struct hrtimer *timer)
1128} 1036}
1129 1037
1130#ifdef CONFIG_SMP 1038#ifdef CONFIG_SMP
1131static void hotplug_hrtick_disable(int cpu) 1039/*
1040 * called from hardirq (IPI) context
1041 */
1042static void __hrtick_start(void *arg)
1132{ 1043{
1133 struct rq *rq = cpu_rq(cpu); 1044 struct rq *rq = arg;
1134 unsigned long flags;
1135 1045
1136 spin_lock_irqsave(&rq->lock, flags); 1046 spin_lock(&rq->lock);
1137 rq->hrtick_flags = 0; 1047 hrtimer_restart(&rq->hrtick_timer);
1138 __set_bit(HRTICK_BLOCK, &rq->hrtick_flags); 1048 rq->hrtick_csd_pending = 0;
1139 spin_unlock_irqrestore(&rq->lock, flags); 1049 spin_unlock(&rq->lock);
1140
1141 hrtick_clear(rq);
1142} 1050}
1143 1051
1144static void hotplug_hrtick_enable(int cpu) 1052/*
1053 * Called to set the hrtick timer state.
1054 *
1055 * called with rq->lock held and irqs disabled
1056 */
1057static void hrtick_start(struct rq *rq, u64 delay)
1145{ 1058{
1146 struct rq *rq = cpu_rq(cpu); 1059 struct hrtimer *timer = &rq->hrtick_timer;
1147 unsigned long flags; 1060 ktime_t time = ktime_add_ns(timer->base->get_time(), delay);
1148 1061
1149 spin_lock_irqsave(&rq->lock, flags); 1062 timer->expires = time;
1150 __clear_bit(HRTICK_BLOCK, &rq->hrtick_flags); 1063
1151 spin_unlock_irqrestore(&rq->lock, flags); 1064 if (rq == this_rq()) {
1065 hrtimer_restart(timer);
1066 } else if (!rq->hrtick_csd_pending) {
1067 __smp_call_function_single(cpu_of(rq), &rq->hrtick_csd);
1068 rq->hrtick_csd_pending = 1;
1069 }
1152} 1070}
1153 1071
1154static int 1072static int
@@ -1163,16 +1081,7 @@ hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu)
1163 case CPU_DOWN_PREPARE_FROZEN: 1081 case CPU_DOWN_PREPARE_FROZEN:
1164 case CPU_DEAD: 1082 case CPU_DEAD:
1165 case CPU_DEAD_FROZEN: 1083 case CPU_DEAD_FROZEN:
1166 hotplug_hrtick_disable(cpu); 1084 hrtick_clear(cpu_rq(cpu));
1167 return NOTIFY_OK;
1168
1169 case CPU_UP_PREPARE:
1170 case CPU_UP_PREPARE_FROZEN:
1171 case CPU_DOWN_FAILED:
1172 case CPU_DOWN_FAILED_FROZEN:
1173 case CPU_ONLINE:
1174 case CPU_ONLINE_FROZEN:
1175 hotplug_hrtick_enable(cpu);
1176 return NOTIFY_OK; 1085 return NOTIFY_OK;
1177 } 1086 }
1178 1087
@@ -1183,46 +1092,45 @@ static void init_hrtick(void)
1183{ 1092{
1184 hotcpu_notifier(hotplug_hrtick, 0); 1093 hotcpu_notifier(hotplug_hrtick, 0);
1185} 1094}
1186#endif /* CONFIG_SMP */ 1095#else
1096/*
1097 * Called to set the hrtick timer state.
1098 *
1099 * called with rq->lock held and irqs disabled
1100 */
1101static void hrtick_start(struct rq *rq, u64 delay)
1102{
1103 hrtimer_start(&rq->hrtick_timer, ns_to_ktime(delay), HRTIMER_MODE_REL);
1104}
1187 1105
1188static void init_rq_hrtick(struct rq *rq) 1106static void init_hrtick(void)
1189{ 1107{
1190 rq->hrtick_flags = 0;
1191 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1192 rq->hrtick_timer.function = hrtick;
1193 rq->hrtick_timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ;
1194} 1108}
1109#endif /* CONFIG_SMP */
1195 1110
1196void hrtick_resched(void) 1111static void init_rq_hrtick(struct rq *rq)
1197{ 1112{
1198 struct rq *rq; 1113#ifdef CONFIG_SMP
1199 unsigned long flags; 1114 rq->hrtick_csd_pending = 0;
1200 1115
1201 if (!test_thread_flag(TIF_HRTICK_RESCHED)) 1116 rq->hrtick_csd.flags = 0;
1202 return; 1117 rq->hrtick_csd.func = __hrtick_start;
1118 rq->hrtick_csd.info = rq;
1119#endif
1203 1120
1204 local_irq_save(flags); 1121 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1205 rq = cpu_rq(smp_processor_id()); 1122 rq->hrtick_timer.function = hrtick;
1206 hrtick_set(rq); 1123 rq->hrtick_timer.cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ;
1207 local_irq_restore(flags);
1208} 1124}
1209#else 1125#else
1210static inline void hrtick_clear(struct rq *rq) 1126static inline void hrtick_clear(struct rq *rq)
1211{ 1127{
1212} 1128}
1213 1129
1214static inline void hrtick_set(struct rq *rq)
1215{
1216}
1217
1218static inline void init_rq_hrtick(struct rq *rq) 1130static inline void init_rq_hrtick(struct rq *rq)
1219{ 1131{
1220} 1132}
1221 1133
1222void hrtick_resched(void)
1223{
1224}
1225
1226static inline void init_hrtick(void) 1134static inline void init_hrtick(void)
1227{ 1135{
1228} 1136}
@@ -1241,16 +1149,16 @@ static inline void init_hrtick(void)
1241#define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG) 1149#define tsk_is_polling(t) test_tsk_thread_flag(t, TIF_POLLING_NRFLAG)
1242#endif 1150#endif
1243 1151
1244static void __resched_task(struct task_struct *p, int tif_bit) 1152static void resched_task(struct task_struct *p)
1245{ 1153{
1246 int cpu; 1154 int cpu;
1247 1155
1248 assert_spin_locked(&task_rq(p)->lock); 1156 assert_spin_locked(&task_rq(p)->lock);
1249 1157
1250 if (unlikely(test_tsk_thread_flag(p, tif_bit))) 1158 if (unlikely(test_tsk_thread_flag(p, TIF_NEED_RESCHED)))
1251 return; 1159 return;
1252 1160
1253 set_tsk_thread_flag(p, tif_bit); 1161 set_tsk_thread_flag(p, TIF_NEED_RESCHED);
1254 1162
1255 cpu = task_cpu(p); 1163 cpu = task_cpu(p);
1256 if (cpu == smp_processor_id()) 1164 if (cpu == smp_processor_id())
@@ -1313,15 +1221,15 @@ void wake_up_idle_cpu(int cpu)
1313 if (!tsk_is_polling(rq->idle)) 1221 if (!tsk_is_polling(rq->idle))
1314 smp_send_reschedule(cpu); 1222 smp_send_reschedule(cpu);
1315} 1223}
1316#endif 1224#endif /* CONFIG_NO_HZ */
1317 1225
1318#else 1226#else /* !CONFIG_SMP */
1319static void __resched_task(struct task_struct *p, int tif_bit) 1227static void resched_task(struct task_struct *p)
1320{ 1228{
1321 assert_spin_locked(&task_rq(p)->lock); 1229 assert_spin_locked(&task_rq(p)->lock);
1322 set_tsk_thread_flag(p, tif_bit); 1230 set_tsk_need_resched(p);
1323} 1231}
1324#endif 1232#endif /* CONFIG_SMP */
1325 1233
1326#if BITS_PER_LONG == 32 1234#if BITS_PER_LONG == 32
1327# define WMULT_CONST (~0UL) 1235# define WMULT_CONST (~0UL)
@@ -1336,6 +1244,9 @@ static void __resched_task(struct task_struct *p, int tif_bit)
1336 */ 1244 */
1337#define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y)) 1245#define SRR(x, y) (((x) + (1UL << ((y) - 1))) >> (y))
1338 1246
1247/*
1248 * delta *= weight / lw
1249 */
1339static unsigned long 1250static unsigned long
1340calc_delta_mine(unsigned long delta_exec, unsigned long weight, 1251calc_delta_mine(unsigned long delta_exec, unsigned long weight,
1341 struct load_weight *lw) 1252 struct load_weight *lw)
@@ -1363,12 +1274,6 @@ calc_delta_mine(unsigned long delta_exec, unsigned long weight,
1363 return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX); 1274 return (unsigned long)min(tmp, (u64)(unsigned long)LONG_MAX);
1364} 1275}
1365 1276
1366static inline unsigned long
1367calc_delta_fair(unsigned long delta_exec, struct load_weight *lw)
1368{
1369 return calc_delta_mine(delta_exec, NICE_0_LOAD, lw);
1370}
1371
1372static inline void update_load_add(struct load_weight *lw, unsigned long inc) 1277static inline void update_load_add(struct load_weight *lw, unsigned long inc)
1373{ 1278{
1374 lw->weight += inc; 1279 lw->weight += inc;
@@ -1479,17 +1384,211 @@ static inline void dec_cpu_load(struct rq *rq, unsigned long load)
1479#ifdef CONFIG_SMP 1384#ifdef CONFIG_SMP
1480static unsigned long source_load(int cpu, int type); 1385static unsigned long source_load(int cpu, int type);
1481static unsigned long target_load(int cpu, int type); 1386static unsigned long target_load(int cpu, int type);
1482static unsigned long cpu_avg_load_per_task(int cpu);
1483static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd); 1387static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);
1484#else /* CONFIG_SMP */ 1388
1389static unsigned long cpu_avg_load_per_task(int cpu)
1390{
1391 struct rq *rq = cpu_rq(cpu);
1392
1393 if (rq->nr_running)
1394 rq->avg_load_per_task = rq->load.weight / rq->nr_running;
1395
1396 return rq->avg_load_per_task;
1397}
1485 1398
1486#ifdef CONFIG_FAIR_GROUP_SCHED 1399#ifdef CONFIG_FAIR_GROUP_SCHED
1487static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares) 1400
1401typedef void (*tg_visitor)(struct task_group *, int, struct sched_domain *);
1402
1403/*
1404 * Iterate the full tree, calling @down when first entering a node and @up when
1405 * leaving it for the final time.
1406 */
1407static void
1408walk_tg_tree(tg_visitor down, tg_visitor up, int cpu, struct sched_domain *sd)
1409{
1410 struct task_group *parent, *child;
1411
1412 rcu_read_lock();
1413 parent = &root_task_group;
1414down:
1415 (*down)(parent, cpu, sd);
1416 list_for_each_entry_rcu(child, &parent->children, siblings) {
1417 parent = child;
1418 goto down;
1419
1420up:
1421 continue;
1422 }
1423 (*up)(parent, cpu, sd);
1424
1425 child = parent;
1426 parent = parent->parent;
1427 if (parent)
1428 goto up;
1429 rcu_read_unlock();
1430}
1431
1432static void __set_se_shares(struct sched_entity *se, unsigned long shares);
1433
1434/*
1435 * Calculate and set the cpu's group shares.
1436 */
1437static void
1438__update_group_shares_cpu(struct task_group *tg, int cpu,
1439 unsigned long sd_shares, unsigned long sd_rq_weight)
1488{ 1440{
1441 int boost = 0;
1442 unsigned long shares;
1443 unsigned long rq_weight;
1444
1445 if (!tg->se[cpu])
1446 return;
1447
1448 rq_weight = tg->cfs_rq[cpu]->load.weight;
1449
1450 /*
1451 * If there are currently no tasks on the cpu pretend there is one of
1452 * average load so that when a new task gets to run here it will not
1453 * get delayed by group starvation.
1454 */
1455 if (!rq_weight) {
1456 boost = 1;
1457 rq_weight = NICE_0_LOAD;
1458 }
1459
1460 if (unlikely(rq_weight > sd_rq_weight))
1461 rq_weight = sd_rq_weight;
1462
1463 /*
1464 * \Sum shares * rq_weight
1465 * shares = -----------------------
1466 * \Sum rq_weight
1467 *
1468 */
1469 shares = (sd_shares * rq_weight) / (sd_rq_weight + 1);
1470
1471 /*
1472 * record the actual number of shares, not the boosted amount.
1473 */
1474 tg->cfs_rq[cpu]->shares = boost ? 0 : shares;
1475 tg->cfs_rq[cpu]->rq_weight = rq_weight;
1476
1477 if (shares < MIN_SHARES)
1478 shares = MIN_SHARES;
1479 else if (shares > MAX_SHARES)
1480 shares = MAX_SHARES;
1481
1482 __set_se_shares(tg->se[cpu], shares);
1483}
1484
1485/*
1486 * Re-compute the task group their per cpu shares over the given domain.
1487 * This needs to be done in a bottom-up fashion because the rq weight of a
1488 * parent group depends on the shares of its child groups.
1489 */
1490static void
1491tg_shares_up(struct task_group *tg, int cpu, struct sched_domain *sd)
1492{
1493 unsigned long rq_weight = 0;
1494 unsigned long shares = 0;
1495 int i;
1496
1497 for_each_cpu_mask(i, sd->span) {
1498 rq_weight += tg->cfs_rq[i]->load.weight;
1499 shares += tg->cfs_rq[i]->shares;
1500 }
1501
1502 if ((!shares && rq_weight) || shares > tg->shares)
1503 shares = tg->shares;
1504
1505 if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE))
1506 shares = tg->shares;
1507
1508 if (!rq_weight)
1509 rq_weight = cpus_weight(sd->span) * NICE_0_LOAD;
1510
1511 for_each_cpu_mask(i, sd->span) {
1512 struct rq *rq = cpu_rq(i);
1513 unsigned long flags;
1514
1515 spin_lock_irqsave(&rq->lock, flags);
1516 __update_group_shares_cpu(tg, i, shares, rq_weight);
1517 spin_unlock_irqrestore(&rq->lock, flags);
1518 }
1489} 1519}
1520
1521/*
1522 * Compute the cpu's hierarchical load factor for each task group.
1523 * This needs to be done in a top-down fashion because the load of a child
1524 * group is a fraction of its parents load.
1525 */
1526static void
1527tg_load_down(struct task_group *tg, int cpu, struct sched_domain *sd)
1528{
1529 unsigned long load;
1530
1531 if (!tg->parent) {
1532 load = cpu_rq(cpu)->load.weight;
1533 } else {
1534 load = tg->parent->cfs_rq[cpu]->h_load;
1535 load *= tg->cfs_rq[cpu]->shares;
1536 load /= tg->parent->cfs_rq[cpu]->load.weight + 1;
1537 }
1538
1539 tg->cfs_rq[cpu]->h_load = load;
1540}
1541
1542static void
1543tg_nop(struct task_group *tg, int cpu, struct sched_domain *sd)
1544{
1545}
1546
1547static void update_shares(struct sched_domain *sd)
1548{
1549 u64 now = cpu_clock(raw_smp_processor_id());
1550 s64 elapsed = now - sd->last_update;
1551
1552 if (elapsed >= (s64)(u64)sysctl_sched_shares_ratelimit) {
1553 sd->last_update = now;
1554 walk_tg_tree(tg_nop, tg_shares_up, 0, sd);
1555 }
1556}
1557
1558static void update_shares_locked(struct rq *rq, struct sched_domain *sd)
1559{
1560 spin_unlock(&rq->lock);
1561 update_shares(sd);
1562 spin_lock(&rq->lock);
1563}
1564
1565static void update_h_load(int cpu)
1566{
1567 walk_tg_tree(tg_load_down, tg_nop, cpu, NULL);
1568}
1569
1570#else
1571
1572static inline void update_shares(struct sched_domain *sd)
1573{
1574}
1575
1576static inline void update_shares_locked(struct rq *rq, struct sched_domain *sd)
1577{
1578}
1579
1490#endif 1580#endif
1491 1581
1492#endif /* CONFIG_SMP */ 1582#endif
1583
1584#ifdef CONFIG_FAIR_GROUP_SCHED
1585static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
1586{
1587#ifdef CONFIG_SMP
1588 cfs_rq->shares = shares;
1589#endif
1590}
1591#endif
1493 1592
1494#include "sched_stats.h" 1593#include "sched_stats.h"
1495#include "sched_idletask.c" 1594#include "sched_idletask.c"
@@ -1500,27 +1599,17 @@ static void cfs_rq_set_shares(struct cfs_rq *cfs_rq, unsigned long shares)
1500#endif 1599#endif
1501 1600
1502#define sched_class_highest (&rt_sched_class) 1601#define sched_class_highest (&rt_sched_class)
1602#define for_each_class(class) \
1603 for (class = sched_class_highest; class; class = class->next)
1503 1604
1504static inline void inc_load(struct rq *rq, const struct task_struct *p) 1605static void inc_nr_running(struct rq *rq)
1505{
1506 update_load_add(&rq->load, p->se.load.weight);
1507}
1508
1509static inline void dec_load(struct rq *rq, const struct task_struct *p)
1510{
1511 update_load_sub(&rq->load, p->se.load.weight);
1512}
1513
1514static void inc_nr_running(struct task_struct *p, struct rq *rq)
1515{ 1606{
1516 rq->nr_running++; 1607 rq->nr_running++;
1517 inc_load(rq, p);
1518} 1608}
1519 1609
1520static void dec_nr_running(struct task_struct *p, struct rq *rq) 1610static void dec_nr_running(struct rq *rq)
1521{ 1611{
1522 rq->nr_running--; 1612 rq->nr_running--;
1523 dec_load(rq, p);
1524} 1613}
1525 1614
1526static void set_load_weight(struct task_struct *p) 1615static void set_load_weight(struct task_struct *p)
@@ -1544,6 +1633,12 @@ static void set_load_weight(struct task_struct *p)
1544 p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO]; 1633 p->se.load.inv_weight = prio_to_wmult[p->static_prio - MAX_RT_PRIO];
1545} 1634}
1546 1635
1636static void update_avg(u64 *avg, u64 sample)
1637{
1638 s64 diff = sample - *avg;
1639 *avg += diff >> 3;
1640}
1641
1547static void enqueue_task(struct rq *rq, struct task_struct *p, int wakeup) 1642static void enqueue_task(struct rq *rq, struct task_struct *p, int wakeup)
1548{ 1643{
1549 sched_info_queued(p); 1644 sched_info_queued(p);
@@ -1553,6 +1648,13 @@ static void enqueue_task(struct rq *rq, struct task_struct *p, int wakeup)
1553 1648
1554static void dequeue_task(struct rq *rq, struct task_struct *p, int sleep) 1649static void dequeue_task(struct rq *rq, struct task_struct *p, int sleep)
1555{ 1650{
1651 if (sleep && p->se.last_wakeup) {
1652 update_avg(&p->se.avg_overlap,
1653 p->se.sum_exec_runtime - p->se.last_wakeup);
1654 p->se.last_wakeup = 0;
1655 }
1656
1657 sched_info_dequeued(p);
1556 p->sched_class->dequeue_task(rq, p, sleep); 1658 p->sched_class->dequeue_task(rq, p, sleep);
1557 p->se.on_rq = 0; 1659 p->se.on_rq = 0;
1558} 1660}
@@ -1612,7 +1714,7 @@ static void activate_task(struct rq *rq, struct task_struct *p, int wakeup)
1612 rq->nr_uninterruptible--; 1714 rq->nr_uninterruptible--;
1613 1715
1614 enqueue_task(rq, p, wakeup); 1716 enqueue_task(rq, p, wakeup);
1615 inc_nr_running(p, rq); 1717 inc_nr_running(rq);
1616} 1718}
1617 1719
1618/* 1720/*
@@ -1624,7 +1726,7 @@ static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep)
1624 rq->nr_uninterruptible++; 1726 rq->nr_uninterruptible++;
1625 1727
1626 dequeue_task(rq, p, sleep); 1728 dequeue_task(rq, p, sleep);
1627 dec_nr_running(p, rq); 1729 dec_nr_running(rq);
1628} 1730}
1629 1731
1630/** 1732/**
@@ -1636,12 +1738,6 @@ inline int task_curr(const struct task_struct *p)
1636 return cpu_curr(task_cpu(p)) == p; 1738 return cpu_curr(task_cpu(p)) == p;
1637} 1739}
1638 1740
1639/* Used instead of source_load when we know the type == 0 */
1640unsigned long weighted_cpuload(const int cpu)
1641{
1642 return cpu_rq(cpu)->load.weight;
1643}
1644
1645static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu) 1741static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
1646{ 1742{
1647 set_task_rq(p, cpu); 1743 set_task_rq(p, cpu);
@@ -1670,6 +1766,12 @@ static inline void check_class_changed(struct rq *rq, struct task_struct *p,
1670 1766
1671#ifdef CONFIG_SMP 1767#ifdef CONFIG_SMP
1672 1768
1769/* Used instead of source_load when we know the type == 0 */
1770static unsigned long weighted_cpuload(const int cpu)
1771{
1772 return cpu_rq(cpu)->load.weight;
1773}
1774
1673/* 1775/*
1674 * Is this task likely cache-hot: 1776 * Is this task likely cache-hot:
1675 */ 1777 */
@@ -1765,16 +1867,24 @@ migrate_task(struct task_struct *p, int dest_cpu, struct migration_req *req)
1765/* 1867/*
1766 * wait_task_inactive - wait for a thread to unschedule. 1868 * wait_task_inactive - wait for a thread to unschedule.
1767 * 1869 *
1870 * If @match_state is nonzero, it's the @p->state value just checked and
1871 * not expected to change. If it changes, i.e. @p might have woken up,
1872 * then return zero. When we succeed in waiting for @p to be off its CPU,
1873 * we return a positive number (its total switch count). If a second call
1874 * a short while later returns the same number, the caller can be sure that
1875 * @p has remained unscheduled the whole time.
1876 *
1768 * The caller must ensure that the task *will* unschedule sometime soon, 1877 * The caller must ensure that the task *will* unschedule sometime soon,
1769 * else this function might spin for a *long* time. This function can't 1878 * else this function might spin for a *long* time. This function can't
1770 * be called with interrupts off, or it may introduce deadlock with 1879 * be called with interrupts off, or it may introduce deadlock with
1771 * smp_call_function() if an IPI is sent by the same process we are 1880 * smp_call_function() if an IPI is sent by the same process we are
1772 * waiting to become inactive. 1881 * waiting to become inactive.
1773 */ 1882 */
1774void wait_task_inactive(struct task_struct *p) 1883unsigned long wait_task_inactive(struct task_struct *p, long match_state)
1775{ 1884{
1776 unsigned long flags; 1885 unsigned long flags;
1777 int running, on_rq; 1886 int running, on_rq;
1887 unsigned long ncsw;
1778 struct rq *rq; 1888 struct rq *rq;
1779 1889
1780 for (;;) { 1890 for (;;) {
@@ -1797,8 +1907,11 @@ void wait_task_inactive(struct task_struct *p)
1797 * return false if the runqueue has changed and p 1907 * return false if the runqueue has changed and p
1798 * is actually now running somewhere else! 1908 * is actually now running somewhere else!
1799 */ 1909 */
1800 while (task_running(rq, p)) 1910 while (task_running(rq, p)) {
1911 if (match_state && unlikely(p->state != match_state))
1912 return 0;
1801 cpu_relax(); 1913 cpu_relax();
1914 }
1802 1915
1803 /* 1916 /*
1804 * Ok, time to look more closely! We need the rq 1917 * Ok, time to look more closely! We need the rq
@@ -1808,9 +1921,21 @@ void wait_task_inactive(struct task_struct *p)
1808 rq = task_rq_lock(p, &flags); 1921 rq = task_rq_lock(p, &flags);
1809 running = task_running(rq, p); 1922 running = task_running(rq, p);
1810 on_rq = p->se.on_rq; 1923 on_rq = p->se.on_rq;
1924 ncsw = 0;
1925 if (!match_state || p->state == match_state) {
1926 ncsw = p->nivcsw + p->nvcsw;
1927 if (unlikely(!ncsw))
1928 ncsw = 1;
1929 }
1811 task_rq_unlock(rq, &flags); 1930 task_rq_unlock(rq, &flags);
1812 1931
1813 /* 1932 /*
1933 * If it changed from the expected state, bail out now.
1934 */
1935 if (unlikely(!ncsw))
1936 break;
1937
1938 /*
1814 * Was it really running after all now that we 1939 * Was it really running after all now that we
1815 * checked with the proper locks actually held? 1940 * checked with the proper locks actually held?
1816 * 1941 *
@@ -1842,6 +1967,8 @@ void wait_task_inactive(struct task_struct *p)
1842 */ 1967 */
1843 break; 1968 break;
1844 } 1969 }
1970
1971 return ncsw;
1845} 1972}
1846 1973
1847/*** 1974/***
@@ -1880,7 +2007,7 @@ static unsigned long source_load(int cpu, int type)
1880 struct rq *rq = cpu_rq(cpu); 2007 struct rq *rq = cpu_rq(cpu);
1881 unsigned long total = weighted_cpuload(cpu); 2008 unsigned long total = weighted_cpuload(cpu);
1882 2009
1883 if (type == 0) 2010 if (type == 0 || !sched_feat(LB_BIAS))
1884 return total; 2011 return total;
1885 2012
1886 return min(rq->cpu_load[type-1], total); 2013 return min(rq->cpu_load[type-1], total);
@@ -1895,25 +2022,13 @@ static unsigned long target_load(int cpu, int type)
1895 struct rq *rq = cpu_rq(cpu); 2022 struct rq *rq = cpu_rq(cpu);
1896 unsigned long total = weighted_cpuload(cpu); 2023 unsigned long total = weighted_cpuload(cpu);
1897 2024
1898 if (type == 0) 2025 if (type == 0 || !sched_feat(LB_BIAS))
1899 return total; 2026 return total;
1900 2027
1901 return max(rq->cpu_load[type-1], total); 2028 return max(rq->cpu_load[type-1], total);
1902} 2029}
1903 2030
1904/* 2031/*
1905 * Return the average load per task on the cpu's run queue
1906 */
1907static unsigned long cpu_avg_load_per_task(int cpu)
1908{
1909 struct rq *rq = cpu_rq(cpu);
1910 unsigned long total = weighted_cpuload(cpu);
1911 unsigned long n = rq->nr_running;
1912
1913 return n ? total / n : SCHED_LOAD_SCALE;
1914}
1915
1916/*
1917 * find_idlest_group finds and returns the least busy CPU group within the 2032 * find_idlest_group finds and returns the least busy CPU group within the
1918 * domain. 2033 * domain.
1919 */ 2034 */
@@ -1939,7 +2054,7 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
1939 /* Tally up the load of all CPUs in the group */ 2054 /* Tally up the load of all CPUs in the group */
1940 avg_load = 0; 2055 avg_load = 0;
1941 2056
1942 for_each_cpu_mask(i, group->cpumask) { 2057 for_each_cpu_mask_nr(i, group->cpumask) {
1943 /* Bias balancing toward cpus of our domain */ 2058 /* Bias balancing toward cpus of our domain */
1944 if (local_group) 2059 if (local_group)
1945 load = source_load(i, load_idx); 2060 load = source_load(i, load_idx);
@@ -1981,7 +2096,7 @@ find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu,
1981 /* Traverse only the allowed CPUs */ 2096 /* Traverse only the allowed CPUs */
1982 cpus_and(*tmp, group->cpumask, p->cpus_allowed); 2097 cpus_and(*tmp, group->cpumask, p->cpus_allowed);
1983 2098
1984 for_each_cpu_mask(i, *tmp) { 2099 for_each_cpu_mask_nr(i, *tmp) {
1985 load = weighted_cpuload(i); 2100 load = weighted_cpuload(i);
1986 2101
1987 if (load < min_load || (load == min_load && i == this_cpu)) { 2102 if (load < min_load || (load == min_load && i == this_cpu)) {
@@ -2019,6 +2134,9 @@ static int sched_balance_self(int cpu, int flag)
2019 sd = tmp; 2134 sd = tmp;
2020 } 2135 }
2021 2136
2137 if (sd)
2138 update_shares(sd);
2139
2022 while (sd) { 2140 while (sd) {
2023 cpumask_t span, tmpmask; 2141 cpumask_t span, tmpmask;
2024 struct sched_group *group; 2142 struct sched_group *group;
@@ -2085,6 +2203,22 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
2085 if (!sched_feat(SYNC_WAKEUPS)) 2203 if (!sched_feat(SYNC_WAKEUPS))
2086 sync = 0; 2204 sync = 0;
2087 2205
2206#ifdef CONFIG_SMP
2207 if (sched_feat(LB_WAKEUP_UPDATE)) {
2208 struct sched_domain *sd;
2209
2210 this_cpu = raw_smp_processor_id();
2211 cpu = task_cpu(p);
2212
2213 for_each_domain(this_cpu, sd) {
2214 if (cpu_isset(cpu, sd->span)) {
2215 update_shares(sd);
2216 break;
2217 }
2218 }
2219 }
2220#endif
2221
2088 smp_wmb(); 2222 smp_wmb();
2089 rq = task_rq_lock(p, &flags); 2223 rq = task_rq_lock(p, &flags);
2090 old_state = p->state; 2224 old_state = p->state;
@@ -2131,7 +2265,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
2131 } 2265 }
2132 } 2266 }
2133 } 2267 }
2134#endif 2268#endif /* CONFIG_SCHEDSTATS */
2135 2269
2136out_activate: 2270out_activate:
2137#endif /* CONFIG_SMP */ 2271#endif /* CONFIG_SMP */
@@ -2149,6 +2283,9 @@ out_activate:
2149 success = 1; 2283 success = 1;
2150 2284
2151out_running: 2285out_running:
2286 trace_mark(kernel_sched_wakeup,
2287 "pid %d state %ld ## rq %p task %p rq->curr %p",
2288 p->pid, p->state, rq, p, rq->curr);
2152 check_preempt_curr(rq, p); 2289 check_preempt_curr(rq, p);
2153 2290
2154 p->state = TASK_RUNNING; 2291 p->state = TASK_RUNNING;
@@ -2157,6 +2294,8 @@ out_running:
2157 p->sched_class->task_wake_up(rq, p); 2294 p->sched_class->task_wake_up(rq, p);
2158#endif 2295#endif
2159out: 2296out:
2297 current->se.last_wakeup = current->se.sum_exec_runtime;
2298
2160 task_rq_unlock(rq, &flags); 2299 task_rq_unlock(rq, &flags);
2161 2300
2162 return success; 2301 return success;
@@ -2277,8 +2416,11 @@ void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
2277 * management (if any): 2416 * management (if any):
2278 */ 2417 */
2279 p->sched_class->task_new(rq, p); 2418 p->sched_class->task_new(rq, p);
2280 inc_nr_running(p, rq); 2419 inc_nr_running(rq);
2281 } 2420 }
2421 trace_mark(kernel_sched_wakeup_new,
2422 "pid %d state %ld ## rq %p task %p rq->curr %p",
2423 p->pid, p->state, rq, p, rq->curr);
2282 check_preempt_curr(rq, p); 2424 check_preempt_curr(rq, p);
2283#ifdef CONFIG_SMP 2425#ifdef CONFIG_SMP
2284 if (p->sched_class->task_wake_up) 2426 if (p->sched_class->task_wake_up)
@@ -2331,7 +2473,7 @@ fire_sched_out_preempt_notifiers(struct task_struct *curr,
2331 notifier->ops->sched_out(notifier, next); 2473 notifier->ops->sched_out(notifier, next);
2332} 2474}
2333 2475
2334#else 2476#else /* !CONFIG_PREEMPT_NOTIFIERS */
2335 2477
2336static void fire_sched_in_preempt_notifiers(struct task_struct *curr) 2478static void fire_sched_in_preempt_notifiers(struct task_struct *curr)
2337{ 2479{
@@ -2343,7 +2485,7 @@ fire_sched_out_preempt_notifiers(struct task_struct *curr,
2343{ 2485{
2344} 2486}
2345 2487
2346#endif 2488#endif /* CONFIG_PREEMPT_NOTIFIERS */
2347 2489
2348/** 2490/**
2349 * prepare_task_switch - prepare to switch tasks 2491 * prepare_task_switch - prepare to switch tasks
@@ -2451,6 +2593,11 @@ context_switch(struct rq *rq, struct task_struct *prev,
2451 struct mm_struct *mm, *oldmm; 2593 struct mm_struct *mm, *oldmm;
2452 2594
2453 prepare_task_switch(rq, prev, next); 2595 prepare_task_switch(rq, prev, next);
2596 trace_mark(kernel_sched_schedule,
2597 "prev_pid %d next_pid %d prev_state %ld "
2598 "## rq %p prev %p next %p",
2599 prev->pid, next->pid, prev->state,
2600 rq, prev, next);
2454 mm = next->mm; 2601 mm = next->mm;
2455 oldmm = prev->active_mm; 2602 oldmm = prev->active_mm;
2456 /* 2603 /*
@@ -2680,7 +2827,7 @@ static void sched_migrate_task(struct task_struct *p, int dest_cpu)
2680 2827
2681 rq = task_rq_lock(p, &flags); 2828 rq = task_rq_lock(p, &flags);
2682 if (!cpu_isset(dest_cpu, p->cpus_allowed) 2829 if (!cpu_isset(dest_cpu, p->cpus_allowed)
2683 || unlikely(cpu_is_offline(dest_cpu))) 2830 || unlikely(!cpu_active(dest_cpu)))
2684 goto out; 2831 goto out;
2685 2832
2686 /* force the process onto the specified CPU */ 2833 /* force the process onto the specified CPU */
@@ -2785,7 +2932,7 @@ balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
2785 enum cpu_idle_type idle, int *all_pinned, 2932 enum cpu_idle_type idle, int *all_pinned,
2786 int *this_best_prio, struct rq_iterator *iterator) 2933 int *this_best_prio, struct rq_iterator *iterator)
2787{ 2934{
2788 int loops = 0, pulled = 0, pinned = 0, skip_for_load; 2935 int loops = 0, pulled = 0, pinned = 0;
2789 struct task_struct *p; 2936 struct task_struct *p;
2790 long rem_load_move = max_load_move; 2937 long rem_load_move = max_load_move;
2791 2938
@@ -2801,14 +2948,8 @@ balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
2801next: 2948next:
2802 if (!p || loops++ > sysctl_sched_nr_migrate) 2949 if (!p || loops++ > sysctl_sched_nr_migrate)
2803 goto out; 2950 goto out;
2804 /* 2951
2805 * To help distribute high priority tasks across CPUs we don't 2952 if ((p->se.load.weight >> 1) > rem_load_move ||
2806 * skip a task if it will be the highest priority task (i.e. smallest
2807 * prio value) on its new queue regardless of its load weight
2808 */
2809 skip_for_load = (p->se.load.weight >> 1) > rem_load_move +
2810 SCHED_LOAD_SCALE_FUZZ;
2811 if ((skip_for_load && p->prio >= *this_best_prio) ||
2812 !can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) { 2953 !can_migrate_task(p, busiest, this_cpu, sd, idle, &pinned)) {
2813 p = iterator->next(iterator->arg); 2954 p = iterator->next(iterator->arg);
2814 goto next; 2955 goto next;
@@ -2863,6 +3004,10 @@ static int move_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
2863 max_load_move - total_load_moved, 3004 max_load_move - total_load_moved,
2864 sd, idle, all_pinned, &this_best_prio); 3005 sd, idle, all_pinned, &this_best_prio);
2865 class = class->next; 3006 class = class->next;
3007
3008 if (idle == CPU_NEWLY_IDLE && this_rq->nr_running)
3009 break;
3010
2866 } while (class && max_load_move > total_load_moved); 3011 } while (class && max_load_move > total_load_moved);
2867 3012
2868 return total_load_moved > 0; 3013 return total_load_moved > 0;
@@ -2939,6 +3084,7 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
2939 max_load = this_load = total_load = total_pwr = 0; 3084 max_load = this_load = total_load = total_pwr = 0;
2940 busiest_load_per_task = busiest_nr_running = 0; 3085 busiest_load_per_task = busiest_nr_running = 0;
2941 this_load_per_task = this_nr_running = 0; 3086 this_load_per_task = this_nr_running = 0;
3087
2942 if (idle == CPU_NOT_IDLE) 3088 if (idle == CPU_NOT_IDLE)
2943 load_idx = sd->busy_idx; 3089 load_idx = sd->busy_idx;
2944 else if (idle == CPU_NEWLY_IDLE) 3090 else if (idle == CPU_NEWLY_IDLE)
@@ -2953,6 +3099,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
2953 int __group_imb = 0; 3099 int __group_imb = 0;
2954 unsigned int balance_cpu = -1, first_idle_cpu = 0; 3100 unsigned int balance_cpu = -1, first_idle_cpu = 0;
2955 unsigned long sum_nr_running, sum_weighted_load; 3101 unsigned long sum_nr_running, sum_weighted_load;
3102 unsigned long sum_avg_load_per_task;
3103 unsigned long avg_load_per_task;
2956 3104
2957 local_group = cpu_isset(this_cpu, group->cpumask); 3105 local_group = cpu_isset(this_cpu, group->cpumask);
2958 3106
@@ -2961,10 +3109,12 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
2961 3109
2962 /* Tally up the load of all CPUs in the group */ 3110 /* Tally up the load of all CPUs in the group */
2963 sum_weighted_load = sum_nr_running = avg_load = 0; 3111 sum_weighted_load = sum_nr_running = avg_load = 0;
3112 sum_avg_load_per_task = avg_load_per_task = 0;
3113
2964 max_cpu_load = 0; 3114 max_cpu_load = 0;
2965 min_cpu_load = ~0UL; 3115 min_cpu_load = ~0UL;
2966 3116
2967 for_each_cpu_mask(i, group->cpumask) { 3117 for_each_cpu_mask_nr(i, group->cpumask) {
2968 struct rq *rq; 3118 struct rq *rq;
2969 3119
2970 if (!cpu_isset(i, *cpus)) 3120 if (!cpu_isset(i, *cpus))
@@ -2994,6 +3144,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
2994 avg_load += load; 3144 avg_load += load;
2995 sum_nr_running += rq->nr_running; 3145 sum_nr_running += rq->nr_running;
2996 sum_weighted_load += weighted_cpuload(i); 3146 sum_weighted_load += weighted_cpuload(i);
3147
3148 sum_avg_load_per_task += cpu_avg_load_per_task(i);
2997 } 3149 }
2998 3150
2999 /* 3151 /*
@@ -3015,7 +3167,20 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3015 avg_load = sg_div_cpu_power(group, 3167 avg_load = sg_div_cpu_power(group,
3016 avg_load * SCHED_LOAD_SCALE); 3168 avg_load * SCHED_LOAD_SCALE);
3017 3169
3018 if ((max_cpu_load - min_cpu_load) > SCHED_LOAD_SCALE) 3170
3171 /*
3172 * Consider the group unbalanced when the imbalance is larger
3173 * than the average weight of two tasks.
3174 *
3175 * APZ: with cgroup the avg task weight can vary wildly and
3176 * might not be a suitable number - should we keep a
3177 * normalized nr_running number somewhere that negates
3178 * the hierarchy?
3179 */
3180 avg_load_per_task = sg_div_cpu_power(group,
3181 sum_avg_load_per_task * SCHED_LOAD_SCALE);
3182
3183 if ((max_cpu_load - min_cpu_load) > 2*avg_load_per_task)
3019 __group_imb = 1; 3184 __group_imb = 1;
3020 3185
3021 group_capacity = group->__cpu_power / SCHED_LOAD_SCALE; 3186 group_capacity = group->__cpu_power / SCHED_LOAD_SCALE;
@@ -3156,9 +3321,9 @@ small_imbalance:
3156 if (busiest_load_per_task > this_load_per_task) 3321 if (busiest_load_per_task > this_load_per_task)
3157 imbn = 1; 3322 imbn = 1;
3158 } else 3323 } else
3159 this_load_per_task = SCHED_LOAD_SCALE; 3324 this_load_per_task = cpu_avg_load_per_task(this_cpu);
3160 3325
3161 if (max_load - this_load + SCHED_LOAD_SCALE_FUZZ >= 3326 if (max_load - this_load + 2*busiest_load_per_task >=
3162 busiest_load_per_task * imbn) { 3327 busiest_load_per_task * imbn) {
3163 *imbalance = busiest_load_per_task; 3328 *imbalance = busiest_load_per_task;
3164 return busiest; 3329 return busiest;
@@ -3228,7 +3393,7 @@ find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
3228 unsigned long max_load = 0; 3393 unsigned long max_load = 0;
3229 int i; 3394 int i;
3230 3395
3231 for_each_cpu_mask(i, group->cpumask) { 3396 for_each_cpu_mask_nr(i, group->cpumask) {
3232 unsigned long wl; 3397 unsigned long wl;
3233 3398
3234 if (!cpu_isset(i, *cpus)) 3399 if (!cpu_isset(i, *cpus))
@@ -3284,6 +3449,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
3284 schedstat_inc(sd, lb_count[idle]); 3449 schedstat_inc(sd, lb_count[idle]);
3285 3450
3286redo: 3451redo:
3452 update_shares(sd);
3287 group = find_busiest_group(sd, this_cpu, &imbalance, idle, &sd_idle, 3453 group = find_busiest_group(sd, this_cpu, &imbalance, idle, &sd_idle,
3288 cpus, balance); 3454 cpus, balance);
3289 3455
@@ -3386,8 +3552,9 @@ redo:
3386 3552
3387 if (!ld_moved && !sd_idle && sd->flags & SD_SHARE_CPUPOWER && 3553 if (!ld_moved && !sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
3388 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE)) 3554 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
3389 return -1; 3555 ld_moved = -1;
3390 return ld_moved; 3556
3557 goto out;
3391 3558
3392out_balanced: 3559out_balanced:
3393 schedstat_inc(sd, lb_balanced[idle]); 3560 schedstat_inc(sd, lb_balanced[idle]);
@@ -3402,8 +3569,13 @@ out_one_pinned:
3402 3569
3403 if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER && 3570 if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
3404 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE)) 3571 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
3405 return -1; 3572 ld_moved = -1;
3406 return 0; 3573 else
3574 ld_moved = 0;
3575out:
3576 if (ld_moved)
3577 update_shares(sd);
3578 return ld_moved;
3407} 3579}
3408 3580
3409/* 3581/*
@@ -3438,6 +3610,7 @@ load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
3438 3610
3439 schedstat_inc(sd, lb_count[CPU_NEWLY_IDLE]); 3611 schedstat_inc(sd, lb_count[CPU_NEWLY_IDLE]);
3440redo: 3612redo:
3613 update_shares_locked(this_rq, sd);
3441 group = find_busiest_group(sd, this_cpu, &imbalance, CPU_NEWLY_IDLE, 3614 group = find_busiest_group(sd, this_cpu, &imbalance, CPU_NEWLY_IDLE,
3442 &sd_idle, cpus, NULL); 3615 &sd_idle, cpus, NULL);
3443 if (!group) { 3616 if (!group) {
@@ -3481,6 +3654,7 @@ redo:
3481 } else 3654 } else
3482 sd->nr_balance_failed = 0; 3655 sd->nr_balance_failed = 0;
3483 3656
3657 update_shares_locked(this_rq, sd);
3484 return ld_moved; 3658 return ld_moved;
3485 3659
3486out_balanced: 3660out_balanced:
@@ -3621,7 +3795,7 @@ int select_nohz_load_balancer(int stop_tick)
3621 /* 3795 /*
3622 * If we are going offline and still the leader, give up! 3796 * If we are going offline and still the leader, give up!
3623 */ 3797 */
3624 if (cpu_is_offline(cpu) && 3798 if (!cpu_active(cpu) &&
3625 atomic_read(&nohz.load_balancer) == cpu) { 3799 atomic_read(&nohz.load_balancer) == cpu) {
3626 if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu) 3800 if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
3627 BUG(); 3801 BUG();
@@ -3672,6 +3846,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3672 /* Earliest time when we have to do rebalance again */ 3846 /* Earliest time when we have to do rebalance again */
3673 unsigned long next_balance = jiffies + 60*HZ; 3847 unsigned long next_balance = jiffies + 60*HZ;
3674 int update_next_balance = 0; 3848 int update_next_balance = 0;
3849 int need_serialize;
3675 cpumask_t tmp; 3850 cpumask_t tmp;
3676 3851
3677 for_each_domain(cpu, sd) { 3852 for_each_domain(cpu, sd) {
@@ -3689,8 +3864,9 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3689 if (interval > HZ*NR_CPUS/10) 3864 if (interval > HZ*NR_CPUS/10)
3690 interval = HZ*NR_CPUS/10; 3865 interval = HZ*NR_CPUS/10;
3691 3866
3867 need_serialize = sd->flags & SD_SERIALIZE;
3692 3868
3693 if (sd->flags & SD_SERIALIZE) { 3869 if (need_serialize) {
3694 if (!spin_trylock(&balancing)) 3870 if (!spin_trylock(&balancing))
3695 goto out; 3871 goto out;
3696 } 3872 }
@@ -3706,7 +3882,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3706 } 3882 }
3707 sd->last_balance = jiffies; 3883 sd->last_balance = jiffies;
3708 } 3884 }
3709 if (sd->flags & SD_SERIALIZE) 3885 if (need_serialize)
3710 spin_unlock(&balancing); 3886 spin_unlock(&balancing);
3711out: 3887out:
3712 if (time_after(next_balance, sd->last_balance + interval)) { 3888 if (time_after(next_balance, sd->last_balance + interval)) {
@@ -3759,7 +3935,7 @@ static void run_rebalance_domains(struct softirq_action *h)
3759 int balance_cpu; 3935 int balance_cpu;
3760 3936
3761 cpu_clear(this_cpu, cpus); 3937 cpu_clear(this_cpu, cpus);
3762 for_each_cpu_mask(balance_cpu, cpus) { 3938 for_each_cpu_mask_nr(balance_cpu, cpus) {
3763 /* 3939 /*
3764 * If this cpu gets work to do, stop the load balancing 3940 * If this cpu gets work to do, stop the load balancing
3765 * work being done for other cpus. Next load 3941 * work being done for other cpus. Next load
@@ -3895,6 +4071,8 @@ void account_user_time(struct task_struct *p, cputime_t cputime)
3895 cpustat->nice = cputime64_add(cpustat->nice, tmp); 4071 cpustat->nice = cputime64_add(cpustat->nice, tmp);
3896 else 4072 else
3897 cpustat->user = cputime64_add(cpustat->user, tmp); 4073 cpustat->user = cputime64_add(cpustat->user, tmp);
4074 /* Account for user time used */
4075 acct_update_integrals(p);
3898} 4076}
3899 4077
3900/* 4078/*
@@ -4021,26 +4199,44 @@ void scheduler_tick(void)
4021#endif 4199#endif
4022} 4200}
4023 4201
4024#if defined(CONFIG_PREEMPT) && defined(CONFIG_DEBUG_PREEMPT) 4202#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \
4203 defined(CONFIG_PREEMPT_TRACER))
4204
4205static inline unsigned long get_parent_ip(unsigned long addr)
4206{
4207 if (in_lock_functions(addr)) {
4208 addr = CALLER_ADDR2;
4209 if (in_lock_functions(addr))
4210 addr = CALLER_ADDR3;
4211 }
4212 return addr;
4213}
4025 4214
4026void __kprobes add_preempt_count(int val) 4215void __kprobes add_preempt_count(int val)
4027{ 4216{
4217#ifdef CONFIG_DEBUG_PREEMPT
4028 /* 4218 /*
4029 * Underflow? 4219 * Underflow?
4030 */ 4220 */
4031 if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) 4221 if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
4032 return; 4222 return;
4223#endif
4033 preempt_count() += val; 4224 preempt_count() += val;
4225#ifdef CONFIG_DEBUG_PREEMPT
4034 /* 4226 /*
4035 * Spinlock count overflowing soon? 4227 * Spinlock count overflowing soon?
4036 */ 4228 */
4037 DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= 4229 DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
4038 PREEMPT_MASK - 10); 4230 PREEMPT_MASK - 10);
4231#endif
4232 if (preempt_count() == val)
4233 trace_preempt_off(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
4039} 4234}
4040EXPORT_SYMBOL(add_preempt_count); 4235EXPORT_SYMBOL(add_preempt_count);
4041 4236
4042void __kprobes sub_preempt_count(int val) 4237void __kprobes sub_preempt_count(int val)
4043{ 4238{
4239#ifdef CONFIG_DEBUG_PREEMPT
4044 /* 4240 /*
4045 * Underflow? 4241 * Underflow?
4046 */ 4242 */
@@ -4052,7 +4248,10 @@ void __kprobes sub_preempt_count(int val)
4052 if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && 4248 if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
4053 !(preempt_count() & PREEMPT_MASK))) 4249 !(preempt_count() & PREEMPT_MASK)))
4054 return; 4250 return;
4251#endif
4055 4252
4253 if (preempt_count() == val)
4254 trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1));
4056 preempt_count() -= val; 4255 preempt_count() -= val;
4057} 4256}
4058EXPORT_SYMBOL(sub_preempt_count); 4257EXPORT_SYMBOL(sub_preempt_count);
@@ -4070,6 +4269,7 @@ static noinline void __schedule_bug(struct task_struct *prev)
4070 prev->comm, prev->pid, preempt_count()); 4269 prev->comm, prev->pid, preempt_count());
4071 4270
4072 debug_show_held_locks(prev); 4271 debug_show_held_locks(prev);
4272 print_modules();
4073 if (irqs_disabled()) 4273 if (irqs_disabled())
4074 print_irqtrace_events(prev); 4274 print_irqtrace_events(prev);
4075 4275
@@ -4158,7 +4358,8 @@ need_resched_nonpreemptible:
4158 4358
4159 schedule_debug(prev); 4359 schedule_debug(prev);
4160 4360
4161 hrtick_clear(rq); 4361 if (sched_feat(HRTICK))
4362 hrtick_clear(rq);
4162 4363
4163 /* 4364 /*
4164 * Do the rq-clock update outside the rq lock: 4365 * Do the rq-clock update outside the rq lock:
@@ -4204,8 +4405,6 @@ need_resched_nonpreemptible:
4204 } else 4405 } else
4205 spin_unlock_irq(&rq->lock); 4406 spin_unlock_irq(&rq->lock);
4206 4407
4207 hrtick_set(rq);
4208
4209 if (unlikely(reacquire_kernel_lock(current) < 0)) 4408 if (unlikely(reacquire_kernel_lock(current) < 0))
4210 goto need_resched_nonpreemptible; 4409 goto need_resched_nonpreemptible;
4211 4410
@@ -4586,10 +4785,8 @@ void set_user_nice(struct task_struct *p, long nice)
4586 goto out_unlock; 4785 goto out_unlock;
4587 } 4786 }
4588 on_rq = p->se.on_rq; 4787 on_rq = p->se.on_rq;
4589 if (on_rq) { 4788 if (on_rq)
4590 dequeue_task(rq, p, 0); 4789 dequeue_task(rq, p, 0);
4591 dec_load(rq, p);
4592 }
4593 4790
4594 p->static_prio = NICE_TO_PRIO(nice); 4791 p->static_prio = NICE_TO_PRIO(nice);
4595 set_load_weight(p); 4792 set_load_weight(p);
@@ -4599,7 +4796,6 @@ void set_user_nice(struct task_struct *p, long nice)
4599 4796
4600 if (on_rq) { 4797 if (on_rq) {
4601 enqueue_task(rq, p, 0); 4798 enqueue_task(rq, p, 0);
4602 inc_load(rq, p);
4603 /* 4799 /*
4604 * If the task increased its priority or is running and 4800 * If the task increased its priority or is running and
4605 * lowered its priority, then reschedule its CPU: 4801 * lowered its priority, then reschedule its CPU:
@@ -4744,16 +4940,8 @@ __setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
4744 set_load_weight(p); 4940 set_load_weight(p);
4745} 4941}
4746 4942
4747/** 4943static int __sched_setscheduler(struct task_struct *p, int policy,
4748 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread. 4944 struct sched_param *param, bool user)
4749 * @p: the task in question.
4750 * @policy: new policy.
4751 * @param: structure containing the new RT priority.
4752 *
4753 * NOTE that the task may be already dead.
4754 */
4755int sched_setscheduler(struct task_struct *p, int policy,
4756 struct sched_param *param)
4757{ 4945{
4758 int retval, oldprio, oldpolicy = -1, on_rq, running; 4946 int retval, oldprio, oldpolicy = -1, on_rq, running;
4759 unsigned long flags; 4947 unsigned long flags;
@@ -4785,7 +4973,7 @@ recheck:
4785 /* 4973 /*
4786 * Allow unprivileged RT tasks to decrease priority: 4974 * Allow unprivileged RT tasks to decrease priority:
4787 */ 4975 */
4788 if (!capable(CAP_SYS_NICE)) { 4976 if (user && !capable(CAP_SYS_NICE)) {
4789 if (rt_policy(policy)) { 4977 if (rt_policy(policy)) {
4790 unsigned long rlim_rtprio; 4978 unsigned long rlim_rtprio;
4791 4979
@@ -4821,7 +5009,8 @@ recheck:
4821 * Do not allow realtime tasks into groups that have no runtime 5009 * Do not allow realtime tasks into groups that have no runtime
4822 * assigned. 5010 * assigned.
4823 */ 5011 */
4824 if (rt_policy(policy) && task_group(p)->rt_bandwidth.rt_runtime == 0) 5012 if (user
5013 && rt_policy(policy) && task_group(p)->rt_bandwidth.rt_runtime == 0)
4825 return -EPERM; 5014 return -EPERM;
4826#endif 5015#endif
4827 5016
@@ -4870,8 +5059,39 @@ recheck:
4870 5059
4871 return 0; 5060 return 0;
4872} 5061}
5062
5063/**
5064 * sched_setscheduler - change the scheduling policy and/or RT priority of a thread.
5065 * @p: the task in question.
5066 * @policy: new policy.
5067 * @param: structure containing the new RT priority.
5068 *
5069 * NOTE that the task may be already dead.
5070 */
5071int sched_setscheduler(struct task_struct *p, int policy,
5072 struct sched_param *param)
5073{
5074 return __sched_setscheduler(p, policy, param, true);
5075}
4873EXPORT_SYMBOL_GPL(sched_setscheduler); 5076EXPORT_SYMBOL_GPL(sched_setscheduler);
4874 5077
5078/**
5079 * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace.
5080 * @p: the task in question.
5081 * @policy: new policy.
5082 * @param: structure containing the new RT priority.
5083 *
5084 * Just like sched_setscheduler, only don't bother checking if the
5085 * current context has permission. For example, this is needed in
5086 * stop_machine(): we create temporary high priority worker threads,
5087 * but our caller might not have that capability.
5088 */
5089int sched_setscheduler_nocheck(struct task_struct *p, int policy,
5090 struct sched_param *param)
5091{
5092 return __sched_setscheduler(p, policy, param, false);
5093}
5094
4875static int 5095static int
4876do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) 5096do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param)
4877{ 5097{
@@ -5070,24 +5290,6 @@ asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
5070 return sched_setaffinity(pid, &new_mask); 5290 return sched_setaffinity(pid, &new_mask);
5071} 5291}
5072 5292
5073/*
5074 * Represents all cpu's present in the system
5075 * In systems capable of hotplug, this map could dynamically grow
5076 * as new cpu's are detected in the system via any platform specific
5077 * method, such as ACPI for e.g.
5078 */
5079
5080cpumask_t cpu_present_map __read_mostly;
5081EXPORT_SYMBOL(cpu_present_map);
5082
5083#ifndef CONFIG_SMP
5084cpumask_t cpu_online_map __read_mostly = CPU_MASK_ALL;
5085EXPORT_SYMBOL(cpu_online_map);
5086
5087cpumask_t cpu_possible_map __read_mostly = CPU_MASK_ALL;
5088EXPORT_SYMBOL(cpu_possible_map);
5089#endif
5090
5091long sched_getaffinity(pid_t pid, cpumask_t *mask) 5293long sched_getaffinity(pid_t pid, cpumask_t *mask)
5092{ 5294{
5093 struct task_struct *p; 5295 struct task_struct *p;
@@ -5384,7 +5586,7 @@ out_unlock:
5384 return retval; 5586 return retval;
5385} 5587}
5386 5588
5387static const char stat_nam[] = "RSDTtZX"; 5589static const char stat_nam[] = TASK_STATE_TO_CHAR_STR;
5388 5590
5389void sched_show_task(struct task_struct *p) 5591void sched_show_task(struct task_struct *p)
5390{ 5592{
@@ -5571,6 +5773,12 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
5571 goto out; 5773 goto out;
5572 } 5774 }
5573 5775
5776 if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
5777 !cpus_equal(p->cpus_allowed, *new_mask))) {
5778 ret = -EINVAL;
5779 goto out;
5780 }
5781
5574 if (p->sched_class->set_cpus_allowed) 5782 if (p->sched_class->set_cpus_allowed)
5575 p->sched_class->set_cpus_allowed(p, new_mask); 5783 p->sched_class->set_cpus_allowed(p, new_mask);
5576 else { 5784 else {
@@ -5613,7 +5821,7 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
5613 struct rq *rq_dest, *rq_src; 5821 struct rq *rq_dest, *rq_src;
5614 int ret = 0, on_rq; 5822 int ret = 0, on_rq;
5615 5823
5616 if (unlikely(cpu_is_offline(dest_cpu))) 5824 if (unlikely(!cpu_active(dest_cpu)))
5617 return ret; 5825 return ret;
5618 5826
5619 rq_src = cpu_rq(src_cpu); 5827 rq_src = cpu_rq(src_cpu);
@@ -6060,6 +6268,36 @@ static void unregister_sched_domain_sysctl(void)
6060} 6268}
6061#endif 6269#endif
6062 6270
6271static void set_rq_online(struct rq *rq)
6272{
6273 if (!rq->online) {
6274 const struct sched_class *class;
6275
6276 cpu_set(rq->cpu, rq->rd->online);
6277 rq->online = 1;
6278
6279 for_each_class(class) {
6280 if (class->rq_online)
6281 class->rq_online(rq);
6282 }
6283 }
6284}
6285
6286static void set_rq_offline(struct rq *rq)
6287{
6288 if (rq->online) {
6289 const struct sched_class *class;
6290
6291 for_each_class(class) {
6292 if (class->rq_offline)
6293 class->rq_offline(rq);
6294 }
6295
6296 cpu_clear(rq->cpu, rq->rd->online);
6297 rq->online = 0;
6298 }
6299}
6300
6063/* 6301/*
6064 * migration_call - callback that gets triggered when a CPU is added. 6302 * migration_call - callback that gets triggered when a CPU is added.
6065 * Here we can start up the necessary migration thread for the new CPU. 6303 * Here we can start up the necessary migration thread for the new CPU.
@@ -6097,7 +6335,8 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6097 spin_lock_irqsave(&rq->lock, flags); 6335 spin_lock_irqsave(&rq->lock, flags);
6098 if (rq->rd) { 6336 if (rq->rd) {
6099 BUG_ON(!cpu_isset(cpu, rq->rd->span)); 6337 BUG_ON(!cpu_isset(cpu, rq->rd->span));
6100 cpu_set(cpu, rq->rd->online); 6338
6339 set_rq_online(rq);
6101 } 6340 }
6102 spin_unlock_irqrestore(&rq->lock, flags); 6341 spin_unlock_irqrestore(&rq->lock, flags);
6103 break; 6342 break;
@@ -6158,7 +6397,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6158 spin_lock_irqsave(&rq->lock, flags); 6397 spin_lock_irqsave(&rq->lock, flags);
6159 if (rq->rd) { 6398 if (rq->rd) {
6160 BUG_ON(!cpu_isset(cpu, rq->rd->span)); 6399 BUG_ON(!cpu_isset(cpu, rq->rd->span));
6161 cpu_clear(cpu, rq->rd->online); 6400 set_rq_offline(rq);
6162 } 6401 }
6163 spin_unlock_irqrestore(&rq->lock, flags); 6402 spin_unlock_irqrestore(&rq->lock, flags);
6164 break; 6403 break;
@@ -6175,7 +6414,7 @@ static struct notifier_block __cpuinitdata migration_notifier = {
6175 .priority = 10 6414 .priority = 10
6176}; 6415};
6177 6416
6178void __init migration_init(void) 6417static int __init migration_init(void)
6179{ 6418{
6180 void *cpu = (void *)(long)smp_processor_id(); 6419 void *cpu = (void *)(long)smp_processor_id();
6181 int err; 6420 int err;
@@ -6185,13 +6424,38 @@ void __init migration_init(void)
6185 BUG_ON(err == NOTIFY_BAD); 6424 BUG_ON(err == NOTIFY_BAD);
6186 migration_call(&migration_notifier, CPU_ONLINE, cpu); 6425 migration_call(&migration_notifier, CPU_ONLINE, cpu);
6187 register_cpu_notifier(&migration_notifier); 6426 register_cpu_notifier(&migration_notifier);
6427
6428 return err;
6188} 6429}
6430early_initcall(migration_init);
6189#endif 6431#endif
6190 6432
6191#ifdef CONFIG_SMP 6433#ifdef CONFIG_SMP
6192 6434
6193#ifdef CONFIG_SCHED_DEBUG 6435#ifdef CONFIG_SCHED_DEBUG
6194 6436
6437static inline const char *sd_level_to_string(enum sched_domain_level lvl)
6438{
6439 switch (lvl) {
6440 case SD_LV_NONE:
6441 return "NONE";
6442 case SD_LV_SIBLING:
6443 return "SIBLING";
6444 case SD_LV_MC:
6445 return "MC";
6446 case SD_LV_CPU:
6447 return "CPU";
6448 case SD_LV_NODE:
6449 return "NODE";
6450 case SD_LV_ALLNODES:
6451 return "ALLNODES";
6452 case SD_LV_MAX:
6453 return "MAX";
6454
6455 }
6456 return "MAX";
6457}
6458
6195static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, 6459static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6196 cpumask_t *groupmask) 6460 cpumask_t *groupmask)
6197{ 6461{
@@ -6211,7 +6475,8 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6211 return -1; 6475 return -1;
6212 } 6476 }
6213 6477
6214 printk(KERN_CONT "span %s\n", str); 6478 printk(KERN_CONT "span %s level %s\n",
6479 str, sd_level_to_string(sd->level));
6215 6480
6216 if (!cpu_isset(cpu, sd->span)) { 6481 if (!cpu_isset(cpu, sd->span)) {
6217 printk(KERN_ERR "ERROR: domain->span does not contain " 6482 printk(KERN_ERR "ERROR: domain->span does not contain "
@@ -6295,9 +6560,9 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6295 } 6560 }
6296 kfree(groupmask); 6561 kfree(groupmask);
6297} 6562}
6298#else 6563#else /* !CONFIG_SCHED_DEBUG */
6299# define sched_domain_debug(sd, cpu) do { } while (0) 6564# define sched_domain_debug(sd, cpu) do { } while (0)
6300#endif 6565#endif /* CONFIG_SCHED_DEBUG */
6301 6566
6302static int sd_degenerate(struct sched_domain *sd) 6567static int sd_degenerate(struct sched_domain *sd)
6303{ 6568{
@@ -6357,20 +6622,16 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6357static void rq_attach_root(struct rq *rq, struct root_domain *rd) 6622static void rq_attach_root(struct rq *rq, struct root_domain *rd)
6358{ 6623{
6359 unsigned long flags; 6624 unsigned long flags;
6360 const struct sched_class *class;
6361 6625
6362 spin_lock_irqsave(&rq->lock, flags); 6626 spin_lock_irqsave(&rq->lock, flags);
6363 6627
6364 if (rq->rd) { 6628 if (rq->rd) {
6365 struct root_domain *old_rd = rq->rd; 6629 struct root_domain *old_rd = rq->rd;
6366 6630
6367 for (class = sched_class_highest; class; class = class->next) { 6631 if (cpu_isset(rq->cpu, old_rd->online))
6368 if (class->leave_domain) 6632 set_rq_offline(rq);
6369 class->leave_domain(rq);
6370 }
6371 6633
6372 cpu_clear(rq->cpu, old_rd->span); 6634 cpu_clear(rq->cpu, old_rd->span);
6373 cpu_clear(rq->cpu, old_rd->online);
6374 6635
6375 if (atomic_dec_and_test(&old_rd->refcount)) 6636 if (atomic_dec_and_test(&old_rd->refcount))
6376 kfree(old_rd); 6637 kfree(old_rd);
@@ -6381,12 +6642,7 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
6381 6642
6382 cpu_set(rq->cpu, rd->span); 6643 cpu_set(rq->cpu, rd->span);
6383 if (cpu_isset(rq->cpu, cpu_online_map)) 6644 if (cpu_isset(rq->cpu, cpu_online_map))
6384 cpu_set(rq->cpu, rd->online); 6645 set_rq_online(rq);
6385
6386 for (class = sched_class_highest; class; class = class->next) {
6387 if (class->join_domain)
6388 class->join_domain(rq);
6389 }
6390 6646
6391 spin_unlock_irqrestore(&rq->lock, flags); 6647 spin_unlock_irqrestore(&rq->lock, flags);
6392} 6648}
@@ -6397,6 +6653,8 @@ static void init_rootdomain(struct root_domain *rd)
6397 6653
6398 cpus_clear(rd->span); 6654 cpus_clear(rd->span);
6399 cpus_clear(rd->online); 6655 cpus_clear(rd->online);
6656
6657 cpupri_init(&rd->cpupri);
6400} 6658}
6401 6659
6402static void init_defrootdomain(void) 6660static void init_defrootdomain(void)
@@ -6458,7 +6716,8 @@ static cpumask_t cpu_isolated_map = CPU_MASK_NONE;
6458/* Setup the mask of cpus configured for isolated domains */ 6716/* Setup the mask of cpus configured for isolated domains */
6459static int __init isolated_cpu_setup(char *str) 6717static int __init isolated_cpu_setup(char *str)
6460{ 6718{
6461 int ints[NR_CPUS], i; 6719 static int __initdata ints[NR_CPUS];
6720 int i;
6462 6721
6463 str = get_options(str, ARRAY_SIZE(ints), ints); 6722 str = get_options(str, ARRAY_SIZE(ints), ints);
6464 cpus_clear(cpu_isolated_map); 6723 cpus_clear(cpu_isolated_map);
@@ -6492,7 +6751,7 @@ init_sched_build_groups(const cpumask_t *span, const cpumask_t *cpu_map,
6492 6751
6493 cpus_clear(*covered); 6752 cpus_clear(*covered);
6494 6753
6495 for_each_cpu_mask(i, *span) { 6754 for_each_cpu_mask_nr(i, *span) {
6496 struct sched_group *sg; 6755 struct sched_group *sg;
6497 int group = group_fn(i, cpu_map, &sg, tmpmask); 6756 int group = group_fn(i, cpu_map, &sg, tmpmask);
6498 int j; 6757 int j;
@@ -6503,7 +6762,7 @@ init_sched_build_groups(const cpumask_t *span, const cpumask_t *cpu_map,
6503 cpus_clear(sg->cpumask); 6762 cpus_clear(sg->cpumask);
6504 sg->__cpu_power = 0; 6763 sg->__cpu_power = 0;
6505 6764
6506 for_each_cpu_mask(j, *span) { 6765 for_each_cpu_mask_nr(j, *span) {
6507 if (group_fn(j, cpu_map, NULL, tmpmask) != group) 6766 if (group_fn(j, cpu_map, NULL, tmpmask) != group)
6508 continue; 6767 continue;
6509 6768
@@ -6539,9 +6798,9 @@ static int find_next_best_node(int node, nodemask_t *used_nodes)
6539 6798
6540 min_val = INT_MAX; 6799 min_val = INT_MAX;
6541 6800
6542 for (i = 0; i < MAX_NUMNODES; i++) { 6801 for (i = 0; i < nr_node_ids; i++) {
6543 /* Start at @node */ 6802 /* Start at @node */
6544 n = (node + i) % MAX_NUMNODES; 6803 n = (node + i) % nr_node_ids;
6545 6804
6546 if (!nr_cpus_node(n)) 6805 if (!nr_cpus_node(n))
6547 continue; 6806 continue;
@@ -6591,7 +6850,7 @@ static void sched_domain_node_span(int node, cpumask_t *span)
6591 cpus_or(*span, *span, *nodemask); 6850 cpus_or(*span, *span, *nodemask);
6592 } 6851 }
6593} 6852}
6594#endif 6853#endif /* CONFIG_NUMA */
6595 6854
6596int sched_smt_power_savings = 0, sched_mc_power_savings = 0; 6855int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
6597 6856
@@ -6610,7 +6869,7 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
6610 *sg = &per_cpu(sched_group_cpus, cpu); 6869 *sg = &per_cpu(sched_group_cpus, cpu);
6611 return cpu; 6870 return cpu;
6612} 6871}
6613#endif 6872#endif /* CONFIG_SCHED_SMT */
6614 6873
6615/* 6874/*
6616 * multi-core sched-domains: 6875 * multi-core sched-domains:
@@ -6618,7 +6877,7 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
6618#ifdef CONFIG_SCHED_MC 6877#ifdef CONFIG_SCHED_MC
6619static DEFINE_PER_CPU(struct sched_domain, core_domains); 6878static DEFINE_PER_CPU(struct sched_domain, core_domains);
6620static DEFINE_PER_CPU(struct sched_group, sched_group_core); 6879static DEFINE_PER_CPU(struct sched_group, sched_group_core);
6621#endif 6880#endif /* CONFIG_SCHED_MC */
6622 6881
6623#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT) 6882#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
6624static int 6883static int
@@ -6703,7 +6962,7 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
6703 if (!sg) 6962 if (!sg)
6704 return; 6963 return;
6705 do { 6964 do {
6706 for_each_cpu_mask(j, sg->cpumask) { 6965 for_each_cpu_mask_nr(j, sg->cpumask) {
6707 struct sched_domain *sd; 6966 struct sched_domain *sd;
6708 6967
6709 sd = &per_cpu(phys_domains, j); 6968 sd = &per_cpu(phys_domains, j);
@@ -6720,7 +6979,7 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
6720 sg = sg->next; 6979 sg = sg->next;
6721 } while (sg != group_head); 6980 } while (sg != group_head);
6722} 6981}
6723#endif 6982#endif /* CONFIG_NUMA */
6724 6983
6725#ifdef CONFIG_NUMA 6984#ifdef CONFIG_NUMA
6726/* Free memory allocated for various sched_group structures */ 6985/* Free memory allocated for various sched_group structures */
@@ -6728,14 +6987,14 @@ static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
6728{ 6987{
6729 int cpu, i; 6988 int cpu, i;
6730 6989
6731 for_each_cpu_mask(cpu, *cpu_map) { 6990 for_each_cpu_mask_nr(cpu, *cpu_map) {
6732 struct sched_group **sched_group_nodes 6991 struct sched_group **sched_group_nodes
6733 = sched_group_nodes_bycpu[cpu]; 6992 = sched_group_nodes_bycpu[cpu];
6734 6993
6735 if (!sched_group_nodes) 6994 if (!sched_group_nodes)
6736 continue; 6995 continue;
6737 6996
6738 for (i = 0; i < MAX_NUMNODES; i++) { 6997 for (i = 0; i < nr_node_ids; i++) {
6739 struct sched_group *oldsg, *sg = sched_group_nodes[i]; 6998 struct sched_group *oldsg, *sg = sched_group_nodes[i];
6740 6999
6741 *nodemask = node_to_cpumask(i); 7000 *nodemask = node_to_cpumask(i);
@@ -6757,11 +7016,11 @@ next_sg:
6757 sched_group_nodes_bycpu[cpu] = NULL; 7016 sched_group_nodes_bycpu[cpu] = NULL;
6758 } 7017 }
6759} 7018}
6760#else 7019#else /* !CONFIG_NUMA */
6761static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask) 7020static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
6762{ 7021{
6763} 7022}
6764#endif 7023#endif /* CONFIG_NUMA */
6765 7024
6766/* 7025/*
6767 * Initialize sched groups cpu_power. 7026 * Initialize sched groups cpu_power.
@@ -6928,7 +7187,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
6928 /* 7187 /*
6929 * Allocate the per-node list of sched groups 7188 * Allocate the per-node list of sched groups
6930 */ 7189 */
6931 sched_group_nodes = kcalloc(MAX_NUMNODES, sizeof(struct sched_group *), 7190 sched_group_nodes = kcalloc(nr_node_ids, sizeof(struct sched_group *),
6932 GFP_KERNEL); 7191 GFP_KERNEL);
6933 if (!sched_group_nodes) { 7192 if (!sched_group_nodes) {
6934 printk(KERN_WARNING "Can not alloc sched group node list\n"); 7193 printk(KERN_WARNING "Can not alloc sched group node list\n");
@@ -6967,7 +7226,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
6967 /* 7226 /*
6968 * Set up domains for cpus specified by the cpu_map. 7227 * Set up domains for cpus specified by the cpu_map.
6969 */ 7228 */
6970 for_each_cpu_mask(i, *cpu_map) { 7229 for_each_cpu_mask_nr(i, *cpu_map) {
6971 struct sched_domain *sd = NULL, *p; 7230 struct sched_domain *sd = NULL, *p;
6972 SCHED_CPUMASK_VAR(nodemask, allmasks); 7231 SCHED_CPUMASK_VAR(nodemask, allmasks);
6973 7232
@@ -7034,7 +7293,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7034 7293
7035#ifdef CONFIG_SCHED_SMT 7294#ifdef CONFIG_SCHED_SMT
7036 /* Set up CPU (sibling) groups */ 7295 /* Set up CPU (sibling) groups */
7037 for_each_cpu_mask(i, *cpu_map) { 7296 for_each_cpu_mask_nr(i, *cpu_map) {
7038 SCHED_CPUMASK_VAR(this_sibling_map, allmasks); 7297 SCHED_CPUMASK_VAR(this_sibling_map, allmasks);
7039 SCHED_CPUMASK_VAR(send_covered, allmasks); 7298 SCHED_CPUMASK_VAR(send_covered, allmasks);
7040 7299
@@ -7051,7 +7310,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7051 7310
7052#ifdef CONFIG_SCHED_MC 7311#ifdef CONFIG_SCHED_MC
7053 /* Set up multi-core groups */ 7312 /* Set up multi-core groups */
7054 for_each_cpu_mask(i, *cpu_map) { 7313 for_each_cpu_mask_nr(i, *cpu_map) {
7055 SCHED_CPUMASK_VAR(this_core_map, allmasks); 7314 SCHED_CPUMASK_VAR(this_core_map, allmasks);
7056 SCHED_CPUMASK_VAR(send_covered, allmasks); 7315 SCHED_CPUMASK_VAR(send_covered, allmasks);
7057 7316
@@ -7067,7 +7326,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7067#endif 7326#endif
7068 7327
7069 /* Set up physical groups */ 7328 /* Set up physical groups */
7070 for (i = 0; i < MAX_NUMNODES; i++) { 7329 for (i = 0; i < nr_node_ids; i++) {
7071 SCHED_CPUMASK_VAR(nodemask, allmasks); 7330 SCHED_CPUMASK_VAR(nodemask, allmasks);
7072 SCHED_CPUMASK_VAR(send_covered, allmasks); 7331 SCHED_CPUMASK_VAR(send_covered, allmasks);
7073 7332
@@ -7091,7 +7350,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7091 send_covered, tmpmask); 7350 send_covered, tmpmask);
7092 } 7351 }
7093 7352
7094 for (i = 0; i < MAX_NUMNODES; i++) { 7353 for (i = 0; i < nr_node_ids; i++) {
7095 /* Set up node groups */ 7354 /* Set up node groups */
7096 struct sched_group *sg, *prev; 7355 struct sched_group *sg, *prev;
7097 SCHED_CPUMASK_VAR(nodemask, allmasks); 7356 SCHED_CPUMASK_VAR(nodemask, allmasks);
@@ -7118,7 +7377,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7118 goto error; 7377 goto error;
7119 } 7378 }
7120 sched_group_nodes[i] = sg; 7379 sched_group_nodes[i] = sg;
7121 for_each_cpu_mask(j, *nodemask) { 7380 for_each_cpu_mask_nr(j, *nodemask) {
7122 struct sched_domain *sd; 7381 struct sched_domain *sd;
7123 7382
7124 sd = &per_cpu(node_domains, j); 7383 sd = &per_cpu(node_domains, j);
@@ -7130,9 +7389,9 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7130 cpus_or(*covered, *covered, *nodemask); 7389 cpus_or(*covered, *covered, *nodemask);
7131 prev = sg; 7390 prev = sg;
7132 7391
7133 for (j = 0; j < MAX_NUMNODES; j++) { 7392 for (j = 0; j < nr_node_ids; j++) {
7134 SCHED_CPUMASK_VAR(notcovered, allmasks); 7393 SCHED_CPUMASK_VAR(notcovered, allmasks);
7135 int n = (i + j) % MAX_NUMNODES; 7394 int n = (i + j) % nr_node_ids;
7136 node_to_cpumask_ptr(pnodemask, n); 7395 node_to_cpumask_ptr(pnodemask, n);
7137 7396
7138 cpus_complement(*notcovered, *covered); 7397 cpus_complement(*notcovered, *covered);
@@ -7164,28 +7423,28 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7164 7423
7165 /* Calculate CPU power for physical packages and nodes */ 7424 /* Calculate CPU power for physical packages and nodes */
7166#ifdef CONFIG_SCHED_SMT 7425#ifdef CONFIG_SCHED_SMT
7167 for_each_cpu_mask(i, *cpu_map) { 7426 for_each_cpu_mask_nr(i, *cpu_map) {
7168 struct sched_domain *sd = &per_cpu(cpu_domains, i); 7427 struct sched_domain *sd = &per_cpu(cpu_domains, i);
7169 7428
7170 init_sched_groups_power(i, sd); 7429 init_sched_groups_power(i, sd);
7171 } 7430 }
7172#endif 7431#endif
7173#ifdef CONFIG_SCHED_MC 7432#ifdef CONFIG_SCHED_MC
7174 for_each_cpu_mask(i, *cpu_map) { 7433 for_each_cpu_mask_nr(i, *cpu_map) {
7175 struct sched_domain *sd = &per_cpu(core_domains, i); 7434 struct sched_domain *sd = &per_cpu(core_domains, i);
7176 7435
7177 init_sched_groups_power(i, sd); 7436 init_sched_groups_power(i, sd);
7178 } 7437 }
7179#endif 7438#endif
7180 7439
7181 for_each_cpu_mask(i, *cpu_map) { 7440 for_each_cpu_mask_nr(i, *cpu_map) {
7182 struct sched_domain *sd = &per_cpu(phys_domains, i); 7441 struct sched_domain *sd = &per_cpu(phys_domains, i);
7183 7442
7184 init_sched_groups_power(i, sd); 7443 init_sched_groups_power(i, sd);
7185 } 7444 }
7186 7445
7187#ifdef CONFIG_NUMA 7446#ifdef CONFIG_NUMA
7188 for (i = 0; i < MAX_NUMNODES; i++) 7447 for (i = 0; i < nr_node_ids; i++)
7189 init_numa_sched_groups_power(sched_group_nodes[i]); 7448 init_numa_sched_groups_power(sched_group_nodes[i]);
7190 7449
7191 if (sd_allnodes) { 7450 if (sd_allnodes) {
@@ -7198,7 +7457,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7198#endif 7457#endif
7199 7458
7200 /* Attach the domains */ 7459 /* Attach the domains */
7201 for_each_cpu_mask(i, *cpu_map) { 7460 for_each_cpu_mask_nr(i, *cpu_map) {
7202 struct sched_domain *sd; 7461 struct sched_domain *sd;
7203#ifdef CONFIG_SCHED_SMT 7462#ifdef CONFIG_SCHED_SMT
7204 sd = &per_cpu(cpu_domains, i); 7463 sd = &per_cpu(cpu_domains, i);
@@ -7243,18 +7502,6 @@ void __attribute__((weak)) arch_update_cpu_topology(void)
7243} 7502}
7244 7503
7245/* 7504/*
7246 * Free current domain masks.
7247 * Called after all cpus are attached to NULL domain.
7248 */
7249static void free_sched_domains(void)
7250{
7251 ndoms_cur = 0;
7252 if (doms_cur != &fallback_doms)
7253 kfree(doms_cur);
7254 doms_cur = &fallback_doms;
7255}
7256
7257/*
7258 * Set up scheduler domains and groups. Callers must hold the hotplug lock. 7505 * Set up scheduler domains and groups. Callers must hold the hotplug lock.
7259 * For now this just excludes isolated cpus, but could be used to 7506 * For now this just excludes isolated cpus, but could be used to
7260 * exclude other special cases in the future. 7507 * exclude other special cases in the future.
@@ -7293,7 +7540,7 @@ static void detach_destroy_domains(const cpumask_t *cpu_map)
7293 7540
7294 unregister_sched_domain_sysctl(); 7541 unregister_sched_domain_sysctl();
7295 7542
7296 for_each_cpu_mask(i, *cpu_map) 7543 for_each_cpu_mask_nr(i, *cpu_map)
7297 cpu_attach_domain(NULL, &def_root_domain, i); 7544 cpu_attach_domain(NULL, &def_root_domain, i);
7298 synchronize_sched(); 7545 synchronize_sched();
7299 arch_destroy_sched_domains(cpu_map, &tmpmask); 7546 arch_destroy_sched_domains(cpu_map, &tmpmask);
@@ -7332,7 +7579,7 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
7332 * ownership of it and will kfree it when done with it. If the caller 7579 * ownership of it and will kfree it when done with it. If the caller
7333 * failed the kmalloc call, then it can pass in doms_new == NULL, 7580 * failed the kmalloc call, then it can pass in doms_new == NULL,
7334 * and partition_sched_domains() will fallback to the single partition 7581 * and partition_sched_domains() will fallback to the single partition
7335 * 'fallback_doms'. 7582 * 'fallback_doms', it also forces the domains to be rebuilt.
7336 * 7583 *
7337 * Call with hotplug lock held 7584 * Call with hotplug lock held
7338 */ 7585 */
@@ -7346,12 +7593,8 @@ void partition_sched_domains(int ndoms_new, cpumask_t *doms_new,
7346 /* always unregister in case we don't destroy any domains */ 7593 /* always unregister in case we don't destroy any domains */
7347 unregister_sched_domain_sysctl(); 7594 unregister_sched_domain_sysctl();
7348 7595
7349 if (doms_new == NULL) { 7596 if (doms_new == NULL)
7350 ndoms_new = 1; 7597 ndoms_new = 0;
7351 doms_new = &fallback_doms;
7352 cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map);
7353 dattr_new = NULL;
7354 }
7355 7598
7356 /* Destroy deleted domains */ 7599 /* Destroy deleted domains */
7357 for (i = 0; i < ndoms_cur; i++) { 7600 for (i = 0; i < ndoms_cur; i++) {
@@ -7366,6 +7609,14 @@ match1:
7366 ; 7609 ;
7367 } 7610 }
7368 7611
7612 if (doms_new == NULL) {
7613 ndoms_cur = 0;
7614 ndoms_new = 1;
7615 doms_new = &fallback_doms;
7616 cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map);
7617 dattr_new = NULL;
7618 }
7619
7369 /* Build new domains */ 7620 /* Build new domains */
7370 for (i = 0; i < ndoms_new; i++) { 7621 for (i = 0; i < ndoms_new; i++) {
7371 for (j = 0; j < ndoms_cur; j++) { 7622 for (j = 0; j < ndoms_cur; j++) {
@@ -7396,17 +7647,10 @@ match2:
7396#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT) 7647#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
7397int arch_reinit_sched_domains(void) 7648int arch_reinit_sched_domains(void)
7398{ 7649{
7399 int err;
7400
7401 get_online_cpus(); 7650 get_online_cpus();
7402 mutex_lock(&sched_domains_mutex); 7651 rebuild_sched_domains();
7403 detach_destroy_domains(&cpu_online_map);
7404 free_sched_domains();
7405 err = arch_init_sched_domains(&cpu_online_map);
7406 mutex_unlock(&sched_domains_mutex);
7407 put_online_cpus(); 7652 put_online_cpus();
7408 7653 return 0;
7409 return err;
7410} 7654}
7411 7655
7412static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt) 7656static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
@@ -7427,11 +7671,13 @@ static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
7427} 7671}
7428 7672
7429#ifdef CONFIG_SCHED_MC 7673#ifdef CONFIG_SCHED_MC
7430static ssize_t sched_mc_power_savings_show(struct sys_device *dev, char *page) 7674static ssize_t sched_mc_power_savings_show(struct sys_device *dev,
7675 struct sysdev_attribute *attr, char *page)
7431{ 7676{
7432 return sprintf(page, "%u\n", sched_mc_power_savings); 7677 return sprintf(page, "%u\n", sched_mc_power_savings);
7433} 7678}
7434static ssize_t sched_mc_power_savings_store(struct sys_device *dev, 7679static ssize_t sched_mc_power_savings_store(struct sys_device *dev,
7680 struct sysdev_attribute *attr,
7435 const char *buf, size_t count) 7681 const char *buf, size_t count)
7436{ 7682{
7437 return sched_power_savings_store(buf, count, 0); 7683 return sched_power_savings_store(buf, count, 0);
@@ -7441,11 +7687,13 @@ static SYSDEV_ATTR(sched_mc_power_savings, 0644, sched_mc_power_savings_show,
7441#endif 7687#endif
7442 7688
7443#ifdef CONFIG_SCHED_SMT 7689#ifdef CONFIG_SCHED_SMT
7444static ssize_t sched_smt_power_savings_show(struct sys_device *dev, char *page) 7690static ssize_t sched_smt_power_savings_show(struct sys_device *dev,
7691 struct sysdev_attribute *attr, char *page)
7445{ 7692{
7446 return sprintf(page, "%u\n", sched_smt_power_savings); 7693 return sprintf(page, "%u\n", sched_smt_power_savings);
7447} 7694}
7448static ssize_t sched_smt_power_savings_store(struct sys_device *dev, 7695static ssize_t sched_smt_power_savings_store(struct sys_device *dev,
7696 struct sysdev_attribute *attr,
7449 const char *buf, size_t count) 7697 const char *buf, size_t count)
7450{ 7698{
7451 return sched_power_savings_store(buf, count, 1); 7699 return sched_power_savings_store(buf, count, 1);
@@ -7470,54 +7718,51 @@ int sched_create_sysfs_power_savings_entries(struct sysdev_class *cls)
7470#endif 7718#endif
7471 return err; 7719 return err;
7472} 7720}
7473#endif 7721#endif /* CONFIG_SCHED_MC || CONFIG_SCHED_SMT */
7474 7722
7723#ifndef CONFIG_CPUSETS
7475/* 7724/*
7476 * Force a reinitialization of the sched domains hierarchy. The domains 7725 * Add online and remove offline CPUs from the scheduler domains.
7477 * and groups cannot be updated in place without racing with the balancing 7726 * When cpusets are enabled they take over this function.
7478 * code, so we temporarily attach all running cpus to the NULL domain
7479 * which will prevent rebalancing while the sched domains are recalculated.
7480 */ 7727 */
7481static int update_sched_domains(struct notifier_block *nfb, 7728static int update_sched_domains(struct notifier_block *nfb,
7482 unsigned long action, void *hcpu) 7729 unsigned long action, void *hcpu)
7483{ 7730{
7484 switch (action) { 7731 switch (action) {
7485 case CPU_UP_PREPARE: 7732 case CPU_ONLINE:
7486 case CPU_UP_PREPARE_FROZEN: 7733 case CPU_ONLINE_FROZEN:
7734 case CPU_DEAD:
7735 case CPU_DEAD_FROZEN:
7736 partition_sched_domains(0, NULL, NULL);
7737 return NOTIFY_OK;
7738
7739 default:
7740 return NOTIFY_DONE;
7741 }
7742}
7743#endif
7744
7745static int update_runtime(struct notifier_block *nfb,
7746 unsigned long action, void *hcpu)
7747{
7748 int cpu = (int)(long)hcpu;
7749
7750 switch (action) {
7487 case CPU_DOWN_PREPARE: 7751 case CPU_DOWN_PREPARE:
7488 case CPU_DOWN_PREPARE_FROZEN: 7752 case CPU_DOWN_PREPARE_FROZEN:
7489 detach_destroy_domains(&cpu_online_map); 7753 disable_runtime(cpu_rq(cpu));
7490 free_sched_domains();
7491 return NOTIFY_OK; 7754 return NOTIFY_OK;
7492 7755
7493 case CPU_UP_CANCELED:
7494 case CPU_UP_CANCELED_FROZEN:
7495 case CPU_DOWN_FAILED: 7756 case CPU_DOWN_FAILED:
7496 case CPU_DOWN_FAILED_FROZEN: 7757 case CPU_DOWN_FAILED_FROZEN:
7497 case CPU_ONLINE: 7758 case CPU_ONLINE:
7498 case CPU_ONLINE_FROZEN: 7759 case CPU_ONLINE_FROZEN:
7499 case CPU_DEAD: 7760 enable_runtime(cpu_rq(cpu));
7500 case CPU_DEAD_FROZEN: 7761 return NOTIFY_OK;
7501 /* 7762
7502 * Fall through and re-initialise the domains.
7503 */
7504 break;
7505 default: 7763 default:
7506 return NOTIFY_DONE; 7764 return NOTIFY_DONE;
7507 } 7765 }
7508
7509#ifndef CONFIG_CPUSETS
7510 /*
7511 * Create default domain partitioning if cpusets are disabled.
7512 * Otherwise we let cpusets rebuild the domains based on the
7513 * current setup.
7514 */
7515
7516 /* The hotplug lock is already held by cpu_up/cpu_down */
7517 arch_init_sched_domains(&cpu_online_map);
7518#endif
7519
7520 return NOTIFY_OK;
7521} 7766}
7522 7767
7523void __init sched_init_smp(void) 7768void __init sched_init_smp(void)
@@ -7537,8 +7782,15 @@ void __init sched_init_smp(void)
7537 cpu_set(smp_processor_id(), non_isolated_cpus); 7782 cpu_set(smp_processor_id(), non_isolated_cpus);
7538 mutex_unlock(&sched_domains_mutex); 7783 mutex_unlock(&sched_domains_mutex);
7539 put_online_cpus(); 7784 put_online_cpus();
7785
7786#ifndef CONFIG_CPUSETS
7540 /* XXX: Theoretical race here - CPU may be hotplugged now */ 7787 /* XXX: Theoretical race here - CPU may be hotplugged now */
7541 hotcpu_notifier(update_sched_domains, 0); 7788 hotcpu_notifier(update_sched_domains, 0);
7789#endif
7790
7791 /* RT runtime code needs to handle some hotplug events */
7792 hotcpu_notifier(update_runtime, 0);
7793
7542 init_hrtick(); 7794 init_hrtick();
7543 7795
7544 /* Move init over to a non-isolated CPU */ 7796 /* Move init over to a non-isolated CPU */
@@ -7695,8 +7947,8 @@ void __init sched_init(void)
7695 7947
7696 root_task_group.cfs_rq = (struct cfs_rq **)ptr; 7948 root_task_group.cfs_rq = (struct cfs_rq **)ptr;
7697 ptr += nr_cpu_ids * sizeof(void **); 7949 ptr += nr_cpu_ids * sizeof(void **);
7698#endif 7950#endif /* CONFIG_USER_SCHED */
7699#endif 7951#endif /* CONFIG_FAIR_GROUP_SCHED */
7700#ifdef CONFIG_RT_GROUP_SCHED 7952#ifdef CONFIG_RT_GROUP_SCHED
7701 init_task_group.rt_se = (struct sched_rt_entity **)ptr; 7953 init_task_group.rt_se = (struct sched_rt_entity **)ptr;
7702 ptr += nr_cpu_ids * sizeof(void **); 7954 ptr += nr_cpu_ids * sizeof(void **);
@@ -7710,8 +7962,8 @@ void __init sched_init(void)
7710 7962
7711 root_task_group.rt_rq = (struct rt_rq **)ptr; 7963 root_task_group.rt_rq = (struct rt_rq **)ptr;
7712 ptr += nr_cpu_ids * sizeof(void **); 7964 ptr += nr_cpu_ids * sizeof(void **);
7713#endif 7965#endif /* CONFIG_USER_SCHED */
7714#endif 7966#endif /* CONFIG_RT_GROUP_SCHED */
7715 } 7967 }
7716 7968
7717#ifdef CONFIG_SMP 7969#ifdef CONFIG_SMP
@@ -7727,8 +7979,8 @@ void __init sched_init(void)
7727#ifdef CONFIG_USER_SCHED 7979#ifdef CONFIG_USER_SCHED
7728 init_rt_bandwidth(&root_task_group.rt_bandwidth, 7980 init_rt_bandwidth(&root_task_group.rt_bandwidth,
7729 global_rt_period(), RUNTIME_INF); 7981 global_rt_period(), RUNTIME_INF);
7730#endif 7982#endif /* CONFIG_USER_SCHED */
7731#endif 7983#endif /* CONFIG_RT_GROUP_SCHED */
7732 7984
7733#ifdef CONFIG_GROUP_SCHED 7985#ifdef CONFIG_GROUP_SCHED
7734 list_add(&init_task_group.list, &task_groups); 7986 list_add(&init_task_group.list, &task_groups);
@@ -7738,8 +7990,8 @@ void __init sched_init(void)
7738 INIT_LIST_HEAD(&root_task_group.children); 7990 INIT_LIST_HEAD(&root_task_group.children);
7739 init_task_group.parent = &root_task_group; 7991 init_task_group.parent = &root_task_group;
7740 list_add(&init_task_group.siblings, &root_task_group.children); 7992 list_add(&init_task_group.siblings, &root_task_group.children);
7741#endif 7993#endif /* CONFIG_USER_SCHED */
7742#endif 7994#endif /* CONFIG_GROUP_SCHED */
7743 7995
7744 for_each_possible_cpu(i) { 7996 for_each_possible_cpu(i) {
7745 struct rq *rq; 7997 struct rq *rq;
@@ -7819,6 +8071,7 @@ void __init sched_init(void)
7819 rq->next_balance = jiffies; 8071 rq->next_balance = jiffies;
7820 rq->push_cpu = 0; 8072 rq->push_cpu = 0;
7821 rq->cpu = i; 8073 rq->cpu = i;
8074 rq->online = 0;
7822 rq->migration_thread = NULL; 8075 rq->migration_thread = NULL;
7823 INIT_LIST_HEAD(&rq->migration_queue); 8076 INIT_LIST_HEAD(&rq->migration_queue);
7824 rq_attach_root(rq, &def_root_domain); 8077 rq_attach_root(rq, &def_root_domain);
@@ -7834,7 +8087,7 @@ void __init sched_init(void)
7834#endif 8087#endif
7835 8088
7836#ifdef CONFIG_SMP 8089#ifdef CONFIG_SMP
7837 open_softirq(SCHED_SOFTIRQ, run_rebalance_domains, NULL); 8090 open_softirq(SCHED_SOFTIRQ, run_rebalance_domains);
7838#endif 8091#endif
7839 8092
7840#ifdef CONFIG_RT_MUTEXES 8093#ifdef CONFIG_RT_MUTEXES
@@ -8058,7 +8311,7 @@ static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
8058{ 8311{
8059 list_del_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list); 8312 list_del_rcu(&tg->cfs_rq[cpu]->leaf_cfs_rq_list);
8060} 8313}
8061#else 8314#else /* !CONFG_FAIR_GROUP_SCHED */
8062static inline void free_fair_sched_group(struct task_group *tg) 8315static inline void free_fair_sched_group(struct task_group *tg)
8063{ 8316{
8064} 8317}
@@ -8076,7 +8329,7 @@ static inline void register_fair_sched_group(struct task_group *tg, int cpu)
8076static inline void unregister_fair_sched_group(struct task_group *tg, int cpu) 8329static inline void unregister_fair_sched_group(struct task_group *tg, int cpu)
8077{ 8330{
8078} 8331}
8079#endif 8332#endif /* CONFIG_FAIR_GROUP_SCHED */
8080 8333
8081#ifdef CONFIG_RT_GROUP_SCHED 8334#ifdef CONFIG_RT_GROUP_SCHED
8082static void free_rt_sched_group(struct task_group *tg) 8335static void free_rt_sched_group(struct task_group *tg)
@@ -8147,7 +8400,7 @@ static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
8147{ 8400{
8148 list_del_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list); 8401 list_del_rcu(&tg->rt_rq[cpu]->leaf_rt_rq_list);
8149} 8402}
8150#else 8403#else /* !CONFIG_RT_GROUP_SCHED */
8151static inline void free_rt_sched_group(struct task_group *tg) 8404static inline void free_rt_sched_group(struct task_group *tg)
8152{ 8405{
8153} 8406}
@@ -8165,7 +8418,7 @@ static inline void register_rt_sched_group(struct task_group *tg, int cpu)
8165static inline void unregister_rt_sched_group(struct task_group *tg, int cpu) 8418static inline void unregister_rt_sched_group(struct task_group *tg, int cpu)
8166{ 8419{
8167} 8420}
8168#endif 8421#endif /* CONFIG_RT_GROUP_SCHED */
8169 8422
8170#ifdef CONFIG_GROUP_SCHED 8423#ifdef CONFIG_GROUP_SCHED
8171static void free_sched_group(struct task_group *tg) 8424static void free_sched_group(struct task_group *tg)
@@ -8276,17 +8529,14 @@ void sched_move_task(struct task_struct *tsk)
8276 8529
8277 task_rq_unlock(rq, &flags); 8530 task_rq_unlock(rq, &flags);
8278} 8531}
8279#endif 8532#endif /* CONFIG_GROUP_SCHED */
8280 8533
8281#ifdef CONFIG_FAIR_GROUP_SCHED 8534#ifdef CONFIG_FAIR_GROUP_SCHED
8282static void set_se_shares(struct sched_entity *se, unsigned long shares) 8535static void __set_se_shares(struct sched_entity *se, unsigned long shares)
8283{ 8536{
8284 struct cfs_rq *cfs_rq = se->cfs_rq; 8537 struct cfs_rq *cfs_rq = se->cfs_rq;
8285 struct rq *rq = cfs_rq->rq;
8286 int on_rq; 8538 int on_rq;
8287 8539
8288 spin_lock_irq(&rq->lock);
8289
8290 on_rq = se->on_rq; 8540 on_rq = se->on_rq;
8291 if (on_rq) 8541 if (on_rq)
8292 dequeue_entity(cfs_rq, se, 0); 8542 dequeue_entity(cfs_rq, se, 0);
@@ -8296,8 +8546,17 @@ static void set_se_shares(struct sched_entity *se, unsigned long shares)
8296 8546
8297 if (on_rq) 8547 if (on_rq)
8298 enqueue_entity(cfs_rq, se, 0); 8548 enqueue_entity(cfs_rq, se, 0);
8549}
8299 8550
8300 spin_unlock_irq(&rq->lock); 8551static void set_se_shares(struct sched_entity *se, unsigned long shares)
8552{
8553 struct cfs_rq *cfs_rq = se->cfs_rq;
8554 struct rq *rq = cfs_rq->rq;
8555 unsigned long flags;
8556
8557 spin_lock_irqsave(&rq->lock, flags);
8558 __set_se_shares(se, shares);
8559 spin_unlock_irqrestore(&rq->lock, flags);
8301} 8560}
8302 8561
8303static DEFINE_MUTEX(shares_mutex); 8562static DEFINE_MUTEX(shares_mutex);
@@ -8336,8 +8595,13 @@ int sched_group_set_shares(struct task_group *tg, unsigned long shares)
8336 * w/o tripping rebalance_share or load_balance_fair. 8595 * w/o tripping rebalance_share or load_balance_fair.
8337 */ 8596 */
8338 tg->shares = shares; 8597 tg->shares = shares;
8339 for_each_possible_cpu(i) 8598 for_each_possible_cpu(i) {
8599 /*
8600 * force a rebalance
8601 */
8602 cfs_rq_set_shares(tg->cfs_rq[i], 0);
8340 set_se_shares(tg->se[i], shares); 8603 set_se_shares(tg->se[i], shares);
8604 }
8341 8605
8342 /* 8606 /*
8343 * Enable load balance activity on this group, by inserting it back on 8607 * Enable load balance activity on this group, by inserting it back on
@@ -8376,7 +8640,7 @@ static unsigned long to_ratio(u64 period, u64 runtime)
8376#ifdef CONFIG_CGROUP_SCHED 8640#ifdef CONFIG_CGROUP_SCHED
8377static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime) 8641static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
8378{ 8642{
8379 struct task_group *tgi, *parent = tg ? tg->parent : NULL; 8643 struct task_group *tgi, *parent = tg->parent;
8380 unsigned long total = 0; 8644 unsigned long total = 0;
8381 8645
8382 if (!parent) { 8646 if (!parent) {
@@ -8400,7 +8664,7 @@ static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime)
8400 } 8664 }
8401 rcu_read_unlock(); 8665 rcu_read_unlock();
8402 8666
8403 return total + to_ratio(period, runtime) < 8667 return total + to_ratio(period, runtime) <=
8404 to_ratio(ktime_to_ns(parent->rt_bandwidth.rt_period), 8668 to_ratio(ktime_to_ns(parent->rt_bandwidth.rt_period),
8405 parent->rt_bandwidth.rt_runtime); 8669 parent->rt_bandwidth.rt_runtime);
8406} 8670}
@@ -8520,16 +8784,21 @@ long sched_group_rt_period(struct task_group *tg)
8520 8784
8521static int sched_rt_global_constraints(void) 8785static int sched_rt_global_constraints(void)
8522{ 8786{
8787 struct task_group *tg = &root_task_group;
8788 u64 rt_runtime, rt_period;
8523 int ret = 0; 8789 int ret = 0;
8524 8790
8791 rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period);
8792 rt_runtime = tg->rt_bandwidth.rt_runtime;
8793
8525 mutex_lock(&rt_constraints_mutex); 8794 mutex_lock(&rt_constraints_mutex);
8526 if (!__rt_schedulable(NULL, 1, 0)) 8795 if (!__rt_schedulable(tg, rt_period, rt_runtime))
8527 ret = -EINVAL; 8796 ret = -EINVAL;
8528 mutex_unlock(&rt_constraints_mutex); 8797 mutex_unlock(&rt_constraints_mutex);
8529 8798
8530 return ret; 8799 return ret;
8531} 8800}
8532#else 8801#else /* !CONFIG_RT_GROUP_SCHED */
8533static int sched_rt_global_constraints(void) 8802static int sched_rt_global_constraints(void)
8534{ 8803{
8535 unsigned long flags; 8804 unsigned long flags;
@@ -8547,7 +8816,7 @@ static int sched_rt_global_constraints(void)
8547 8816
8548 return 0; 8817 return 0;
8549} 8818}
8550#endif 8819#endif /* CONFIG_RT_GROUP_SCHED */
8551 8820
8552int sched_rt_handler(struct ctl_table *table, int write, 8821int sched_rt_handler(struct ctl_table *table, int write,
8553 struct file *filp, void __user *buffer, size_t *lenp, 8822 struct file *filp, void __user *buffer, size_t *lenp,
@@ -8655,7 +8924,7 @@ static u64 cpu_shares_read_u64(struct cgroup *cgrp, struct cftype *cft)
8655 8924
8656 return (u64) tg->shares; 8925 return (u64) tg->shares;
8657} 8926}
8658#endif 8927#endif /* CONFIG_FAIR_GROUP_SCHED */
8659 8928
8660#ifdef CONFIG_RT_GROUP_SCHED 8929#ifdef CONFIG_RT_GROUP_SCHED
8661static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft, 8930static int cpu_rt_runtime_write(struct cgroup *cgrp, struct cftype *cft,
@@ -8679,7 +8948,7 @@ static u64 cpu_rt_period_read_uint(struct cgroup *cgrp, struct cftype *cft)
8679{ 8948{
8680 return sched_group_rt_period(cgroup_tg(cgrp)); 8949 return sched_group_rt_period(cgroup_tg(cgrp));
8681} 8950}
8682#endif 8951#endif /* CONFIG_RT_GROUP_SCHED */
8683 8952
8684static struct cftype cpu_files[] = { 8953static struct cftype cpu_files[] = {
8685#ifdef CONFIG_FAIR_GROUP_SCHED 8954#ifdef CONFIG_FAIR_GROUP_SCHED
diff --git a/kernel/sched_clock.c b/kernel/sched_clock.c
index ce05271219ab..22ed55d1167f 100644
--- a/kernel/sched_clock.c
+++ b/kernel/sched_clock.c
@@ -3,6 +3,9 @@
3 * 3 *
4 * Copyright (C) 2008 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> 4 * Copyright (C) 2008 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
5 * 5 *
6 * Updates and enhancements:
7 * Copyright (C) 2008 Red Hat, Inc. Steven Rostedt <srostedt@redhat.com>
8 *
6 * Based on code by: 9 * Based on code by:
7 * Ingo Molnar <mingo@redhat.com> 10 * Ingo Molnar <mingo@redhat.com>
8 * Guillaume Chazarain <guichaz@gmail.com> 11 * Guillaume Chazarain <guichaz@gmail.com>
@@ -32,6 +35,11 @@
32 35
33#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK 36#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
34 37
38#define MULTI_SHIFT 15
39/* Max is double, Min is 1/2 */
40#define MAX_MULTI (2LL << MULTI_SHIFT)
41#define MIN_MULTI (1LL << (MULTI_SHIFT-1))
42
35struct sched_clock_data { 43struct sched_clock_data {
36 /* 44 /*
37 * Raw spinlock - this is a special case: this might be called 45 * Raw spinlock - this is a special case: this might be called
@@ -40,11 +48,15 @@ struct sched_clock_data {
40 */ 48 */
41 raw_spinlock_t lock; 49 raw_spinlock_t lock;
42 50
43 unsigned long prev_jiffies; 51 unsigned long tick_jiffies;
44 u64 prev_raw; 52 u64 prev_raw;
45 u64 tick_raw; 53 u64 tick_raw;
46 u64 tick_gtod; 54 u64 tick_gtod;
47 u64 clock; 55 u64 clock;
56 s64 multi;
57#ifdef CONFIG_NO_HZ
58 int check_max;
59#endif
48}; 60};
49 61
50static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data); 62static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data);
@@ -71,41 +83,91 @@ void sched_clock_init(void)
71 struct sched_clock_data *scd = cpu_sdc(cpu); 83 struct sched_clock_data *scd = cpu_sdc(cpu);
72 84
73 scd->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED; 85 scd->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
74 scd->prev_jiffies = now_jiffies; 86 scd->tick_jiffies = now_jiffies;
75 scd->prev_raw = 0; 87 scd->prev_raw = 0;
76 scd->tick_raw = 0; 88 scd->tick_raw = 0;
77 scd->tick_gtod = ktime_now; 89 scd->tick_gtod = ktime_now;
78 scd->clock = ktime_now; 90 scd->clock = ktime_now;
91 scd->multi = 1 << MULTI_SHIFT;
92#ifdef CONFIG_NO_HZ
93 scd->check_max = 1;
94#endif
79 } 95 }
80 96
81 sched_clock_running = 1; 97 sched_clock_running = 1;
82} 98}
83 99
100#ifdef CONFIG_NO_HZ
101/*
102 * The dynamic ticks makes the delta jiffies inaccurate. This
103 * prevents us from checking the maximum time update.
104 * Disable the maximum check during stopped ticks.
105 */
106void sched_clock_tick_stop(int cpu)
107{
108 struct sched_clock_data *scd = cpu_sdc(cpu);
109
110 scd->check_max = 0;
111}
112
113void sched_clock_tick_start(int cpu)
114{
115 struct sched_clock_data *scd = cpu_sdc(cpu);
116
117 scd->check_max = 1;
118}
119
120static int check_max(struct sched_clock_data *scd)
121{
122 return scd->check_max;
123}
124#else
125static int check_max(struct sched_clock_data *scd)
126{
127 return 1;
128}
129#endif /* CONFIG_NO_HZ */
130
84/* 131/*
85 * update the percpu scd from the raw @now value 132 * update the percpu scd from the raw @now value
86 * 133 *
87 * - filter out backward motion 134 * - filter out backward motion
88 * - use jiffies to generate a min,max window to clip the raw values 135 * - use jiffies to generate a min,max window to clip the raw values
89 */ 136 */
90static void __update_sched_clock(struct sched_clock_data *scd, u64 now) 137static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *time)
91{ 138{
92 unsigned long now_jiffies = jiffies; 139 unsigned long now_jiffies = jiffies;
93 long delta_jiffies = now_jiffies - scd->prev_jiffies; 140 long delta_jiffies = now_jiffies - scd->tick_jiffies;
94 u64 clock = scd->clock; 141 u64 clock = scd->clock;
95 u64 min_clock, max_clock; 142 u64 min_clock, max_clock;
96 s64 delta = now - scd->prev_raw; 143 s64 delta = now - scd->prev_raw;
97 144
98 WARN_ON_ONCE(!irqs_disabled()); 145 WARN_ON_ONCE(!irqs_disabled());
99 min_clock = scd->tick_gtod + delta_jiffies * TICK_NSEC; 146
147 /*
148 * At schedule tick the clock can be just under the gtod. We don't
149 * want to push it too prematurely.
150 */
151 min_clock = scd->tick_gtod + (delta_jiffies * TICK_NSEC);
152 if (min_clock > TICK_NSEC)
153 min_clock -= TICK_NSEC / 2;
100 154
101 if (unlikely(delta < 0)) { 155 if (unlikely(delta < 0)) {
102 clock++; 156 clock++;
103 goto out; 157 goto out;
104 } 158 }
105 159
106 max_clock = min_clock + TICK_NSEC; 160 /*
161 * The clock must stay within a jiffie of the gtod.
162 * But since we may be at the start of a jiffy or the end of one
163 * we add another jiffy buffer.
164 */
165 max_clock = scd->tick_gtod + (2 + delta_jiffies) * TICK_NSEC;
166
167 delta *= scd->multi;
168 delta >>= MULTI_SHIFT;
107 169
108 if (unlikely(clock + delta > max_clock)) { 170 if (unlikely(clock + delta > max_clock) && check_max(scd)) {
109 if (clock < max_clock) 171 if (clock < max_clock)
110 clock = max_clock; 172 clock = max_clock;
111 else 173 else
@@ -118,9 +180,12 @@ static void __update_sched_clock(struct sched_clock_data *scd, u64 now)
118 if (unlikely(clock < min_clock)) 180 if (unlikely(clock < min_clock))
119 clock = min_clock; 181 clock = min_clock;
120 182
121 scd->prev_raw = now; 183 if (time)
122 scd->prev_jiffies = now_jiffies; 184 *time = clock;
123 scd->clock = clock; 185 else {
186 scd->prev_raw = now;
187 scd->clock = clock;
188 }
124} 189}
125 190
126static void lock_double_clock(struct sched_clock_data *data1, 191static void lock_double_clock(struct sched_clock_data *data1,
@@ -160,25 +225,30 @@ u64 sched_clock_cpu(int cpu)
160 now -= my_scd->tick_raw; 225 now -= my_scd->tick_raw;
161 now += scd->tick_raw; 226 now += scd->tick_raw;
162 227
163 now -= my_scd->tick_gtod; 228 now += my_scd->tick_gtod;
164 now += scd->tick_gtod; 229 now -= scd->tick_gtod;
165 230
166 __raw_spin_unlock(&my_scd->lock); 231 __raw_spin_unlock(&my_scd->lock);
232
233 __update_sched_clock(scd, now, &clock);
234
235 __raw_spin_unlock(&scd->lock);
236
167 } else { 237 } else {
168 __raw_spin_lock(&scd->lock); 238 __raw_spin_lock(&scd->lock);
239 __update_sched_clock(scd, now, NULL);
240 clock = scd->clock;
241 __raw_spin_unlock(&scd->lock);
169 } 242 }
170 243
171 __update_sched_clock(scd, now);
172 clock = scd->clock;
173
174 __raw_spin_unlock(&scd->lock);
175
176 return clock; 244 return clock;
177} 245}
178 246
179void sched_clock_tick(void) 247void sched_clock_tick(void)
180{ 248{
181 struct sched_clock_data *scd = this_scd(); 249 struct sched_clock_data *scd = this_scd();
250 unsigned long now_jiffies = jiffies;
251 s64 mult, delta_gtod, delta_raw;
182 u64 now, now_gtod; 252 u64 now, now_gtod;
183 253
184 if (unlikely(!sched_clock_running)) 254 if (unlikely(!sched_clock_running))
@@ -186,18 +256,33 @@ void sched_clock_tick(void)
186 256
187 WARN_ON_ONCE(!irqs_disabled()); 257 WARN_ON_ONCE(!irqs_disabled());
188 258
189 now = sched_clock();
190 now_gtod = ktime_to_ns(ktime_get()); 259 now_gtod = ktime_to_ns(ktime_get());
260 now = sched_clock();
191 261
192 __raw_spin_lock(&scd->lock); 262 __raw_spin_lock(&scd->lock);
193 __update_sched_clock(scd, now); 263 __update_sched_clock(scd, now, NULL);
194 /* 264 /*
195 * update tick_gtod after __update_sched_clock() because that will 265 * update tick_gtod after __update_sched_clock() because that will
196 * already observe 1 new jiffy; adding a new tick_gtod to that would 266 * already observe 1 new jiffy; adding a new tick_gtod to that would
197 * increase the clock 2 jiffies. 267 * increase the clock 2 jiffies.
198 */ 268 */
269 delta_gtod = now_gtod - scd->tick_gtod;
270 delta_raw = now - scd->tick_raw;
271
272 if ((long)delta_raw > 0) {
273 mult = delta_gtod << MULTI_SHIFT;
274 do_div(mult, delta_raw);
275 scd->multi = mult;
276 if (scd->multi > MAX_MULTI)
277 scd->multi = MAX_MULTI;
278 else if (scd->multi < MIN_MULTI)
279 scd->multi = MIN_MULTI;
280 } else
281 scd->multi = 1 << MULTI_SHIFT;
282
199 scd->tick_raw = now; 283 scd->tick_raw = now;
200 scd->tick_gtod = now_gtod; 284 scd->tick_gtod = now_gtod;
285 scd->tick_jiffies = now_jiffies;
201 __raw_spin_unlock(&scd->lock); 286 __raw_spin_unlock(&scd->lock);
202} 287}
203 288
@@ -227,6 +312,7 @@ void sched_clock_idle_wakeup_event(u64 delta_ns)
227 __raw_spin_lock(&scd->lock); 312 __raw_spin_lock(&scd->lock);
228 scd->prev_raw = now; 313 scd->prev_raw = now;
229 scd->clock += delta_ns; 314 scd->clock += delta_ns;
315 scd->multi = 1 << MULTI_SHIFT;
230 __raw_spin_unlock(&scd->lock); 316 __raw_spin_unlock(&scd->lock);
231 317
232 touch_softlockup_watchdog(); 318 touch_softlockup_watchdog();
@@ -244,3 +330,16 @@ unsigned long long __attribute__((weak)) sched_clock(void)
244{ 330{
245 return (unsigned long long)jiffies * (NSEC_PER_SEC / HZ); 331 return (unsigned long long)jiffies * (NSEC_PER_SEC / HZ);
246} 332}
333
334unsigned long long cpu_clock(int cpu)
335{
336 unsigned long long clock;
337 unsigned long flags;
338
339 local_irq_save(flags);
340 clock = sched_clock_cpu(cpu);
341 local_irq_restore(flags);
342
343 return clock;
344}
345EXPORT_SYMBOL_GPL(cpu_clock);
diff --git a/kernel/sched_cpupri.c b/kernel/sched_cpupri.c
new file mode 100644
index 000000000000..52154fefab7e
--- /dev/null
+++ b/kernel/sched_cpupri.c
@@ -0,0 +1,174 @@
1/*
2 * kernel/sched_cpupri.c
3 *
4 * CPU priority management
5 *
6 * Copyright (C) 2007-2008 Novell
7 *
8 * Author: Gregory Haskins <ghaskins@novell.com>
9 *
10 * This code tracks the priority of each CPU so that global migration
11 * decisions are easy to calculate. Each CPU can be in a state as follows:
12 *
13 * (INVALID), IDLE, NORMAL, RT1, ... RT99
14 *
15 * going from the lowest priority to the highest. CPUs in the INVALID state
16 * are not eligible for routing. The system maintains this state with
17 * a 2 dimensional bitmap (the first for priority class, the second for cpus
18 * in that class). Therefore a typical application without affinity
19 * restrictions can find a suitable CPU with O(1) complexity (e.g. two bit
20 * searches). For tasks with affinity restrictions, the algorithm has a
21 * worst case complexity of O(min(102, nr_domcpus)), though the scenario that
22 * yields the worst case search is fairly contrived.
23 *
24 * This program is free software; you can redistribute it and/or
25 * modify it under the terms of the GNU General Public License
26 * as published by the Free Software Foundation; version 2
27 * of the License.
28 */
29
30#include "sched_cpupri.h"
31
32/* Convert between a 140 based task->prio, and our 102 based cpupri */
33static int convert_prio(int prio)
34{
35 int cpupri;
36
37 if (prio == CPUPRI_INVALID)
38 cpupri = CPUPRI_INVALID;
39 else if (prio == MAX_PRIO)
40 cpupri = CPUPRI_IDLE;
41 else if (prio >= MAX_RT_PRIO)
42 cpupri = CPUPRI_NORMAL;
43 else
44 cpupri = MAX_RT_PRIO - prio + 1;
45
46 return cpupri;
47}
48
49#define for_each_cpupri_active(array, idx) \
50 for (idx = find_first_bit(array, CPUPRI_NR_PRIORITIES); \
51 idx < CPUPRI_NR_PRIORITIES; \
52 idx = find_next_bit(array, CPUPRI_NR_PRIORITIES, idx+1))
53
54/**
55 * cpupri_find - find the best (lowest-pri) CPU in the system
56 * @cp: The cpupri context
57 * @p: The task
58 * @lowest_mask: A mask to fill in with selected CPUs
59 *
60 * Note: This function returns the recommended CPUs as calculated during the
61 * current invokation. By the time the call returns, the CPUs may have in
62 * fact changed priorities any number of times. While not ideal, it is not
63 * an issue of correctness since the normal rebalancer logic will correct
64 * any discrepancies created by racing against the uncertainty of the current
65 * priority configuration.
66 *
67 * Returns: (int)bool - CPUs were found
68 */
69int cpupri_find(struct cpupri *cp, struct task_struct *p,
70 cpumask_t *lowest_mask)
71{
72 int idx = 0;
73 int task_pri = convert_prio(p->prio);
74
75 for_each_cpupri_active(cp->pri_active, idx) {
76 struct cpupri_vec *vec = &cp->pri_to_cpu[idx];
77 cpumask_t mask;
78
79 if (idx >= task_pri)
80 break;
81
82 cpus_and(mask, p->cpus_allowed, vec->mask);
83
84 if (cpus_empty(mask))
85 continue;
86
87 *lowest_mask = mask;
88 return 1;
89 }
90
91 return 0;
92}
93
94/**
95 * cpupri_set - update the cpu priority setting
96 * @cp: The cpupri context
97 * @cpu: The target cpu
98 * @pri: The priority (INVALID-RT99) to assign to this CPU
99 *
100 * Note: Assumes cpu_rq(cpu)->lock is locked
101 *
102 * Returns: (void)
103 */
104void cpupri_set(struct cpupri *cp, int cpu, int newpri)
105{
106 int *currpri = &cp->cpu_to_pri[cpu];
107 int oldpri = *currpri;
108 unsigned long flags;
109
110 newpri = convert_prio(newpri);
111
112 BUG_ON(newpri >= CPUPRI_NR_PRIORITIES);
113
114 if (newpri == oldpri)
115 return;
116
117 /*
118 * If the cpu was currently mapped to a different value, we
119 * first need to unmap the old value
120 */
121 if (likely(oldpri != CPUPRI_INVALID)) {
122 struct cpupri_vec *vec = &cp->pri_to_cpu[oldpri];
123
124 spin_lock_irqsave(&vec->lock, flags);
125
126 vec->count--;
127 if (!vec->count)
128 clear_bit(oldpri, cp->pri_active);
129 cpu_clear(cpu, vec->mask);
130
131 spin_unlock_irqrestore(&vec->lock, flags);
132 }
133
134 if (likely(newpri != CPUPRI_INVALID)) {
135 struct cpupri_vec *vec = &cp->pri_to_cpu[newpri];
136
137 spin_lock_irqsave(&vec->lock, flags);
138
139 cpu_set(cpu, vec->mask);
140 vec->count++;
141 if (vec->count == 1)
142 set_bit(newpri, cp->pri_active);
143
144 spin_unlock_irqrestore(&vec->lock, flags);
145 }
146
147 *currpri = newpri;
148}
149
150/**
151 * cpupri_init - initialize the cpupri structure
152 * @cp: The cpupri context
153 *
154 * Returns: (void)
155 */
156void cpupri_init(struct cpupri *cp)
157{
158 int i;
159
160 memset(cp, 0, sizeof(*cp));
161
162 for (i = 0; i < CPUPRI_NR_PRIORITIES; i++) {
163 struct cpupri_vec *vec = &cp->pri_to_cpu[i];
164
165 spin_lock_init(&vec->lock);
166 vec->count = 0;
167 cpus_clear(vec->mask);
168 }
169
170 for_each_possible_cpu(i)
171 cp->cpu_to_pri[i] = CPUPRI_INVALID;
172}
173
174
diff --git a/kernel/sched_cpupri.h b/kernel/sched_cpupri.h
new file mode 100644
index 000000000000..f25811b0f931
--- /dev/null
+++ b/kernel/sched_cpupri.h
@@ -0,0 +1,36 @@
1#ifndef _LINUX_CPUPRI_H
2#define _LINUX_CPUPRI_H
3
4#include <linux/sched.h>
5
6#define CPUPRI_NR_PRIORITIES (MAX_RT_PRIO + 2)
7#define CPUPRI_NR_PRI_WORDS BITS_TO_LONGS(CPUPRI_NR_PRIORITIES)
8
9#define CPUPRI_INVALID -1
10#define CPUPRI_IDLE 0
11#define CPUPRI_NORMAL 1
12/* values 2-101 are RT priorities 0-99 */
13
14struct cpupri_vec {
15 spinlock_t lock;
16 int count;
17 cpumask_t mask;
18};
19
20struct cpupri {
21 struct cpupri_vec pri_to_cpu[CPUPRI_NR_PRIORITIES];
22 long pri_active[CPUPRI_NR_PRI_WORDS];
23 int cpu_to_pri[NR_CPUS];
24};
25
26#ifdef CONFIG_SMP
27int cpupri_find(struct cpupri *cp,
28 struct task_struct *p, cpumask_t *lowest_mask);
29void cpupri_set(struct cpupri *cp, int cpu, int pri);
30void cpupri_init(struct cpupri *cp);
31#else
32#define cpupri_set(cp, cpu, pri) do { } while (0)
33#define cpupri_init() do { } while (0)
34#endif
35
36#endif /* _LINUX_CPUPRI_H */
diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c
index 8bb713040ac9..bbe6b31c3c56 100644
--- a/kernel/sched_debug.c
+++ b/kernel/sched_debug.c
@@ -119,9 +119,7 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
119 struct sched_entity *last; 119 struct sched_entity *last;
120 unsigned long flags; 120 unsigned long flags;
121 121
122#if !defined(CONFIG_CGROUP_SCHED) || !defined(CONFIG_USER_SCHED) 122#if defined(CONFIG_CGROUP_SCHED) && defined(CONFIG_FAIR_GROUP_SCHED)
123 SEQ_printf(m, "\ncfs_rq[%d]:\n", cpu);
124#else
125 char path[128] = ""; 123 char path[128] = "";
126 struct cgroup *cgroup = NULL; 124 struct cgroup *cgroup = NULL;
127 struct task_group *tg = cfs_rq->tg; 125 struct task_group *tg = cfs_rq->tg;
@@ -133,6 +131,8 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
133 cgroup_path(cgroup, path, sizeof(path)); 131 cgroup_path(cgroup, path, sizeof(path));
134 132
135 SEQ_printf(m, "\ncfs_rq[%d]:%s\n", cpu, path); 133 SEQ_printf(m, "\ncfs_rq[%d]:%s\n", cpu, path);
134#else
135 SEQ_printf(m, "\ncfs_rq[%d]:\n", cpu);
136#endif 136#endif
137 137
138 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "exec_clock", 138 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "exec_clock",
@@ -162,11 +162,64 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
162 SEQ_printf(m, " .%-30s: %ld\n", "nr_running", cfs_rq->nr_running); 162 SEQ_printf(m, " .%-30s: %ld\n", "nr_running", cfs_rq->nr_running);
163 SEQ_printf(m, " .%-30s: %ld\n", "load", cfs_rq->load.weight); 163 SEQ_printf(m, " .%-30s: %ld\n", "load", cfs_rq->load.weight);
164#ifdef CONFIG_SCHEDSTATS 164#ifdef CONFIG_SCHEDSTATS
165 SEQ_printf(m, " .%-30s: %d\n", "bkl_count", 165#define P(n) SEQ_printf(m, " .%-30s: %d\n", #n, rq->n);
166 rq->bkl_count); 166
167 P(yld_exp_empty);
168 P(yld_act_empty);
169 P(yld_both_empty);
170 P(yld_count);
171
172 P(sched_switch);
173 P(sched_count);
174 P(sched_goidle);
175
176 P(ttwu_count);
177 P(ttwu_local);
178
179 P(bkl_count);
180
181#undef P
167#endif 182#endif
168 SEQ_printf(m, " .%-30s: %ld\n", "nr_spread_over", 183 SEQ_printf(m, " .%-30s: %ld\n", "nr_spread_over",
169 cfs_rq->nr_spread_over); 184 cfs_rq->nr_spread_over);
185#ifdef CONFIG_FAIR_GROUP_SCHED
186#ifdef CONFIG_SMP
187 SEQ_printf(m, " .%-30s: %lu\n", "shares", cfs_rq->shares);
188#endif
189#endif
190}
191
192void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq)
193{
194#if defined(CONFIG_CGROUP_SCHED) && defined(CONFIG_RT_GROUP_SCHED)
195 char path[128] = "";
196 struct cgroup *cgroup = NULL;
197 struct task_group *tg = rt_rq->tg;
198
199 if (tg)
200 cgroup = tg->css.cgroup;
201
202 if (cgroup)
203 cgroup_path(cgroup, path, sizeof(path));
204
205 SEQ_printf(m, "\nrt_rq[%d]:%s\n", cpu, path);
206#else
207 SEQ_printf(m, "\nrt_rq[%d]:\n", cpu);
208#endif
209
210
211#define P(x) \
212 SEQ_printf(m, " .%-30s: %Ld\n", #x, (long long)(rt_rq->x))
213#define PN(x) \
214 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", #x, SPLIT_NS(rt_rq->x))
215
216 P(rt_nr_running);
217 P(rt_throttled);
218 PN(rt_time);
219 PN(rt_runtime);
220
221#undef PN
222#undef P
170} 223}
171 224
172static void print_cpu(struct seq_file *m, int cpu) 225static void print_cpu(struct seq_file *m, int cpu)
@@ -208,6 +261,7 @@ static void print_cpu(struct seq_file *m, int cpu)
208#undef PN 261#undef PN
209 262
210 print_cfs_stats(m, cpu); 263 print_cfs_stats(m, cpu);
264 print_rt_stats(m, cpu);
211 265
212 print_rq(m, rq, cpu); 266 print_rq(m, rq, cpu);
213} 267}
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 08ae848b71d4..cf2cd6ce4cb2 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -63,13 +63,13 @@ unsigned int __read_mostly sysctl_sched_compat_yield;
63 63
64/* 64/*
65 * SCHED_OTHER wake-up granularity. 65 * SCHED_OTHER wake-up granularity.
66 * (default: 10 msec * (1 + ilog(ncpus)), units: nanoseconds) 66 * (default: 5 msec * (1 + ilog(ncpus)), units: nanoseconds)
67 * 67 *
68 * This option delays the preemption effects of decoupled workloads 68 * This option delays the preemption effects of decoupled workloads
69 * and reduces their over-scheduling. Synchronous workloads will still 69 * and reduces their over-scheduling. Synchronous workloads will still
70 * have immediate wakeup/sleep latencies. 70 * have immediate wakeup/sleep latencies.
71 */ 71 */
72unsigned int sysctl_sched_wakeup_granularity = 10000000UL; 72unsigned int sysctl_sched_wakeup_granularity = 5000000UL;
73 73
74const_debug unsigned int sysctl_sched_migration_cost = 500000UL; 74const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
75 75
@@ -334,6 +334,34 @@ int sched_nr_latency_handler(struct ctl_table *table, int write,
334#endif 334#endif
335 335
336/* 336/*
337 * delta *= w / rw
338 */
339static inline unsigned long
340calc_delta_weight(unsigned long delta, struct sched_entity *se)
341{
342 for_each_sched_entity(se) {
343 delta = calc_delta_mine(delta,
344 se->load.weight, &cfs_rq_of(se)->load);
345 }
346
347 return delta;
348}
349
350/*
351 * delta *= rw / w
352 */
353static inline unsigned long
354calc_delta_fair(unsigned long delta, struct sched_entity *se)
355{
356 for_each_sched_entity(se) {
357 delta = calc_delta_mine(delta,
358 cfs_rq_of(se)->load.weight, &se->load);
359 }
360
361 return delta;
362}
363
364/*
337 * The idea is to set a period in which each task runs once. 365 * The idea is to set a period in which each task runs once.
338 * 366 *
339 * When there are too many tasks (sysctl_sched_nr_latency) we have to stretch 367 * When there are too many tasks (sysctl_sched_nr_latency) we have to stretch
@@ -362,47 +390,80 @@ static u64 __sched_period(unsigned long nr_running)
362 */ 390 */
363static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) 391static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se)
364{ 392{
365 u64 slice = __sched_period(cfs_rq->nr_running); 393 return calc_delta_weight(__sched_period(cfs_rq->nr_running), se);
366
367 for_each_sched_entity(se) {
368 cfs_rq = cfs_rq_of(se);
369
370 slice *= se->load.weight;
371 do_div(slice, cfs_rq->load.weight);
372 }
373
374
375 return slice;
376} 394}
377 395
378/* 396/*
379 * We calculate the vruntime slice of a to be inserted task 397 * We calculate the vruntime slice of a to be inserted task
380 * 398 *
381 * vs = s/w = p/rw 399 * vs = s*rw/w = p
382 */ 400 */
383static u64 sched_vslice_add(struct cfs_rq *cfs_rq, struct sched_entity *se) 401static u64 sched_vslice_add(struct cfs_rq *cfs_rq, struct sched_entity *se)
384{ 402{
385 unsigned long nr_running = cfs_rq->nr_running; 403 unsigned long nr_running = cfs_rq->nr_running;
386 unsigned long weight;
387 u64 vslice;
388 404
389 if (!se->on_rq) 405 if (!se->on_rq)
390 nr_running++; 406 nr_running++;
391 407
392 vslice = __sched_period(nr_running); 408 return __sched_period(nr_running);
409}
410
411/*
412 * The goal of calc_delta_asym() is to be asymmetrically around NICE_0_LOAD, in
413 * that it favours >=0 over <0.
414 *
415 * -20 |
416 * |
417 * 0 --------+-------
418 * .'
419 * 19 .'
420 *
421 */
422static unsigned long
423calc_delta_asym(unsigned long delta, struct sched_entity *se)
424{
425 struct load_weight lw = {
426 .weight = NICE_0_LOAD,
427 .inv_weight = 1UL << (WMULT_SHIFT-NICE_0_SHIFT)
428 };
393 429
394 for_each_sched_entity(se) { 430 for_each_sched_entity(se) {
395 cfs_rq = cfs_rq_of(se); 431 struct load_weight *se_lw = &se->load;
432 unsigned long rw = cfs_rq_of(se)->load.weight;
433
434#ifdef CONFIG_FAIR_SCHED_GROUP
435 struct cfs_rq *cfs_rq = se->my_q;
436 struct task_group *tg = NULL
437
438 if (cfs_rq)
439 tg = cfs_rq->tg;
440
441 if (tg && tg->shares < NICE_0_LOAD) {
442 /*
443 * scale shares to what it would have been had
444 * tg->weight been NICE_0_LOAD:
445 *
446 * weight = 1024 * shares / tg->weight
447 */
448 lw.weight *= se->load.weight;
449 lw.weight /= tg->shares;
450
451 lw.inv_weight = 0;
452
453 se_lw = &lw;
454 rw += lw.weight - se->load.weight;
455 } else
456#endif
396 457
397 weight = cfs_rq->load.weight; 458 if (se->load.weight < NICE_0_LOAD) {
398 if (!se->on_rq) 459 se_lw = &lw;
399 weight += se->load.weight; 460 rw += NICE_0_LOAD - se->load.weight;
461 }
400 462
401 vslice *= NICE_0_LOAD; 463 delta = calc_delta_mine(delta, rw, se_lw);
402 do_div(vslice, weight);
403 } 464 }
404 465
405 return vslice; 466 return delta;
406} 467}
407 468
408/* 469/*
@@ -419,11 +480,7 @@ __update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr,
419 480
420 curr->sum_exec_runtime += delta_exec; 481 curr->sum_exec_runtime += delta_exec;
421 schedstat_add(cfs_rq, exec_clock, delta_exec); 482 schedstat_add(cfs_rq, exec_clock, delta_exec);
422 delta_exec_weighted = delta_exec; 483 delta_exec_weighted = calc_delta_fair(delta_exec, curr);
423 if (unlikely(curr->load.weight != NICE_0_LOAD)) {
424 delta_exec_weighted = calc_delta_fair(delta_exec_weighted,
425 &curr->load);
426 }
427 curr->vruntime += delta_exec_weighted; 484 curr->vruntime += delta_exec_weighted;
428} 485}
429 486
@@ -510,10 +567,27 @@ update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
510 * Scheduling class queueing methods: 567 * Scheduling class queueing methods:
511 */ 568 */
512 569
570#if defined CONFIG_SMP && defined CONFIG_FAIR_GROUP_SCHED
571static void
572add_cfs_task_weight(struct cfs_rq *cfs_rq, unsigned long weight)
573{
574 cfs_rq->task_weight += weight;
575}
576#else
577static inline void
578add_cfs_task_weight(struct cfs_rq *cfs_rq, unsigned long weight)
579{
580}
581#endif
582
513static void 583static void
514account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se) 584account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
515{ 585{
516 update_load_add(&cfs_rq->load, se->load.weight); 586 update_load_add(&cfs_rq->load, se->load.weight);
587 if (!parent_entity(se))
588 inc_cpu_load(rq_of(cfs_rq), se->load.weight);
589 if (entity_is_task(se))
590 add_cfs_task_weight(cfs_rq, se->load.weight);
517 cfs_rq->nr_running++; 591 cfs_rq->nr_running++;
518 se->on_rq = 1; 592 se->on_rq = 1;
519 list_add(&se->group_node, &cfs_rq->tasks); 593 list_add(&se->group_node, &cfs_rq->tasks);
@@ -523,6 +597,10 @@ static void
523account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) 597account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
524{ 598{
525 update_load_sub(&cfs_rq->load, se->load.weight); 599 update_load_sub(&cfs_rq->load, se->load.weight);
600 if (!parent_entity(se))
601 dec_cpu_load(rq_of(cfs_rq), se->load.weight);
602 if (entity_is_task(se))
603 add_cfs_task_weight(cfs_rq, -se->load.weight);
526 cfs_rq->nr_running--; 604 cfs_rq->nr_running--;
527 se->on_rq = 0; 605 se->on_rq = 0;
528 list_del_init(&se->group_node); 606 list_del_init(&se->group_node);
@@ -609,8 +687,17 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
609 687
610 if (!initial) { 688 if (!initial) {
611 /* sleeps upto a single latency don't count. */ 689 /* sleeps upto a single latency don't count. */
612 if (sched_feat(NEW_FAIR_SLEEPERS)) 690 if (sched_feat(NEW_FAIR_SLEEPERS)) {
613 vruntime -= sysctl_sched_latency; 691 unsigned long thresh = sysctl_sched_latency;
692
693 /*
694 * convert the sleeper threshold into virtual time
695 */
696 if (sched_feat(NORMALIZED_SLEEPER))
697 thresh = calc_delta_fair(thresh, se);
698
699 vruntime -= thresh;
700 }
614 701
615 /* ensure we never gain time by being placed backwards. */ 702 /* ensure we never gain time by being placed backwards. */
616 vruntime = max_vruntime(se->vruntime, vruntime); 703 vruntime = max_vruntime(se->vruntime, vruntime);
@@ -639,21 +726,6 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int wakeup)
639 __enqueue_entity(cfs_rq, se); 726 __enqueue_entity(cfs_rq, se);
640} 727}
641 728
642static void update_avg(u64 *avg, u64 sample)
643{
644 s64 diff = sample - *avg;
645 *avg += diff >> 3;
646}
647
648static void update_avg_stats(struct cfs_rq *cfs_rq, struct sched_entity *se)
649{
650 if (!se->last_wakeup)
651 return;
652
653 update_avg(&se->avg_overlap, se->sum_exec_runtime - se->last_wakeup);
654 se->last_wakeup = 0;
655}
656
657static void 729static void
658dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep) 730dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep)
659{ 731{
@@ -664,7 +736,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep)
664 736
665 update_stats_dequeue(cfs_rq, se); 737 update_stats_dequeue(cfs_rq, se);
666 if (sleep) { 738 if (sleep) {
667 update_avg_stats(cfs_rq, se);
668#ifdef CONFIG_SCHEDSTATS 739#ifdef CONFIG_SCHEDSTATS
669 if (entity_is_task(se)) { 740 if (entity_is_task(se)) {
670 struct task_struct *tsk = task_of(se); 741 struct task_struct *tsk = task_of(se);
@@ -726,17 +797,16 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
726 se->prev_sum_exec_runtime = se->sum_exec_runtime; 797 se->prev_sum_exec_runtime = se->sum_exec_runtime;
727} 798}
728 799
729static int
730wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se);
731
732static struct sched_entity * 800static struct sched_entity *
733pick_next(struct cfs_rq *cfs_rq, struct sched_entity *se) 801pick_next(struct cfs_rq *cfs_rq, struct sched_entity *se)
734{ 802{
735 if (!cfs_rq->next) 803 struct rq *rq = rq_of(cfs_rq);
736 return se; 804 u64 pair_slice = rq->clock - cfs_rq->pair_start;
737 805
738 if (wakeup_preempt_entity(cfs_rq->next, se) != 0) 806 if (!cfs_rq->next || pair_slice > sched_slice(cfs_rq, cfs_rq->next)) {
807 cfs_rq->pair_start = rq->clock;
739 return se; 808 return se;
809 }
740 810
741 return cfs_rq->next; 811 return cfs_rq->next;
742} 812}
@@ -808,7 +878,6 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
808#ifdef CONFIG_SCHED_HRTICK 878#ifdef CONFIG_SCHED_HRTICK
809static void hrtick_start_fair(struct rq *rq, struct task_struct *p) 879static void hrtick_start_fair(struct rq *rq, struct task_struct *p)
810{ 880{
811 int requeue = rq->curr == p;
812 struct sched_entity *se = &p->se; 881 struct sched_entity *se = &p->se;
813 struct cfs_rq *cfs_rq = cfs_rq_of(se); 882 struct cfs_rq *cfs_rq = cfs_rq_of(se);
814 883
@@ -829,13 +898,13 @@ static void hrtick_start_fair(struct rq *rq, struct task_struct *p)
829 * Don't schedule slices shorter than 10000ns, that just 898 * Don't schedule slices shorter than 10000ns, that just
830 * doesn't make sense. Rely on vruntime for fairness. 899 * doesn't make sense. Rely on vruntime for fairness.
831 */ 900 */
832 if (!requeue) 901 if (rq->curr != p)
833 delta = max(10000LL, delta); 902 delta = max(10000LL, delta);
834 903
835 hrtick_start(rq, delta, requeue); 904 hrtick_start(rq, delta);
836 } 905 }
837} 906}
838#else 907#else /* !CONFIG_SCHED_HRTICK */
839static inline void 908static inline void
840hrtick_start_fair(struct rq *rq, struct task_struct *p) 909hrtick_start_fair(struct rq *rq, struct task_struct *p)
841{ 910{
@@ -934,6 +1003,8 @@ static void yield_task_fair(struct rq *rq)
934 * not idle and an idle cpu is available. The span of cpus to 1003 * not idle and an idle cpu is available. The span of cpus to
935 * search starts with cpus closest then further out as needed, 1004 * search starts with cpus closest then further out as needed,
936 * so we always favor a closer, idle cpu. 1005 * so we always favor a closer, idle cpu.
1006 * Domains may include CPUs that are not usable for migration,
1007 * hence we need to mask them out (cpu_active_map)
937 * 1008 *
938 * Returns the CPU we should wake onto. 1009 * Returns the CPU we should wake onto.
939 */ 1010 */
@@ -961,7 +1032,8 @@ static int wake_idle(int cpu, struct task_struct *p)
961 || ((sd->flags & SD_WAKE_IDLE_FAR) 1032 || ((sd->flags & SD_WAKE_IDLE_FAR)
962 && !task_hot(p, task_rq(p)->clock, sd))) { 1033 && !task_hot(p, task_rq(p)->clock, sd))) {
963 cpus_and(tmp, sd->span, p->cpus_allowed); 1034 cpus_and(tmp, sd->span, p->cpus_allowed);
964 for_each_cpu_mask(i, tmp) { 1035 cpus_and(tmp, tmp, cpu_active_map);
1036 for_each_cpu_mask_nr(i, tmp) {
965 if (idle_cpu(i)) { 1037 if (idle_cpu(i)) {
966 if (i != task_cpu(p)) { 1038 if (i != task_cpu(p)) {
967 schedstat_inc(p, 1039 schedstat_inc(p,
@@ -976,7 +1048,7 @@ static int wake_idle(int cpu, struct task_struct *p)
976 } 1048 }
977 return cpu; 1049 return cpu;
978} 1050}
979#else 1051#else /* !ARCH_HAS_SCHED_WAKE_IDLE*/
980static inline int wake_idle(int cpu, struct task_struct *p) 1052static inline int wake_idle(int cpu, struct task_struct *p)
981{ 1053{
982 return cpu; 1054 return cpu;
@@ -987,6 +1059,89 @@ static inline int wake_idle(int cpu, struct task_struct *p)
987 1059
988static const struct sched_class fair_sched_class; 1060static const struct sched_class fair_sched_class;
989 1061
1062#ifdef CONFIG_FAIR_GROUP_SCHED
1063/*
1064 * effective_load() calculates the load change as seen from the root_task_group
1065 *
1066 * Adding load to a group doesn't make a group heavier, but can cause movement
1067 * of group shares between cpus. Assuming the shares were perfectly aligned one
1068 * can calculate the shift in shares.
1069 *
1070 * The problem is that perfectly aligning the shares is rather expensive, hence
1071 * we try to avoid doing that too often - see update_shares(), which ratelimits
1072 * this change.
1073 *
1074 * We compensate this by not only taking the current delta into account, but
1075 * also considering the delta between when the shares were last adjusted and
1076 * now.
1077 *
1078 * We still saw a performance dip, some tracing learned us that between
1079 * cgroup:/ and cgroup:/foo balancing the number of affine wakeups increased
1080 * significantly. Therefore try to bias the error in direction of failing
1081 * the affine wakeup.
1082 *
1083 */
1084static long effective_load(struct task_group *tg, int cpu,
1085 long wl, long wg)
1086{
1087 struct sched_entity *se = tg->se[cpu];
1088 long more_w;
1089
1090 if (!tg->parent)
1091 return wl;
1092
1093 /*
1094 * By not taking the decrease of shares on the other cpu into
1095 * account our error leans towards reducing the affine wakeups.
1096 */
1097 if (!wl && sched_feat(ASYM_EFF_LOAD))
1098 return wl;
1099
1100 /*
1101 * Instead of using this increment, also add the difference
1102 * between when the shares were last updated and now.
1103 */
1104 more_w = se->my_q->load.weight - se->my_q->rq_weight;
1105 wl += more_w;
1106 wg += more_w;
1107
1108 for_each_sched_entity(se) {
1109#define D(n) (likely(n) ? (n) : 1)
1110
1111 long S, rw, s, a, b;
1112
1113 S = se->my_q->tg->shares;
1114 s = se->my_q->shares;
1115 rw = se->my_q->rq_weight;
1116
1117 a = S*(rw + wl);
1118 b = S*rw + s*wg;
1119
1120 wl = s*(a-b)/D(b);
1121 /*
1122 * Assume the group is already running and will
1123 * thus already be accounted for in the weight.
1124 *
1125 * That is, moving shares between CPUs, does not
1126 * alter the group weight.
1127 */
1128 wg = 0;
1129#undef D
1130 }
1131
1132 return wl;
1133}
1134
1135#else
1136
1137static inline unsigned long effective_load(struct task_group *tg, int cpu,
1138 unsigned long wl, unsigned long wg)
1139{
1140 return wl;
1141}
1142
1143#endif
1144
990static int 1145static int
991wake_affine(struct rq *rq, struct sched_domain *this_sd, struct rq *this_rq, 1146wake_affine(struct rq *rq, struct sched_domain *this_sd, struct rq *this_rq,
992 struct task_struct *p, int prev_cpu, int this_cpu, int sync, 1147 struct task_struct *p, int prev_cpu, int this_cpu, int sync,
@@ -994,8 +1149,10 @@ wake_affine(struct rq *rq, struct sched_domain *this_sd, struct rq *this_rq,
994 unsigned int imbalance) 1149 unsigned int imbalance)
995{ 1150{
996 struct task_struct *curr = this_rq->curr; 1151 struct task_struct *curr = this_rq->curr;
1152 struct task_group *tg;
997 unsigned long tl = this_load; 1153 unsigned long tl = this_load;
998 unsigned long tl_per_task; 1154 unsigned long tl_per_task;
1155 unsigned long weight;
999 int balanced; 1156 int balanced;
1000 1157
1001 if (!(this_sd->flags & SD_WAKE_AFFINE) || !sched_feat(AFFINE_WAKEUPS)) 1158 if (!(this_sd->flags & SD_WAKE_AFFINE) || !sched_feat(AFFINE_WAKEUPS))
@@ -1006,19 +1163,28 @@ wake_affine(struct rq *rq, struct sched_domain *this_sd, struct rq *this_rq,
1006 * effect of the currently running task from the load 1163 * effect of the currently running task from the load
1007 * of the current CPU: 1164 * of the current CPU:
1008 */ 1165 */
1009 if (sync) 1166 if (sync) {
1010 tl -= current->se.load.weight; 1167 tg = task_group(current);
1168 weight = current->se.load.weight;
1169
1170 tl += effective_load(tg, this_cpu, -weight, -weight);
1171 load += effective_load(tg, prev_cpu, 0, -weight);
1172 }
1011 1173
1012 balanced = 100*(tl + p->se.load.weight) <= imbalance*load; 1174 tg = task_group(p);
1175 weight = p->se.load.weight;
1176
1177 balanced = 100*(tl + effective_load(tg, this_cpu, weight, weight)) <=
1178 imbalance*(load + effective_load(tg, prev_cpu, 0, weight));
1013 1179
1014 /* 1180 /*
1015 * If the currently running task will sleep within 1181 * If the currently running task will sleep within
1016 * a reasonable amount of time then attract this newly 1182 * a reasonable amount of time then attract this newly
1017 * woken task: 1183 * woken task:
1018 */ 1184 */
1019 if (sync && balanced && curr->sched_class == &fair_sched_class) { 1185 if (sync && balanced) {
1020 if (curr->se.avg_overlap < sysctl_sched_migration_cost && 1186 if (curr->se.avg_overlap < sysctl_sched_migration_cost &&
1021 p->se.avg_overlap < sysctl_sched_migration_cost) 1187 p->se.avg_overlap < sysctl_sched_migration_cost)
1022 return 1; 1188 return 1;
1023 } 1189 }
1024 1190
@@ -1111,11 +1277,13 @@ static unsigned long wakeup_gran(struct sched_entity *se)
1111 unsigned long gran = sysctl_sched_wakeup_granularity; 1277 unsigned long gran = sysctl_sched_wakeup_granularity;
1112 1278
1113 /* 1279 /*
1114 * More easily preempt - nice tasks, while not making 1280 * More easily preempt - nice tasks, while not making it harder for
1115 * it harder for + nice tasks. 1281 * + nice tasks.
1116 */ 1282 */
1117 if (unlikely(se->load.weight > NICE_0_LOAD)) 1283 if (sched_feat(ASYM_GRAN))
1118 gran = calc_delta_fair(gran, &se->load); 1284 gran = calc_delta_asym(sysctl_sched_wakeup_granularity, se);
1285 else
1286 gran = calc_delta_fair(sysctl_sched_wakeup_granularity, se);
1119 1287
1120 return gran; 1288 return gran;
1121} 1289}
@@ -1177,7 +1345,6 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p)
1177 return; 1345 return;
1178 } 1346 }
1179 1347
1180 se->last_wakeup = se->sum_exec_runtime;
1181 if (unlikely(se == pse)) 1348 if (unlikely(se == pse))
1182 return; 1349 return;
1183 1350
@@ -1275,23 +1442,18 @@ __load_balance_iterator(struct cfs_rq *cfs_rq, struct list_head *next)
1275 struct task_struct *p = NULL; 1442 struct task_struct *p = NULL;
1276 struct sched_entity *se; 1443 struct sched_entity *se;
1277 1444
1278 if (next == &cfs_rq->tasks) 1445 while (next != &cfs_rq->tasks) {
1279 return NULL;
1280
1281 /* Skip over entities that are not tasks */
1282 do {
1283 se = list_entry(next, struct sched_entity, group_node); 1446 se = list_entry(next, struct sched_entity, group_node);
1284 next = next->next; 1447 next = next->next;
1285 } while (next != &cfs_rq->tasks && !entity_is_task(se));
1286 1448
1287 if (next == &cfs_rq->tasks) 1449 /* Skip over entities that are not tasks */
1288 return NULL; 1450 if (entity_is_task(se)) {
1451 p = task_of(se);
1452 break;
1453 }
1454 }
1289 1455
1290 cfs_rq->balance_iterator = next; 1456 cfs_rq->balance_iterator = next;
1291
1292 if (entity_is_task(se))
1293 p = task_of(se);
1294
1295 return p; 1457 return p;
1296} 1458}
1297 1459
@@ -1309,75 +1471,82 @@ static struct task_struct *load_balance_next_fair(void *arg)
1309 return __load_balance_iterator(cfs_rq, cfs_rq->balance_iterator); 1471 return __load_balance_iterator(cfs_rq, cfs_rq->balance_iterator);
1310} 1472}
1311 1473
1312#ifdef CONFIG_FAIR_GROUP_SCHED 1474static unsigned long
1313static int cfs_rq_best_prio(struct cfs_rq *cfs_rq) 1475__load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
1476 unsigned long max_load_move, struct sched_domain *sd,
1477 enum cpu_idle_type idle, int *all_pinned, int *this_best_prio,
1478 struct cfs_rq *cfs_rq)
1314{ 1479{
1315 struct sched_entity *curr; 1480 struct rq_iterator cfs_rq_iterator;
1316 struct task_struct *p;
1317
1318 if (!cfs_rq->nr_running || !first_fair(cfs_rq))
1319 return MAX_PRIO;
1320
1321 curr = cfs_rq->curr;
1322 if (!curr)
1323 curr = __pick_next_entity(cfs_rq);
1324 1481
1325 p = task_of(curr); 1482 cfs_rq_iterator.start = load_balance_start_fair;
1483 cfs_rq_iterator.next = load_balance_next_fair;
1484 cfs_rq_iterator.arg = cfs_rq;
1326 1485
1327 return p->prio; 1486 return balance_tasks(this_rq, this_cpu, busiest,
1487 max_load_move, sd, idle, all_pinned,
1488 this_best_prio, &cfs_rq_iterator);
1328} 1489}
1329#endif
1330 1490
1491#ifdef CONFIG_FAIR_GROUP_SCHED
1331static unsigned long 1492static unsigned long
1332load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest, 1493load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
1333 unsigned long max_load_move, 1494 unsigned long max_load_move,
1334 struct sched_domain *sd, enum cpu_idle_type idle, 1495 struct sched_domain *sd, enum cpu_idle_type idle,
1335 int *all_pinned, int *this_best_prio) 1496 int *all_pinned, int *this_best_prio)
1336{ 1497{
1337 struct cfs_rq *busy_cfs_rq;
1338 long rem_load_move = max_load_move; 1498 long rem_load_move = max_load_move;
1339 struct rq_iterator cfs_rq_iterator; 1499 int busiest_cpu = cpu_of(busiest);
1340 1500 struct task_group *tg;
1341 cfs_rq_iterator.start = load_balance_start_fair;
1342 cfs_rq_iterator.next = load_balance_next_fair;
1343 1501
1344 for_each_leaf_cfs_rq(busiest, busy_cfs_rq) { 1502 rcu_read_lock();
1345#ifdef CONFIG_FAIR_GROUP_SCHED 1503 update_h_load(busiest_cpu);
1346 struct cfs_rq *this_cfs_rq;
1347 long imbalance;
1348 unsigned long maxload;
1349 1504
1350 this_cfs_rq = cpu_cfs_rq(busy_cfs_rq, this_cpu); 1505 list_for_each_entry(tg, &task_groups, list) {
1506 struct cfs_rq *busiest_cfs_rq = tg->cfs_rq[busiest_cpu];
1507 unsigned long busiest_h_load = busiest_cfs_rq->h_load;
1508 unsigned long busiest_weight = busiest_cfs_rq->load.weight;
1509 u64 rem_load, moved_load;
1351 1510
1352 imbalance = busy_cfs_rq->load.weight - this_cfs_rq->load.weight; 1511 /*
1353 /* Don't pull if this_cfs_rq has more load than busy_cfs_rq */ 1512 * empty group
1354 if (imbalance <= 0) 1513 */
1514 if (!busiest_cfs_rq->task_weight)
1355 continue; 1515 continue;
1356 1516
1357 /* Don't pull more than imbalance/2 */ 1517 rem_load = (u64)rem_load_move * busiest_weight;
1358 imbalance /= 2; 1518 rem_load = div_u64(rem_load, busiest_h_load + 1);
1359 maxload = min(rem_load_move, imbalance);
1360 1519
1361 *this_best_prio = cfs_rq_best_prio(this_cfs_rq); 1520 moved_load = __load_balance_fair(this_rq, this_cpu, busiest,
1362#else 1521 rem_load, sd, idle, all_pinned, this_best_prio,
1363# define maxload rem_load_move 1522 tg->cfs_rq[busiest_cpu]);
1364#endif 1523
1365 /* 1524 if (!moved_load)
1366 * pass busy_cfs_rq argument into 1525 continue;
1367 * load_balance_[start|next]_fair iterators 1526
1368 */ 1527 moved_load *= busiest_h_load;
1369 cfs_rq_iterator.arg = busy_cfs_rq; 1528 moved_load = div_u64(moved_load, busiest_weight + 1);
1370 rem_load_move -= balance_tasks(this_rq, this_cpu, busiest,
1371 maxload, sd, idle, all_pinned,
1372 this_best_prio,
1373 &cfs_rq_iterator);
1374 1529
1375 if (rem_load_move <= 0) 1530 rem_load_move -= moved_load;
1531 if (rem_load_move < 0)
1376 break; 1532 break;
1377 } 1533 }
1534 rcu_read_unlock();
1378 1535
1379 return max_load_move - rem_load_move; 1536 return max_load_move - rem_load_move;
1380} 1537}
1538#else
1539static unsigned long
1540load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
1541 unsigned long max_load_move,
1542 struct sched_domain *sd, enum cpu_idle_type idle,
1543 int *all_pinned, int *this_best_prio)
1544{
1545 return __load_balance_fair(this_rq, this_cpu, busiest,
1546 max_load_move, sd, idle, all_pinned,
1547 this_best_prio, &busiest->cfs);
1548}
1549#endif
1381 1550
1382static int 1551static int
1383move_one_task_fair(struct rq *this_rq, int this_cpu, struct rq *busiest, 1552move_one_task_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
@@ -1402,7 +1571,7 @@ move_one_task_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
1402 1571
1403 return 0; 1572 return 0;
1404} 1573}
1405#endif 1574#endif /* CONFIG_SMP */
1406 1575
1407/* 1576/*
1408 * scheduler tick hitting a task of our scheduling class: 1577 * scheduler tick hitting a task of our scheduling class:
diff --git a/kernel/sched_features.h b/kernel/sched_features.h
index 1c7283cb9581..862b06bd560a 100644
--- a/kernel/sched_features.h
+++ b/kernel/sched_features.h
@@ -1,4 +1,5 @@
1SCHED_FEAT(NEW_FAIR_SLEEPERS, 1) 1SCHED_FEAT(NEW_FAIR_SLEEPERS, 1)
2SCHED_FEAT(NORMALIZED_SLEEPER, 1)
2SCHED_FEAT(WAKEUP_PREEMPT, 1) 3SCHED_FEAT(WAKEUP_PREEMPT, 1)
3SCHED_FEAT(START_DEBIT, 1) 4SCHED_FEAT(START_DEBIT, 1)
4SCHED_FEAT(AFFINE_WAKEUPS, 1) 5SCHED_FEAT(AFFINE_WAKEUPS, 1)
@@ -6,5 +7,7 @@ SCHED_FEAT(CACHE_HOT_BUDDY, 1)
6SCHED_FEAT(SYNC_WAKEUPS, 1) 7SCHED_FEAT(SYNC_WAKEUPS, 1)
7SCHED_FEAT(HRTICK, 1) 8SCHED_FEAT(HRTICK, 1)
8SCHED_FEAT(DOUBLE_TICK, 0) 9SCHED_FEAT(DOUBLE_TICK, 0)
9SCHED_FEAT(NORMALIZED_SLEEPER, 1) 10SCHED_FEAT(ASYM_GRAN, 1)
10SCHED_FEAT(DEADLINE, 1) 11SCHED_FEAT(LB_BIAS, 0)
12SCHED_FEAT(LB_WAKEUP_UPDATE, 1)
13SCHED_FEAT(ASYM_EFF_LOAD, 1)
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c
index 0f3c19197fa4..908c04f9dad0 100644
--- a/kernel/sched_rt.c
+++ b/kernel/sched_rt.c
@@ -12,6 +12,9 @@ static inline int rt_overloaded(struct rq *rq)
12 12
13static inline void rt_set_overload(struct rq *rq) 13static inline void rt_set_overload(struct rq *rq)
14{ 14{
15 if (!rq->online)
16 return;
17
15 cpu_set(rq->cpu, rq->rd->rto_mask); 18 cpu_set(rq->cpu, rq->rd->rto_mask);
16 /* 19 /*
17 * Make sure the mask is visible before we set 20 * Make sure the mask is visible before we set
@@ -26,6 +29,9 @@ static inline void rt_set_overload(struct rq *rq)
26 29
27static inline void rt_clear_overload(struct rq *rq) 30static inline void rt_clear_overload(struct rq *rq)
28{ 31{
32 if (!rq->online)
33 return;
34
29 /* the order here really doesn't matter */ 35 /* the order here really doesn't matter */
30 atomic_dec(&rq->rd->rto_count); 36 atomic_dec(&rq->rd->rto_count);
31 cpu_clear(rq->cpu, rq->rd->rto_mask); 37 cpu_clear(rq->cpu, rq->rd->rto_mask);
@@ -155,7 +161,7 @@ static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)
155 return &rt_rq->tg->rt_bandwidth; 161 return &rt_rq->tg->rt_bandwidth;
156} 162}
157 163
158#else 164#else /* !CONFIG_RT_GROUP_SCHED */
159 165
160static inline u64 sched_rt_runtime(struct rt_rq *rt_rq) 166static inline u64 sched_rt_runtime(struct rt_rq *rt_rq)
161{ 167{
@@ -220,7 +226,160 @@ static inline struct rt_bandwidth *sched_rt_bandwidth(struct rt_rq *rt_rq)
220 return &def_rt_bandwidth; 226 return &def_rt_bandwidth;
221} 227}
222 228
223#endif 229#endif /* CONFIG_RT_GROUP_SCHED */
230
231#ifdef CONFIG_SMP
232static int do_balance_runtime(struct rt_rq *rt_rq)
233{
234 struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
235 struct root_domain *rd = cpu_rq(smp_processor_id())->rd;
236 int i, weight, more = 0;
237 u64 rt_period;
238
239 weight = cpus_weight(rd->span);
240
241 spin_lock(&rt_b->rt_runtime_lock);
242 rt_period = ktime_to_ns(rt_b->rt_period);
243 for_each_cpu_mask_nr(i, rd->span) {
244 struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
245 s64 diff;
246
247 if (iter == rt_rq)
248 continue;
249
250 spin_lock(&iter->rt_runtime_lock);
251 if (iter->rt_runtime == RUNTIME_INF)
252 goto next;
253
254 diff = iter->rt_runtime - iter->rt_time;
255 if (diff > 0) {
256 diff = div_u64((u64)diff, weight);
257 if (rt_rq->rt_runtime + diff > rt_period)
258 diff = rt_period - rt_rq->rt_runtime;
259 iter->rt_runtime -= diff;
260 rt_rq->rt_runtime += diff;
261 more = 1;
262 if (rt_rq->rt_runtime == rt_period) {
263 spin_unlock(&iter->rt_runtime_lock);
264 break;
265 }
266 }
267next:
268 spin_unlock(&iter->rt_runtime_lock);
269 }
270 spin_unlock(&rt_b->rt_runtime_lock);
271
272 return more;
273}
274
275static void __disable_runtime(struct rq *rq)
276{
277 struct root_domain *rd = rq->rd;
278 struct rt_rq *rt_rq;
279
280 if (unlikely(!scheduler_running))
281 return;
282
283 for_each_leaf_rt_rq(rt_rq, rq) {
284 struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
285 s64 want;
286 int i;
287
288 spin_lock(&rt_b->rt_runtime_lock);
289 spin_lock(&rt_rq->rt_runtime_lock);
290 if (rt_rq->rt_runtime == RUNTIME_INF ||
291 rt_rq->rt_runtime == rt_b->rt_runtime)
292 goto balanced;
293 spin_unlock(&rt_rq->rt_runtime_lock);
294
295 want = rt_b->rt_runtime - rt_rq->rt_runtime;
296
297 for_each_cpu_mask(i, rd->span) {
298 struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
299 s64 diff;
300
301 if (iter == rt_rq)
302 continue;
303
304 spin_lock(&iter->rt_runtime_lock);
305 if (want > 0) {
306 diff = min_t(s64, iter->rt_runtime, want);
307 iter->rt_runtime -= diff;
308 want -= diff;
309 } else {
310 iter->rt_runtime -= want;
311 want -= want;
312 }
313 spin_unlock(&iter->rt_runtime_lock);
314
315 if (!want)
316 break;
317 }
318
319 spin_lock(&rt_rq->rt_runtime_lock);
320 BUG_ON(want);
321balanced:
322 rt_rq->rt_runtime = RUNTIME_INF;
323 spin_unlock(&rt_rq->rt_runtime_lock);
324 spin_unlock(&rt_b->rt_runtime_lock);
325 }
326}
327
328static void disable_runtime(struct rq *rq)
329{
330 unsigned long flags;
331
332 spin_lock_irqsave(&rq->lock, flags);
333 __disable_runtime(rq);
334 spin_unlock_irqrestore(&rq->lock, flags);
335}
336
337static void __enable_runtime(struct rq *rq)
338{
339 struct rt_rq *rt_rq;
340
341 if (unlikely(!scheduler_running))
342 return;
343
344 for_each_leaf_rt_rq(rt_rq, rq) {
345 struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
346
347 spin_lock(&rt_b->rt_runtime_lock);
348 spin_lock(&rt_rq->rt_runtime_lock);
349 rt_rq->rt_runtime = rt_b->rt_runtime;
350 rt_rq->rt_time = 0;
351 spin_unlock(&rt_rq->rt_runtime_lock);
352 spin_unlock(&rt_b->rt_runtime_lock);
353 }
354}
355
356static void enable_runtime(struct rq *rq)
357{
358 unsigned long flags;
359
360 spin_lock_irqsave(&rq->lock, flags);
361 __enable_runtime(rq);
362 spin_unlock_irqrestore(&rq->lock, flags);
363}
364
365static int balance_runtime(struct rt_rq *rt_rq)
366{
367 int more = 0;
368
369 if (rt_rq->rt_time > rt_rq->rt_runtime) {
370 spin_unlock(&rt_rq->rt_runtime_lock);
371 more = do_balance_runtime(rt_rq);
372 spin_lock(&rt_rq->rt_runtime_lock);
373 }
374
375 return more;
376}
377#else /* !CONFIG_SMP */
378static inline int balance_runtime(struct rt_rq *rt_rq)
379{
380 return 0;
381}
382#endif /* CONFIG_SMP */
224 383
225static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) 384static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)
226{ 385{
@@ -241,6 +400,8 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)
241 u64 runtime; 400 u64 runtime;
242 401
243 spin_lock(&rt_rq->rt_runtime_lock); 402 spin_lock(&rt_rq->rt_runtime_lock);
403 if (rt_rq->rt_throttled)
404 balance_runtime(rt_rq);
244 runtime = rt_rq->rt_runtime; 405 runtime = rt_rq->rt_runtime;
245 rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime); 406 rt_rq->rt_time -= min(rt_rq->rt_time, overrun*runtime);
246 if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) { 407 if (rt_rq->rt_throttled && rt_rq->rt_time < runtime) {
@@ -261,47 +422,6 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)
261 return idle; 422 return idle;
262} 423}
263 424
264#ifdef CONFIG_SMP
265static int balance_runtime(struct rt_rq *rt_rq)
266{
267 struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
268 struct root_domain *rd = cpu_rq(smp_processor_id())->rd;
269 int i, weight, more = 0;
270 u64 rt_period;
271
272 weight = cpus_weight(rd->span);
273
274 spin_lock(&rt_b->rt_runtime_lock);
275 rt_period = ktime_to_ns(rt_b->rt_period);
276 for_each_cpu_mask(i, rd->span) {
277 struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
278 s64 diff;
279
280 if (iter == rt_rq)
281 continue;
282
283 spin_lock(&iter->rt_runtime_lock);
284 diff = iter->rt_runtime - iter->rt_time;
285 if (diff > 0) {
286 do_div(diff, weight);
287 if (rt_rq->rt_runtime + diff > rt_period)
288 diff = rt_period - rt_rq->rt_runtime;
289 iter->rt_runtime -= diff;
290 rt_rq->rt_runtime += diff;
291 more = 1;
292 if (rt_rq->rt_runtime == rt_period) {
293 spin_unlock(&iter->rt_runtime_lock);
294 break;
295 }
296 }
297 spin_unlock(&iter->rt_runtime_lock);
298 }
299 spin_unlock(&rt_b->rt_runtime_lock);
300
301 return more;
302}
303#endif
304
305static inline int rt_se_prio(struct sched_rt_entity *rt_se) 425static inline int rt_se_prio(struct sched_rt_entity *rt_se)
306{ 426{
307#ifdef CONFIG_RT_GROUP_SCHED 427#ifdef CONFIG_RT_GROUP_SCHED
@@ -327,18 +447,10 @@ static int sched_rt_runtime_exceeded(struct rt_rq *rt_rq)
327 if (sched_rt_runtime(rt_rq) >= sched_rt_period(rt_rq)) 447 if (sched_rt_runtime(rt_rq) >= sched_rt_period(rt_rq))
328 return 0; 448 return 0;
329 449
330#ifdef CONFIG_SMP 450 balance_runtime(rt_rq);
331 if (rt_rq->rt_time > runtime) { 451 runtime = sched_rt_runtime(rt_rq);
332 int more; 452 if (runtime == RUNTIME_INF)
333 453 return 0;
334 spin_unlock(&rt_rq->rt_runtime_lock);
335 more = balance_runtime(rt_rq);
336 spin_lock(&rt_rq->rt_runtime_lock);
337
338 if (more)
339 runtime = sched_rt_runtime(rt_rq);
340 }
341#endif
342 454
343 if (rt_rq->rt_time > runtime) { 455 if (rt_rq->rt_time > runtime) {
344 rt_rq->rt_throttled = 1; 456 rt_rq->rt_throttled = 1;
@@ -392,12 +504,23 @@ void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
392 WARN_ON(!rt_prio(rt_se_prio(rt_se))); 504 WARN_ON(!rt_prio(rt_se_prio(rt_se)));
393 rt_rq->rt_nr_running++; 505 rt_rq->rt_nr_running++;
394#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED 506#if defined CONFIG_SMP || defined CONFIG_RT_GROUP_SCHED
395 if (rt_se_prio(rt_se) < rt_rq->highest_prio) 507 if (rt_se_prio(rt_se) < rt_rq->highest_prio) {
508#ifdef CONFIG_SMP
509 struct rq *rq = rq_of_rt_rq(rt_rq);
510#endif
511
396 rt_rq->highest_prio = rt_se_prio(rt_se); 512 rt_rq->highest_prio = rt_se_prio(rt_se);
513#ifdef CONFIG_SMP
514 if (rq->online)
515 cpupri_set(&rq->rd->cpupri, rq->cpu,
516 rt_se_prio(rt_se));
517#endif
518 }
397#endif 519#endif
398#ifdef CONFIG_SMP 520#ifdef CONFIG_SMP
399 if (rt_se->nr_cpus_allowed > 1) { 521 if (rt_se->nr_cpus_allowed > 1) {
400 struct rq *rq = rq_of_rt_rq(rt_rq); 522 struct rq *rq = rq_of_rt_rq(rt_rq);
523
401 rq->rt.rt_nr_migratory++; 524 rq->rt.rt_nr_migratory++;
402 } 525 }
403 526
@@ -417,6 +540,10 @@ void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
417static inline 540static inline
418void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq) 541void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
419{ 542{
543#ifdef CONFIG_SMP
544 int highest_prio = rt_rq->highest_prio;
545#endif
546
420 WARN_ON(!rt_prio(rt_se_prio(rt_se))); 547 WARN_ON(!rt_prio(rt_se_prio(rt_se)));
421 WARN_ON(!rt_rq->rt_nr_running); 548 WARN_ON(!rt_rq->rt_nr_running);
422 rt_rq->rt_nr_running--; 549 rt_rq->rt_nr_running--;
@@ -440,6 +567,14 @@ void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
440 rq->rt.rt_nr_migratory--; 567 rq->rt.rt_nr_migratory--;
441 } 568 }
442 569
570 if (rt_rq->highest_prio != highest_prio) {
571 struct rq *rq = rq_of_rt_rq(rt_rq);
572
573 if (rq->online)
574 cpupri_set(&rq->rd->cpupri, rq->cpu,
575 rt_rq->highest_prio);
576 }
577
443 update_rt_migration(rq_of_rt_rq(rt_rq)); 578 update_rt_migration(rq_of_rt_rq(rt_rq));
444#endif /* CONFIG_SMP */ 579#endif /* CONFIG_SMP */
445#ifdef CONFIG_RT_GROUP_SCHED 580#ifdef CONFIG_RT_GROUP_SCHED
@@ -455,6 +590,7 @@ static void __enqueue_rt_entity(struct sched_rt_entity *rt_se)
455 struct rt_rq *rt_rq = rt_rq_of_se(rt_se); 590 struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
456 struct rt_prio_array *array = &rt_rq->active; 591 struct rt_prio_array *array = &rt_rq->active;
457 struct rt_rq *group_rq = group_rt_rq(rt_se); 592 struct rt_rq *group_rq = group_rt_rq(rt_se);
593 struct list_head *queue = array->queue + rt_se_prio(rt_se);
458 594
459 /* 595 /*
460 * Don't enqueue the group if its throttled, or when empty. 596 * Don't enqueue the group if its throttled, or when empty.
@@ -465,7 +601,7 @@ static void __enqueue_rt_entity(struct sched_rt_entity *rt_se)
465 if (group_rq && (rt_rq_throttled(group_rq) || !group_rq->rt_nr_running)) 601 if (group_rq && (rt_rq_throttled(group_rq) || !group_rq->rt_nr_running))
466 return; 602 return;
467 603
468 list_add_tail(&rt_se->run_list, array->queue + rt_se_prio(rt_se)); 604 list_add_tail(&rt_se->run_list, queue);
469 __set_bit(rt_se_prio(rt_se), array->bitmap); 605 __set_bit(rt_se_prio(rt_se), array->bitmap);
470 606
471 inc_rt_tasks(rt_se, rt_rq); 607 inc_rt_tasks(rt_se, rt_rq);
@@ -532,6 +668,8 @@ static void enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup)
532 rt_se->timeout = 0; 668 rt_se->timeout = 0;
533 669
534 enqueue_rt_entity(rt_se); 670 enqueue_rt_entity(rt_se);
671
672 inc_cpu_load(rq, p->se.load.weight);
535} 673}
536 674
537static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep) 675static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep)
@@ -540,36 +678,42 @@ static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep)
540 678
541 update_curr_rt(rq); 679 update_curr_rt(rq);
542 dequeue_rt_entity(rt_se); 680 dequeue_rt_entity(rt_se);
681
682 dec_cpu_load(rq, p->se.load.weight);
543} 683}
544 684
545/* 685/*
546 * Put task to the end of the run list without the overhead of dequeue 686 * Put task to the end of the run list without the overhead of dequeue
547 * followed by enqueue. 687 * followed by enqueue.
548 */ 688 */
549static 689static void
550void requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se) 690requeue_rt_entity(struct rt_rq *rt_rq, struct sched_rt_entity *rt_se, int head)
551{ 691{
552 struct rt_prio_array *array = &rt_rq->active; 692 if (on_rt_rq(rt_se)) {
553 struct list_head *queue = array->queue + rt_se_prio(rt_se); 693 struct rt_prio_array *array = &rt_rq->active;
694 struct list_head *queue = array->queue + rt_se_prio(rt_se);
554 695
555 if (on_rt_rq(rt_se)) 696 if (head)
556 list_move_tail(&rt_se->run_list, queue); 697 list_move(&rt_se->run_list, queue);
698 else
699 list_move_tail(&rt_se->run_list, queue);
700 }
557} 701}
558 702
559static void requeue_task_rt(struct rq *rq, struct task_struct *p) 703static void requeue_task_rt(struct rq *rq, struct task_struct *p, int head)
560{ 704{
561 struct sched_rt_entity *rt_se = &p->rt; 705 struct sched_rt_entity *rt_se = &p->rt;
562 struct rt_rq *rt_rq; 706 struct rt_rq *rt_rq;
563 707
564 for_each_sched_rt_entity(rt_se) { 708 for_each_sched_rt_entity(rt_se) {
565 rt_rq = rt_rq_of_se(rt_se); 709 rt_rq = rt_rq_of_se(rt_se);
566 requeue_rt_entity(rt_rq, rt_se); 710 requeue_rt_entity(rt_rq, rt_se, head);
567 } 711 }
568} 712}
569 713
570static void yield_task_rt(struct rq *rq) 714static void yield_task_rt(struct rq *rq)
571{ 715{
572 requeue_task_rt(rq, rq->curr); 716 requeue_task_rt(rq, rq->curr, 0);
573} 717}
574 718
575#ifdef CONFIG_SMP 719#ifdef CONFIG_SMP
@@ -609,6 +753,30 @@ static int select_task_rq_rt(struct task_struct *p, int sync)
609 */ 753 */
610 return task_cpu(p); 754 return task_cpu(p);
611} 755}
756
757static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
758{
759 cpumask_t mask;
760
761 if (rq->curr->rt.nr_cpus_allowed == 1)
762 return;
763
764 if (p->rt.nr_cpus_allowed != 1
765 && cpupri_find(&rq->rd->cpupri, p, &mask))
766 return;
767
768 if (!cpupri_find(&rq->rd->cpupri, rq->curr, &mask))
769 return;
770
771 /*
772 * There appears to be other cpus that can accept
773 * current and none to run 'p', so lets reschedule
774 * to try and push current away:
775 */
776 requeue_task_rt(rq, p, 1);
777 resched_task(rq->curr);
778}
779
612#endif /* CONFIG_SMP */ 780#endif /* CONFIG_SMP */
613 781
614/* 782/*
@@ -616,8 +784,27 @@ static int select_task_rq_rt(struct task_struct *p, int sync)
616 */ 784 */
617static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p) 785static void check_preempt_curr_rt(struct rq *rq, struct task_struct *p)
618{ 786{
619 if (p->prio < rq->curr->prio) 787 if (p->prio < rq->curr->prio) {
620 resched_task(rq->curr); 788 resched_task(rq->curr);
789 return;
790 }
791
792#ifdef CONFIG_SMP
793 /*
794 * If:
795 *
796 * - the newly woken task is of equal priority to the current task
797 * - the newly woken task is non-migratable while current is migratable
798 * - current will be preempted on the next reschedule
799 *
800 * we should check to see if current can readily move to a different
801 * cpu. If so, we will reschedule to allow the push logic to try
802 * to move current somewhere else, making room for our non-migratable
803 * task.
804 */
805 if (p->prio == rq->curr->prio && !need_resched())
806 check_preempt_equal_prio(rq, p);
807#endif
621} 808}
622 809
623static struct sched_rt_entity *pick_next_rt_entity(struct rq *rq, 810static struct sched_rt_entity *pick_next_rt_entity(struct rq *rq,
@@ -720,73 +907,6 @@ static struct task_struct *pick_next_highest_task_rt(struct rq *rq, int cpu)
720 907
721static DEFINE_PER_CPU(cpumask_t, local_cpu_mask); 908static DEFINE_PER_CPU(cpumask_t, local_cpu_mask);
722 909
723static int find_lowest_cpus(struct task_struct *task, cpumask_t *lowest_mask)
724{
725 int lowest_prio = -1;
726 int lowest_cpu = -1;
727 int count = 0;
728 int cpu;
729
730 cpus_and(*lowest_mask, task_rq(task)->rd->online, task->cpus_allowed);
731
732 /*
733 * Scan each rq for the lowest prio.
734 */
735 for_each_cpu_mask(cpu, *lowest_mask) {
736 struct rq *rq = cpu_rq(cpu);
737
738 /* We look for lowest RT prio or non-rt CPU */
739 if (rq->rt.highest_prio >= MAX_RT_PRIO) {
740 /*
741 * if we already found a low RT queue
742 * and now we found this non-rt queue
743 * clear the mask and set our bit.
744 * Otherwise just return the queue as is
745 * and the count==1 will cause the algorithm
746 * to use the first bit found.
747 */
748 if (lowest_cpu != -1) {
749 cpus_clear(*lowest_mask);
750 cpu_set(rq->cpu, *lowest_mask);
751 }
752 return 1;
753 }
754
755 /* no locking for now */
756 if ((rq->rt.highest_prio > task->prio)
757 && (rq->rt.highest_prio >= lowest_prio)) {
758 if (rq->rt.highest_prio > lowest_prio) {
759 /* new low - clear old data */
760 lowest_prio = rq->rt.highest_prio;
761 lowest_cpu = cpu;
762 count = 0;
763 }
764 count++;
765 } else
766 cpu_clear(cpu, *lowest_mask);
767 }
768
769 /*
770 * Clear out all the set bits that represent
771 * runqueues that were of higher prio than
772 * the lowest_prio.
773 */
774 if (lowest_cpu > 0) {
775 /*
776 * Perhaps we could add another cpumask op to
777 * zero out bits. Like cpu_zero_bits(cpumask, nrbits);
778 * Then that could be optimized to use memset and such.
779 */
780 for_each_cpu_mask(cpu, *lowest_mask) {
781 if (cpu >= lowest_cpu)
782 break;
783 cpu_clear(cpu, *lowest_mask);
784 }
785 }
786
787 return count;
788}
789
790static inline int pick_optimal_cpu(int this_cpu, cpumask_t *mask) 910static inline int pick_optimal_cpu(int this_cpu, cpumask_t *mask)
791{ 911{
792 int first; 912 int first;
@@ -808,17 +928,19 @@ static int find_lowest_rq(struct task_struct *task)
808 cpumask_t *lowest_mask = &__get_cpu_var(local_cpu_mask); 928 cpumask_t *lowest_mask = &__get_cpu_var(local_cpu_mask);
809 int this_cpu = smp_processor_id(); 929 int this_cpu = smp_processor_id();
810 int cpu = task_cpu(task); 930 int cpu = task_cpu(task);
811 int count = find_lowest_cpus(task, lowest_mask);
812 931
813 if (!count) 932 if (task->rt.nr_cpus_allowed == 1)
933 return -1; /* No other targets possible */
934
935 if (!cpupri_find(&task_rq(task)->rd->cpupri, task, lowest_mask))
814 return -1; /* No targets found */ 936 return -1; /* No targets found */
815 937
816 /* 938 /*
817 * There is no sense in performing an optimal search if only one 939 * Only consider CPUs that are usable for migration.
818 * target is found. 940 * I guess we might want to change cpupri_find() to ignore those
941 * in the first place.
819 */ 942 */
820 if (count == 1) 943 cpus_and(*lowest_mask, *lowest_mask, cpu_active_map);
821 return first_cpu(*lowest_mask);
822 944
823 /* 945 /*
824 * At this point we have built a mask of cpus representing the 946 * At this point we have built a mask of cpus representing the
@@ -1006,7 +1128,7 @@ static int pull_rt_task(struct rq *this_rq)
1006 1128
1007 next = pick_next_task_rt(this_rq); 1129 next = pick_next_task_rt(this_rq);
1008 1130
1009 for_each_cpu_mask(cpu, this_rq->rd->rto_mask) { 1131 for_each_cpu_mask_nr(cpu, this_rq->rd->rto_mask) {
1010 if (this_cpu == cpu) 1132 if (this_cpu == cpu)
1011 continue; 1133 continue;
1012 1134
@@ -1163,17 +1285,25 @@ static void set_cpus_allowed_rt(struct task_struct *p,
1163} 1285}
1164 1286
1165/* Assumes rq->lock is held */ 1287/* Assumes rq->lock is held */
1166static void join_domain_rt(struct rq *rq) 1288static void rq_online_rt(struct rq *rq)
1167{ 1289{
1168 if (rq->rt.overloaded) 1290 if (rq->rt.overloaded)
1169 rt_set_overload(rq); 1291 rt_set_overload(rq);
1292
1293 __enable_runtime(rq);
1294
1295 cpupri_set(&rq->rd->cpupri, rq->cpu, rq->rt.highest_prio);
1170} 1296}
1171 1297
1172/* Assumes rq->lock is held */ 1298/* Assumes rq->lock is held */
1173static void leave_domain_rt(struct rq *rq) 1299static void rq_offline_rt(struct rq *rq)
1174{ 1300{
1175 if (rq->rt.overloaded) 1301 if (rq->rt.overloaded)
1176 rt_clear_overload(rq); 1302 rt_clear_overload(rq);
1303
1304 __disable_runtime(rq);
1305
1306 cpupri_set(&rq->rd->cpupri, rq->cpu, CPUPRI_INVALID);
1177} 1307}
1178 1308
1179/* 1309/*
@@ -1306,7 +1436,7 @@ static void task_tick_rt(struct rq *rq, struct task_struct *p, int queued)
1306 * on the queue: 1436 * on the queue:
1307 */ 1437 */
1308 if (p->rt.run_list.prev != p->rt.run_list.next) { 1438 if (p->rt.run_list.prev != p->rt.run_list.next) {
1309 requeue_task_rt(rq, p); 1439 requeue_task_rt(rq, p, 0);
1310 set_tsk_need_resched(p); 1440 set_tsk_need_resched(p);
1311 } 1441 }
1312} 1442}
@@ -1336,8 +1466,8 @@ static const struct sched_class rt_sched_class = {
1336 .load_balance = load_balance_rt, 1466 .load_balance = load_balance_rt,
1337 .move_one_task = move_one_task_rt, 1467 .move_one_task = move_one_task_rt,
1338 .set_cpus_allowed = set_cpus_allowed_rt, 1468 .set_cpus_allowed = set_cpus_allowed_rt,
1339 .join_domain = join_domain_rt, 1469 .rq_online = rq_online_rt,
1340 .leave_domain = leave_domain_rt, 1470 .rq_offline = rq_offline_rt,
1341 .pre_schedule = pre_schedule_rt, 1471 .pre_schedule = pre_schedule_rt,
1342 .post_schedule = post_schedule_rt, 1472 .post_schedule = post_schedule_rt,
1343 .task_wake_up = task_wake_up_rt, 1473 .task_wake_up = task_wake_up_rt,
@@ -1350,3 +1480,17 @@ static const struct sched_class rt_sched_class = {
1350 .prio_changed = prio_changed_rt, 1480 .prio_changed = prio_changed_rt,
1351 .switched_to = switched_to_rt, 1481 .switched_to = switched_to_rt,
1352}; 1482};
1483
1484#ifdef CONFIG_SCHED_DEBUG
1485extern void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq);
1486
1487static void print_rt_stats(struct seq_file *m, int cpu)
1488{
1489 struct rt_rq *rt_rq;
1490
1491 rcu_read_lock();
1492 for_each_leaf_rt_rq(rt_rq, cpu_rq(cpu))
1493 print_rt_rq(m, cpu, rt_rq);
1494 rcu_read_unlock();
1495}
1496#endif /* CONFIG_SCHED_DEBUG */
diff --git a/kernel/sched_stats.h b/kernel/sched_stats.h
index 80179ef7450e..8385d43987e2 100644
--- a/kernel/sched_stats.h
+++ b/kernel/sched_stats.h
@@ -118,6 +118,13 @@ rq_sched_info_depart(struct rq *rq, unsigned long long delta)
118 if (rq) 118 if (rq)
119 rq->rq_sched_info.cpu_time += delta; 119 rq->rq_sched_info.cpu_time += delta;
120} 120}
121
122static inline void
123rq_sched_info_dequeued(struct rq *rq, unsigned long long delta)
124{
125 if (rq)
126 rq->rq_sched_info.run_delay += delta;
127}
121# define schedstat_inc(rq, field) do { (rq)->field++; } while (0) 128# define schedstat_inc(rq, field) do { (rq)->field++; } while (0)
122# define schedstat_add(rq, field, amt) do { (rq)->field += (amt); } while (0) 129# define schedstat_add(rq, field, amt) do { (rq)->field += (amt); } while (0)
123# define schedstat_set(var, val) do { var = (val); } while (0) 130# define schedstat_set(var, val) do { var = (val); } while (0)
@@ -126,6 +133,9 @@ static inline void
126rq_sched_info_arrive(struct rq *rq, unsigned long long delta) 133rq_sched_info_arrive(struct rq *rq, unsigned long long delta)
127{} 134{}
128static inline void 135static inline void
136rq_sched_info_dequeued(struct rq *rq, unsigned long long delta)
137{}
138static inline void
129rq_sched_info_depart(struct rq *rq, unsigned long long delta) 139rq_sched_info_depart(struct rq *rq, unsigned long long delta)
130{} 140{}
131# define schedstat_inc(rq, field) do { } while (0) 141# define schedstat_inc(rq, field) do { } while (0)
@@ -134,6 +144,11 @@ rq_sched_info_depart(struct rq *rq, unsigned long long delta)
134#endif 144#endif
135 145
136#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) 146#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT)
147static inline void sched_info_reset_dequeued(struct task_struct *t)
148{
149 t->sched_info.last_queued = 0;
150}
151
137/* 152/*
138 * Called when a process is dequeued from the active array and given 153 * Called when a process is dequeued from the active array and given
139 * the cpu. We should note that with the exception of interactive 154 * the cpu. We should note that with the exception of interactive
@@ -143,15 +158,22 @@ rq_sched_info_depart(struct rq *rq, unsigned long long delta)
143 * active queue, thus delaying tasks in the expired queue from running; 158 * active queue, thus delaying tasks in the expired queue from running;
144 * see scheduler_tick()). 159 * see scheduler_tick()).
145 * 160 *
146 * This function is only called from sched_info_arrive(), rather than 161 * Though we are interested in knowing how long it was from the *first* time a
147 * dequeue_task(). Even though a task may be queued and dequeued multiple 162 * task was queued to the time that it finally hit a cpu, we call this routine
148 * times as it is shuffled about, we're really interested in knowing how 163 * from dequeue_task() to account for possible rq->clock skew across cpus. The
149 * long it was from the *first* time it was queued to the time that it 164 * delta taken on each cpu would annul the skew.
150 * finally hit a cpu.
151 */ 165 */
152static inline void sched_info_dequeued(struct task_struct *t) 166static inline void sched_info_dequeued(struct task_struct *t)
153{ 167{
154 t->sched_info.last_queued = 0; 168 unsigned long long now = task_rq(t)->clock, delta = 0;
169
170 if (unlikely(sched_info_on()))
171 if (t->sched_info.last_queued)
172 delta = now - t->sched_info.last_queued;
173 sched_info_reset_dequeued(t);
174 t->sched_info.run_delay += delta;
175
176 rq_sched_info_dequeued(task_rq(t), delta);
155} 177}
156 178
157/* 179/*
@@ -165,7 +187,7 @@ static void sched_info_arrive(struct task_struct *t)
165 187
166 if (t->sched_info.last_queued) 188 if (t->sched_info.last_queued)
167 delta = now - t->sched_info.last_queued; 189 delta = now - t->sched_info.last_queued;
168 sched_info_dequeued(t); 190 sched_info_reset_dequeued(t);
169 t->sched_info.run_delay += delta; 191 t->sched_info.run_delay += delta;
170 t->sched_info.last_arrival = now; 192 t->sched_info.last_arrival = now;
171 t->sched_info.pcount++; 193 t->sched_info.pcount++;
@@ -242,7 +264,9 @@ sched_info_switch(struct task_struct *prev, struct task_struct *next)
242 __sched_info_switch(prev, next); 264 __sched_info_switch(prev, next);
243} 265}
244#else 266#else
245#define sched_info_queued(t) do { } while (0) 267#define sched_info_queued(t) do { } while (0)
246#define sched_info_switch(t, next) do { } while (0) 268#define sched_info_reset_dequeued(t) do { } while (0)
269#define sched_info_dequeued(t) do { } while (0)
270#define sched_info_switch(t, next) do { } while (0)
247#endif /* CONFIG_SCHEDSTATS || CONFIG_TASK_DELAY_ACCT */ 271#endif /* CONFIG_SCHEDSTATS || CONFIG_TASK_DELAY_ACCT */
248 272
diff --git a/kernel/semaphore.c b/kernel/semaphore.c
index 5c2942e768cd..aaaeae8244e7 100644
--- a/kernel/semaphore.c
+++ b/kernel/semaphore.c
@@ -31,6 +31,7 @@
31#include <linux/sched.h> 31#include <linux/sched.h>
32#include <linux/semaphore.h> 32#include <linux/semaphore.h>
33#include <linux/spinlock.h> 33#include <linux/spinlock.h>
34#include <linux/ftrace.h>
34 35
35static noinline void __down(struct semaphore *sem); 36static noinline void __down(struct semaphore *sem);
36static noinline int __down_interruptible(struct semaphore *sem); 37static noinline int __down_interruptible(struct semaphore *sem);
diff --git a/kernel/signal.c b/kernel/signal.c
index 6c0958e52ea7..954f77d7e3bc 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -22,6 +22,7 @@
22#include <linux/ptrace.h> 22#include <linux/ptrace.h>
23#include <linux/signal.h> 23#include <linux/signal.h>
24#include <linux/signalfd.h> 24#include <linux/signalfd.h>
25#include <linux/tracehook.h>
25#include <linux/capability.h> 26#include <linux/capability.h>
26#include <linux/freezer.h> 27#include <linux/freezer.h>
27#include <linux/pid_namespace.h> 28#include <linux/pid_namespace.h>
@@ -39,24 +40,21 @@
39 40
40static struct kmem_cache *sigqueue_cachep; 41static struct kmem_cache *sigqueue_cachep;
41 42
42static int __sig_ignored(struct task_struct *t, int sig) 43static void __user *sig_handler(struct task_struct *t, int sig)
43{ 44{
44 void __user *handler; 45 return t->sighand->action[sig - 1].sa.sa_handler;
46}
45 47
48static int sig_handler_ignored(void __user *handler, int sig)
49{
46 /* Is it explicitly or implicitly ignored? */ 50 /* Is it explicitly or implicitly ignored? */
47
48 handler = t->sighand->action[sig - 1].sa.sa_handler;
49 return handler == SIG_IGN || 51 return handler == SIG_IGN ||
50 (handler == SIG_DFL && sig_kernel_ignore(sig)); 52 (handler == SIG_DFL && sig_kernel_ignore(sig));
51} 53}
52 54
53static int sig_ignored(struct task_struct *t, int sig) 55static int sig_ignored(struct task_struct *t, int sig)
54{ 56{
55 /* 57 void __user *handler;
56 * Tracers always want to know about signals..
57 */
58 if (t->ptrace & PT_PTRACED)
59 return 0;
60 58
61 /* 59 /*
62 * Blocked signals are never ignored, since the 60 * Blocked signals are never ignored, since the
@@ -66,7 +64,14 @@ static int sig_ignored(struct task_struct *t, int sig)
66 if (sigismember(&t->blocked, sig) || sigismember(&t->real_blocked, sig)) 64 if (sigismember(&t->blocked, sig) || sigismember(&t->real_blocked, sig))
67 return 0; 65 return 0;
68 66
69 return __sig_ignored(t, sig); 67 handler = sig_handler(t, sig);
68 if (!sig_handler_ignored(handler, sig))
69 return 0;
70
71 /*
72 * Tracers may want to know about even ignored signals.
73 */
74 return !tracehook_consider_ignored_signal(t, sig, handler);
70} 75}
71 76
72/* 77/*
@@ -129,7 +134,9 @@ void recalc_sigpending_and_wake(struct task_struct *t)
129 134
130void recalc_sigpending(void) 135void recalc_sigpending(void)
131{ 136{
132 if (!recalc_sigpending_tsk(current) && !freezing(current)) 137 if (unlikely(tracehook_force_sigpending()))
138 set_thread_flag(TIF_SIGPENDING);
139 else if (!recalc_sigpending_tsk(current) && !freezing(current))
133 clear_thread_flag(TIF_SIGPENDING); 140 clear_thread_flag(TIF_SIGPENDING);
134 141
135} 142}
@@ -295,12 +302,12 @@ flush_signal_handlers(struct task_struct *t, int force_default)
295 302
296int unhandled_signal(struct task_struct *tsk, int sig) 303int unhandled_signal(struct task_struct *tsk, int sig)
297{ 304{
305 void __user *handler = tsk->sighand->action[sig-1].sa.sa_handler;
298 if (is_global_init(tsk)) 306 if (is_global_init(tsk))
299 return 1; 307 return 1;
300 if (tsk->ptrace & PT_PTRACED) 308 if (handler != SIG_IGN && handler != SIG_DFL)
301 return 0; 309 return 0;
302 return (tsk->sighand->action[sig-1].sa.sa_handler == SIG_IGN) || 310 return !tracehook_consider_fatal_signal(tsk, sig, handler);
303 (tsk->sighand->action[sig-1].sa.sa_handler == SIG_DFL);
304} 311}
305 312
306 313
@@ -338,13 +345,9 @@ unblock_all_signals(void)
338 spin_unlock_irqrestore(&current->sighand->siglock, flags); 345 spin_unlock_irqrestore(&current->sighand->siglock, flags);
339} 346}
340 347
341static int collect_signal(int sig, struct sigpending *list, siginfo_t *info) 348static void collect_signal(int sig, struct sigpending *list, siginfo_t *info)
342{ 349{
343 struct sigqueue *q, *first = NULL; 350 struct sigqueue *q, *first = NULL;
344 int still_pending = 0;
345
346 if (unlikely(!sigismember(&list->signal, sig)))
347 return 0;
348 351
349 /* 352 /*
350 * Collect the siginfo appropriate to this signal. Check if 353 * Collect the siginfo appropriate to this signal. Check if
@@ -352,33 +355,30 @@ static int collect_signal(int sig, struct sigpending *list, siginfo_t *info)
352 */ 355 */
353 list_for_each_entry(q, &list->list, list) { 356 list_for_each_entry(q, &list->list, list) {
354 if (q->info.si_signo == sig) { 357 if (q->info.si_signo == sig) {
355 if (first) { 358 if (first)
356 still_pending = 1; 359 goto still_pending;
357 break;
358 }
359 first = q; 360 first = q;
360 } 361 }
361 } 362 }
363
364 sigdelset(&list->signal, sig);
365
362 if (first) { 366 if (first) {
367still_pending:
363 list_del_init(&first->list); 368 list_del_init(&first->list);
364 copy_siginfo(info, &first->info); 369 copy_siginfo(info, &first->info);
365 __sigqueue_free(first); 370 __sigqueue_free(first);
366 if (!still_pending)
367 sigdelset(&list->signal, sig);
368 } else { 371 } else {
369
370 /* Ok, it wasn't in the queue. This must be 372 /* Ok, it wasn't in the queue. This must be
371 a fast-pathed signal or we must have been 373 a fast-pathed signal or we must have been
372 out of queue space. So zero out the info. 374 out of queue space. So zero out the info.
373 */ 375 */
374 sigdelset(&list->signal, sig);
375 info->si_signo = sig; 376 info->si_signo = sig;
376 info->si_errno = 0; 377 info->si_errno = 0;
377 info->si_code = 0; 378 info->si_code = 0;
378 info->si_pid = 0; 379 info->si_pid = 0;
379 info->si_uid = 0; 380 info->si_uid = 0;
380 } 381 }
381 return 1;
382} 382}
383 383
384static int __dequeue_signal(struct sigpending *pending, sigset_t *mask, 384static int __dequeue_signal(struct sigpending *pending, sigset_t *mask,
@@ -396,8 +396,7 @@ static int __dequeue_signal(struct sigpending *pending, sigset_t *mask,
396 } 396 }
397 } 397 }
398 398
399 if (!collect_signal(sig, pending, info)) 399 collect_signal(sig, pending, info);
400 sig = 0;
401 } 400 }
402 401
403 return sig; 402 return sig;
@@ -462,8 +461,7 @@ int dequeue_signal(struct task_struct *tsk, sigset_t *mask, siginfo_t *info)
462 * is to alert stop-signal processing code when another 461 * is to alert stop-signal processing code when another
463 * processor has come along and cleared the flag. 462 * processor has come along and cleared the flag.
464 */ 463 */
465 if (!(tsk->signal->flags & SIGNAL_GROUP_EXIT)) 464 tsk->signal->flags |= SIGNAL_STOP_DEQUEUED;
466 tsk->signal->flags |= SIGNAL_STOP_DEQUEUED;
467 } 465 }
468 if ((info->si_code & __SI_MASK) == __SI_TIMER && info->si_sys_private) { 466 if ((info->si_code & __SI_MASK) == __SI_TIMER && info->si_sys_private) {
469 /* 467 /*
@@ -600,9 +598,6 @@ static int check_kill_permission(int sig, struct siginfo *info,
600 return security_task_kill(t, info, sig, 0); 598 return security_task_kill(t, info, sig, 0);
601} 599}
602 600
603/* forward decl */
604static void do_notify_parent_cldstop(struct task_struct *tsk, int why);
605
606/* 601/*
607 * Handle magic process-wide effects of stop/continue signals. Unlike 602 * Handle magic process-wide effects of stop/continue signals. Unlike
608 * the signal actions, these happen immediately at signal-generation 603 * the signal actions, these happen immediately at signal-generation
@@ -765,7 +760,8 @@ static void complete_signal(int sig, struct task_struct *p, int group)
765 if (sig_fatal(p, sig) && 760 if (sig_fatal(p, sig) &&
766 !(signal->flags & (SIGNAL_UNKILLABLE | SIGNAL_GROUP_EXIT)) && 761 !(signal->flags & (SIGNAL_UNKILLABLE | SIGNAL_GROUP_EXIT)) &&
767 !sigismember(&t->real_blocked, sig) && 762 !sigismember(&t->real_blocked, sig) &&
768 (sig == SIGKILL || !(t->ptrace & PT_PTRACED))) { 763 (sig == SIGKILL ||
764 !tracehook_consider_fatal_signal(t, sig, SIG_DFL))) {
769 /* 765 /*
770 * This signal will be fatal to the whole group. 766 * This signal will be fatal to the whole group.
771 */ 767 */
@@ -1125,7 +1121,7 @@ EXPORT_SYMBOL_GPL(kill_pid_info_as_uid);
1125 * is probably wrong. Should make it like BSD or SYSV. 1121 * is probably wrong. Should make it like BSD or SYSV.
1126 */ 1122 */
1127 1123
1128static int kill_something_info(int sig, struct siginfo *info, int pid) 1124static int kill_something_info(int sig, struct siginfo *info, pid_t pid)
1129{ 1125{
1130 int ret; 1126 int ret;
1131 1127
@@ -1237,17 +1233,6 @@ int kill_pid(struct pid *pid, int sig, int priv)
1237} 1233}
1238EXPORT_SYMBOL(kill_pid); 1234EXPORT_SYMBOL(kill_pid);
1239 1235
1240int
1241kill_proc(pid_t pid, int sig, int priv)
1242{
1243 int ret;
1244
1245 rcu_read_lock();
1246 ret = kill_pid_info(sig, __si_special(priv), find_pid(pid));
1247 rcu_read_unlock();
1248 return ret;
1249}
1250
1251/* 1236/*
1252 * These functions support sending signals using preallocated sigqueue 1237 * These functions support sending signals using preallocated sigqueue
1253 * structures. This is needed "because realtime applications cannot 1238 * structures. This is needed "because realtime applications cannot
@@ -1343,9 +1328,11 @@ static inline void __wake_up_parent(struct task_struct *p,
1343/* 1328/*
1344 * Let a parent know about the death of a child. 1329 * Let a parent know about the death of a child.
1345 * For a stopped/continued status change, use do_notify_parent_cldstop instead. 1330 * For a stopped/continued status change, use do_notify_parent_cldstop instead.
1331 *
1332 * Returns -1 if our parent ignored us and so we've switched to
1333 * self-reaping, or else @sig.
1346 */ 1334 */
1347 1335int do_notify_parent(struct task_struct *tsk, int sig)
1348void do_notify_parent(struct task_struct *tsk, int sig)
1349{ 1336{
1350 struct siginfo info; 1337 struct siginfo info;
1351 unsigned long flags; 1338 unsigned long flags;
@@ -1379,10 +1366,9 @@ void do_notify_parent(struct task_struct *tsk, int sig)
1379 1366
1380 info.si_uid = tsk->uid; 1367 info.si_uid = tsk->uid;
1381 1368
1382 /* FIXME: find out whether or not this is supposed to be c*time. */ 1369 info.si_utime = cputime_to_clock_t(cputime_add(tsk->utime,
1383 info.si_utime = cputime_to_jiffies(cputime_add(tsk->utime,
1384 tsk->signal->utime)); 1370 tsk->signal->utime));
1385 info.si_stime = cputime_to_jiffies(cputime_add(tsk->stime, 1371 info.si_stime = cputime_to_clock_t(cputime_add(tsk->stime,
1386 tsk->signal->stime)); 1372 tsk->signal->stime));
1387 1373
1388 info.si_status = tsk->exit_code & 0x7f; 1374 info.si_status = tsk->exit_code & 0x7f;
@@ -1417,12 +1403,14 @@ void do_notify_parent(struct task_struct *tsk, int sig)
1417 */ 1403 */
1418 tsk->exit_signal = -1; 1404 tsk->exit_signal = -1;
1419 if (psig->action[SIGCHLD-1].sa.sa_handler == SIG_IGN) 1405 if (psig->action[SIGCHLD-1].sa.sa_handler == SIG_IGN)
1420 sig = 0; 1406 sig = -1;
1421 } 1407 }
1422 if (valid_signal(sig) && sig > 0) 1408 if (valid_signal(sig) && sig > 0)
1423 __group_send_sig_info(sig, &info, tsk->parent); 1409 __group_send_sig_info(sig, &info, tsk->parent);
1424 __wake_up_parent(tsk, tsk->parent); 1410 __wake_up_parent(tsk, tsk->parent);
1425 spin_unlock_irqrestore(&psig->siglock, flags); 1411 spin_unlock_irqrestore(&psig->siglock, flags);
1412
1413 return sig;
1426} 1414}
1427 1415
1428static void do_notify_parent_cldstop(struct task_struct *tsk, int why) 1416static void do_notify_parent_cldstop(struct task_struct *tsk, int why)
@@ -1450,9 +1438,8 @@ static void do_notify_parent_cldstop(struct task_struct *tsk, int why)
1450 1438
1451 info.si_uid = tsk->uid; 1439 info.si_uid = tsk->uid;
1452 1440
1453 /* FIXME: find out whether or not this is supposed to be c*time. */ 1441 info.si_utime = cputime_to_clock_t(tsk->utime);
1454 info.si_utime = cputime_to_jiffies(tsk->utime); 1442 info.si_stime = cputime_to_clock_t(tsk->stime);
1455 info.si_stime = cputime_to_jiffies(tsk->stime);
1456 1443
1457 info.si_code = why; 1444 info.si_code = why;
1458 switch (why) { 1445 switch (why) {
@@ -1491,10 +1478,10 @@ static inline int may_ptrace_stop(void)
1491 * is a deadlock situation, and pointless because our tracer 1478 * is a deadlock situation, and pointless because our tracer
1492 * is dead so don't allow us to stop. 1479 * is dead so don't allow us to stop.
1493 * If SIGKILL was already sent before the caller unlocked 1480 * If SIGKILL was already sent before the caller unlocked
1494 * ->siglock we must see ->core_waiters != 0. Otherwise it 1481 * ->siglock we must see ->core_state != NULL. Otherwise it
1495 * is safe to enter schedule(). 1482 * is safe to enter schedule().
1496 */ 1483 */
1497 if (unlikely(current->mm->core_waiters) && 1484 if (unlikely(current->mm->core_state) &&
1498 unlikely(current->mm == current->parent->mm)) 1485 unlikely(current->mm == current->parent->mm))
1499 return 0; 1486 return 0;
1500 1487
@@ -1507,9 +1494,8 @@ static inline int may_ptrace_stop(void)
1507 */ 1494 */
1508static int sigkill_pending(struct task_struct *tsk) 1495static int sigkill_pending(struct task_struct *tsk)
1509{ 1496{
1510 return ((sigismember(&tsk->pending.signal, SIGKILL) || 1497 return sigismember(&tsk->pending.signal, SIGKILL) ||
1511 sigismember(&tsk->signal->shared_pending.signal, SIGKILL)) && 1498 sigismember(&tsk->signal->shared_pending.signal, SIGKILL);
1512 !unlikely(sigismember(&tsk->blocked, SIGKILL)));
1513} 1499}
1514 1500
1515/* 1501/*
@@ -1525,8 +1511,6 @@ static int sigkill_pending(struct task_struct *tsk)
1525 */ 1511 */
1526static void ptrace_stop(int exit_code, int clear_code, siginfo_t *info) 1512static void ptrace_stop(int exit_code, int clear_code, siginfo_t *info)
1527{ 1513{
1528 int killed = 0;
1529
1530 if (arch_ptrace_stop_needed(exit_code, info)) { 1514 if (arch_ptrace_stop_needed(exit_code, info)) {
1531 /* 1515 /*
1532 * The arch code has something special to do before a 1516 * The arch code has something special to do before a
@@ -1542,7 +1526,8 @@ static void ptrace_stop(int exit_code, int clear_code, siginfo_t *info)
1542 spin_unlock_irq(&current->sighand->siglock); 1526 spin_unlock_irq(&current->sighand->siglock);
1543 arch_ptrace_stop(exit_code, info); 1527 arch_ptrace_stop(exit_code, info);
1544 spin_lock_irq(&current->sighand->siglock); 1528 spin_lock_irq(&current->sighand->siglock);
1545 killed = sigkill_pending(current); 1529 if (sigkill_pending(current))
1530 return;
1546 } 1531 }
1547 1532
1548 /* 1533 /*
@@ -1559,7 +1544,7 @@ static void ptrace_stop(int exit_code, int clear_code, siginfo_t *info)
1559 __set_current_state(TASK_TRACED); 1544 __set_current_state(TASK_TRACED);
1560 spin_unlock_irq(&current->sighand->siglock); 1545 spin_unlock_irq(&current->sighand->siglock);
1561 read_lock(&tasklist_lock); 1546 read_lock(&tasklist_lock);
1562 if (!unlikely(killed) && may_ptrace_stop()) { 1547 if (may_ptrace_stop()) {
1563 do_notify_parent_cldstop(current, CLD_TRAPPED); 1548 do_notify_parent_cldstop(current, CLD_TRAPPED);
1564 read_unlock(&tasklist_lock); 1549 read_unlock(&tasklist_lock);
1565 schedule(); 1550 schedule();
@@ -1623,7 +1608,7 @@ finish_stop(int stop_count)
1623 * a group stop in progress and we are the last to stop, 1608 * a group stop in progress and we are the last to stop,
1624 * report to the parent. When ptraced, every thread reports itself. 1609 * report to the parent. When ptraced, every thread reports itself.
1625 */ 1610 */
1626 if (stop_count == 0 || (current->ptrace & PT_PTRACED)) { 1611 if (tracehook_notify_jctl(stop_count == 0, CLD_STOPPED)) {
1627 read_lock(&tasklist_lock); 1612 read_lock(&tasklist_lock);
1628 do_notify_parent_cldstop(current, CLD_STOPPED); 1613 do_notify_parent_cldstop(current, CLD_STOPPED);
1629 read_unlock(&tasklist_lock); 1614 read_unlock(&tasklist_lock);
@@ -1658,8 +1643,7 @@ static int do_signal_stop(int signr)
1658 } else { 1643 } else {
1659 struct task_struct *t; 1644 struct task_struct *t;
1660 1645
1661 if (unlikely((sig->flags & (SIGNAL_STOP_DEQUEUED | SIGNAL_UNKILLABLE)) 1646 if (!likely(sig->flags & SIGNAL_STOP_DEQUEUED) ||
1662 != SIGNAL_STOP_DEQUEUED) ||
1663 unlikely(signal_group_exit(sig))) 1647 unlikely(signal_group_exit(sig)))
1664 return 0; 1648 return 0;
1665 /* 1649 /*
@@ -1760,6 +1744,9 @@ relock:
1760 signal->flags &= ~SIGNAL_CLD_MASK; 1744 signal->flags &= ~SIGNAL_CLD_MASK;
1761 spin_unlock_irq(&sighand->siglock); 1745 spin_unlock_irq(&sighand->siglock);
1762 1746
1747 if (unlikely(!tracehook_notify_jctl(1, why)))
1748 goto relock;
1749
1763 read_lock(&tasklist_lock); 1750 read_lock(&tasklist_lock);
1764 do_notify_parent_cldstop(current->group_leader, why); 1751 do_notify_parent_cldstop(current->group_leader, why);
1765 read_unlock(&tasklist_lock); 1752 read_unlock(&tasklist_lock);
@@ -1773,17 +1760,33 @@ relock:
1773 do_signal_stop(0)) 1760 do_signal_stop(0))
1774 goto relock; 1761 goto relock;
1775 1762
1776 signr = dequeue_signal(current, &current->blocked, info); 1763 /*
1777 if (!signr) 1764 * Tracing can induce an artifical signal and choose sigaction.
1778 break; /* will return 0 */ 1765 * The return value in @signr determines the default action,
1766 * but @info->si_signo is the signal number we will report.
1767 */
1768 signr = tracehook_get_signal(current, regs, info, return_ka);
1769 if (unlikely(signr < 0))
1770 goto relock;
1771 if (unlikely(signr != 0))
1772 ka = return_ka;
1773 else {
1774 signr = dequeue_signal(current, &current->blocked,
1775 info);
1779 1776
1780 if (signr != SIGKILL) {
1781 signr = ptrace_signal(signr, info, regs, cookie);
1782 if (!signr) 1777 if (!signr)
1783 continue; 1778 break; /* will return 0 */
1779
1780 if (signr != SIGKILL) {
1781 signr = ptrace_signal(signr, info,
1782 regs, cookie);
1783 if (!signr)
1784 continue;
1785 }
1786
1787 ka = &sighand->action[signr-1];
1784 } 1788 }
1785 1789
1786 ka = &sighand->action[signr-1];
1787 if (ka->sa.sa_handler == SIG_IGN) /* Do nothing. */ 1790 if (ka->sa.sa_handler == SIG_IGN) /* Do nothing. */
1788 continue; 1791 continue;
1789 if (ka->sa.sa_handler != SIG_DFL) { 1792 if (ka->sa.sa_handler != SIG_DFL) {
@@ -1831,7 +1834,7 @@ relock:
1831 spin_lock_irq(&sighand->siglock); 1834 spin_lock_irq(&sighand->siglock);
1832 } 1835 }
1833 1836
1834 if (likely(do_signal_stop(signr))) { 1837 if (likely(do_signal_stop(info->si_signo))) {
1835 /* It released the siglock. */ 1838 /* It released the siglock. */
1836 goto relock; 1839 goto relock;
1837 } 1840 }
@@ -1852,7 +1855,7 @@ relock:
1852 1855
1853 if (sig_kernel_coredump(signr)) { 1856 if (sig_kernel_coredump(signr)) {
1854 if (print_fatal_signals) 1857 if (print_fatal_signals)
1855 print_fatal_signal(regs, signr); 1858 print_fatal_signal(regs, info->si_signo);
1856 /* 1859 /*
1857 * If it was able to dump core, this kills all 1860 * If it was able to dump core, this kills all
1858 * other threads in the group and synchronizes with 1861 * other threads in the group and synchronizes with
@@ -1861,13 +1864,13 @@ relock:
1861 * first and our do_group_exit call below will use 1864 * first and our do_group_exit call below will use
1862 * that value and ignore the one we pass it. 1865 * that value and ignore the one we pass it.
1863 */ 1866 */
1864 do_coredump((long)signr, signr, regs); 1867 do_coredump(info->si_signo, info->si_signo, regs);
1865 } 1868 }
1866 1869
1867 /* 1870 /*
1868 * Death signals, no core dump. 1871 * Death signals, no core dump.
1869 */ 1872 */
1870 do_group_exit(signr); 1873 do_group_exit(info->si_signo);
1871 /* NOTREACHED */ 1874 /* NOTREACHED */
1872 } 1875 }
1873 spin_unlock_irq(&sighand->siglock); 1876 spin_unlock_irq(&sighand->siglock);
@@ -1909,7 +1912,7 @@ void exit_signals(struct task_struct *tsk)
1909out: 1912out:
1910 spin_unlock_irq(&tsk->sighand->siglock); 1913 spin_unlock_irq(&tsk->sighand->siglock);
1911 1914
1912 if (unlikely(group_stop)) { 1915 if (unlikely(group_stop) && tracehook_notify_jctl(1, CLD_STOPPED)) {
1913 read_lock(&tasklist_lock); 1916 read_lock(&tasklist_lock);
1914 do_notify_parent_cldstop(tsk, CLD_STOPPED); 1917 do_notify_parent_cldstop(tsk, CLD_STOPPED);
1915 read_unlock(&tasklist_lock); 1918 read_unlock(&tasklist_lock);
@@ -1920,8 +1923,6 @@ EXPORT_SYMBOL(recalc_sigpending);
1920EXPORT_SYMBOL_GPL(dequeue_signal); 1923EXPORT_SYMBOL_GPL(dequeue_signal);
1921EXPORT_SYMBOL(flush_signals); 1924EXPORT_SYMBOL(flush_signals);
1922EXPORT_SYMBOL(force_sig); 1925EXPORT_SYMBOL(force_sig);
1923EXPORT_SYMBOL(kill_proc);
1924EXPORT_SYMBOL(ptrace_notify);
1925EXPORT_SYMBOL(send_sig); 1926EXPORT_SYMBOL(send_sig);
1926EXPORT_SYMBOL(send_sig_info); 1927EXPORT_SYMBOL(send_sig_info);
1927EXPORT_SYMBOL(sigprocmask); 1928EXPORT_SYMBOL(sigprocmask);
@@ -2196,7 +2197,7 @@ sys_rt_sigtimedwait(const sigset_t __user *uthese,
2196} 2197}
2197 2198
2198asmlinkage long 2199asmlinkage long
2199sys_kill(int pid, int sig) 2200sys_kill(pid_t pid, int sig)
2200{ 2201{
2201 struct siginfo info; 2202 struct siginfo info;
2202 2203
@@ -2209,7 +2210,7 @@ sys_kill(int pid, int sig)
2209 return kill_something_info(sig, &info, pid); 2210 return kill_something_info(sig, &info, pid);
2210} 2211}
2211 2212
2212static int do_tkill(int tgid, int pid, int sig) 2213static int do_tkill(pid_t tgid, pid_t pid, int sig)
2213{ 2214{
2214 int error; 2215 int error;
2215 struct siginfo info; 2216 struct siginfo info;
@@ -2255,7 +2256,7 @@ static int do_tkill(int tgid, int pid, int sig)
2255 * exists but it's not belonging to the target process anymore. This 2256 * exists but it's not belonging to the target process anymore. This
2256 * method solves the problem of threads exiting and PIDs getting reused. 2257 * method solves the problem of threads exiting and PIDs getting reused.
2257 */ 2258 */
2258asmlinkage long sys_tgkill(int tgid, int pid, int sig) 2259asmlinkage long sys_tgkill(pid_t tgid, pid_t pid, int sig)
2259{ 2260{
2260 /* This is only valid for single tasks */ 2261 /* This is only valid for single tasks */
2261 if (pid <= 0 || tgid <= 0) 2262 if (pid <= 0 || tgid <= 0)
@@ -2268,7 +2269,7 @@ asmlinkage long sys_tgkill(int tgid, int pid, int sig)
2268 * Send a signal to only one task, even if it's a CLONE_THREAD task. 2269 * Send a signal to only one task, even if it's a CLONE_THREAD task.
2269 */ 2270 */
2270asmlinkage long 2271asmlinkage long
2271sys_tkill(int pid, int sig) 2272sys_tkill(pid_t pid, int sig)
2272{ 2273{
2273 /* This is only valid for single tasks */ 2274 /* This is only valid for single tasks */
2274 if (pid <= 0) 2275 if (pid <= 0)
@@ -2278,7 +2279,7 @@ sys_tkill(int pid, int sig)
2278} 2279}
2279 2280
2280asmlinkage long 2281asmlinkage long
2281sys_rt_sigqueueinfo(int pid, int sig, siginfo_t __user *uinfo) 2282sys_rt_sigqueueinfo(pid_t pid, int sig, siginfo_t __user *uinfo)
2282{ 2283{
2283 siginfo_t info; 2284 siginfo_t info;
2284 2285
@@ -2325,7 +2326,7 @@ int do_sigaction(int sig, struct k_sigaction *act, struct k_sigaction *oact)
2325 * (for example, SIGCHLD), shall cause the pending signal to 2326 * (for example, SIGCHLD), shall cause the pending signal to
2326 * be discarded, whether or not it is blocked" 2327 * be discarded, whether or not it is blocked"
2327 */ 2328 */
2328 if (__sig_ignored(t, sig)) { 2329 if (sig_handler_ignored(sig_handler(t, sig), sig)) {
2329 sigemptyset(&mask); 2330 sigemptyset(&mask);
2330 sigaddset(&mask, sig); 2331 sigaddset(&mask, sig);
2331 rm_from_queue_full(&mask, &t->signal->shared_pending); 2332 rm_from_queue_full(&mask, &t->signal->shared_pending);
diff --git a/kernel/smp.c b/kernel/smp.c
new file mode 100644
index 000000000000..96fc7c0edc59
--- /dev/null
+++ b/kernel/smp.c
@@ -0,0 +1,385 @@
1/*
2 * Generic helpers for smp ipi calls
3 *
4 * (C) Jens Axboe <jens.axboe@oracle.com> 2008
5 *
6 */
7#include <linux/init.h>
8#include <linux/module.h>
9#include <linux/percpu.h>
10#include <linux/rcupdate.h>
11#include <linux/rculist.h>
12#include <linux/smp.h>
13
14static DEFINE_PER_CPU(struct call_single_queue, call_single_queue);
15static LIST_HEAD(call_function_queue);
16__cacheline_aligned_in_smp DEFINE_SPINLOCK(call_function_lock);
17
18enum {
19 CSD_FLAG_WAIT = 0x01,
20 CSD_FLAG_ALLOC = 0x02,
21};
22
23struct call_function_data {
24 struct call_single_data csd;
25 spinlock_t lock;
26 unsigned int refs;
27 cpumask_t cpumask;
28 struct rcu_head rcu_head;
29};
30
31struct call_single_queue {
32 struct list_head list;
33 spinlock_t lock;
34};
35
36static int __cpuinit init_call_single_data(void)
37{
38 int i;
39
40 for_each_possible_cpu(i) {
41 struct call_single_queue *q = &per_cpu(call_single_queue, i);
42
43 spin_lock_init(&q->lock);
44 INIT_LIST_HEAD(&q->list);
45 }
46 return 0;
47}
48early_initcall(init_call_single_data);
49
50static void csd_flag_wait(struct call_single_data *data)
51{
52 /* Wait for response */
53 do {
54 /*
55 * We need to see the flags store in the IPI handler
56 */
57 smp_mb();
58 if (!(data->flags & CSD_FLAG_WAIT))
59 break;
60 cpu_relax();
61 } while (1);
62}
63
64/*
65 * Insert a previously allocated call_single_data element for execution
66 * on the given CPU. data must already have ->func, ->info, and ->flags set.
67 */
68static void generic_exec_single(int cpu, struct call_single_data *data)
69{
70 struct call_single_queue *dst = &per_cpu(call_single_queue, cpu);
71 int wait = data->flags & CSD_FLAG_WAIT, ipi;
72 unsigned long flags;
73
74 spin_lock_irqsave(&dst->lock, flags);
75 ipi = list_empty(&dst->list);
76 list_add_tail(&data->list, &dst->list);
77 spin_unlock_irqrestore(&dst->lock, flags);
78
79 if (ipi)
80 arch_send_call_function_single_ipi(cpu);
81
82 if (wait)
83 csd_flag_wait(data);
84}
85
86static void rcu_free_call_data(struct rcu_head *head)
87{
88 struct call_function_data *data;
89
90 data = container_of(head, struct call_function_data, rcu_head);
91
92 kfree(data);
93}
94
95/*
96 * Invoked by arch to handle an IPI for call function. Must be called with
97 * interrupts disabled.
98 */
99void generic_smp_call_function_interrupt(void)
100{
101 struct call_function_data *data;
102 int cpu = get_cpu();
103
104 /*
105 * It's ok to use list_for_each_rcu() here even though we may delete
106 * 'pos', since list_del_rcu() doesn't clear ->next
107 */
108 rcu_read_lock();
109 list_for_each_entry_rcu(data, &call_function_queue, csd.list) {
110 int refs;
111
112 if (!cpu_isset(cpu, data->cpumask))
113 continue;
114
115 data->csd.func(data->csd.info);
116
117 spin_lock(&data->lock);
118 cpu_clear(cpu, data->cpumask);
119 WARN_ON(data->refs == 0);
120 data->refs--;
121 refs = data->refs;
122 spin_unlock(&data->lock);
123
124 if (refs)
125 continue;
126
127 spin_lock(&call_function_lock);
128 list_del_rcu(&data->csd.list);
129 spin_unlock(&call_function_lock);
130
131 if (data->csd.flags & CSD_FLAG_WAIT) {
132 /*
133 * serialize stores to data with the flag clear
134 * and wakeup
135 */
136 smp_wmb();
137 data->csd.flags &= ~CSD_FLAG_WAIT;
138 } else
139 call_rcu(&data->rcu_head, rcu_free_call_data);
140 }
141 rcu_read_unlock();
142
143 put_cpu();
144}
145
146/*
147 * Invoked by arch to handle an IPI for call function single. Must be called
148 * from the arch with interrupts disabled.
149 */
150void generic_smp_call_function_single_interrupt(void)
151{
152 struct call_single_queue *q = &__get_cpu_var(call_single_queue);
153 LIST_HEAD(list);
154
155 /*
156 * Need to see other stores to list head for checking whether
157 * list is empty without holding q->lock
158 */
159 smp_mb();
160 while (!list_empty(&q->list)) {
161 unsigned int data_flags;
162
163 spin_lock(&q->lock);
164 list_replace_init(&q->list, &list);
165 spin_unlock(&q->lock);
166
167 while (!list_empty(&list)) {
168 struct call_single_data *data;
169
170 data = list_entry(list.next, struct call_single_data,
171 list);
172 list_del(&data->list);
173
174 /*
175 * 'data' can be invalid after this call if
176 * flags == 0 (when called through
177 * generic_exec_single(), so save them away before
178 * making the call.
179 */
180 data_flags = data->flags;
181
182 data->func(data->info);
183
184 if (data_flags & CSD_FLAG_WAIT) {
185 smp_wmb();
186 data->flags &= ~CSD_FLAG_WAIT;
187 } else if (data_flags & CSD_FLAG_ALLOC)
188 kfree(data);
189 }
190 /*
191 * See comment on outer loop
192 */
193 smp_mb();
194 }
195}
196
197/*
198 * smp_call_function_single - Run a function on a specific CPU
199 * @func: The function to run. This must be fast and non-blocking.
200 * @info: An arbitrary pointer to pass to the function.
201 * @wait: If true, wait until function has completed on other CPUs.
202 *
203 * Returns 0 on success, else a negative status code. Note that @wait
204 * will be implicitly turned on in case of allocation failures, since
205 * we fall back to on-stack allocation.
206 */
207int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
208 int wait)
209{
210 struct call_single_data d;
211 unsigned long flags;
212 /* prevent preemption and reschedule on another processor */
213 int me = get_cpu();
214
215 /* Can deadlock when called with interrupts disabled */
216 WARN_ON(irqs_disabled());
217
218 if (cpu == me) {
219 local_irq_save(flags);
220 func(info);
221 local_irq_restore(flags);
222 } else {
223 struct call_single_data *data = NULL;
224
225 if (!wait) {
226 data = kmalloc(sizeof(*data), GFP_ATOMIC);
227 if (data)
228 data->flags = CSD_FLAG_ALLOC;
229 }
230 if (!data) {
231 data = &d;
232 data->flags = CSD_FLAG_WAIT;
233 }
234
235 data->func = func;
236 data->info = info;
237 generic_exec_single(cpu, data);
238 }
239
240 put_cpu();
241 return 0;
242}
243EXPORT_SYMBOL(smp_call_function_single);
244
245/**
246 * __smp_call_function_single(): Run a function on another CPU
247 * @cpu: The CPU to run on.
248 * @data: Pre-allocated and setup data structure
249 *
250 * Like smp_call_function_single(), but allow caller to pass in a pre-allocated
251 * data structure. Useful for embedding @data inside other structures, for
252 * instance.
253 *
254 */
255void __smp_call_function_single(int cpu, struct call_single_data *data)
256{
257 /* Can deadlock when called with interrupts disabled */
258 WARN_ON((data->flags & CSD_FLAG_WAIT) && irqs_disabled());
259
260 generic_exec_single(cpu, data);
261}
262
263/**
264 * smp_call_function_mask(): Run a function on a set of other CPUs.
265 * @mask: The set of cpus to run on.
266 * @func: The function to run. This must be fast and non-blocking.
267 * @info: An arbitrary pointer to pass to the function.
268 * @wait: If true, wait (atomically) until function has completed on other CPUs.
269 *
270 * Returns 0 on success, else a negative status code.
271 *
272 * If @wait is true, then returns once @func has returned. Note that @wait
273 * will be implicitly turned on in case of allocation failures, since
274 * we fall back to on-stack allocation.
275 *
276 * You must not call this function with disabled interrupts or from a
277 * hardware interrupt handler or from a bottom half handler. Preemption
278 * must be disabled when calling this function.
279 */
280int smp_call_function_mask(cpumask_t mask, void (*func)(void *), void *info,
281 int wait)
282{
283 struct call_function_data d;
284 struct call_function_data *data = NULL;
285 cpumask_t allbutself;
286 unsigned long flags;
287 int cpu, num_cpus;
288
289 /* Can deadlock when called with interrupts disabled */
290 WARN_ON(irqs_disabled());
291
292 cpu = smp_processor_id();
293 allbutself = cpu_online_map;
294 cpu_clear(cpu, allbutself);
295 cpus_and(mask, mask, allbutself);
296 num_cpus = cpus_weight(mask);
297
298 /*
299 * If zero CPUs, return. If just a single CPU, turn this request
300 * into a targetted single call instead since it's faster.
301 */
302 if (!num_cpus)
303 return 0;
304 else if (num_cpus == 1) {
305 cpu = first_cpu(mask);
306 return smp_call_function_single(cpu, func, info, wait);
307 }
308
309 if (!wait) {
310 data = kmalloc(sizeof(*data), GFP_ATOMIC);
311 if (data)
312 data->csd.flags = CSD_FLAG_ALLOC;
313 }
314 if (!data) {
315 data = &d;
316 data->csd.flags = CSD_FLAG_WAIT;
317 wait = 1;
318 }
319
320 spin_lock_init(&data->lock);
321 data->csd.func = func;
322 data->csd.info = info;
323 data->refs = num_cpus;
324 data->cpumask = mask;
325
326 spin_lock_irqsave(&call_function_lock, flags);
327 list_add_tail_rcu(&data->csd.list, &call_function_queue);
328 spin_unlock_irqrestore(&call_function_lock, flags);
329
330 /* Send a message to all CPUs in the map */
331 arch_send_call_function_ipi(mask);
332
333 /* optionally wait for the CPUs to complete */
334 if (wait)
335 csd_flag_wait(&data->csd);
336
337 return 0;
338}
339EXPORT_SYMBOL(smp_call_function_mask);
340
341/**
342 * smp_call_function(): Run a function on all other CPUs.
343 * @func: The function to run. This must be fast and non-blocking.
344 * @info: An arbitrary pointer to pass to the function.
345 * @wait: If true, wait (atomically) until function has completed on other CPUs.
346 *
347 * Returns 0 on success, else a negative status code.
348 *
349 * If @wait is true, then returns once @func has returned; otherwise
350 * it returns just before the target cpu calls @func. In case of allocation
351 * failure, @wait will be implicitly turned on.
352 *
353 * You must not call this function with disabled interrupts or from a
354 * hardware interrupt handler or from a bottom half handler.
355 */
356int smp_call_function(void (*func)(void *), void *info, int wait)
357{
358 int ret;
359
360 preempt_disable();
361 ret = smp_call_function_mask(cpu_online_map, func, info, wait);
362 preempt_enable();
363 return ret;
364}
365EXPORT_SYMBOL(smp_call_function);
366
367void ipi_call_lock(void)
368{
369 spin_lock(&call_function_lock);
370}
371
372void ipi_call_unlock(void)
373{
374 spin_unlock(&call_function_lock);
375}
376
377void ipi_call_lock_irq(void)
378{
379 spin_lock_irq(&call_function_lock);
380}
381
382void ipi_call_unlock_irq(void)
383{
384 spin_unlock_irq(&call_function_lock);
385}
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 36e061740047..c506f266a6b9 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -131,23 +131,17 @@ void _local_bh_enable(void)
131 131
132EXPORT_SYMBOL(_local_bh_enable); 132EXPORT_SYMBOL(_local_bh_enable);
133 133
134void local_bh_enable(void) 134static inline void _local_bh_enable_ip(unsigned long ip)
135{ 135{
136 WARN_ON_ONCE(in_irq() || irqs_disabled());
136#ifdef CONFIG_TRACE_IRQFLAGS 137#ifdef CONFIG_TRACE_IRQFLAGS
137 unsigned long flags; 138 local_irq_disable();
138
139 WARN_ON_ONCE(in_irq());
140#endif
141 WARN_ON_ONCE(irqs_disabled());
142
143#ifdef CONFIG_TRACE_IRQFLAGS
144 local_irq_save(flags);
145#endif 139#endif
146 /* 140 /*
147 * Are softirqs going to be turned on now: 141 * Are softirqs going to be turned on now:
148 */ 142 */
149 if (softirq_count() == SOFTIRQ_OFFSET) 143 if (softirq_count() == SOFTIRQ_OFFSET)
150 trace_softirqs_on((unsigned long)__builtin_return_address(0)); 144 trace_softirqs_on(ip);
151 /* 145 /*
152 * Keep preemption disabled until we are done with 146 * Keep preemption disabled until we are done with
153 * softirq processing: 147 * softirq processing:
@@ -159,40 +153,20 @@ void local_bh_enable(void)
159 153
160 dec_preempt_count(); 154 dec_preempt_count();
161#ifdef CONFIG_TRACE_IRQFLAGS 155#ifdef CONFIG_TRACE_IRQFLAGS
162 local_irq_restore(flags); 156 local_irq_enable();
163#endif 157#endif
164 preempt_check_resched(); 158 preempt_check_resched();
165} 159}
160
161void local_bh_enable(void)
162{
163 _local_bh_enable_ip((unsigned long)__builtin_return_address(0));
164}
166EXPORT_SYMBOL(local_bh_enable); 165EXPORT_SYMBOL(local_bh_enable);
167 166
168void local_bh_enable_ip(unsigned long ip) 167void local_bh_enable_ip(unsigned long ip)
169{ 168{
170#ifdef CONFIG_TRACE_IRQFLAGS 169 _local_bh_enable_ip(ip);
171 unsigned long flags;
172
173 WARN_ON_ONCE(in_irq());
174
175 local_irq_save(flags);
176#endif
177 /*
178 * Are softirqs going to be turned on now:
179 */
180 if (softirq_count() == SOFTIRQ_OFFSET)
181 trace_softirqs_on(ip);
182 /*
183 * Keep preemption disabled until we are done with
184 * softirq processing:
185 */
186 sub_preempt_count(SOFTIRQ_OFFSET - 1);
187
188 if (unlikely(!in_interrupt() && local_softirq_pending()))
189 do_softirq();
190
191 dec_preempt_count();
192#ifdef CONFIG_TRACE_IRQFLAGS
193 local_irq_restore(flags);
194#endif
195 preempt_check_resched();
196} 170}
197EXPORT_SYMBOL(local_bh_enable_ip); 171EXPORT_SYMBOL(local_bh_enable_ip);
198 172
@@ -312,7 +286,7 @@ void irq_exit(void)
312#ifdef CONFIG_NO_HZ 286#ifdef CONFIG_NO_HZ
313 /* Make sure that timer wheel updates are propagated */ 287 /* Make sure that timer wheel updates are propagated */
314 if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched()) 288 if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
315 tick_nohz_stop_sched_tick(); 289 tick_nohz_stop_sched_tick(0);
316 rcu_irq_exit(); 290 rcu_irq_exit();
317#endif 291#endif
318 preempt_enable_no_resched(); 292 preempt_enable_no_resched();
@@ -347,9 +321,8 @@ void raise_softirq(unsigned int nr)
347 local_irq_restore(flags); 321 local_irq_restore(flags);
348} 322}
349 323
350void open_softirq(int nr, void (*action)(struct softirq_action*), void *data) 324void open_softirq(int nr, void (*action)(struct softirq_action *))
351{ 325{
352 softirq_vec[nr].data = data;
353 softirq_vec[nr].action = action; 326 softirq_vec[nr].action = action;
354} 327}
355 328
@@ -360,10 +333,8 @@ struct tasklet_head
360 struct tasklet_struct **tail; 333 struct tasklet_struct **tail;
361}; 334};
362 335
363/* Some compilers disobey section attribute on statics when not 336static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
364 initialized -- RR */ 337static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
365static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec) = { NULL };
366static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec) = { NULL };
367 338
368void __tasklet_schedule(struct tasklet_struct *t) 339void __tasklet_schedule(struct tasklet_struct *t)
369{ 340{
@@ -503,8 +474,8 @@ void __init softirq_init(void)
503 &per_cpu(tasklet_hi_vec, cpu).head; 474 &per_cpu(tasklet_hi_vec, cpu).head;
504 } 475 }
505 476
506 open_softirq(TASKLET_SOFTIRQ, tasklet_action, NULL); 477 open_softirq(TASKLET_SOFTIRQ, tasklet_action);
507 open_softirq(HI_SOFTIRQ, tasklet_hi_action, NULL); 478 open_softirq(HI_SOFTIRQ, tasklet_hi_action);
508} 479}
509 480
510static int ksoftirqd(void * __bind_cpu) 481static int ksoftirqd(void * __bind_cpu)
@@ -645,7 +616,7 @@ static int __cpuinit cpu_callback(struct notifier_block *nfb,
645 616
646 p = per_cpu(ksoftirqd, hotcpu); 617 p = per_cpu(ksoftirqd, hotcpu);
647 per_cpu(ksoftirqd, hotcpu) = NULL; 618 per_cpu(ksoftirqd, hotcpu) = NULL;
648 sched_setscheduler(p, SCHED_FIFO, &param); 619 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
649 kthread_stop(p); 620 kthread_stop(p);
650 takeover_tasklets(hotcpu); 621 takeover_tasklets(hotcpu);
651 break; 622 break;
@@ -659,7 +630,7 @@ static struct notifier_block __cpuinitdata cpu_nfb = {
659 .notifier_call = cpu_callback 630 .notifier_call = cpu_callback
660}; 631};
661 632
662__init int spawn_ksoftirqd(void) 633static __init int spawn_ksoftirqd(void)
663{ 634{
664 void *cpu = (void *)(long)smp_processor_id(); 635 void *cpu = (void *)(long)smp_processor_id();
665 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu); 636 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
@@ -669,17 +640,18 @@ __init int spawn_ksoftirqd(void)
669 register_cpu_notifier(&cpu_nfb); 640 register_cpu_notifier(&cpu_nfb);
670 return 0; 641 return 0;
671} 642}
643early_initcall(spawn_ksoftirqd);
672 644
673#ifdef CONFIG_SMP 645#ifdef CONFIG_SMP
674/* 646/*
675 * Call a function on all processors 647 * Call a function on all processors
676 */ 648 */
677int on_each_cpu(void (*func) (void *info), void *info, int retry, int wait) 649int on_each_cpu(void (*func) (void *info), void *info, int wait)
678{ 650{
679 int ret = 0; 651 int ret = 0;
680 652
681 preempt_disable(); 653 preempt_disable();
682 ret = smp_call_function(func, info, retry, wait); 654 ret = smp_call_function(func, info, wait);
683 local_irq_disable(); 655 local_irq_disable();
684 func(info); 656 func(info);
685 local_irq_enable(); 657 local_irq_enable();
diff --git a/kernel/softlockup.c b/kernel/softlockup.c
index a272d78185eb..b75b492fbfcf 100644
--- a/kernel/softlockup.c
+++ b/kernel/softlockup.c
@@ -13,6 +13,7 @@
13#include <linux/delay.h> 13#include <linux/delay.h>
14#include <linux/freezer.h> 14#include <linux/freezer.h>
15#include <linux/kthread.h> 15#include <linux/kthread.h>
16#include <linux/lockdep.h>
16#include <linux/notifier.h> 17#include <linux/notifier.h>
17#include <linux/module.h> 18#include <linux/module.h>
18 19
@@ -25,7 +26,22 @@ static DEFINE_PER_CPU(unsigned long, print_timestamp);
25static DEFINE_PER_CPU(struct task_struct *, watchdog_task); 26static DEFINE_PER_CPU(struct task_struct *, watchdog_task);
26 27
27static int __read_mostly did_panic; 28static int __read_mostly did_panic;
28unsigned long __read_mostly softlockup_thresh = 60; 29int __read_mostly softlockup_thresh = 60;
30
31/*
32 * Should we panic (and reboot, if panic_timeout= is set) when a
33 * soft-lockup occurs:
34 */
35unsigned int __read_mostly softlockup_panic =
36 CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE;
37
38static int __init softlockup_panic_setup(char *str)
39{
40 softlockup_panic = simple_strtoul(str, NULL, 0);
41
42 return 1;
43}
44__setup("softlockup_panic=", softlockup_panic_setup);
29 45
30static int 46static int
31softlock_panic(struct notifier_block *this, unsigned long event, void *ptr) 47softlock_panic(struct notifier_block *this, unsigned long event, void *ptr)
@@ -84,6 +100,14 @@ void softlockup_tick(void)
84 struct pt_regs *regs = get_irq_regs(); 100 struct pt_regs *regs = get_irq_regs();
85 unsigned long now; 101 unsigned long now;
86 102
103 /* Is detection switched off? */
104 if (!per_cpu(watchdog_task, this_cpu) || softlockup_thresh <= 0) {
105 /* Be sure we don't false trigger if switched back on */
106 if (touch_timestamp)
107 per_cpu(touch_timestamp, this_cpu) = 0;
108 return;
109 }
110
87 if (touch_timestamp == 0) { 111 if (touch_timestamp == 0) {
88 __touch_softlockup_watchdog(); 112 __touch_softlockup_watchdog();
89 return; 113 return;
@@ -92,11 +116,8 @@ void softlockup_tick(void)
92 print_timestamp = per_cpu(print_timestamp, this_cpu); 116 print_timestamp = per_cpu(print_timestamp, this_cpu);
93 117
94 /* report at most once a second */ 118 /* report at most once a second */
95 if ((print_timestamp >= touch_timestamp && 119 if (print_timestamp == touch_timestamp || did_panic)
96 print_timestamp < (touch_timestamp + 1)) ||
97 did_panic || !per_cpu(watchdog_task, this_cpu)) {
98 return; 120 return;
99 }
100 121
101 /* do not print during early bootup: */ 122 /* do not print during early bootup: */
102 if (unlikely(system_state != SYSTEM_RUNNING)) { 123 if (unlikely(system_state != SYSTEM_RUNNING)) {
@@ -106,8 +127,11 @@ void softlockup_tick(void)
106 127
107 now = get_timestamp(this_cpu); 128 now = get_timestamp(this_cpu);
108 129
109 /* Wake up the high-prio watchdog task every second: */ 130 /*
110 if (now > (touch_timestamp + 1)) 131 * Wake up the high-prio watchdog task twice per
132 * threshold timespan.
133 */
134 if (now > touch_timestamp + softlockup_thresh/2)
111 wake_up_process(per_cpu(watchdog_task, this_cpu)); 135 wake_up_process(per_cpu(watchdog_task, this_cpu));
112 136
113 /* Warn about unreasonable delays: */ 137 /* Warn about unreasonable delays: */
@@ -121,11 +145,15 @@ void softlockup_tick(void)
121 this_cpu, now - touch_timestamp, 145 this_cpu, now - touch_timestamp,
122 current->comm, task_pid_nr(current)); 146 current->comm, task_pid_nr(current));
123 print_modules(); 147 print_modules();
148 print_irqtrace_events(current);
124 if (regs) 149 if (regs)
125 show_regs(regs); 150 show_regs(regs);
126 else 151 else
127 dump_stack(); 152 dump_stack();
128 spin_unlock(&print_lock); 153 spin_unlock(&print_lock);
154
155 if (softlockup_panic)
156 panic("softlockup: hung tasks");
129} 157}
130 158
131/* 159/*
@@ -178,6 +206,9 @@ static void check_hung_task(struct task_struct *t, unsigned long now)
178 206
179 t->last_switch_timestamp = now; 207 t->last_switch_timestamp = now;
180 touch_nmi_watchdog(); 208 touch_nmi_watchdog();
209
210 if (softlockup_panic)
211 panic("softlockup: blocked tasks");
181} 212}
182 213
183/* 214/*
@@ -307,14 +338,33 @@ static struct notifier_block __cpuinitdata cpu_nfb = {
307 .notifier_call = cpu_callback 338 .notifier_call = cpu_callback
308}; 339};
309 340
310__init void spawn_softlockup_task(void) 341static int __initdata nosoftlockup;
342
343static int __init nosoftlockup_setup(char *str)
344{
345 nosoftlockup = 1;
346 return 1;
347}
348__setup("nosoftlockup", nosoftlockup_setup);
349
350static int __init spawn_softlockup_task(void)
311{ 351{
312 void *cpu = (void *)(long)smp_processor_id(); 352 void *cpu = (void *)(long)smp_processor_id();
313 int err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu); 353 int err;
354
355 if (nosoftlockup)
356 return 0;
314 357
315 BUG_ON(err == NOTIFY_BAD); 358 err = cpu_callback(&cpu_nfb, CPU_UP_PREPARE, cpu);
359 if (err == NOTIFY_BAD) {
360 BUG();
361 return 1;
362 }
316 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu); 363 cpu_callback(&cpu_nfb, CPU_ONLINE, cpu);
317 register_cpu_notifier(&cpu_nfb); 364 register_cpu_notifier(&cpu_nfb);
318 365
319 atomic_notifier_chain_register(&panic_notifier_list, &panic_block); 366 atomic_notifier_chain_register(&panic_notifier_list, &panic_block);
367
368 return 0;
320} 369}
370early_initcall(spawn_softlockup_task);
diff --git a/kernel/spinlock.c b/kernel/spinlock.c
index ae28c8245123..a1fb54c93cdd 100644
--- a/kernel/spinlock.c
+++ b/kernel/spinlock.c
@@ -436,7 +436,7 @@ int __lockfunc _spin_trylock_bh(spinlock_t *lock)
436} 436}
437EXPORT_SYMBOL(_spin_trylock_bh); 437EXPORT_SYMBOL(_spin_trylock_bh);
438 438
439int in_lock_functions(unsigned long addr) 439notrace int in_lock_functions(unsigned long addr)
440{ 440{
441 /* Linker adds these: start and end of __lockfunc functions */ 441 /* Linker adds these: start and end of __lockfunc functions */
442 extern char __lock_text_start[], __lock_text_end[]; 442 extern char __lock_text_start[], __lock_text_end[];
diff --git a/kernel/stacktrace.c b/kernel/stacktrace.c
index b71816e47a30..94b527ef1d1e 100644
--- a/kernel/stacktrace.c
+++ b/kernel/stacktrace.c
@@ -6,19 +6,21 @@
6 * Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> 6 * Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
7 */ 7 */
8#include <linux/sched.h> 8#include <linux/sched.h>
9#include <linux/module.h>
9#include <linux/kallsyms.h> 10#include <linux/kallsyms.h>
10#include <linux/stacktrace.h> 11#include <linux/stacktrace.h>
11 12
12void print_stack_trace(struct stack_trace *trace, int spaces) 13void print_stack_trace(struct stack_trace *trace, int spaces)
13{ 14{
14 int i, j; 15 int i;
15 16
16 for (i = 0; i < trace->nr_entries; i++) { 17 if (WARN_ON(!trace->entries))
17 unsigned long ip = trace->entries[i]; 18 return;
18 19
19 for (j = 0; j < spaces + 1; j++) 20 for (i = 0; i < trace->nr_entries; i++) {
20 printk(" "); 21 printk("%*c", 1 + spaces, ' ');
21 print_ip_sym(ip); 22 print_ip_sym(trace->entries[i]);
22 } 23 }
23} 24}
25EXPORT_SYMBOL_GPL(print_stack_trace);
24 26
diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c
index b7350bbfb076..738b411ff2d3 100644
--- a/kernel/stop_machine.c
+++ b/kernel/stop_machine.c
@@ -33,8 +33,9 @@ static int stopmachine(void *cpu)
33{ 33{
34 int irqs_disabled = 0; 34 int irqs_disabled = 0;
35 int prepared = 0; 35 int prepared = 0;
36 cpumask_of_cpu_ptr(cpumask, (int)(long)cpu);
36 37
37 set_cpus_allowed_ptr(current, &cpumask_of_cpu((int)(long)cpu)); 38 set_cpus_allowed_ptr(current, cpumask);
38 39
39 /* Ack: we are alive */ 40 /* Ack: we are alive */
40 smp_mb(); /* Theoretically the ack = 0 might not be on this CPU yet. */ 41 smp_mb(); /* Theoretically the ack = 0 might not be on this CPU yet. */
@@ -187,7 +188,7 @@ struct task_struct *__stop_machine_run(int (*fn)(void *), void *data,
187 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; 188 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
188 189
189 /* One high-prio thread per cpu. We'll do this one. */ 190 /* One high-prio thread per cpu. We'll do this one. */
190 sched_setscheduler(p, SCHED_FIFO, &param); 191 sched_setscheduler_nocheck(p, SCHED_FIFO, &param);
191 kthread_bind(p, cpu); 192 kthread_bind(p, cpu);
192 wake_up_process(p); 193 wake_up_process(p);
193 wait_for_completion(&smdata.done); 194 wait_for_completion(&smdata.done);
diff --git a/kernel/sys.c b/kernel/sys.c
index 14e97282eb6c..c01858090a98 100644
--- a/kernel/sys.c
+++ b/kernel/sys.c
@@ -301,26 +301,6 @@ void kernel_restart(char *cmd)
301} 301}
302EXPORT_SYMBOL_GPL(kernel_restart); 302EXPORT_SYMBOL_GPL(kernel_restart);
303 303
304/**
305 * kernel_kexec - reboot the system
306 *
307 * Move into place and start executing a preloaded standalone
308 * executable. If nothing was preloaded return an error.
309 */
310static void kernel_kexec(void)
311{
312#ifdef CONFIG_KEXEC
313 struct kimage *image;
314 image = xchg(&kexec_image, NULL);
315 if (!image)
316 return;
317 kernel_restart_prepare(NULL);
318 printk(KERN_EMERG "Starting new kernel\n");
319 machine_shutdown();
320 machine_kexec(image);
321#endif
322}
323
324static void kernel_shutdown_prepare(enum system_states state) 304static void kernel_shutdown_prepare(enum system_states state)
325{ 305{
326 blocking_notifier_call_chain(&reboot_notifier_list, 306 blocking_notifier_call_chain(&reboot_notifier_list,
@@ -425,10 +405,15 @@ asmlinkage long sys_reboot(int magic1, int magic2, unsigned int cmd, void __user
425 kernel_restart(buffer); 405 kernel_restart(buffer);
426 break; 406 break;
427 407
408#ifdef CONFIG_KEXEC
428 case LINUX_REBOOT_CMD_KEXEC: 409 case LINUX_REBOOT_CMD_KEXEC:
429 kernel_kexec(); 410 {
430 unlock_kernel(); 411 int ret;
431 return -EINVAL; 412 ret = kernel_kexec();
413 unlock_kernel();
414 return ret;
415 }
416#endif
432 417
433#ifdef CONFIG_HIBERNATION 418#ifdef CONFIG_HIBERNATION
434 case LINUX_REBOOT_CMD_SW_SUSPEND: 419 case LINUX_REBOOT_CMD_SW_SUSPEND:
@@ -1343,8 +1328,6 @@ EXPORT_SYMBOL(in_egroup_p);
1343 1328
1344DECLARE_RWSEM(uts_sem); 1329DECLARE_RWSEM(uts_sem);
1345 1330
1346EXPORT_SYMBOL(uts_sem);
1347
1348asmlinkage long sys_newuname(struct new_utsname __user * name) 1331asmlinkage long sys_newuname(struct new_utsname __user * name)
1349{ 1332{
1350 int errno = 0; 1333 int errno = 0;
@@ -1795,7 +1778,7 @@ int orderly_poweroff(bool force)
1795 goto out; 1778 goto out;
1796 } 1779 }
1797 1780
1798 info = call_usermodehelper_setup(argv[0], argv, envp); 1781 info = call_usermodehelper_setup(argv[0], argv, envp, GFP_ATOMIC);
1799 if (info == NULL) { 1782 if (info == NULL) {
1800 argv_free(argv); 1783 argv_free(argv);
1801 goto out; 1784 goto out;
diff --git a/kernel/sys_ni.c b/kernel/sys_ni.c
index 5b9b467de070..08d6e1bb99ac 100644
--- a/kernel/sys_ni.c
+++ b/kernel/sys_ni.c
@@ -31,6 +31,7 @@ cond_syscall(sys_socketpair);
31cond_syscall(sys_bind); 31cond_syscall(sys_bind);
32cond_syscall(sys_listen); 32cond_syscall(sys_listen);
33cond_syscall(sys_accept); 33cond_syscall(sys_accept);
34cond_syscall(sys_paccept);
34cond_syscall(sys_connect); 35cond_syscall(sys_connect);
35cond_syscall(sys_getsockname); 36cond_syscall(sys_getsockname);
36cond_syscall(sys_getpeername); 37cond_syscall(sys_getpeername);
@@ -56,9 +57,11 @@ cond_syscall(compat_sys_set_robust_list);
56cond_syscall(sys_get_robust_list); 57cond_syscall(sys_get_robust_list);
57cond_syscall(compat_sys_get_robust_list); 58cond_syscall(compat_sys_get_robust_list);
58cond_syscall(sys_epoll_create); 59cond_syscall(sys_epoll_create);
60cond_syscall(sys_epoll_create1);
59cond_syscall(sys_epoll_ctl); 61cond_syscall(sys_epoll_ctl);
60cond_syscall(sys_epoll_wait); 62cond_syscall(sys_epoll_wait);
61cond_syscall(sys_epoll_pwait); 63cond_syscall(sys_epoll_pwait);
64cond_syscall(compat_sys_epoll_pwait);
62cond_syscall(sys_semget); 65cond_syscall(sys_semget);
63cond_syscall(sys_semop); 66cond_syscall(sys_semop);
64cond_syscall(sys_semtimedop); 67cond_syscall(sys_semtimedop);
@@ -94,6 +97,7 @@ cond_syscall(sys_keyctl);
94cond_syscall(compat_sys_keyctl); 97cond_syscall(compat_sys_keyctl);
95cond_syscall(compat_sys_socketcall); 98cond_syscall(compat_sys_socketcall);
96cond_syscall(sys_inotify_init); 99cond_syscall(sys_inotify_init);
100cond_syscall(sys_inotify_init1);
97cond_syscall(sys_inotify_add_watch); 101cond_syscall(sys_inotify_add_watch);
98cond_syscall(sys_inotify_rm_watch); 102cond_syscall(sys_inotify_rm_watch);
99cond_syscall(sys_migrate_pages); 103cond_syscall(sys_migrate_pages);
@@ -154,10 +158,13 @@ cond_syscall(sys_ioprio_get);
154 158
155/* New file descriptors */ 159/* New file descriptors */
156cond_syscall(sys_signalfd); 160cond_syscall(sys_signalfd);
161cond_syscall(sys_signalfd4);
157cond_syscall(compat_sys_signalfd); 162cond_syscall(compat_sys_signalfd);
163cond_syscall(compat_sys_signalfd4);
158cond_syscall(sys_timerfd_create); 164cond_syscall(sys_timerfd_create);
159cond_syscall(sys_timerfd_settime); 165cond_syscall(sys_timerfd_settime);
160cond_syscall(sys_timerfd_gettime); 166cond_syscall(sys_timerfd_gettime);
161cond_syscall(compat_sys_timerfd_settime); 167cond_syscall(compat_sys_timerfd_settime);
162cond_syscall(compat_sys_timerfd_gettime); 168cond_syscall(compat_sys_timerfd_gettime);
163cond_syscall(sys_eventfd); 169cond_syscall(sys_eventfd);
170cond_syscall(sys_eventfd2);
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 29116652dca8..fe4713347275 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -43,9 +43,11 @@
43#include <linux/limits.h> 43#include <linux/limits.h>
44#include <linux/dcache.h> 44#include <linux/dcache.h>
45#include <linux/syscalls.h> 45#include <linux/syscalls.h>
46#include <linux/vmstat.h>
46#include <linux/nfs_fs.h> 47#include <linux/nfs_fs.h>
47#include <linux/acpi.h> 48#include <linux/acpi.h>
48#include <linux/reboot.h> 49#include <linux/reboot.h>
50#include <linux/ftrace.h>
49 51
50#include <asm/uaccess.h> 52#include <asm/uaccess.h>
51#include <asm/processor.h> 53#include <asm/processor.h>
@@ -79,17 +81,20 @@ extern int sysctl_drop_caches;
79extern int percpu_pagelist_fraction; 81extern int percpu_pagelist_fraction;
80extern int compat_log; 82extern int compat_log;
81extern int maps_protect; 83extern int maps_protect;
82extern int sysctl_stat_interval;
83extern int latencytop_enabled; 84extern int latencytop_enabled;
84extern int sysctl_nr_open_min, sysctl_nr_open_max; 85extern int sysctl_nr_open_min, sysctl_nr_open_max;
86#ifdef CONFIG_RCU_TORTURE_TEST
87extern int rcutorture_runnable;
88#endif /* #ifdef CONFIG_RCU_TORTURE_TEST */
85 89
86/* Constants used for minimum and maximum */ 90/* Constants used for minimum and maximum */
87#if defined(CONFIG_DETECT_SOFTLOCKUP) || defined(CONFIG_HIGHMEM) 91#if defined(CONFIG_HIGHMEM) || defined(CONFIG_DETECT_SOFTLOCKUP)
88static int one = 1; 92static int one = 1;
89#endif 93#endif
90 94
91#ifdef CONFIG_DETECT_SOFTLOCKUP 95#ifdef CONFIG_DETECT_SOFTLOCKUP
92static int sixty = 60; 96static int sixty = 60;
97static int neg_one = -1;
93#endif 98#endif
94 99
95#ifdef CONFIG_MMU 100#ifdef CONFIG_MMU
@@ -106,7 +111,7 @@ static int min_percpu_pagelist_fract = 8;
106 111
107static int ngroups_max = NGROUPS_MAX; 112static int ngroups_max = NGROUPS_MAX;
108 113
109#ifdef CONFIG_KMOD 114#ifdef CONFIG_MODULES
110extern char modprobe_path[]; 115extern char modprobe_path[];
111#endif 116#endif
112#ifdef CONFIG_CHR_DEV_SG 117#ifdef CONFIG_CHR_DEV_SG
@@ -132,8 +137,6 @@ extern int sysctl_userprocess_debug;
132extern int spin_retry; 137extern int spin_retry;
133#endif 138#endif
134 139
135extern int sysctl_hz_timer;
136
137#ifdef CONFIG_BSD_PROCESS_ACCT 140#ifdef CONFIG_BSD_PROCESS_ACCT
138extern int acct_parm[]; 141extern int acct_parm[];
139#endif 142#endif
@@ -157,12 +160,13 @@ static struct ctl_table root_table[];
157static struct ctl_table_root sysctl_table_root; 160static struct ctl_table_root sysctl_table_root;
158static struct ctl_table_header root_table_header = { 161static struct ctl_table_header root_table_header = {
159 .ctl_table = root_table, 162 .ctl_table = root_table,
160 .ctl_entry = LIST_HEAD_INIT(sysctl_table_root.header_list), 163 .ctl_entry = LIST_HEAD_INIT(sysctl_table_root.default_set.list),
161 .root = &sysctl_table_root, 164 .root = &sysctl_table_root,
165 .set = &sysctl_table_root.default_set,
162}; 166};
163static struct ctl_table_root sysctl_table_root = { 167static struct ctl_table_root sysctl_table_root = {
164 .root_list = LIST_HEAD_INIT(sysctl_table_root.root_list), 168 .root_list = LIST_HEAD_INIT(sysctl_table_root.root_list),
165 .header_list = LIST_HEAD_INIT(root_table_header.ctl_entry), 169 .default_set.list = LIST_HEAD_INIT(root_table_header.ctl_entry),
166}; 170};
167 171
168static struct ctl_table kern_table[]; 172static struct ctl_table kern_table[];
@@ -266,6 +270,14 @@ static struct ctl_table kern_table[] = {
266 }, 270 },
267 { 271 {
268 .ctl_name = CTL_UNNUMBERED, 272 .ctl_name = CTL_UNNUMBERED,
273 .procname = "sched_shares_ratelimit",
274 .data = &sysctl_sched_shares_ratelimit,
275 .maxlen = sizeof(unsigned int),
276 .mode = 0644,
277 .proc_handler = &proc_dointvec,
278 },
279 {
280 .ctl_name = CTL_UNNUMBERED,
269 .procname = "sched_child_runs_first", 281 .procname = "sched_child_runs_first",
270 .data = &sysctl_sched_child_runs_first, 282 .data = &sysctl_sched_child_runs_first,
271 .maxlen = sizeof(unsigned int), 283 .maxlen = sizeof(unsigned int),
@@ -455,7 +467,17 @@ static struct ctl_table kern_table[] = {
455 .mode = 0644, 467 .mode = 0644,
456 .proc_handler = &proc_dointvec, 468 .proc_handler = &proc_dointvec,
457 }, 469 },
458#ifdef CONFIG_KMOD 470#ifdef CONFIG_FTRACE
471 {
472 .ctl_name = CTL_UNNUMBERED,
473 .procname = "ftrace_enabled",
474 .data = &ftrace_enabled,
475 .maxlen = sizeof(int),
476 .mode = 0644,
477 .proc_handler = &ftrace_enable_sysctl,
478 },
479#endif
480#ifdef CONFIG_MODULES
459 { 481 {
460 .ctl_name = KERN_MODPROBE, 482 .ctl_name = KERN_MODPROBE,
461 .procname = "modprobe", 483 .procname = "modprobe",
@@ -563,16 +585,6 @@ static struct ctl_table kern_table[] = {
563 .proc_handler = &proc_dointvec, 585 .proc_handler = &proc_dointvec,
564 }, 586 },
565#endif 587#endif
566#ifdef CONFIG_NO_IDLE_HZ
567 {
568 .ctl_name = KERN_HZ_TIMER,
569 .procname = "hz_timer",
570 .data = &sysctl_hz_timer,
571 .maxlen = sizeof(int),
572 .mode = 0644,
573 .proc_handler = &proc_dointvec,
574 },
575#endif
576 { 588 {
577 .ctl_name = KERN_S390_USER_DEBUG_LOGGING, 589 .ctl_name = KERN_S390_USER_DEBUG_LOGGING,
578 .procname = "userprocess_debug", 590 .procname = "userprocess_debug",
@@ -613,7 +625,7 @@ static struct ctl_table kern_table[] = {
613 { 625 {
614 .ctl_name = KERN_PRINTK_RATELIMIT, 626 .ctl_name = KERN_PRINTK_RATELIMIT,
615 .procname = "printk_ratelimit", 627 .procname = "printk_ratelimit",
616 .data = &printk_ratelimit_jiffies, 628 .data = &printk_ratelimit_state.interval,
617 .maxlen = sizeof(int), 629 .maxlen = sizeof(int),
618 .mode = 0644, 630 .mode = 0644,
619 .proc_handler = &proc_dointvec_jiffies, 631 .proc_handler = &proc_dointvec_jiffies,
@@ -622,7 +634,7 @@ static struct ctl_table kern_table[] = {
622 { 634 {
623 .ctl_name = KERN_PRINTK_RATELIMIT_BURST, 635 .ctl_name = KERN_PRINTK_RATELIMIT_BURST,
624 .procname = "printk_ratelimit_burst", 636 .procname = "printk_ratelimit_burst",
625 .data = &printk_ratelimit_burst, 637 .data = &printk_ratelimit_state.burst,
626 .maxlen = sizeof(int), 638 .maxlen = sizeof(int),
627 .mode = 0644, 639 .mode = 0644,
628 .proc_handler = &proc_dointvec, 640 .proc_handler = &proc_dointvec,
@@ -729,13 +741,24 @@ static struct ctl_table kern_table[] = {
729#ifdef CONFIG_DETECT_SOFTLOCKUP 741#ifdef CONFIG_DETECT_SOFTLOCKUP
730 { 742 {
731 .ctl_name = CTL_UNNUMBERED, 743 .ctl_name = CTL_UNNUMBERED,
744 .procname = "softlockup_panic",
745 .data = &softlockup_panic,
746 .maxlen = sizeof(int),
747 .mode = 0644,
748 .proc_handler = &proc_dointvec_minmax,
749 .strategy = &sysctl_intvec,
750 .extra1 = &zero,
751 .extra2 = &one,
752 },
753 {
754 .ctl_name = CTL_UNNUMBERED,
732 .procname = "softlockup_thresh", 755 .procname = "softlockup_thresh",
733 .data = &softlockup_thresh, 756 .data = &softlockup_thresh,
734 .maxlen = sizeof(unsigned long), 757 .maxlen = sizeof(int),
735 .mode = 0644, 758 .mode = 0644,
736 .proc_handler = &proc_doulongvec_minmax, 759 .proc_handler = &proc_dointvec_minmax,
737 .strategy = &sysctl_intvec, 760 .strategy = &sysctl_intvec,
738 .extra1 = &one, 761 .extra1 = &neg_one,
739 .extra2 = &sixty, 762 .extra2 = &sixty,
740 }, 763 },
741 { 764 {
@@ -813,6 +836,16 @@ static struct ctl_table kern_table[] = {
813 .child = key_sysctls, 836 .child = key_sysctls,
814 }, 837 },
815#endif 838#endif
839#ifdef CONFIG_RCU_TORTURE_TEST
840 {
841 .ctl_name = CTL_UNNUMBERED,
842 .procname = "rcutorture_runnable",
843 .data = &rcutorture_runnable,
844 .maxlen = sizeof(int),
845 .mode = 0644,
846 .proc_handler = &proc_dointvec,
847 },
848#endif
816/* 849/*
817 * NOTE: do not add new entries to this table unless you have read 850 * NOTE: do not add new entries to this table unless you have read
818 * Documentation/sysctl/ctl_unnumbered.txt 851 * Documentation/sysctl/ctl_unnumbered.txt
@@ -927,7 +960,7 @@ static struct ctl_table vm_table[] = {
927#ifdef CONFIG_HUGETLB_PAGE 960#ifdef CONFIG_HUGETLB_PAGE
928 { 961 {
929 .procname = "nr_hugepages", 962 .procname = "nr_hugepages",
930 .data = &max_huge_pages, 963 .data = NULL,
931 .maxlen = sizeof(unsigned long), 964 .maxlen = sizeof(unsigned long),
932 .mode = 0644, 965 .mode = 0644,
933 .proc_handler = &hugetlb_sysctl_handler, 966 .proc_handler = &hugetlb_sysctl_handler,
@@ -953,10 +986,12 @@ static struct ctl_table vm_table[] = {
953 { 986 {
954 .ctl_name = CTL_UNNUMBERED, 987 .ctl_name = CTL_UNNUMBERED,
955 .procname = "nr_overcommit_hugepages", 988 .procname = "nr_overcommit_hugepages",
956 .data = &sysctl_overcommit_huge_pages, 989 .data = NULL,
957 .maxlen = sizeof(sysctl_overcommit_huge_pages), 990 .maxlen = sizeof(unsigned long),
958 .mode = 0644, 991 .mode = 0644,
959 .proc_handler = &hugetlb_overcommit_handler, 992 .proc_handler = &hugetlb_overcommit_handler,
993 .extra1 = (void *)&hugetlb_zero,
994 .extra2 = (void *)&hugetlb_infinity,
960 }, 995 },
961#endif 996#endif
962 { 997 {
@@ -1352,6 +1387,9 @@ static void start_unregistering(struct ctl_table_header *p)
1352 spin_unlock(&sysctl_lock); 1387 spin_unlock(&sysctl_lock);
1353 wait_for_completion(&wait); 1388 wait_for_completion(&wait);
1354 spin_lock(&sysctl_lock); 1389 spin_lock(&sysctl_lock);
1390 } else {
1391 /* anything non-NULL; we'll never dereference it */
1392 p->unregistering = ERR_PTR(-EINVAL);
1355 } 1393 }
1356 /* 1394 /*
1357 * do not remove from the list until nobody holds it; walking the 1395 * do not remove from the list until nobody holds it; walking the
@@ -1360,6 +1398,32 @@ static void start_unregistering(struct ctl_table_header *p)
1360 list_del_init(&p->ctl_entry); 1398 list_del_init(&p->ctl_entry);
1361} 1399}
1362 1400
1401void sysctl_head_get(struct ctl_table_header *head)
1402{
1403 spin_lock(&sysctl_lock);
1404 head->count++;
1405 spin_unlock(&sysctl_lock);
1406}
1407
1408void sysctl_head_put(struct ctl_table_header *head)
1409{
1410 spin_lock(&sysctl_lock);
1411 if (!--head->count)
1412 kfree(head);
1413 spin_unlock(&sysctl_lock);
1414}
1415
1416struct ctl_table_header *sysctl_head_grab(struct ctl_table_header *head)
1417{
1418 if (!head)
1419 BUG();
1420 spin_lock(&sysctl_lock);
1421 if (!use_table(head))
1422 head = ERR_PTR(-ENOENT);
1423 spin_unlock(&sysctl_lock);
1424 return head;
1425}
1426
1363void sysctl_head_finish(struct ctl_table_header *head) 1427void sysctl_head_finish(struct ctl_table_header *head)
1364{ 1428{
1365 if (!head) 1429 if (!head)
@@ -1369,14 +1433,20 @@ void sysctl_head_finish(struct ctl_table_header *head)
1369 spin_unlock(&sysctl_lock); 1433 spin_unlock(&sysctl_lock);
1370} 1434}
1371 1435
1436static struct ctl_table_set *
1437lookup_header_set(struct ctl_table_root *root, struct nsproxy *namespaces)
1438{
1439 struct ctl_table_set *set = &root->default_set;
1440 if (root->lookup)
1441 set = root->lookup(root, namespaces);
1442 return set;
1443}
1444
1372static struct list_head * 1445static struct list_head *
1373lookup_header_list(struct ctl_table_root *root, struct nsproxy *namespaces) 1446lookup_header_list(struct ctl_table_root *root, struct nsproxy *namespaces)
1374{ 1447{
1375 struct list_head *header_list; 1448 struct ctl_table_set *set = lookup_header_set(root, namespaces);
1376 header_list = &root->header_list; 1449 return &set->list;
1377 if (root->lookup)
1378 header_list = root->lookup(root, namespaces);
1379 return header_list;
1380} 1450}
1381 1451
1382struct ctl_table_header *__sysctl_head_next(struct nsproxy *namespaces, 1452struct ctl_table_header *__sysctl_head_next(struct nsproxy *namespaces,
@@ -1446,9 +1516,9 @@ static int do_sysctl_strategy(struct ctl_table_root *root,
1446 int op = 0, rc; 1516 int op = 0, rc;
1447 1517
1448 if (oldval) 1518 if (oldval)
1449 op |= 004; 1519 op |= MAY_READ;
1450 if (newval) 1520 if (newval)
1451 op |= 002; 1521 op |= MAY_WRITE;
1452 if (sysctl_perm(root, table, op)) 1522 if (sysctl_perm(root, table, op))
1453 return -EPERM; 1523 return -EPERM;
1454 1524
@@ -1490,7 +1560,7 @@ repeat:
1490 if (n == table->ctl_name) { 1560 if (n == table->ctl_name) {
1491 int error; 1561 int error;
1492 if (table->child) { 1562 if (table->child) {
1493 if (sysctl_perm(root, table, 001)) 1563 if (sysctl_perm(root, table, MAY_EXEC))
1494 return -EPERM; 1564 return -EPERM;
1495 name++; 1565 name++;
1496 nlen--; 1566 nlen--;
@@ -1565,7 +1635,7 @@ static int test_perm(int mode, int op)
1565 mode >>= 6; 1635 mode >>= 6;
1566 else if (in_egroup_p(0)) 1636 else if (in_egroup_p(0))
1567 mode >>= 3; 1637 mode >>= 3;
1568 if ((mode & op & 0007) == op) 1638 if ((op & ~mode & (MAY_READ|MAY_WRITE|MAY_EXEC)) == 0)
1569 return 0; 1639 return 0;
1570 return -EACCES; 1640 return -EACCES;
1571} 1641}
@@ -1575,7 +1645,7 @@ int sysctl_perm(struct ctl_table_root *root, struct ctl_table *table, int op)
1575 int error; 1645 int error;
1576 int mode; 1646 int mode;
1577 1647
1578 error = security_sysctl(table, op); 1648 error = security_sysctl(table, op & (MAY_READ | MAY_WRITE | MAY_EXEC));
1579 if (error) 1649 if (error)
1580 return error; 1650 return error;
1581 1651
@@ -1610,6 +1680,54 @@ static __init int sysctl_init(void)
1610 1680
1611core_initcall(sysctl_init); 1681core_initcall(sysctl_init);
1612 1682
1683static struct ctl_table *is_branch_in(struct ctl_table *branch,
1684 struct ctl_table *table)
1685{
1686 struct ctl_table *p;
1687 const char *s = branch->procname;
1688
1689 /* branch should have named subdirectory as its first element */
1690 if (!s || !branch->child)
1691 return NULL;
1692
1693 /* ... and nothing else */
1694 if (branch[1].procname || branch[1].ctl_name)
1695 return NULL;
1696
1697 /* table should contain subdirectory with the same name */
1698 for (p = table; p->procname || p->ctl_name; p++) {
1699 if (!p->child)
1700 continue;
1701 if (p->procname && strcmp(p->procname, s) == 0)
1702 return p;
1703 }
1704 return NULL;
1705}
1706
1707/* see if attaching q to p would be an improvement */
1708static void try_attach(struct ctl_table_header *p, struct ctl_table_header *q)
1709{
1710 struct ctl_table *to = p->ctl_table, *by = q->ctl_table;
1711 struct ctl_table *next;
1712 int is_better = 0;
1713 int not_in_parent = !p->attached_by;
1714
1715 while ((next = is_branch_in(by, to)) != NULL) {
1716 if (by == q->attached_by)
1717 is_better = 1;
1718 if (to == p->attached_by)
1719 not_in_parent = 1;
1720 by = by->child;
1721 to = next->child;
1722 }
1723
1724 if (is_better && not_in_parent) {
1725 q->attached_by = by;
1726 q->attached_to = to;
1727 q->parent = p;
1728 }
1729}
1730
1613/** 1731/**
1614 * __register_sysctl_paths - register a sysctl hierarchy 1732 * __register_sysctl_paths - register a sysctl hierarchy
1615 * @root: List of sysctl headers to register on 1733 * @root: List of sysctl headers to register on
@@ -1686,10 +1804,10 @@ struct ctl_table_header *__register_sysctl_paths(
1686 struct nsproxy *namespaces, 1804 struct nsproxy *namespaces,
1687 const struct ctl_path *path, struct ctl_table *table) 1805 const struct ctl_path *path, struct ctl_table *table)
1688{ 1806{
1689 struct list_head *header_list;
1690 struct ctl_table_header *header; 1807 struct ctl_table_header *header;
1691 struct ctl_table *new, **prevp; 1808 struct ctl_table *new, **prevp;
1692 unsigned int n, npath; 1809 unsigned int n, npath;
1810 struct ctl_table_set *set;
1693 1811
1694 /* Count the path components */ 1812 /* Count the path components */
1695 for (npath = 0; path[npath].ctl_name || path[npath].procname; ++npath) 1813 for (npath = 0; path[npath].ctl_name || path[npath].procname; ++npath)
@@ -1731,6 +1849,7 @@ struct ctl_table_header *__register_sysctl_paths(
1731 header->unregistering = NULL; 1849 header->unregistering = NULL;
1732 header->root = root; 1850 header->root = root;
1733 sysctl_set_parent(NULL, header->ctl_table); 1851 sysctl_set_parent(NULL, header->ctl_table);
1852 header->count = 1;
1734#ifdef CONFIG_SYSCTL_SYSCALL_CHECK 1853#ifdef CONFIG_SYSCTL_SYSCALL_CHECK
1735 if (sysctl_check_table(namespaces, header->ctl_table)) { 1854 if (sysctl_check_table(namespaces, header->ctl_table)) {
1736 kfree(header); 1855 kfree(header);
@@ -1738,8 +1857,20 @@ struct ctl_table_header *__register_sysctl_paths(
1738 } 1857 }
1739#endif 1858#endif
1740 spin_lock(&sysctl_lock); 1859 spin_lock(&sysctl_lock);
1741 header_list = lookup_header_list(root, namespaces); 1860 header->set = lookup_header_set(root, namespaces);
1742 list_add_tail(&header->ctl_entry, header_list); 1861 header->attached_by = header->ctl_table;
1862 header->attached_to = root_table;
1863 header->parent = &root_table_header;
1864 for (set = header->set; set; set = set->parent) {
1865 struct ctl_table_header *p;
1866 list_for_each_entry(p, &set->list, ctl_entry) {
1867 if (p->unregistering)
1868 continue;
1869 try_attach(p, header);
1870 }
1871 }
1872 header->parent->count++;
1873 list_add_tail(&header->ctl_entry, &header->set->list);
1743 spin_unlock(&sysctl_lock); 1874 spin_unlock(&sysctl_lock);
1744 1875
1745 return header; 1876 return header;
@@ -1794,8 +1925,37 @@ void unregister_sysctl_table(struct ctl_table_header * header)
1794 1925
1795 spin_lock(&sysctl_lock); 1926 spin_lock(&sysctl_lock);
1796 start_unregistering(header); 1927 start_unregistering(header);
1928 if (!--header->parent->count) {
1929 WARN_ON(1);
1930 kfree(header->parent);
1931 }
1932 if (!--header->count)
1933 kfree(header);
1934 spin_unlock(&sysctl_lock);
1935}
1936
1937int sysctl_is_seen(struct ctl_table_header *p)
1938{
1939 struct ctl_table_set *set = p->set;
1940 int res;
1941 spin_lock(&sysctl_lock);
1942 if (p->unregistering)
1943 res = 0;
1944 else if (!set->is_seen)
1945 res = 1;
1946 else
1947 res = set->is_seen(set);
1797 spin_unlock(&sysctl_lock); 1948 spin_unlock(&sysctl_lock);
1798 kfree(header); 1949 return res;
1950}
1951
1952void setup_sysctl_set(struct ctl_table_set *p,
1953 struct ctl_table_set *parent,
1954 int (*is_seen)(struct ctl_table_set *))
1955{
1956 INIT_LIST_HEAD(&p->list);
1957 p->parent = parent ? parent : &sysctl_table_root.default_set;
1958 p->is_seen = is_seen;
1799} 1959}
1800 1960
1801#else /* !CONFIG_SYSCTL */ 1961#else /* !CONFIG_SYSCTL */
@@ -1814,6 +1974,16 @@ void unregister_sysctl_table(struct ctl_table_header * table)
1814{ 1974{
1815} 1975}
1816 1976
1977void setup_sysctl_set(struct ctl_table_set *p,
1978 struct ctl_table_set *parent,
1979 int (*is_seen)(struct ctl_table_set *))
1980{
1981}
1982
1983void sysctl_head_put(struct ctl_table_header *head)
1984{
1985}
1986
1817#endif /* CONFIG_SYSCTL */ 1987#endif /* CONFIG_SYSCTL */
1818 1988
1819/* 1989/*
diff --git a/kernel/sysctl_check.c b/kernel/sysctl_check.c
index c09350d564f2..c35da23ab8fb 100644
--- a/kernel/sysctl_check.c
+++ b/kernel/sysctl_check.c
@@ -1532,6 +1532,8 @@ int sysctl_check_table(struct nsproxy *namespaces, struct ctl_table *table)
1532 sysctl_check_leaf(namespaces, table, &fail); 1532 sysctl_check_leaf(namespaces, table, &fail);
1533 } 1533 }
1534 sysctl_check_bin_path(table, &fail); 1534 sysctl_check_bin_path(table, &fail);
1535 if (table->mode > 0777)
1536 set_fail(&fail, table, "bogus .mode");
1535 if (fail) { 1537 if (fail) {
1536 set_fail(&fail, table, NULL); 1538 set_fail(&fail, table, NULL);
1537 error = -EINVAL; 1539 error = -EINVAL;
diff --git a/kernel/taskstats.c b/kernel/taskstats.c
index 4a23517169a6..bd6be76303cf 100644
--- a/kernel/taskstats.c
+++ b/kernel/taskstats.c
@@ -35,7 +35,7 @@
35 */ 35 */
36#define TASKSTATS_CPUMASK_MAXLEN (100+6*NR_CPUS) 36#define TASKSTATS_CPUMASK_MAXLEN (100+6*NR_CPUS)
37 37
38static DEFINE_PER_CPU(__u32, taskstats_seqnum) = { 0 }; 38static DEFINE_PER_CPU(__u32, taskstats_seqnum);
39static int family_registered; 39static int family_registered;
40struct kmem_cache *taskstats_cache; 40struct kmem_cache *taskstats_cache;
41 41
@@ -301,7 +301,7 @@ static int add_del_listener(pid_t pid, cpumask_t *maskp, int isadd)
301 return -EINVAL; 301 return -EINVAL;
302 302
303 if (isadd == REGISTER) { 303 if (isadd == REGISTER) {
304 for_each_cpu_mask(cpu, mask) { 304 for_each_cpu_mask_nr(cpu, mask) {
305 s = kmalloc_node(sizeof(struct listener), GFP_KERNEL, 305 s = kmalloc_node(sizeof(struct listener), GFP_KERNEL,
306 cpu_to_node(cpu)); 306 cpu_to_node(cpu));
307 if (!s) 307 if (!s)
@@ -320,7 +320,7 @@ static int add_del_listener(pid_t pid, cpumask_t *maskp, int isadd)
320 320
321 /* Deregister or cleanup */ 321 /* Deregister or cleanup */
322cleanup: 322cleanup:
323 for_each_cpu_mask(cpu, mask) { 323 for_each_cpu_mask_nr(cpu, mask) {
324 listeners = &per_cpu(listener_array, cpu); 324 listeners = &per_cpu(listener_array, cpu);
325 down_write(&listeners->sem); 325 down_write(&listeners->sem);
326 list_for_each_entry_safe(s, tmp, &listeners->list, list) { 326 list_for_each_entry_safe(s, tmp, &listeners->list, list) {
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index dadde5361f32..093d4acf993b 100644
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -145,9 +145,9 @@ static void clocksource_watchdog(unsigned long data)
145 * Cycle through CPUs to check if the CPUs stay 145 * Cycle through CPUs to check if the CPUs stay
146 * synchronized to each other. 146 * synchronized to each other.
147 */ 147 */
148 int next_cpu = next_cpu(raw_smp_processor_id(), cpu_online_map); 148 int next_cpu = next_cpu_nr(raw_smp_processor_id(), cpu_online_map);
149 149
150 if (next_cpu >= NR_CPUS) 150 if (next_cpu >= nr_cpu_ids)
151 next_cpu = first_cpu(cpu_online_map); 151 next_cpu = first_cpu(cpu_online_map);
152 watchdog_timer.expires += WATCHDOG_INTERVAL; 152 watchdog_timer.expires += WATCHDOG_INTERVAL;
153 add_timer_on(&watchdog_timer, next_cpu); 153 add_timer_on(&watchdog_timer, next_cpu);
@@ -376,7 +376,8 @@ void clocksource_unregister(struct clocksource *cs)
376 * Provides sysfs interface for listing current clocksource. 376 * Provides sysfs interface for listing current clocksource.
377 */ 377 */
378static ssize_t 378static ssize_t
379sysfs_show_current_clocksources(struct sys_device *dev, char *buf) 379sysfs_show_current_clocksources(struct sys_device *dev,
380 struct sysdev_attribute *attr, char *buf)
380{ 381{
381 ssize_t count = 0; 382 ssize_t count = 0;
382 383
@@ -397,6 +398,7 @@ sysfs_show_current_clocksources(struct sys_device *dev, char *buf)
397 * clocksource selction. 398 * clocksource selction.
398 */ 399 */
399static ssize_t sysfs_override_clocksource(struct sys_device *dev, 400static ssize_t sysfs_override_clocksource(struct sys_device *dev,
401 struct sysdev_attribute *attr,
400 const char *buf, size_t count) 402 const char *buf, size_t count)
401{ 403{
402 struct clocksource *ovr = NULL; 404 struct clocksource *ovr = NULL;
@@ -449,7 +451,9 @@ static ssize_t sysfs_override_clocksource(struct sys_device *dev,
449 * Provides sysfs interface for listing registered clocksources 451 * Provides sysfs interface for listing registered clocksources
450 */ 452 */
451static ssize_t 453static ssize_t
452sysfs_show_available_clocksources(struct sys_device *dev, char *buf) 454sysfs_show_available_clocksources(struct sys_device *dev,
455 struct sysdev_attribute *attr,
456 char *buf)
453{ 457{
454 struct clocksource *src; 458 struct clocksource *src;
455 ssize_t count = 0; 459 ssize_t count = 0;
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c
index 57a1f02e5ec0..31463d370b94 100644
--- a/kernel/time/tick-broadcast.c
+++ b/kernel/time/tick-broadcast.c
@@ -30,6 +30,7 @@
30struct tick_device tick_broadcast_device; 30struct tick_device tick_broadcast_device;
31static cpumask_t tick_broadcast_mask; 31static cpumask_t tick_broadcast_mask;
32static DEFINE_SPINLOCK(tick_broadcast_lock); 32static DEFINE_SPINLOCK(tick_broadcast_lock);
33static int tick_broadcast_force;
33 34
34#ifdef CONFIG_TICK_ONESHOT 35#ifdef CONFIG_TICK_ONESHOT
35static void tick_broadcast_clear_oneshot(int cpu); 36static void tick_broadcast_clear_oneshot(int cpu);
@@ -232,10 +233,11 @@ static void tick_do_broadcast_on_off(void *why)
232 CLOCK_EVT_MODE_SHUTDOWN); 233 CLOCK_EVT_MODE_SHUTDOWN);
233 } 234 }
234 if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE) 235 if (*reason == CLOCK_EVT_NOTIFY_BROADCAST_FORCE)
235 dev->features |= CLOCK_EVT_FEAT_DUMMY; 236 tick_broadcast_force = 1;
236 break; 237 break;
237 case CLOCK_EVT_NOTIFY_BROADCAST_OFF: 238 case CLOCK_EVT_NOTIFY_BROADCAST_OFF:
238 if (cpu_isset(cpu, tick_broadcast_mask)) { 239 if (!tick_broadcast_force &&
240 cpu_isset(cpu, tick_broadcast_mask)) {
239 cpu_clear(cpu, tick_broadcast_mask); 241 cpu_clear(cpu, tick_broadcast_mask);
240 if (td->mode == TICKDEV_MODE_PERIODIC) 242 if (td->mode == TICKDEV_MODE_PERIODIC)
241 tick_setup_periodic(dev, 0); 243 tick_setup_periodic(dev, 0);
@@ -266,7 +268,7 @@ void tick_broadcast_on_off(unsigned long reason, int *oncpu)
266 "offline CPU #%d\n", *oncpu); 268 "offline CPU #%d\n", *oncpu);
267 else 269 else
268 smp_call_function_single(*oncpu, tick_do_broadcast_on_off, 270 smp_call_function_single(*oncpu, tick_do_broadcast_on_off,
269 &reason, 1, 1); 271 &reason, 1);
270} 272}
271 273
272/* 274/*
@@ -397,8 +399,7 @@ again:
397 mask = CPU_MASK_NONE; 399 mask = CPU_MASK_NONE;
398 now = ktime_get(); 400 now = ktime_get();
399 /* Find all expired events */ 401 /* Find all expired events */
400 for (cpu = first_cpu(tick_broadcast_oneshot_mask); cpu != NR_CPUS; 402 for_each_cpu_mask_nr(cpu, tick_broadcast_oneshot_mask) {
401 cpu = next_cpu(cpu, tick_broadcast_oneshot_mask)) {
402 td = &per_cpu(tick_cpu_device, cpu); 403 td = &per_cpu(tick_cpu_device, cpu);
403 if (td->evtdev->next_event.tv64 <= now.tv64) 404 if (td->evtdev->next_event.tv64 <= now.tv64)
404 cpu_set(cpu, mask); 405 cpu_set(cpu, mask);
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c
index 4f3886562b8c..bf43284d6855 100644
--- a/kernel/time/tick-common.c
+++ b/kernel/time/tick-common.c
@@ -135,7 +135,7 @@ void tick_setup_periodic(struct clock_event_device *dev, int broadcast)
135 */ 135 */
136static void tick_setup_device(struct tick_device *td, 136static void tick_setup_device(struct tick_device *td,
137 struct clock_event_device *newdev, int cpu, 137 struct clock_event_device *newdev, int cpu,
138 cpumask_t cpumask) 138 const cpumask_t *cpumask)
139{ 139{
140 ktime_t next_event; 140 ktime_t next_event;
141 void (*handler)(struct clock_event_device *) = NULL; 141 void (*handler)(struct clock_event_device *) = NULL;
@@ -169,8 +169,8 @@ static void tick_setup_device(struct tick_device *td,
169 * When the device is not per cpu, pin the interrupt to the 169 * When the device is not per cpu, pin the interrupt to the
170 * current cpu: 170 * current cpu:
171 */ 171 */
172 if (!cpus_equal(newdev->cpumask, cpumask)) 172 if (!cpus_equal(newdev->cpumask, *cpumask))
173 irq_set_affinity(newdev->irq, cpumask); 173 irq_set_affinity(newdev->irq, *cpumask);
174 174
175 /* 175 /*
176 * When global broadcasting is active, check if the current 176 * When global broadcasting is active, check if the current
@@ -196,20 +196,20 @@ static int tick_check_new_device(struct clock_event_device *newdev)
196 struct tick_device *td; 196 struct tick_device *td;
197 int cpu, ret = NOTIFY_OK; 197 int cpu, ret = NOTIFY_OK;
198 unsigned long flags; 198 unsigned long flags;
199 cpumask_t cpumask; 199 cpumask_of_cpu_ptr_declare(cpumask);
200 200
201 spin_lock_irqsave(&tick_device_lock, flags); 201 spin_lock_irqsave(&tick_device_lock, flags);
202 202
203 cpu = smp_processor_id(); 203 cpu = smp_processor_id();
204 cpumask_of_cpu_ptr_next(cpumask, cpu);
204 if (!cpu_isset(cpu, newdev->cpumask)) 205 if (!cpu_isset(cpu, newdev->cpumask))
205 goto out_bc; 206 goto out_bc;
206 207
207 td = &per_cpu(tick_cpu_device, cpu); 208 td = &per_cpu(tick_cpu_device, cpu);
208 curdev = td->evtdev; 209 curdev = td->evtdev;
209 cpumask = cpumask_of_cpu(cpu);
210 210
211 /* cpu local device ? */ 211 /* cpu local device ? */
212 if (!cpus_equal(newdev->cpumask, cpumask)) { 212 if (!cpus_equal(newdev->cpumask, *cpumask)) {
213 213
214 /* 214 /*
215 * If the cpu affinity of the device interrupt can not 215 * If the cpu affinity of the device interrupt can not
@@ -222,7 +222,7 @@ static int tick_check_new_device(struct clock_event_device *newdev)
222 * If we have a cpu local device already, do not replace it 222 * If we have a cpu local device already, do not replace it
223 * by a non cpu local device 223 * by a non cpu local device
224 */ 224 */
225 if (curdev && cpus_equal(curdev->cpumask, cpumask)) 225 if (curdev && cpus_equal(curdev->cpumask, *cpumask))
226 goto out_bc; 226 goto out_bc;
227 } 227 }
228 228
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index b854a895591e..825b4c00fe44 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -48,6 +48,13 @@ static void tick_do_update_jiffies64(ktime_t now)
48 unsigned long ticks = 0; 48 unsigned long ticks = 0;
49 ktime_t delta; 49 ktime_t delta;
50 50
51 /*
52 * Do a quick check without holding xtime_lock:
53 */
54 delta = ktime_sub(now, last_jiffies_update);
55 if (delta.tv64 < tick_period.tv64)
56 return;
57
51 /* Reevalute with xtime_lock held */ 58 /* Reevalute with xtime_lock held */
52 write_seqlock(&xtime_lock); 59 write_seqlock(&xtime_lock);
53 60
@@ -133,8 +140,6 @@ void tick_nohz_update_jiffies(void)
133 if (!ts->tick_stopped) 140 if (!ts->tick_stopped)
134 return; 141 return;
135 142
136 touch_softlockup_watchdog();
137
138 cpu_clear(cpu, nohz_cpu_mask); 143 cpu_clear(cpu, nohz_cpu_mask);
139 now = ktime_get(); 144 now = ktime_get();
140 ts->idle_waketime = now; 145 ts->idle_waketime = now;
@@ -142,6 +147,8 @@ void tick_nohz_update_jiffies(void)
142 local_irq_save(flags); 147 local_irq_save(flags);
143 tick_do_update_jiffies64(now); 148 tick_do_update_jiffies64(now);
144 local_irq_restore(flags); 149 local_irq_restore(flags);
150
151 touch_softlockup_watchdog();
145} 152}
146 153
147void tick_nohz_stop_idle(int cpu) 154void tick_nohz_stop_idle(int cpu)
@@ -188,7 +195,7 @@ u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
188 * Called either from the idle loop or from irq_exit() when an idle period was 195 * Called either from the idle loop or from irq_exit() when an idle period was
189 * just interrupted by an interrupt which did not cause a reschedule. 196 * just interrupted by an interrupt which did not cause a reschedule.
190 */ 197 */
191void tick_nohz_stop_sched_tick(void) 198void tick_nohz_stop_sched_tick(int inidle)
192{ 199{
193 unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags; 200 unsigned long seq, last_jiffies, next_jiffies, delta_jiffies, flags;
194 struct tick_sched *ts; 201 struct tick_sched *ts;
@@ -217,6 +224,11 @@ void tick_nohz_stop_sched_tick(void)
217 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE)) 224 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
218 goto end; 225 goto end;
219 226
227 if (!inidle && !ts->inidle)
228 goto end;
229
230 ts->inidle = 1;
231
220 if (need_resched()) 232 if (need_resched())
221 goto end; 233 goto end;
222 234
@@ -228,6 +240,7 @@ void tick_nohz_stop_sched_tick(void)
228 local_softirq_pending()); 240 local_softirq_pending());
229 ratelimit++; 241 ratelimit++;
230 } 242 }
243 goto end;
231 } 244 }
232 245
233 ts->idle_calls++; 246 ts->idle_calls++;
@@ -276,6 +289,7 @@ void tick_nohz_stop_sched_tick(void)
276 ts->tick_stopped = 1; 289 ts->tick_stopped = 1;
277 ts->idle_jiffies = last_jiffies; 290 ts->idle_jiffies = last_jiffies;
278 rcu_enter_nohz(); 291 rcu_enter_nohz();
292 sched_clock_tick_stop(cpu);
279 } 293 }
280 294
281 /* 295 /*
@@ -364,17 +378,21 @@ void tick_nohz_restart_sched_tick(void)
364 local_irq_disable(); 378 local_irq_disable();
365 tick_nohz_stop_idle(cpu); 379 tick_nohz_stop_idle(cpu);
366 380
367 if (!ts->tick_stopped) { 381 if (!ts->inidle || !ts->tick_stopped) {
382 ts->inidle = 0;
368 local_irq_enable(); 383 local_irq_enable();
369 return; 384 return;
370 } 385 }
371 386
387 ts->inidle = 0;
388
372 rcu_exit_nohz(); 389 rcu_exit_nohz();
373 390
374 /* Update jiffies first */ 391 /* Update jiffies first */
375 select_nohz_load_balancer(0); 392 select_nohz_load_balancer(0);
376 now = ktime_get(); 393 now = ktime_get();
377 tick_do_update_jiffies64(now); 394 tick_do_update_jiffies64(now);
395 sched_clock_tick_start(cpu);
378 cpu_clear(cpu, nohz_cpu_mask); 396 cpu_clear(cpu, nohz_cpu_mask);
379 397
380 /* 398 /*
diff --git a/kernel/timer.c b/kernel/timer.c
index ceacc6626572..03bc7f1f1593 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -812,7 +812,7 @@ static inline void __run_timers(struct tvec_base *base)
812 spin_unlock_irq(&base->lock); 812 spin_unlock_irq(&base->lock);
813} 813}
814 814
815#if defined(CONFIG_NO_IDLE_HZ) || defined(CONFIG_NO_HZ) 815#ifdef CONFIG_NO_HZ
816/* 816/*
817 * Find out when the next timer event is due to happen. This 817 * Find out when the next timer event is due to happen. This
818 * is used on S/390 to stop all activity when a cpus is idle. 818 * is used on S/390 to stop all activity when a cpus is idle.
@@ -947,14 +947,6 @@ unsigned long get_next_timer_interrupt(unsigned long now)
947 947
948 return cmp_next_hrtimer_event(now, expires); 948 return cmp_next_hrtimer_event(now, expires);
949} 949}
950
951#ifdef CONFIG_NO_IDLE_HZ
952unsigned long next_timer_interrupt(void)
953{
954 return get_next_timer_interrupt(jiffies);
955}
956#endif
957
958#endif 950#endif
959 951
960#ifndef CONFIG_VIRT_CPU_ACCOUNTING 952#ifndef CONFIG_VIRT_CPU_ACCOUNTING
@@ -1502,7 +1494,7 @@ void __init init_timers(void)
1502 1494
1503 BUG_ON(err == NOTIFY_BAD); 1495 BUG_ON(err == NOTIFY_BAD);
1504 register_cpu_notifier(&timers_nb); 1496 register_cpu_notifier(&timers_nb);
1505 open_softirq(TIMER_SOFTIRQ, run_timer_softirq, NULL); 1497 open_softirq(TIMER_SOFTIRQ, run_timer_softirq);
1506} 1498}
1507 1499
1508/** 1500/**
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
new file mode 100644
index 000000000000..263e9e6bbd60
--- /dev/null
+++ b/kernel/trace/Kconfig
@@ -0,0 +1,135 @@
1#
2# Architectures that offer an FTRACE implementation should select HAVE_FTRACE:
3#
4config HAVE_FTRACE
5 bool
6
7config HAVE_DYNAMIC_FTRACE
8 bool
9
10config TRACER_MAX_TRACE
11 bool
12
13config TRACING
14 bool
15 select DEBUG_FS
16 select STACKTRACE
17
18config FTRACE
19 bool "Kernel Function Tracer"
20 depends on HAVE_FTRACE
21 select FRAME_POINTER
22 select TRACING
23 select CONTEXT_SWITCH_TRACER
24 help
25 Enable the kernel to trace every kernel function. This is done
26 by using a compiler feature to insert a small, 5-byte No-Operation
27 instruction to the beginning of every kernel function, which NOP
28 sequence is then dynamically patched into a tracer call when
29 tracing is enabled by the administrator. If it's runtime disabled
30 (the bootup default), then the overhead of the instructions is very
31 small and not measurable even in micro-benchmarks.
32
33config IRQSOFF_TRACER
34 bool "Interrupts-off Latency Tracer"
35 default n
36 depends on TRACE_IRQFLAGS_SUPPORT
37 depends on GENERIC_TIME
38 depends on HAVE_FTRACE
39 select TRACE_IRQFLAGS
40 select TRACING
41 select TRACER_MAX_TRACE
42 help
43 This option measures the time spent in irqs-off critical
44 sections, with microsecond accuracy.
45
46 The default measurement method is a maximum search, which is
47 disabled by default and can be runtime (re-)started
48 via:
49
50 echo 0 > /debugfs/tracing/tracing_max_latency
51
52 (Note that kernel size and overhead increases with this option
53 enabled. This option and the preempt-off timing option can be
54 used together or separately.)
55
56config PREEMPT_TRACER
57 bool "Preemption-off Latency Tracer"
58 default n
59 depends on GENERIC_TIME
60 depends on PREEMPT
61 depends on HAVE_FTRACE
62 select TRACING
63 select TRACER_MAX_TRACE
64 help
65 This option measures the time spent in preemption off critical
66 sections, with microsecond accuracy.
67
68 The default measurement method is a maximum search, which is
69 disabled by default and can be runtime (re-)started
70 via:
71
72 echo 0 > /debugfs/tracing/tracing_max_latency
73
74 (Note that kernel size and overhead increases with this option
75 enabled. This option and the irqs-off timing option can be
76 used together or separately.)
77
78config SYSPROF_TRACER
79 bool "Sysprof Tracer"
80 depends on X86
81 select TRACING
82 help
83 This tracer provides the trace needed by the 'Sysprof' userspace
84 tool.
85
86config SCHED_TRACER
87 bool "Scheduling Latency Tracer"
88 depends on HAVE_FTRACE
89 select TRACING
90 select CONTEXT_SWITCH_TRACER
91 select TRACER_MAX_TRACE
92 help
93 This tracer tracks the latency of the highest priority task
94 to be scheduled in, starting from the point it has woken up.
95
96config CONTEXT_SWITCH_TRACER
97 bool "Trace process context switches"
98 depends on HAVE_FTRACE
99 select TRACING
100 select MARKERS
101 help
102 This tracer gets called from the context switch and records
103 all switching of tasks.
104
105config DYNAMIC_FTRACE
106 bool "enable/disable ftrace tracepoints dynamically"
107 depends on FTRACE
108 depends on HAVE_DYNAMIC_FTRACE
109 default y
110 help
111 This option will modify all the calls to ftrace dynamically
112 (will patch them out of the binary image and replaces them
113 with a No-Op instruction) as they are called. A table is
114 created to dynamically enable them again.
115
116 This way a CONFIG_FTRACE kernel is slightly larger, but otherwise
117 has native performance as long as no tracing is active.
118
119 The changes to the code are done by a kernel thread that
120 wakes up once a second and checks to see if any ftrace calls
121 were made. If so, it runs stop_machine (stops all CPUS)
122 and modifies the code to jump over the call to ftrace.
123
124config FTRACE_SELFTEST
125 bool
126
127config FTRACE_STARTUP_TEST
128 bool "Perform a startup test on ftrace"
129 depends on TRACING
130 select FTRACE_SELFTEST
131 help
132 This option performs a series of startup tests on ftrace. On bootup
133 a series of tests are made to verify that the tracer is
134 functioning properly. It will do tests on all the configured
135 tracers of ftrace.
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile
new file mode 100644
index 000000000000..71d17de17288
--- /dev/null
+++ b/kernel/trace/Makefile
@@ -0,0 +1,24 @@
1
2# Do not instrument the tracer itself:
3
4ifdef CONFIG_FTRACE
5ORIG_CFLAGS := $(KBUILD_CFLAGS)
6KBUILD_CFLAGS = $(subst -pg,,$(ORIG_CFLAGS))
7
8# selftest needs instrumentation
9CFLAGS_trace_selftest_dynamic.o = -pg
10obj-y += trace_selftest_dynamic.o
11endif
12
13obj-$(CONFIG_FTRACE) += libftrace.o
14
15obj-$(CONFIG_TRACING) += trace.o
16obj-$(CONFIG_CONTEXT_SWITCH_TRACER) += trace_sched_switch.o
17obj-$(CONFIG_SYSPROF_TRACER) += trace_sysprof.o
18obj-$(CONFIG_FTRACE) += trace_functions.o
19obj-$(CONFIG_IRQSOFF_TRACER) += trace_irqsoff.o
20obj-$(CONFIG_PREEMPT_TRACER) += trace_irqsoff.o
21obj-$(CONFIG_SCHED_TRACER) += trace_sched_wakeup.o
22obj-$(CONFIG_MMIOTRACE) += trace_mmiotrace.o
23
24libftrace-y := ftrace.o
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
new file mode 100644
index 000000000000..4231a3dc224a
--- /dev/null
+++ b/kernel/trace/ftrace.c
@@ -0,0 +1,1727 @@
1/*
2 * Infrastructure for profiling code inserted by 'gcc -pg'.
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally ported from the -rt patch by:
8 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code in the latency_tracer, that is:
11 *
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 William Lee Irwin III
14 */
15
16#include <linux/stop_machine.h>
17#include <linux/clocksource.h>
18#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
20#include <linux/debugfs.h>
21#include <linux/hardirq.h>
22#include <linux/kthread.h>
23#include <linux/uaccess.h>
24#include <linux/kprobes.h>
25#include <linux/ftrace.h>
26#include <linux/sysctl.h>
27#include <linux/ctype.h>
28#include <linux/hash.h>
29#include <linux/list.h>
30
31#include <asm/ftrace.h>
32
33#include "trace.h"
34
35/* ftrace_enabled is a method to turn ftrace on or off */
36int ftrace_enabled __read_mostly;
37static int last_ftrace_enabled;
38
39/*
40 * ftrace_disabled is set when an anomaly is discovered.
41 * ftrace_disabled is much stronger than ftrace_enabled.
42 */
43static int ftrace_disabled __read_mostly;
44
45static DEFINE_SPINLOCK(ftrace_lock);
46static DEFINE_MUTEX(ftrace_sysctl_lock);
47
48static struct ftrace_ops ftrace_list_end __read_mostly =
49{
50 .func = ftrace_stub,
51};
52
53static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
54ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
55
56static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
57{
58 struct ftrace_ops *op = ftrace_list;
59
60 /* in case someone actually ports this to alpha! */
61 read_barrier_depends();
62
63 while (op != &ftrace_list_end) {
64 /* silly alpha */
65 read_barrier_depends();
66 op->func(ip, parent_ip);
67 op = op->next;
68 };
69}
70
71/**
72 * clear_ftrace_function - reset the ftrace function
73 *
74 * This NULLs the ftrace function and in essence stops
75 * tracing. There may be lag
76 */
77void clear_ftrace_function(void)
78{
79 ftrace_trace_function = ftrace_stub;
80}
81
82static int __register_ftrace_function(struct ftrace_ops *ops)
83{
84 /* Should never be called by interrupts */
85 spin_lock(&ftrace_lock);
86
87 ops->next = ftrace_list;
88 /*
89 * We are entering ops into the ftrace_list but another
90 * CPU might be walking that list. We need to make sure
91 * the ops->next pointer is valid before another CPU sees
92 * the ops pointer included into the ftrace_list.
93 */
94 smp_wmb();
95 ftrace_list = ops;
96
97 if (ftrace_enabled) {
98 /*
99 * For one func, simply call it directly.
100 * For more than one func, call the chain.
101 */
102 if (ops->next == &ftrace_list_end)
103 ftrace_trace_function = ops->func;
104 else
105 ftrace_trace_function = ftrace_list_func;
106 }
107
108 spin_unlock(&ftrace_lock);
109
110 return 0;
111}
112
113static int __unregister_ftrace_function(struct ftrace_ops *ops)
114{
115 struct ftrace_ops **p;
116 int ret = 0;
117
118 spin_lock(&ftrace_lock);
119
120 /*
121 * If we are removing the last function, then simply point
122 * to the ftrace_stub.
123 */
124 if (ftrace_list == ops && ops->next == &ftrace_list_end) {
125 ftrace_trace_function = ftrace_stub;
126 ftrace_list = &ftrace_list_end;
127 goto out;
128 }
129
130 for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
131 if (*p == ops)
132 break;
133
134 if (*p != ops) {
135 ret = -1;
136 goto out;
137 }
138
139 *p = (*p)->next;
140
141 if (ftrace_enabled) {
142 /* If we only have one func left, then call that directly */
143 if (ftrace_list == &ftrace_list_end ||
144 ftrace_list->next == &ftrace_list_end)
145 ftrace_trace_function = ftrace_list->func;
146 }
147
148 out:
149 spin_unlock(&ftrace_lock);
150
151 return ret;
152}
153
154#ifdef CONFIG_DYNAMIC_FTRACE
155
156static struct task_struct *ftraced_task;
157
158enum {
159 FTRACE_ENABLE_CALLS = (1 << 0),
160 FTRACE_DISABLE_CALLS = (1 << 1),
161 FTRACE_UPDATE_TRACE_FUNC = (1 << 2),
162 FTRACE_ENABLE_MCOUNT = (1 << 3),
163 FTRACE_DISABLE_MCOUNT = (1 << 4),
164};
165
166static int ftrace_filtered;
167static int tracing_on;
168static int frozen_record_count;
169
170static struct hlist_head ftrace_hash[FTRACE_HASHSIZE];
171
172static DEFINE_PER_CPU(int, ftrace_shutdown_disable_cpu);
173
174static DEFINE_SPINLOCK(ftrace_shutdown_lock);
175static DEFINE_MUTEX(ftraced_lock);
176static DEFINE_MUTEX(ftrace_regex_lock);
177
178struct ftrace_page {
179 struct ftrace_page *next;
180 unsigned long index;
181 struct dyn_ftrace records[];
182};
183
184#define ENTRIES_PER_PAGE \
185 ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
186
187/* estimate from running different kernels */
188#define NR_TO_INIT 10000
189
190static struct ftrace_page *ftrace_pages_start;
191static struct ftrace_page *ftrace_pages;
192
193static int ftraced_trigger;
194static int ftraced_suspend;
195static int ftraced_stop;
196
197static int ftrace_record_suspend;
198
199static struct dyn_ftrace *ftrace_free_records;
200
201
202#ifdef CONFIG_KPROBES
203static inline void freeze_record(struct dyn_ftrace *rec)
204{
205 if (!(rec->flags & FTRACE_FL_FROZEN)) {
206 rec->flags |= FTRACE_FL_FROZEN;
207 frozen_record_count++;
208 }
209}
210
211static inline void unfreeze_record(struct dyn_ftrace *rec)
212{
213 if (rec->flags & FTRACE_FL_FROZEN) {
214 rec->flags &= ~FTRACE_FL_FROZEN;
215 frozen_record_count--;
216 }
217}
218
219static inline int record_frozen(struct dyn_ftrace *rec)
220{
221 return rec->flags & FTRACE_FL_FROZEN;
222}
223#else
224# define freeze_record(rec) ({ 0; })
225# define unfreeze_record(rec) ({ 0; })
226# define record_frozen(rec) ({ 0; })
227#endif /* CONFIG_KPROBES */
228
229int skip_trace(unsigned long ip)
230{
231 unsigned long fl;
232 struct dyn_ftrace *rec;
233 struct hlist_node *t;
234 struct hlist_head *head;
235
236 if (frozen_record_count == 0)
237 return 0;
238
239 head = &ftrace_hash[hash_long(ip, FTRACE_HASHBITS)];
240 hlist_for_each_entry_rcu(rec, t, head, node) {
241 if (rec->ip == ip) {
242 if (record_frozen(rec)) {
243 if (rec->flags & FTRACE_FL_FAILED)
244 return 1;
245
246 if (!(rec->flags & FTRACE_FL_CONVERTED))
247 return 1;
248
249 if (!tracing_on || !ftrace_enabled)
250 return 1;
251
252 if (ftrace_filtered) {
253 fl = rec->flags & (FTRACE_FL_FILTER |
254 FTRACE_FL_NOTRACE);
255 if (!fl || (fl & FTRACE_FL_NOTRACE))
256 return 1;
257 }
258 }
259 break;
260 }
261 }
262
263 return 0;
264}
265
266static inline int
267ftrace_ip_in_hash(unsigned long ip, unsigned long key)
268{
269 struct dyn_ftrace *p;
270 struct hlist_node *t;
271 int found = 0;
272
273 hlist_for_each_entry_rcu(p, t, &ftrace_hash[key], node) {
274 if (p->ip == ip) {
275 found = 1;
276 break;
277 }
278 }
279
280 return found;
281}
282
283static inline void
284ftrace_add_hash(struct dyn_ftrace *node, unsigned long key)
285{
286 hlist_add_head_rcu(&node->node, &ftrace_hash[key]);
287}
288
289/* called from kstop_machine */
290static inline void ftrace_del_hash(struct dyn_ftrace *node)
291{
292 hlist_del(&node->node);
293}
294
295static void ftrace_free_rec(struct dyn_ftrace *rec)
296{
297 /* no locking, only called from kstop_machine */
298
299 rec->ip = (unsigned long)ftrace_free_records;
300 ftrace_free_records = rec;
301 rec->flags |= FTRACE_FL_FREE;
302}
303
304static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
305{
306 struct dyn_ftrace *rec;
307
308 /* First check for freed records */
309 if (ftrace_free_records) {
310 rec = ftrace_free_records;
311
312 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
313 WARN_ON_ONCE(1);
314 ftrace_free_records = NULL;
315 ftrace_disabled = 1;
316 ftrace_enabled = 0;
317 return NULL;
318 }
319
320 ftrace_free_records = (void *)rec->ip;
321 memset(rec, 0, sizeof(*rec));
322 return rec;
323 }
324
325 if (ftrace_pages->index == ENTRIES_PER_PAGE) {
326 if (!ftrace_pages->next)
327 return NULL;
328 ftrace_pages = ftrace_pages->next;
329 }
330
331 return &ftrace_pages->records[ftrace_pages->index++];
332}
333
334static void
335ftrace_record_ip(unsigned long ip)
336{
337 struct dyn_ftrace *node;
338 unsigned long flags;
339 unsigned long key;
340 int resched;
341 int atomic;
342 int cpu;
343
344 if (!ftrace_enabled || ftrace_disabled)
345 return;
346
347 resched = need_resched();
348 preempt_disable_notrace();
349
350 /*
351 * We simply need to protect against recursion.
352 * Use the the raw version of smp_processor_id and not
353 * __get_cpu_var which can call debug hooks that can
354 * cause a recursive crash here.
355 */
356 cpu = raw_smp_processor_id();
357 per_cpu(ftrace_shutdown_disable_cpu, cpu)++;
358 if (per_cpu(ftrace_shutdown_disable_cpu, cpu) != 1)
359 goto out;
360
361 if (unlikely(ftrace_record_suspend))
362 goto out;
363
364 key = hash_long(ip, FTRACE_HASHBITS);
365
366 WARN_ON_ONCE(key >= FTRACE_HASHSIZE);
367
368 if (ftrace_ip_in_hash(ip, key))
369 goto out;
370
371 atomic = irqs_disabled();
372
373 spin_lock_irqsave(&ftrace_shutdown_lock, flags);
374
375 /* This ip may have hit the hash before the lock */
376 if (ftrace_ip_in_hash(ip, key))
377 goto out_unlock;
378
379 node = ftrace_alloc_dyn_node(ip);
380 if (!node)
381 goto out_unlock;
382
383 node->ip = ip;
384
385 ftrace_add_hash(node, key);
386
387 ftraced_trigger = 1;
388
389 out_unlock:
390 spin_unlock_irqrestore(&ftrace_shutdown_lock, flags);
391 out:
392 per_cpu(ftrace_shutdown_disable_cpu, cpu)--;
393
394 /* prevent recursion with scheduler */
395 if (resched)
396 preempt_enable_no_resched_notrace();
397 else
398 preempt_enable_notrace();
399}
400
401#define FTRACE_ADDR ((long)(ftrace_caller))
402
403static int
404__ftrace_replace_code(struct dyn_ftrace *rec,
405 unsigned char *old, unsigned char *new, int enable)
406{
407 unsigned long ip, fl;
408
409 ip = rec->ip;
410
411 if (ftrace_filtered && enable) {
412 /*
413 * If filtering is on:
414 *
415 * If this record is set to be filtered and
416 * is enabled then do nothing.
417 *
418 * If this record is set to be filtered and
419 * it is not enabled, enable it.
420 *
421 * If this record is not set to be filtered
422 * and it is not enabled do nothing.
423 *
424 * If this record is set not to trace then
425 * do nothing.
426 *
427 * If this record is set not to trace and
428 * it is enabled then disable it.
429 *
430 * If this record is not set to be filtered and
431 * it is enabled, disable it.
432 */
433
434 fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_NOTRACE |
435 FTRACE_FL_ENABLED);
436
437 if ((fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED)) ||
438 (fl == (FTRACE_FL_FILTER | FTRACE_FL_NOTRACE)) ||
439 !fl || (fl == FTRACE_FL_NOTRACE))
440 return 0;
441
442 /*
443 * If it is enabled disable it,
444 * otherwise enable it!
445 */
446 if (fl & FTRACE_FL_ENABLED) {
447 /* swap new and old */
448 new = old;
449 old = ftrace_call_replace(ip, FTRACE_ADDR);
450 rec->flags &= ~FTRACE_FL_ENABLED;
451 } else {
452 new = ftrace_call_replace(ip, FTRACE_ADDR);
453 rec->flags |= FTRACE_FL_ENABLED;
454 }
455 } else {
456
457 if (enable) {
458 /*
459 * If this record is set not to trace and is
460 * not enabled, do nothing.
461 */
462 fl = rec->flags & (FTRACE_FL_NOTRACE | FTRACE_FL_ENABLED);
463 if (fl == FTRACE_FL_NOTRACE)
464 return 0;
465
466 new = ftrace_call_replace(ip, FTRACE_ADDR);
467 } else
468 old = ftrace_call_replace(ip, FTRACE_ADDR);
469
470 if (enable) {
471 if (rec->flags & FTRACE_FL_ENABLED)
472 return 0;
473 rec->flags |= FTRACE_FL_ENABLED;
474 } else {
475 if (!(rec->flags & FTRACE_FL_ENABLED))
476 return 0;
477 rec->flags &= ~FTRACE_FL_ENABLED;
478 }
479 }
480
481 return ftrace_modify_code(ip, old, new);
482}
483
484static void ftrace_replace_code(int enable)
485{
486 int i, failed;
487 unsigned char *new = NULL, *old = NULL;
488 struct dyn_ftrace *rec;
489 struct ftrace_page *pg;
490
491 if (enable)
492 old = ftrace_nop_replace();
493 else
494 new = ftrace_nop_replace();
495
496 for (pg = ftrace_pages_start; pg; pg = pg->next) {
497 for (i = 0; i < pg->index; i++) {
498 rec = &pg->records[i];
499
500 /* don't modify code that has already faulted */
501 if (rec->flags & FTRACE_FL_FAILED)
502 continue;
503
504 /* ignore updates to this record's mcount site */
505 if (get_kprobe((void *)rec->ip)) {
506 freeze_record(rec);
507 continue;
508 } else {
509 unfreeze_record(rec);
510 }
511
512 failed = __ftrace_replace_code(rec, old, new, enable);
513 if (failed && (rec->flags & FTRACE_FL_CONVERTED)) {
514 rec->flags |= FTRACE_FL_FAILED;
515 if ((system_state == SYSTEM_BOOTING) ||
516 !core_kernel_text(rec->ip)) {
517 ftrace_del_hash(rec);
518 ftrace_free_rec(rec);
519 }
520 }
521 }
522 }
523}
524
525static void ftrace_shutdown_replenish(void)
526{
527 if (ftrace_pages->next)
528 return;
529
530 /* allocate another page */
531 ftrace_pages->next = (void *)get_zeroed_page(GFP_KERNEL);
532}
533
534static int
535ftrace_code_disable(struct dyn_ftrace *rec)
536{
537 unsigned long ip;
538 unsigned char *nop, *call;
539 int failed;
540
541 ip = rec->ip;
542
543 nop = ftrace_nop_replace();
544 call = ftrace_call_replace(ip, MCOUNT_ADDR);
545
546 failed = ftrace_modify_code(ip, call, nop);
547 if (failed) {
548 rec->flags |= FTRACE_FL_FAILED;
549 return 0;
550 }
551 return 1;
552}
553
554static int __ftrace_update_code(void *ignore);
555
556static int __ftrace_modify_code(void *data)
557{
558 unsigned long addr;
559 int *command = data;
560
561 if (*command & FTRACE_ENABLE_CALLS) {
562 /*
563 * Update any recorded ips now that we have the
564 * machine stopped
565 */
566 __ftrace_update_code(NULL);
567 ftrace_replace_code(1);
568 tracing_on = 1;
569 } else if (*command & FTRACE_DISABLE_CALLS) {
570 ftrace_replace_code(0);
571 tracing_on = 0;
572 }
573
574 if (*command & FTRACE_UPDATE_TRACE_FUNC)
575 ftrace_update_ftrace_func(ftrace_trace_function);
576
577 if (*command & FTRACE_ENABLE_MCOUNT) {
578 addr = (unsigned long)ftrace_record_ip;
579 ftrace_mcount_set(&addr);
580 } else if (*command & FTRACE_DISABLE_MCOUNT) {
581 addr = (unsigned long)ftrace_stub;
582 ftrace_mcount_set(&addr);
583 }
584
585 return 0;
586}
587
588static void ftrace_run_update_code(int command)
589{
590 stop_machine_run(__ftrace_modify_code, &command, NR_CPUS);
591}
592
593void ftrace_disable_daemon(void)
594{
595 /* Stop the daemon from calling kstop_machine */
596 mutex_lock(&ftraced_lock);
597 ftraced_stop = 1;
598 mutex_unlock(&ftraced_lock);
599
600 ftrace_force_update();
601}
602
603void ftrace_enable_daemon(void)
604{
605 mutex_lock(&ftraced_lock);
606 ftraced_stop = 0;
607 mutex_unlock(&ftraced_lock);
608
609 ftrace_force_update();
610}
611
612static ftrace_func_t saved_ftrace_func;
613
614static void ftrace_startup(void)
615{
616 int command = 0;
617
618 if (unlikely(ftrace_disabled))
619 return;
620
621 mutex_lock(&ftraced_lock);
622 ftraced_suspend++;
623 if (ftraced_suspend == 1)
624 command |= FTRACE_ENABLE_CALLS;
625
626 if (saved_ftrace_func != ftrace_trace_function) {
627 saved_ftrace_func = ftrace_trace_function;
628 command |= FTRACE_UPDATE_TRACE_FUNC;
629 }
630
631 if (!command || !ftrace_enabled)
632 goto out;
633
634 ftrace_run_update_code(command);
635 out:
636 mutex_unlock(&ftraced_lock);
637}
638
639static void ftrace_shutdown(void)
640{
641 int command = 0;
642
643 if (unlikely(ftrace_disabled))
644 return;
645
646 mutex_lock(&ftraced_lock);
647 ftraced_suspend--;
648 if (!ftraced_suspend)
649 command |= FTRACE_DISABLE_CALLS;
650
651 if (saved_ftrace_func != ftrace_trace_function) {
652 saved_ftrace_func = ftrace_trace_function;
653 command |= FTRACE_UPDATE_TRACE_FUNC;
654 }
655
656 if (!command || !ftrace_enabled)
657 goto out;
658
659 ftrace_run_update_code(command);
660 out:
661 mutex_unlock(&ftraced_lock);
662}
663
664static void ftrace_startup_sysctl(void)
665{
666 int command = FTRACE_ENABLE_MCOUNT;
667
668 if (unlikely(ftrace_disabled))
669 return;
670
671 mutex_lock(&ftraced_lock);
672 /* Force update next time */
673 saved_ftrace_func = NULL;
674 /* ftraced_suspend is true if we want ftrace running */
675 if (ftraced_suspend)
676 command |= FTRACE_ENABLE_CALLS;
677
678 ftrace_run_update_code(command);
679 mutex_unlock(&ftraced_lock);
680}
681
682static void ftrace_shutdown_sysctl(void)
683{
684 int command = FTRACE_DISABLE_MCOUNT;
685
686 if (unlikely(ftrace_disabled))
687 return;
688
689 mutex_lock(&ftraced_lock);
690 /* ftraced_suspend is true if ftrace is running */
691 if (ftraced_suspend)
692 command |= FTRACE_DISABLE_CALLS;
693
694 ftrace_run_update_code(command);
695 mutex_unlock(&ftraced_lock);
696}
697
698static cycle_t ftrace_update_time;
699static unsigned long ftrace_update_cnt;
700unsigned long ftrace_update_tot_cnt;
701
702static int __ftrace_update_code(void *ignore)
703{
704 int i, save_ftrace_enabled;
705 cycle_t start, stop;
706 struct dyn_ftrace *p;
707 struct hlist_node *t, *n;
708 struct hlist_head *head, temp_list;
709
710 /* Don't be recording funcs now */
711 ftrace_record_suspend++;
712 save_ftrace_enabled = ftrace_enabled;
713 ftrace_enabled = 0;
714
715 start = ftrace_now(raw_smp_processor_id());
716 ftrace_update_cnt = 0;
717
718 /* No locks needed, the machine is stopped! */
719 for (i = 0; i < FTRACE_HASHSIZE; i++) {
720 INIT_HLIST_HEAD(&temp_list);
721 head = &ftrace_hash[i];
722
723 /* all CPUS are stopped, we are safe to modify code */
724 hlist_for_each_entry_safe(p, t, n, head, node) {
725 /* Skip over failed records which have not been
726 * freed. */
727 if (p->flags & FTRACE_FL_FAILED)
728 continue;
729
730 /* Unconverted records are always at the head of the
731 * hash bucket. Once we encounter a converted record,
732 * simply skip over to the next bucket. Saves ftraced
733 * some processor cycles (ftrace does its bid for
734 * global warming :-p ). */
735 if (p->flags & (FTRACE_FL_CONVERTED))
736 break;
737
738 /* Ignore updates to this record's mcount site.
739 * Reintroduce this record at the head of this
740 * bucket to attempt to "convert" it again if
741 * the kprobe on it is unregistered before the
742 * next run. */
743 if (get_kprobe((void *)p->ip)) {
744 ftrace_del_hash(p);
745 INIT_HLIST_NODE(&p->node);
746 hlist_add_head(&p->node, &temp_list);
747 freeze_record(p);
748 continue;
749 } else {
750 unfreeze_record(p);
751 }
752
753 /* convert record (i.e, patch mcount-call with NOP) */
754 if (ftrace_code_disable(p)) {
755 p->flags |= FTRACE_FL_CONVERTED;
756 ftrace_update_cnt++;
757 } else {
758 if ((system_state == SYSTEM_BOOTING) ||
759 !core_kernel_text(p->ip)) {
760 ftrace_del_hash(p);
761 ftrace_free_rec(p);
762 }
763 }
764 }
765
766 hlist_for_each_entry_safe(p, t, n, &temp_list, node) {
767 hlist_del(&p->node);
768 INIT_HLIST_NODE(&p->node);
769 hlist_add_head(&p->node, head);
770 }
771 }
772
773 stop = ftrace_now(raw_smp_processor_id());
774 ftrace_update_time = stop - start;
775 ftrace_update_tot_cnt += ftrace_update_cnt;
776 ftraced_trigger = 0;
777
778 ftrace_enabled = save_ftrace_enabled;
779 ftrace_record_suspend--;
780
781 return 0;
782}
783
784static int ftrace_update_code(void)
785{
786 if (unlikely(ftrace_disabled) ||
787 !ftrace_enabled || !ftraced_trigger)
788 return 0;
789
790 stop_machine_run(__ftrace_update_code, NULL, NR_CPUS);
791
792 return 1;
793}
794
795static int ftraced(void *ignore)
796{
797 unsigned long usecs;
798
799 while (!kthread_should_stop()) {
800
801 set_current_state(TASK_INTERRUPTIBLE);
802
803 /* check once a second */
804 schedule_timeout(HZ);
805
806 if (unlikely(ftrace_disabled))
807 continue;
808
809 mutex_lock(&ftrace_sysctl_lock);
810 mutex_lock(&ftraced_lock);
811 if (!ftraced_suspend && !ftraced_stop &&
812 ftrace_update_code()) {
813 usecs = nsecs_to_usecs(ftrace_update_time);
814 if (ftrace_update_tot_cnt > 100000) {
815 ftrace_update_tot_cnt = 0;
816 pr_info("hm, dftrace overflow: %lu change%s"
817 " (%lu total) in %lu usec%s\n",
818 ftrace_update_cnt,
819 ftrace_update_cnt != 1 ? "s" : "",
820 ftrace_update_tot_cnt,
821 usecs, usecs != 1 ? "s" : "");
822 ftrace_disabled = 1;
823 WARN_ON_ONCE(1);
824 }
825 }
826 mutex_unlock(&ftraced_lock);
827 mutex_unlock(&ftrace_sysctl_lock);
828
829 ftrace_shutdown_replenish();
830 }
831 __set_current_state(TASK_RUNNING);
832 return 0;
833}
834
835static int __init ftrace_dyn_table_alloc(void)
836{
837 struct ftrace_page *pg;
838 int cnt;
839 int i;
840
841 /* allocate a few pages */
842 ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
843 if (!ftrace_pages_start)
844 return -1;
845
846 /*
847 * Allocate a few more pages.
848 *
849 * TODO: have some parser search vmlinux before
850 * final linking to find all calls to ftrace.
851 * Then we can:
852 * a) know how many pages to allocate.
853 * and/or
854 * b) set up the table then.
855 *
856 * The dynamic code is still necessary for
857 * modules.
858 */
859
860 pg = ftrace_pages = ftrace_pages_start;
861
862 cnt = NR_TO_INIT / ENTRIES_PER_PAGE;
863
864 for (i = 0; i < cnt; i++) {
865 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
866
867 /* If we fail, we'll try later anyway */
868 if (!pg->next)
869 break;
870
871 pg = pg->next;
872 }
873
874 return 0;
875}
876
877enum {
878 FTRACE_ITER_FILTER = (1 << 0),
879 FTRACE_ITER_CONT = (1 << 1),
880 FTRACE_ITER_NOTRACE = (1 << 2),
881 FTRACE_ITER_FAILURES = (1 << 3),
882};
883
884#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
885
886struct ftrace_iterator {
887 loff_t pos;
888 struct ftrace_page *pg;
889 unsigned idx;
890 unsigned flags;
891 unsigned char buffer[FTRACE_BUFF_MAX+1];
892 unsigned buffer_idx;
893 unsigned filtered;
894};
895
896static void *
897t_next(struct seq_file *m, void *v, loff_t *pos)
898{
899 struct ftrace_iterator *iter = m->private;
900 struct dyn_ftrace *rec = NULL;
901
902 (*pos)++;
903
904 retry:
905 if (iter->idx >= iter->pg->index) {
906 if (iter->pg->next) {
907 iter->pg = iter->pg->next;
908 iter->idx = 0;
909 goto retry;
910 }
911 } else {
912 rec = &iter->pg->records[iter->idx++];
913 if ((!(iter->flags & FTRACE_ITER_FAILURES) &&
914 (rec->flags & FTRACE_FL_FAILED)) ||
915
916 ((iter->flags & FTRACE_ITER_FAILURES) &&
917 (!(rec->flags & FTRACE_FL_FAILED) ||
918 (rec->flags & FTRACE_FL_FREE))) ||
919
920 ((iter->flags & FTRACE_ITER_FILTER) &&
921 !(rec->flags & FTRACE_FL_FILTER)) ||
922
923 ((iter->flags & FTRACE_ITER_NOTRACE) &&
924 !(rec->flags & FTRACE_FL_NOTRACE))) {
925 rec = NULL;
926 goto retry;
927 }
928 }
929
930 iter->pos = *pos;
931
932 return rec;
933}
934
935static void *t_start(struct seq_file *m, loff_t *pos)
936{
937 struct ftrace_iterator *iter = m->private;
938 void *p = NULL;
939 loff_t l = -1;
940
941 if (*pos != iter->pos) {
942 for (p = t_next(m, p, &l); p && l < *pos; p = t_next(m, p, &l))
943 ;
944 } else {
945 l = *pos;
946 p = t_next(m, p, &l);
947 }
948
949 return p;
950}
951
952static void t_stop(struct seq_file *m, void *p)
953{
954}
955
956static int t_show(struct seq_file *m, void *v)
957{
958 struct dyn_ftrace *rec = v;
959 char str[KSYM_SYMBOL_LEN];
960
961 if (!rec)
962 return 0;
963
964 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
965
966 seq_printf(m, "%s\n", str);
967
968 return 0;
969}
970
971static struct seq_operations show_ftrace_seq_ops = {
972 .start = t_start,
973 .next = t_next,
974 .stop = t_stop,
975 .show = t_show,
976};
977
978static int
979ftrace_avail_open(struct inode *inode, struct file *file)
980{
981 struct ftrace_iterator *iter;
982 int ret;
983
984 if (unlikely(ftrace_disabled))
985 return -ENODEV;
986
987 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
988 if (!iter)
989 return -ENOMEM;
990
991 iter->pg = ftrace_pages_start;
992 iter->pos = -1;
993
994 ret = seq_open(file, &show_ftrace_seq_ops);
995 if (!ret) {
996 struct seq_file *m = file->private_data;
997
998 m->private = iter;
999 } else {
1000 kfree(iter);
1001 }
1002
1003 return ret;
1004}
1005
1006int ftrace_avail_release(struct inode *inode, struct file *file)
1007{
1008 struct seq_file *m = (struct seq_file *)file->private_data;
1009 struct ftrace_iterator *iter = m->private;
1010
1011 seq_release(inode, file);
1012 kfree(iter);
1013
1014 return 0;
1015}
1016
1017static int
1018ftrace_failures_open(struct inode *inode, struct file *file)
1019{
1020 int ret;
1021 struct seq_file *m;
1022 struct ftrace_iterator *iter;
1023
1024 ret = ftrace_avail_open(inode, file);
1025 if (!ret) {
1026 m = (struct seq_file *)file->private_data;
1027 iter = (struct ftrace_iterator *)m->private;
1028 iter->flags = FTRACE_ITER_FAILURES;
1029 }
1030
1031 return ret;
1032}
1033
1034
1035static void ftrace_filter_reset(int enable)
1036{
1037 struct ftrace_page *pg;
1038 struct dyn_ftrace *rec;
1039 unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1040 unsigned i;
1041
1042 /* keep kstop machine from running */
1043 preempt_disable();
1044 if (enable)
1045 ftrace_filtered = 0;
1046 pg = ftrace_pages_start;
1047 while (pg) {
1048 for (i = 0; i < pg->index; i++) {
1049 rec = &pg->records[i];
1050 if (rec->flags & FTRACE_FL_FAILED)
1051 continue;
1052 rec->flags &= ~type;
1053 }
1054 pg = pg->next;
1055 }
1056 preempt_enable();
1057}
1058
1059static int
1060ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1061{
1062 struct ftrace_iterator *iter;
1063 int ret = 0;
1064
1065 if (unlikely(ftrace_disabled))
1066 return -ENODEV;
1067
1068 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1069 if (!iter)
1070 return -ENOMEM;
1071
1072 mutex_lock(&ftrace_regex_lock);
1073 if ((file->f_mode & FMODE_WRITE) &&
1074 !(file->f_flags & O_APPEND))
1075 ftrace_filter_reset(enable);
1076
1077 if (file->f_mode & FMODE_READ) {
1078 iter->pg = ftrace_pages_start;
1079 iter->pos = -1;
1080 iter->flags = enable ? FTRACE_ITER_FILTER :
1081 FTRACE_ITER_NOTRACE;
1082
1083 ret = seq_open(file, &show_ftrace_seq_ops);
1084 if (!ret) {
1085 struct seq_file *m = file->private_data;
1086 m->private = iter;
1087 } else
1088 kfree(iter);
1089 } else
1090 file->private_data = iter;
1091 mutex_unlock(&ftrace_regex_lock);
1092
1093 return ret;
1094}
1095
1096static int
1097ftrace_filter_open(struct inode *inode, struct file *file)
1098{
1099 return ftrace_regex_open(inode, file, 1);
1100}
1101
1102static int
1103ftrace_notrace_open(struct inode *inode, struct file *file)
1104{
1105 return ftrace_regex_open(inode, file, 0);
1106}
1107
1108static ssize_t
1109ftrace_regex_read(struct file *file, char __user *ubuf,
1110 size_t cnt, loff_t *ppos)
1111{
1112 if (file->f_mode & FMODE_READ)
1113 return seq_read(file, ubuf, cnt, ppos);
1114 else
1115 return -EPERM;
1116}
1117
1118static loff_t
1119ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1120{
1121 loff_t ret;
1122
1123 if (file->f_mode & FMODE_READ)
1124 ret = seq_lseek(file, offset, origin);
1125 else
1126 file->f_pos = ret = 1;
1127
1128 return ret;
1129}
1130
1131enum {
1132 MATCH_FULL,
1133 MATCH_FRONT_ONLY,
1134 MATCH_MIDDLE_ONLY,
1135 MATCH_END_ONLY,
1136};
1137
1138static void
1139ftrace_match(unsigned char *buff, int len, int enable)
1140{
1141 char str[KSYM_SYMBOL_LEN];
1142 char *search = NULL;
1143 struct ftrace_page *pg;
1144 struct dyn_ftrace *rec;
1145 int type = MATCH_FULL;
1146 unsigned long flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1147 unsigned i, match = 0, search_len = 0;
1148
1149 for (i = 0; i < len; i++) {
1150 if (buff[i] == '*') {
1151 if (!i) {
1152 search = buff + i + 1;
1153 type = MATCH_END_ONLY;
1154 search_len = len - (i + 1);
1155 } else {
1156 if (type == MATCH_END_ONLY) {
1157 type = MATCH_MIDDLE_ONLY;
1158 } else {
1159 match = i;
1160 type = MATCH_FRONT_ONLY;
1161 }
1162 buff[i] = 0;
1163 break;
1164 }
1165 }
1166 }
1167
1168 /* keep kstop machine from running */
1169 preempt_disable();
1170 if (enable)
1171 ftrace_filtered = 1;
1172 pg = ftrace_pages_start;
1173 while (pg) {
1174 for (i = 0; i < pg->index; i++) {
1175 int matched = 0;
1176 char *ptr;
1177
1178 rec = &pg->records[i];
1179 if (rec->flags & FTRACE_FL_FAILED)
1180 continue;
1181 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1182 switch (type) {
1183 case MATCH_FULL:
1184 if (strcmp(str, buff) == 0)
1185 matched = 1;
1186 break;
1187 case MATCH_FRONT_ONLY:
1188 if (memcmp(str, buff, match) == 0)
1189 matched = 1;
1190 break;
1191 case MATCH_MIDDLE_ONLY:
1192 if (strstr(str, search))
1193 matched = 1;
1194 break;
1195 case MATCH_END_ONLY:
1196 ptr = strstr(str, search);
1197 if (ptr && (ptr[search_len] == 0))
1198 matched = 1;
1199 break;
1200 }
1201 if (matched)
1202 rec->flags |= flag;
1203 }
1204 pg = pg->next;
1205 }
1206 preempt_enable();
1207}
1208
1209static ssize_t
1210ftrace_regex_write(struct file *file, const char __user *ubuf,
1211 size_t cnt, loff_t *ppos, int enable)
1212{
1213 struct ftrace_iterator *iter;
1214 char ch;
1215 size_t read = 0;
1216 ssize_t ret;
1217
1218 if (!cnt || cnt < 0)
1219 return 0;
1220
1221 mutex_lock(&ftrace_regex_lock);
1222
1223 if (file->f_mode & FMODE_READ) {
1224 struct seq_file *m = file->private_data;
1225 iter = m->private;
1226 } else
1227 iter = file->private_data;
1228
1229 if (!*ppos) {
1230 iter->flags &= ~FTRACE_ITER_CONT;
1231 iter->buffer_idx = 0;
1232 }
1233
1234 ret = get_user(ch, ubuf++);
1235 if (ret)
1236 goto out;
1237 read++;
1238 cnt--;
1239
1240 if (!(iter->flags & ~FTRACE_ITER_CONT)) {
1241 /* skip white space */
1242 while (cnt && isspace(ch)) {
1243 ret = get_user(ch, ubuf++);
1244 if (ret)
1245 goto out;
1246 read++;
1247 cnt--;
1248 }
1249
1250 if (isspace(ch)) {
1251 file->f_pos += read;
1252 ret = read;
1253 goto out;
1254 }
1255
1256 iter->buffer_idx = 0;
1257 }
1258
1259 while (cnt && !isspace(ch)) {
1260 if (iter->buffer_idx < FTRACE_BUFF_MAX)
1261 iter->buffer[iter->buffer_idx++] = ch;
1262 else {
1263 ret = -EINVAL;
1264 goto out;
1265 }
1266 ret = get_user(ch, ubuf++);
1267 if (ret)
1268 goto out;
1269 read++;
1270 cnt--;
1271 }
1272
1273 if (isspace(ch)) {
1274 iter->filtered++;
1275 iter->buffer[iter->buffer_idx] = 0;
1276 ftrace_match(iter->buffer, iter->buffer_idx, enable);
1277 iter->buffer_idx = 0;
1278 } else
1279 iter->flags |= FTRACE_ITER_CONT;
1280
1281
1282 file->f_pos += read;
1283
1284 ret = read;
1285 out:
1286 mutex_unlock(&ftrace_regex_lock);
1287
1288 return ret;
1289}
1290
1291static ssize_t
1292ftrace_filter_write(struct file *file, const char __user *ubuf,
1293 size_t cnt, loff_t *ppos)
1294{
1295 return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
1296}
1297
1298static ssize_t
1299ftrace_notrace_write(struct file *file, const char __user *ubuf,
1300 size_t cnt, loff_t *ppos)
1301{
1302 return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
1303}
1304
1305static void
1306ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
1307{
1308 if (unlikely(ftrace_disabled))
1309 return;
1310
1311 mutex_lock(&ftrace_regex_lock);
1312 if (reset)
1313 ftrace_filter_reset(enable);
1314 if (buf)
1315 ftrace_match(buf, len, enable);
1316 mutex_unlock(&ftrace_regex_lock);
1317}
1318
1319/**
1320 * ftrace_set_filter - set a function to filter on in ftrace
1321 * @buf - the string that holds the function filter text.
1322 * @len - the length of the string.
1323 * @reset - non zero to reset all filters before applying this filter.
1324 *
1325 * Filters denote which functions should be enabled when tracing is enabled.
1326 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
1327 */
1328void ftrace_set_filter(unsigned char *buf, int len, int reset)
1329{
1330 ftrace_set_regex(buf, len, reset, 1);
1331}
1332
1333/**
1334 * ftrace_set_notrace - set a function to not trace in ftrace
1335 * @buf - the string that holds the function notrace text.
1336 * @len - the length of the string.
1337 * @reset - non zero to reset all filters before applying this filter.
1338 *
1339 * Notrace Filters denote which functions should not be enabled when tracing
1340 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
1341 * for tracing.
1342 */
1343void ftrace_set_notrace(unsigned char *buf, int len, int reset)
1344{
1345 ftrace_set_regex(buf, len, reset, 0);
1346}
1347
1348static int
1349ftrace_regex_release(struct inode *inode, struct file *file, int enable)
1350{
1351 struct seq_file *m = (struct seq_file *)file->private_data;
1352 struct ftrace_iterator *iter;
1353
1354 mutex_lock(&ftrace_regex_lock);
1355 if (file->f_mode & FMODE_READ) {
1356 iter = m->private;
1357
1358 seq_release(inode, file);
1359 } else
1360 iter = file->private_data;
1361
1362 if (iter->buffer_idx) {
1363 iter->filtered++;
1364 iter->buffer[iter->buffer_idx] = 0;
1365 ftrace_match(iter->buffer, iter->buffer_idx, enable);
1366 }
1367
1368 mutex_lock(&ftrace_sysctl_lock);
1369 mutex_lock(&ftraced_lock);
1370 if (iter->filtered && ftraced_suspend && ftrace_enabled)
1371 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1372 mutex_unlock(&ftraced_lock);
1373 mutex_unlock(&ftrace_sysctl_lock);
1374
1375 kfree(iter);
1376 mutex_unlock(&ftrace_regex_lock);
1377 return 0;
1378}
1379
1380static int
1381ftrace_filter_release(struct inode *inode, struct file *file)
1382{
1383 return ftrace_regex_release(inode, file, 1);
1384}
1385
1386static int
1387ftrace_notrace_release(struct inode *inode, struct file *file)
1388{
1389 return ftrace_regex_release(inode, file, 0);
1390}
1391
1392static ssize_t
1393ftraced_read(struct file *filp, char __user *ubuf,
1394 size_t cnt, loff_t *ppos)
1395{
1396 /* don't worry about races */
1397 char *buf = ftraced_stop ? "disabled\n" : "enabled\n";
1398 int r = strlen(buf);
1399
1400 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
1401}
1402
1403static ssize_t
1404ftraced_write(struct file *filp, const char __user *ubuf,
1405 size_t cnt, loff_t *ppos)
1406{
1407 char buf[64];
1408 long val;
1409 int ret;
1410
1411 if (cnt >= sizeof(buf))
1412 return -EINVAL;
1413
1414 if (copy_from_user(&buf, ubuf, cnt))
1415 return -EFAULT;
1416
1417 if (strncmp(buf, "enable", 6) == 0)
1418 val = 1;
1419 else if (strncmp(buf, "disable", 7) == 0)
1420 val = 0;
1421 else {
1422 buf[cnt] = 0;
1423
1424 ret = strict_strtoul(buf, 10, &val);
1425 if (ret < 0)
1426 return ret;
1427
1428 val = !!val;
1429 }
1430
1431 if (val)
1432 ftrace_enable_daemon();
1433 else
1434 ftrace_disable_daemon();
1435
1436 filp->f_pos += cnt;
1437
1438 return cnt;
1439}
1440
1441static struct file_operations ftrace_avail_fops = {
1442 .open = ftrace_avail_open,
1443 .read = seq_read,
1444 .llseek = seq_lseek,
1445 .release = ftrace_avail_release,
1446};
1447
1448static struct file_operations ftrace_failures_fops = {
1449 .open = ftrace_failures_open,
1450 .read = seq_read,
1451 .llseek = seq_lseek,
1452 .release = ftrace_avail_release,
1453};
1454
1455static struct file_operations ftrace_filter_fops = {
1456 .open = ftrace_filter_open,
1457 .read = ftrace_regex_read,
1458 .write = ftrace_filter_write,
1459 .llseek = ftrace_regex_lseek,
1460 .release = ftrace_filter_release,
1461};
1462
1463static struct file_operations ftrace_notrace_fops = {
1464 .open = ftrace_notrace_open,
1465 .read = ftrace_regex_read,
1466 .write = ftrace_notrace_write,
1467 .llseek = ftrace_regex_lseek,
1468 .release = ftrace_notrace_release,
1469};
1470
1471static struct file_operations ftraced_fops = {
1472 .open = tracing_open_generic,
1473 .read = ftraced_read,
1474 .write = ftraced_write,
1475};
1476
1477/**
1478 * ftrace_force_update - force an update to all recording ftrace functions
1479 */
1480int ftrace_force_update(void)
1481{
1482 int ret = 0;
1483
1484 if (unlikely(ftrace_disabled))
1485 return -ENODEV;
1486
1487 mutex_lock(&ftrace_sysctl_lock);
1488 mutex_lock(&ftraced_lock);
1489
1490 /*
1491 * If ftraced_trigger is not set, then there is nothing
1492 * to update.
1493 */
1494 if (ftraced_trigger && !ftrace_update_code())
1495 ret = -EBUSY;
1496
1497 mutex_unlock(&ftraced_lock);
1498 mutex_unlock(&ftrace_sysctl_lock);
1499
1500 return ret;
1501}
1502
1503static void ftrace_force_shutdown(void)
1504{
1505 struct task_struct *task;
1506 int command = FTRACE_DISABLE_CALLS | FTRACE_UPDATE_TRACE_FUNC;
1507
1508 mutex_lock(&ftraced_lock);
1509 task = ftraced_task;
1510 ftraced_task = NULL;
1511 ftraced_suspend = -1;
1512 ftrace_run_update_code(command);
1513 mutex_unlock(&ftraced_lock);
1514
1515 if (task)
1516 kthread_stop(task);
1517}
1518
1519static __init int ftrace_init_debugfs(void)
1520{
1521 struct dentry *d_tracer;
1522 struct dentry *entry;
1523
1524 d_tracer = tracing_init_dentry();
1525
1526 entry = debugfs_create_file("available_filter_functions", 0444,
1527 d_tracer, NULL, &ftrace_avail_fops);
1528 if (!entry)
1529 pr_warning("Could not create debugfs "
1530 "'available_filter_functions' entry\n");
1531
1532 entry = debugfs_create_file("failures", 0444,
1533 d_tracer, NULL, &ftrace_failures_fops);
1534 if (!entry)
1535 pr_warning("Could not create debugfs 'failures' entry\n");
1536
1537 entry = debugfs_create_file("set_ftrace_filter", 0644, d_tracer,
1538 NULL, &ftrace_filter_fops);
1539 if (!entry)
1540 pr_warning("Could not create debugfs "
1541 "'set_ftrace_filter' entry\n");
1542
1543 entry = debugfs_create_file("set_ftrace_notrace", 0644, d_tracer,
1544 NULL, &ftrace_notrace_fops);
1545 if (!entry)
1546 pr_warning("Could not create debugfs "
1547 "'set_ftrace_notrace' entry\n");
1548
1549 entry = debugfs_create_file("ftraced_enabled", 0644, d_tracer,
1550 NULL, &ftraced_fops);
1551 if (!entry)
1552 pr_warning("Could not create debugfs "
1553 "'ftraced_enabled' entry\n");
1554 return 0;
1555}
1556
1557fs_initcall(ftrace_init_debugfs);
1558
1559static int __init ftrace_dynamic_init(void)
1560{
1561 struct task_struct *p;
1562 unsigned long addr;
1563 int ret;
1564
1565 addr = (unsigned long)ftrace_record_ip;
1566
1567 stop_machine_run(ftrace_dyn_arch_init, &addr, NR_CPUS);
1568
1569 /* ftrace_dyn_arch_init places the return code in addr */
1570 if (addr) {
1571 ret = (int)addr;
1572 goto failed;
1573 }
1574
1575 ret = ftrace_dyn_table_alloc();
1576 if (ret)
1577 goto failed;
1578
1579 p = kthread_run(ftraced, NULL, "ftraced");
1580 if (IS_ERR(p)) {
1581 ret = -1;
1582 goto failed;
1583 }
1584
1585 last_ftrace_enabled = ftrace_enabled = 1;
1586 ftraced_task = p;
1587
1588 return 0;
1589
1590 failed:
1591 ftrace_disabled = 1;
1592 return ret;
1593}
1594
1595core_initcall(ftrace_dynamic_init);
1596#else
1597# define ftrace_startup() do { } while (0)
1598# define ftrace_shutdown() do { } while (0)
1599# define ftrace_startup_sysctl() do { } while (0)
1600# define ftrace_shutdown_sysctl() do { } while (0)
1601# define ftrace_force_shutdown() do { } while (0)
1602#endif /* CONFIG_DYNAMIC_FTRACE */
1603
1604/**
1605 * ftrace_kill_atomic - kill ftrace from critical sections
1606 *
1607 * This function should be used by panic code. It stops ftrace
1608 * but in a not so nice way. If you need to simply kill ftrace
1609 * from a non-atomic section, use ftrace_kill.
1610 */
1611void ftrace_kill_atomic(void)
1612{
1613 ftrace_disabled = 1;
1614 ftrace_enabled = 0;
1615#ifdef CONFIG_DYNAMIC_FTRACE
1616 ftraced_suspend = -1;
1617#endif
1618 clear_ftrace_function();
1619}
1620
1621/**
1622 * ftrace_kill - totally shutdown ftrace
1623 *
1624 * This is a safety measure. If something was detected that seems
1625 * wrong, calling this function will keep ftrace from doing
1626 * any more modifications, and updates.
1627 * used when something went wrong.
1628 */
1629void ftrace_kill(void)
1630{
1631 mutex_lock(&ftrace_sysctl_lock);
1632 ftrace_disabled = 1;
1633 ftrace_enabled = 0;
1634
1635 clear_ftrace_function();
1636 mutex_unlock(&ftrace_sysctl_lock);
1637
1638 /* Try to totally disable ftrace */
1639 ftrace_force_shutdown();
1640}
1641
1642/**
1643 * register_ftrace_function - register a function for profiling
1644 * @ops - ops structure that holds the function for profiling.
1645 *
1646 * Register a function to be called by all functions in the
1647 * kernel.
1648 *
1649 * Note: @ops->func and all the functions it calls must be labeled
1650 * with "notrace", otherwise it will go into a
1651 * recursive loop.
1652 */
1653int register_ftrace_function(struct ftrace_ops *ops)
1654{
1655 int ret;
1656
1657 if (unlikely(ftrace_disabled))
1658 return -1;
1659
1660 mutex_lock(&ftrace_sysctl_lock);
1661 ret = __register_ftrace_function(ops);
1662 ftrace_startup();
1663 mutex_unlock(&ftrace_sysctl_lock);
1664
1665 return ret;
1666}
1667
1668/**
1669 * unregister_ftrace_function - unresgister a function for profiling.
1670 * @ops - ops structure that holds the function to unregister
1671 *
1672 * Unregister a function that was added to be called by ftrace profiling.
1673 */
1674int unregister_ftrace_function(struct ftrace_ops *ops)
1675{
1676 int ret;
1677
1678 mutex_lock(&ftrace_sysctl_lock);
1679 ret = __unregister_ftrace_function(ops);
1680 ftrace_shutdown();
1681 mutex_unlock(&ftrace_sysctl_lock);
1682
1683 return ret;
1684}
1685
1686int
1687ftrace_enable_sysctl(struct ctl_table *table, int write,
1688 struct file *file, void __user *buffer, size_t *lenp,
1689 loff_t *ppos)
1690{
1691 int ret;
1692
1693 if (unlikely(ftrace_disabled))
1694 return -ENODEV;
1695
1696 mutex_lock(&ftrace_sysctl_lock);
1697
1698 ret = proc_dointvec(table, write, file, buffer, lenp, ppos);
1699
1700 if (ret || !write || (last_ftrace_enabled == ftrace_enabled))
1701 goto out;
1702
1703 last_ftrace_enabled = ftrace_enabled;
1704
1705 if (ftrace_enabled) {
1706
1707 ftrace_startup_sysctl();
1708
1709 /* we are starting ftrace again */
1710 if (ftrace_list != &ftrace_list_end) {
1711 if (ftrace_list->next == &ftrace_list_end)
1712 ftrace_trace_function = ftrace_list->func;
1713 else
1714 ftrace_trace_function = ftrace_list_func;
1715 }
1716
1717 } else {
1718 /* stopping ftrace calls (just send to ftrace_stub) */
1719 ftrace_trace_function = ftrace_stub;
1720
1721 ftrace_shutdown_sysctl();
1722 }
1723
1724 out:
1725 mutex_unlock(&ftrace_sysctl_lock);
1726 return ret;
1727}
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
new file mode 100644
index 000000000000..8f3fb3db61c3
--- /dev/null
+++ b/kernel/trace/trace.c
@@ -0,0 +1,3157 @@
1/*
2 * ring buffer based function tracer
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 William Lee Irwin III
13 */
14#include <linux/utsrelease.h>
15#include <linux/kallsyms.h>
16#include <linux/seq_file.h>
17#include <linux/debugfs.h>
18#include <linux/pagemap.h>
19#include <linux/hardirq.h>
20#include <linux/linkage.h>
21#include <linux/uaccess.h>
22#include <linux/ftrace.h>
23#include <linux/module.h>
24#include <linux/percpu.h>
25#include <linux/ctype.h>
26#include <linux/init.h>
27#include <linux/poll.h>
28#include <linux/gfp.h>
29#include <linux/fs.h>
30#include <linux/kprobes.h>
31#include <linux/writeback.h>
32
33#include <linux/stacktrace.h>
34
35#include "trace.h"
36
37unsigned long __read_mostly tracing_max_latency = (cycle_t)ULONG_MAX;
38unsigned long __read_mostly tracing_thresh;
39
40static unsigned long __read_mostly tracing_nr_buffers;
41static cpumask_t __read_mostly tracing_buffer_mask;
42
43#define for_each_tracing_cpu(cpu) \
44 for_each_cpu_mask(cpu, tracing_buffer_mask)
45
46static int trace_alloc_page(void);
47static int trace_free_page(void);
48
49static int tracing_disabled = 1;
50
51static unsigned long tracing_pages_allocated;
52
53long
54ns2usecs(cycle_t nsec)
55{
56 nsec += 500;
57 do_div(nsec, 1000);
58 return nsec;
59}
60
61cycle_t ftrace_now(int cpu)
62{
63 return cpu_clock(cpu);
64}
65
66/*
67 * The global_trace is the descriptor that holds the tracing
68 * buffers for the live tracing. For each CPU, it contains
69 * a link list of pages that will store trace entries. The
70 * page descriptor of the pages in the memory is used to hold
71 * the link list by linking the lru item in the page descriptor
72 * to each of the pages in the buffer per CPU.
73 *
74 * For each active CPU there is a data field that holds the
75 * pages for the buffer for that CPU. Each CPU has the same number
76 * of pages allocated for its buffer.
77 */
78static struct trace_array global_trace;
79
80static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
81
82/*
83 * The max_tr is used to snapshot the global_trace when a maximum
84 * latency is reached. Some tracers will use this to store a maximum
85 * trace while it continues examining live traces.
86 *
87 * The buffers for the max_tr are set up the same as the global_trace.
88 * When a snapshot is taken, the link list of the max_tr is swapped
89 * with the link list of the global_trace and the buffers are reset for
90 * the global_trace so the tracing can continue.
91 */
92static struct trace_array max_tr;
93
94static DEFINE_PER_CPU(struct trace_array_cpu, max_data);
95
96/* tracer_enabled is used to toggle activation of a tracer */
97static int tracer_enabled = 1;
98
99/* function tracing enabled */
100int ftrace_function_enabled;
101
102/*
103 * trace_nr_entries is the number of entries that is allocated
104 * for a buffer. Note, the number of entries is always rounded
105 * to ENTRIES_PER_PAGE.
106 */
107static unsigned long trace_nr_entries = 65536UL;
108
109/* trace_types holds a link list of available tracers. */
110static struct tracer *trace_types __read_mostly;
111
112/* current_trace points to the tracer that is currently active */
113static struct tracer *current_trace __read_mostly;
114
115/*
116 * max_tracer_type_len is used to simplify the allocating of
117 * buffers to read userspace tracer names. We keep track of
118 * the longest tracer name registered.
119 */
120static int max_tracer_type_len;
121
122/*
123 * trace_types_lock is used to protect the trace_types list.
124 * This lock is also used to keep user access serialized.
125 * Accesses from userspace will grab this lock while userspace
126 * activities happen inside the kernel.
127 */
128static DEFINE_MUTEX(trace_types_lock);
129
130/* trace_wait is a waitqueue for tasks blocked on trace_poll */
131static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
132
133/* trace_flags holds iter_ctrl options */
134unsigned long trace_flags = TRACE_ITER_PRINT_PARENT;
135
136static notrace void no_trace_init(struct trace_array *tr)
137{
138 int cpu;
139
140 ftrace_function_enabled = 0;
141 if(tr->ctrl)
142 for_each_online_cpu(cpu)
143 tracing_reset(tr->data[cpu]);
144 tracer_enabled = 0;
145}
146
147/* dummy trace to disable tracing */
148static struct tracer no_tracer __read_mostly = {
149 .name = "none",
150 .init = no_trace_init
151};
152
153
154/**
155 * trace_wake_up - wake up tasks waiting for trace input
156 *
157 * Simply wakes up any task that is blocked on the trace_wait
158 * queue. These is used with trace_poll for tasks polling the trace.
159 */
160void trace_wake_up(void)
161{
162 /*
163 * The runqueue_is_locked() can fail, but this is the best we
164 * have for now:
165 */
166 if (!(trace_flags & TRACE_ITER_BLOCK) && !runqueue_is_locked())
167 wake_up(&trace_wait);
168}
169
170#define ENTRIES_PER_PAGE (PAGE_SIZE / sizeof(struct trace_entry))
171
172static int __init set_nr_entries(char *str)
173{
174 unsigned long nr_entries;
175 int ret;
176
177 if (!str)
178 return 0;
179 ret = strict_strtoul(str, 0, &nr_entries);
180 /* nr_entries can not be zero */
181 if (ret < 0 || nr_entries == 0)
182 return 0;
183 trace_nr_entries = nr_entries;
184 return 1;
185}
186__setup("trace_entries=", set_nr_entries);
187
188unsigned long nsecs_to_usecs(unsigned long nsecs)
189{
190 return nsecs / 1000;
191}
192
193/*
194 * trace_flag_type is an enumeration that holds different
195 * states when a trace occurs. These are:
196 * IRQS_OFF - interrupts were disabled
197 * NEED_RESCED - reschedule is requested
198 * HARDIRQ - inside an interrupt handler
199 * SOFTIRQ - inside a softirq handler
200 */
201enum trace_flag_type {
202 TRACE_FLAG_IRQS_OFF = 0x01,
203 TRACE_FLAG_NEED_RESCHED = 0x02,
204 TRACE_FLAG_HARDIRQ = 0x04,
205 TRACE_FLAG_SOFTIRQ = 0x08,
206};
207
208/*
209 * TRACE_ITER_SYM_MASK masks the options in trace_flags that
210 * control the output of kernel symbols.
211 */
212#define TRACE_ITER_SYM_MASK \
213 (TRACE_ITER_PRINT_PARENT|TRACE_ITER_SYM_OFFSET|TRACE_ITER_SYM_ADDR)
214
215/* These must match the bit postions in trace_iterator_flags */
216static const char *trace_options[] = {
217 "print-parent",
218 "sym-offset",
219 "sym-addr",
220 "verbose",
221 "raw",
222 "hex",
223 "bin",
224 "block",
225 "stacktrace",
226 "sched-tree",
227 NULL
228};
229
230/*
231 * ftrace_max_lock is used to protect the swapping of buffers
232 * when taking a max snapshot. The buffers themselves are
233 * protected by per_cpu spinlocks. But the action of the swap
234 * needs its own lock.
235 *
236 * This is defined as a raw_spinlock_t in order to help
237 * with performance when lockdep debugging is enabled.
238 */
239static raw_spinlock_t ftrace_max_lock =
240 (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
241
242/*
243 * Copy the new maximum trace into the separate maximum-trace
244 * structure. (this way the maximum trace is permanently saved,
245 * for later retrieval via /debugfs/tracing/latency_trace)
246 */
247static void
248__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
249{
250 struct trace_array_cpu *data = tr->data[cpu];
251
252 max_tr.cpu = cpu;
253 max_tr.time_start = data->preempt_timestamp;
254
255 data = max_tr.data[cpu];
256 data->saved_latency = tracing_max_latency;
257
258 memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
259 data->pid = tsk->pid;
260 data->uid = tsk->uid;
261 data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
262 data->policy = tsk->policy;
263 data->rt_priority = tsk->rt_priority;
264
265 /* record this tasks comm */
266 tracing_record_cmdline(current);
267}
268
269#define CHECK_COND(cond) \
270 if (unlikely(cond)) { \
271 tracing_disabled = 1; \
272 WARN_ON(1); \
273 return -1; \
274 }
275
276/**
277 * check_pages - integrity check of trace buffers
278 *
279 * As a safty measure we check to make sure the data pages have not
280 * been corrupted.
281 */
282int check_pages(struct trace_array_cpu *data)
283{
284 struct page *page, *tmp;
285
286 CHECK_COND(data->trace_pages.next->prev != &data->trace_pages);
287 CHECK_COND(data->trace_pages.prev->next != &data->trace_pages);
288
289 list_for_each_entry_safe(page, tmp, &data->trace_pages, lru) {
290 CHECK_COND(page->lru.next->prev != &page->lru);
291 CHECK_COND(page->lru.prev->next != &page->lru);
292 }
293
294 return 0;
295}
296
297/**
298 * head_page - page address of the first page in per_cpu buffer.
299 *
300 * head_page returns the page address of the first page in
301 * a per_cpu buffer. This also preforms various consistency
302 * checks to make sure the buffer has not been corrupted.
303 */
304void *head_page(struct trace_array_cpu *data)
305{
306 struct page *page;
307
308 if (list_empty(&data->trace_pages))
309 return NULL;
310
311 page = list_entry(data->trace_pages.next, struct page, lru);
312 BUG_ON(&page->lru == &data->trace_pages);
313
314 return page_address(page);
315}
316
317/**
318 * trace_seq_printf - sequence printing of trace information
319 * @s: trace sequence descriptor
320 * @fmt: printf format string
321 *
322 * The tracer may use either sequence operations or its own
323 * copy to user routines. To simplify formating of a trace
324 * trace_seq_printf is used to store strings into a special
325 * buffer (@s). Then the output may be either used by
326 * the sequencer or pulled into another buffer.
327 */
328int
329trace_seq_printf(struct trace_seq *s, const char *fmt, ...)
330{
331 int len = (PAGE_SIZE - 1) - s->len;
332 va_list ap;
333 int ret;
334
335 if (!len)
336 return 0;
337
338 va_start(ap, fmt);
339 ret = vsnprintf(s->buffer + s->len, len, fmt, ap);
340 va_end(ap);
341
342 /* If we can't write it all, don't bother writing anything */
343 if (ret >= len)
344 return 0;
345
346 s->len += ret;
347
348 return len;
349}
350
351/**
352 * trace_seq_puts - trace sequence printing of simple string
353 * @s: trace sequence descriptor
354 * @str: simple string to record
355 *
356 * The tracer may use either the sequence operations or its own
357 * copy to user routines. This function records a simple string
358 * into a special buffer (@s) for later retrieval by a sequencer
359 * or other mechanism.
360 */
361static int
362trace_seq_puts(struct trace_seq *s, const char *str)
363{
364 int len = strlen(str);
365
366 if (len > ((PAGE_SIZE - 1) - s->len))
367 return 0;
368
369 memcpy(s->buffer + s->len, str, len);
370 s->len += len;
371
372 return len;
373}
374
375static int
376trace_seq_putc(struct trace_seq *s, unsigned char c)
377{
378 if (s->len >= (PAGE_SIZE - 1))
379 return 0;
380
381 s->buffer[s->len++] = c;
382
383 return 1;
384}
385
386static int
387trace_seq_putmem(struct trace_seq *s, void *mem, size_t len)
388{
389 if (len > ((PAGE_SIZE - 1) - s->len))
390 return 0;
391
392 memcpy(s->buffer + s->len, mem, len);
393 s->len += len;
394
395 return len;
396}
397
398#define HEX_CHARS 17
399static const char hex2asc[] = "0123456789abcdef";
400
401static int
402trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
403{
404 unsigned char hex[HEX_CHARS];
405 unsigned char *data = mem;
406 unsigned char byte;
407 int i, j;
408
409 BUG_ON(len >= HEX_CHARS);
410
411#ifdef __BIG_ENDIAN
412 for (i = 0, j = 0; i < len; i++) {
413#else
414 for (i = len-1, j = 0; i >= 0; i--) {
415#endif
416 byte = data[i];
417
418 hex[j++] = hex2asc[byte & 0x0f];
419 hex[j++] = hex2asc[byte >> 4];
420 }
421 hex[j++] = ' ';
422
423 return trace_seq_putmem(s, hex, j);
424}
425
426static void
427trace_seq_reset(struct trace_seq *s)
428{
429 s->len = 0;
430 s->readpos = 0;
431}
432
433ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
434{
435 int len;
436 int ret;
437
438 if (s->len <= s->readpos)
439 return -EBUSY;
440
441 len = s->len - s->readpos;
442 if (cnt > len)
443 cnt = len;
444 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
445 if (ret)
446 return -EFAULT;
447
448 s->readpos += len;
449 return cnt;
450}
451
452static void
453trace_print_seq(struct seq_file *m, struct trace_seq *s)
454{
455 int len = s->len >= PAGE_SIZE ? PAGE_SIZE - 1 : s->len;
456
457 s->buffer[len] = 0;
458 seq_puts(m, s->buffer);
459
460 trace_seq_reset(s);
461}
462
463/*
464 * flip the trace buffers between two trace descriptors.
465 * This usually is the buffers between the global_trace and
466 * the max_tr to record a snapshot of a current trace.
467 *
468 * The ftrace_max_lock must be held.
469 */
470static void
471flip_trace(struct trace_array_cpu *tr1, struct trace_array_cpu *tr2)
472{
473 struct list_head flip_pages;
474
475 INIT_LIST_HEAD(&flip_pages);
476
477 memcpy(&tr1->trace_head_idx, &tr2->trace_head_idx,
478 sizeof(struct trace_array_cpu) -
479 offsetof(struct trace_array_cpu, trace_head_idx));
480
481 check_pages(tr1);
482 check_pages(tr2);
483 list_splice_init(&tr1->trace_pages, &flip_pages);
484 list_splice_init(&tr2->trace_pages, &tr1->trace_pages);
485 list_splice_init(&flip_pages, &tr2->trace_pages);
486 BUG_ON(!list_empty(&flip_pages));
487 check_pages(tr1);
488 check_pages(tr2);
489}
490
491/**
492 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
493 * @tr: tracer
494 * @tsk: the task with the latency
495 * @cpu: The cpu that initiated the trace.
496 *
497 * Flip the buffers between the @tr and the max_tr and record information
498 * about which task was the cause of this latency.
499 */
500void
501update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
502{
503 struct trace_array_cpu *data;
504 int i;
505
506 WARN_ON_ONCE(!irqs_disabled());
507 __raw_spin_lock(&ftrace_max_lock);
508 /* clear out all the previous traces */
509 for_each_tracing_cpu(i) {
510 data = tr->data[i];
511 flip_trace(max_tr.data[i], data);
512 tracing_reset(data);
513 }
514
515 __update_max_tr(tr, tsk, cpu);
516 __raw_spin_unlock(&ftrace_max_lock);
517}
518
519/**
520 * update_max_tr_single - only copy one trace over, and reset the rest
521 * @tr - tracer
522 * @tsk - task with the latency
523 * @cpu - the cpu of the buffer to copy.
524 *
525 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
526 */
527void
528update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
529{
530 struct trace_array_cpu *data = tr->data[cpu];
531 int i;
532
533 WARN_ON_ONCE(!irqs_disabled());
534 __raw_spin_lock(&ftrace_max_lock);
535 for_each_tracing_cpu(i)
536 tracing_reset(max_tr.data[i]);
537
538 flip_trace(max_tr.data[cpu], data);
539 tracing_reset(data);
540
541 __update_max_tr(tr, tsk, cpu);
542 __raw_spin_unlock(&ftrace_max_lock);
543}
544
545/**
546 * register_tracer - register a tracer with the ftrace system.
547 * @type - the plugin for the tracer
548 *
549 * Register a new plugin tracer.
550 */
551int register_tracer(struct tracer *type)
552{
553 struct tracer *t;
554 int len;
555 int ret = 0;
556
557 if (!type->name) {
558 pr_info("Tracer must have a name\n");
559 return -1;
560 }
561
562 mutex_lock(&trace_types_lock);
563 for (t = trace_types; t; t = t->next) {
564 if (strcmp(type->name, t->name) == 0) {
565 /* already found */
566 pr_info("Trace %s already registered\n",
567 type->name);
568 ret = -1;
569 goto out;
570 }
571 }
572
573#ifdef CONFIG_FTRACE_STARTUP_TEST
574 if (type->selftest) {
575 struct tracer *saved_tracer = current_trace;
576 struct trace_array_cpu *data;
577 struct trace_array *tr = &global_trace;
578 int saved_ctrl = tr->ctrl;
579 int i;
580 /*
581 * Run a selftest on this tracer.
582 * Here we reset the trace buffer, and set the current
583 * tracer to be this tracer. The tracer can then run some
584 * internal tracing to verify that everything is in order.
585 * If we fail, we do not register this tracer.
586 */
587 for_each_tracing_cpu(i) {
588 data = tr->data[i];
589 if (!head_page(data))
590 continue;
591 tracing_reset(data);
592 }
593 current_trace = type;
594 tr->ctrl = 0;
595 /* the test is responsible for initializing and enabling */
596 pr_info("Testing tracer %s: ", type->name);
597 ret = type->selftest(type, tr);
598 /* the test is responsible for resetting too */
599 current_trace = saved_tracer;
600 tr->ctrl = saved_ctrl;
601 if (ret) {
602 printk(KERN_CONT "FAILED!\n");
603 goto out;
604 }
605 /* Only reset on passing, to avoid touching corrupted buffers */
606 for_each_tracing_cpu(i) {
607 data = tr->data[i];
608 if (!head_page(data))
609 continue;
610 tracing_reset(data);
611 }
612 printk(KERN_CONT "PASSED\n");
613 }
614#endif
615
616 type->next = trace_types;
617 trace_types = type;
618 len = strlen(type->name);
619 if (len > max_tracer_type_len)
620 max_tracer_type_len = len;
621
622 out:
623 mutex_unlock(&trace_types_lock);
624
625 return ret;
626}
627
628void unregister_tracer(struct tracer *type)
629{
630 struct tracer **t;
631 int len;
632
633 mutex_lock(&trace_types_lock);
634 for (t = &trace_types; *t; t = &(*t)->next) {
635 if (*t == type)
636 goto found;
637 }
638 pr_info("Trace %s not registered\n", type->name);
639 goto out;
640
641 found:
642 *t = (*t)->next;
643 if (strlen(type->name) != max_tracer_type_len)
644 goto out;
645
646 max_tracer_type_len = 0;
647 for (t = &trace_types; *t; t = &(*t)->next) {
648 len = strlen((*t)->name);
649 if (len > max_tracer_type_len)
650 max_tracer_type_len = len;
651 }
652 out:
653 mutex_unlock(&trace_types_lock);
654}
655
656void tracing_reset(struct trace_array_cpu *data)
657{
658 data->trace_idx = 0;
659 data->overrun = 0;
660 data->trace_head = data->trace_tail = head_page(data);
661 data->trace_head_idx = 0;
662 data->trace_tail_idx = 0;
663}
664
665#define SAVED_CMDLINES 128
666static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
667static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
668static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
669static int cmdline_idx;
670static DEFINE_SPINLOCK(trace_cmdline_lock);
671
672/* temporary disable recording */
673atomic_t trace_record_cmdline_disabled __read_mostly;
674
675static void trace_init_cmdlines(void)
676{
677 memset(&map_pid_to_cmdline, -1, sizeof(map_pid_to_cmdline));
678 memset(&map_cmdline_to_pid, -1, sizeof(map_cmdline_to_pid));
679 cmdline_idx = 0;
680}
681
682void trace_stop_cmdline_recording(void);
683
684static void trace_save_cmdline(struct task_struct *tsk)
685{
686 unsigned map;
687 unsigned idx;
688
689 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
690 return;
691
692 /*
693 * It's not the end of the world if we don't get
694 * the lock, but we also don't want to spin
695 * nor do we want to disable interrupts,
696 * so if we miss here, then better luck next time.
697 */
698 if (!spin_trylock(&trace_cmdline_lock))
699 return;
700
701 idx = map_pid_to_cmdline[tsk->pid];
702 if (idx >= SAVED_CMDLINES) {
703 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
704
705 map = map_cmdline_to_pid[idx];
706 if (map <= PID_MAX_DEFAULT)
707 map_pid_to_cmdline[map] = (unsigned)-1;
708
709 map_pid_to_cmdline[tsk->pid] = idx;
710
711 cmdline_idx = idx;
712 }
713
714 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
715
716 spin_unlock(&trace_cmdline_lock);
717}
718
719static char *trace_find_cmdline(int pid)
720{
721 char *cmdline = "<...>";
722 unsigned map;
723
724 if (!pid)
725 return "<idle>";
726
727 if (pid > PID_MAX_DEFAULT)
728 goto out;
729
730 map = map_pid_to_cmdline[pid];
731 if (map >= SAVED_CMDLINES)
732 goto out;
733
734 cmdline = saved_cmdlines[map];
735
736 out:
737 return cmdline;
738}
739
740void tracing_record_cmdline(struct task_struct *tsk)
741{
742 if (atomic_read(&trace_record_cmdline_disabled))
743 return;
744
745 trace_save_cmdline(tsk);
746}
747
748static inline struct list_head *
749trace_next_list(struct trace_array_cpu *data, struct list_head *next)
750{
751 /*
752 * Roundrobin - but skip the head (which is not a real page):
753 */
754 next = next->next;
755 if (unlikely(next == &data->trace_pages))
756 next = next->next;
757 BUG_ON(next == &data->trace_pages);
758
759 return next;
760}
761
762static inline void *
763trace_next_page(struct trace_array_cpu *data, void *addr)
764{
765 struct list_head *next;
766 struct page *page;
767
768 page = virt_to_page(addr);
769
770 next = trace_next_list(data, &page->lru);
771 page = list_entry(next, struct page, lru);
772
773 return page_address(page);
774}
775
776static inline struct trace_entry *
777tracing_get_trace_entry(struct trace_array *tr, struct trace_array_cpu *data)
778{
779 unsigned long idx, idx_next;
780 struct trace_entry *entry;
781
782 data->trace_idx++;
783 idx = data->trace_head_idx;
784 idx_next = idx + 1;
785
786 BUG_ON(idx * TRACE_ENTRY_SIZE >= PAGE_SIZE);
787
788 entry = data->trace_head + idx * TRACE_ENTRY_SIZE;
789
790 if (unlikely(idx_next >= ENTRIES_PER_PAGE)) {
791 data->trace_head = trace_next_page(data, data->trace_head);
792 idx_next = 0;
793 }
794
795 if (data->trace_head == data->trace_tail &&
796 idx_next == data->trace_tail_idx) {
797 /* overrun */
798 data->overrun++;
799 data->trace_tail_idx++;
800 if (data->trace_tail_idx >= ENTRIES_PER_PAGE) {
801 data->trace_tail =
802 trace_next_page(data, data->trace_tail);
803 data->trace_tail_idx = 0;
804 }
805 }
806
807 data->trace_head_idx = idx_next;
808
809 return entry;
810}
811
812static inline void
813tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags)
814{
815 struct task_struct *tsk = current;
816 unsigned long pc;
817
818 pc = preempt_count();
819
820 entry->preempt_count = pc & 0xff;
821 entry->pid = (tsk) ? tsk->pid : 0;
822 entry->t = ftrace_now(raw_smp_processor_id());
823 entry->flags = (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
824 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
825 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
826 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
827}
828
829void
830trace_function(struct trace_array *tr, struct trace_array_cpu *data,
831 unsigned long ip, unsigned long parent_ip, unsigned long flags)
832{
833 struct trace_entry *entry;
834 unsigned long irq_flags;
835
836 raw_local_irq_save(irq_flags);
837 __raw_spin_lock(&data->lock);
838 entry = tracing_get_trace_entry(tr, data);
839 tracing_generic_entry_update(entry, flags);
840 entry->type = TRACE_FN;
841 entry->fn.ip = ip;
842 entry->fn.parent_ip = parent_ip;
843 __raw_spin_unlock(&data->lock);
844 raw_local_irq_restore(irq_flags);
845}
846
847void
848ftrace(struct trace_array *tr, struct trace_array_cpu *data,
849 unsigned long ip, unsigned long parent_ip, unsigned long flags)
850{
851 if (likely(!atomic_read(&data->disabled)))
852 trace_function(tr, data, ip, parent_ip, flags);
853}
854
855#ifdef CONFIG_MMIOTRACE
856void __trace_mmiotrace_rw(struct trace_array *tr, struct trace_array_cpu *data,
857 struct mmiotrace_rw *rw)
858{
859 struct trace_entry *entry;
860 unsigned long irq_flags;
861
862 raw_local_irq_save(irq_flags);
863 __raw_spin_lock(&data->lock);
864
865 entry = tracing_get_trace_entry(tr, data);
866 tracing_generic_entry_update(entry, 0);
867 entry->type = TRACE_MMIO_RW;
868 entry->mmiorw = *rw;
869
870 __raw_spin_unlock(&data->lock);
871 raw_local_irq_restore(irq_flags);
872
873 trace_wake_up();
874}
875
876void __trace_mmiotrace_map(struct trace_array *tr, struct trace_array_cpu *data,
877 struct mmiotrace_map *map)
878{
879 struct trace_entry *entry;
880 unsigned long irq_flags;
881
882 raw_local_irq_save(irq_flags);
883 __raw_spin_lock(&data->lock);
884
885 entry = tracing_get_trace_entry(tr, data);
886 tracing_generic_entry_update(entry, 0);
887 entry->type = TRACE_MMIO_MAP;
888 entry->mmiomap = *map;
889
890 __raw_spin_unlock(&data->lock);
891 raw_local_irq_restore(irq_flags);
892
893 trace_wake_up();
894}
895#endif
896
897void __trace_stack(struct trace_array *tr,
898 struct trace_array_cpu *data,
899 unsigned long flags,
900 int skip)
901{
902 struct trace_entry *entry;
903 struct stack_trace trace;
904
905 if (!(trace_flags & TRACE_ITER_STACKTRACE))
906 return;
907
908 entry = tracing_get_trace_entry(tr, data);
909 tracing_generic_entry_update(entry, flags);
910 entry->type = TRACE_STACK;
911
912 memset(&entry->stack, 0, sizeof(entry->stack));
913
914 trace.nr_entries = 0;
915 trace.max_entries = FTRACE_STACK_ENTRIES;
916 trace.skip = skip;
917 trace.entries = entry->stack.caller;
918
919 save_stack_trace(&trace);
920}
921
922void
923__trace_special(void *__tr, void *__data,
924 unsigned long arg1, unsigned long arg2, unsigned long arg3)
925{
926 struct trace_array_cpu *data = __data;
927 struct trace_array *tr = __tr;
928 struct trace_entry *entry;
929 unsigned long irq_flags;
930
931 raw_local_irq_save(irq_flags);
932 __raw_spin_lock(&data->lock);
933 entry = tracing_get_trace_entry(tr, data);
934 tracing_generic_entry_update(entry, 0);
935 entry->type = TRACE_SPECIAL;
936 entry->special.arg1 = arg1;
937 entry->special.arg2 = arg2;
938 entry->special.arg3 = arg3;
939 __trace_stack(tr, data, irq_flags, 4);
940 __raw_spin_unlock(&data->lock);
941 raw_local_irq_restore(irq_flags);
942
943 trace_wake_up();
944}
945
946void
947tracing_sched_switch_trace(struct trace_array *tr,
948 struct trace_array_cpu *data,
949 struct task_struct *prev,
950 struct task_struct *next,
951 unsigned long flags)
952{
953 struct trace_entry *entry;
954 unsigned long irq_flags;
955
956 raw_local_irq_save(irq_flags);
957 __raw_spin_lock(&data->lock);
958 entry = tracing_get_trace_entry(tr, data);
959 tracing_generic_entry_update(entry, flags);
960 entry->type = TRACE_CTX;
961 entry->ctx.prev_pid = prev->pid;
962 entry->ctx.prev_prio = prev->prio;
963 entry->ctx.prev_state = prev->state;
964 entry->ctx.next_pid = next->pid;
965 entry->ctx.next_prio = next->prio;
966 entry->ctx.next_state = next->state;
967 __trace_stack(tr, data, flags, 5);
968 __raw_spin_unlock(&data->lock);
969 raw_local_irq_restore(irq_flags);
970}
971
972void
973tracing_sched_wakeup_trace(struct trace_array *tr,
974 struct trace_array_cpu *data,
975 struct task_struct *wakee,
976 struct task_struct *curr,
977 unsigned long flags)
978{
979 struct trace_entry *entry;
980 unsigned long irq_flags;
981
982 raw_local_irq_save(irq_flags);
983 __raw_spin_lock(&data->lock);
984 entry = tracing_get_trace_entry(tr, data);
985 tracing_generic_entry_update(entry, flags);
986 entry->type = TRACE_WAKE;
987 entry->ctx.prev_pid = curr->pid;
988 entry->ctx.prev_prio = curr->prio;
989 entry->ctx.prev_state = curr->state;
990 entry->ctx.next_pid = wakee->pid;
991 entry->ctx.next_prio = wakee->prio;
992 entry->ctx.next_state = wakee->state;
993 __trace_stack(tr, data, flags, 6);
994 __raw_spin_unlock(&data->lock);
995 raw_local_irq_restore(irq_flags);
996
997 trace_wake_up();
998}
999
1000void
1001ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
1002{
1003 struct trace_array *tr = &global_trace;
1004 struct trace_array_cpu *data;
1005 unsigned long flags;
1006 long disabled;
1007 int cpu;
1008
1009 if (tracing_disabled || current_trace == &no_tracer || !tr->ctrl)
1010 return;
1011
1012 local_irq_save(flags);
1013 cpu = raw_smp_processor_id();
1014 data = tr->data[cpu];
1015 disabled = atomic_inc_return(&data->disabled);
1016
1017 if (likely(disabled == 1))
1018 __trace_special(tr, data, arg1, arg2, arg3);
1019
1020 atomic_dec(&data->disabled);
1021 local_irq_restore(flags);
1022}
1023
1024#ifdef CONFIG_FTRACE
1025static void
1026function_trace_call(unsigned long ip, unsigned long parent_ip)
1027{
1028 struct trace_array *tr = &global_trace;
1029 struct trace_array_cpu *data;
1030 unsigned long flags;
1031 long disabled;
1032 int cpu;
1033
1034 if (unlikely(!ftrace_function_enabled))
1035 return;
1036
1037 if (skip_trace(ip))
1038 return;
1039
1040 local_irq_save(flags);
1041 cpu = raw_smp_processor_id();
1042 data = tr->data[cpu];
1043 disabled = atomic_inc_return(&data->disabled);
1044
1045 if (likely(disabled == 1))
1046 trace_function(tr, data, ip, parent_ip, flags);
1047
1048 atomic_dec(&data->disabled);
1049 local_irq_restore(flags);
1050}
1051
1052static struct ftrace_ops trace_ops __read_mostly =
1053{
1054 .func = function_trace_call,
1055};
1056
1057void tracing_start_function_trace(void)
1058{
1059 ftrace_function_enabled = 0;
1060 register_ftrace_function(&trace_ops);
1061 if (tracer_enabled)
1062 ftrace_function_enabled = 1;
1063}
1064
1065void tracing_stop_function_trace(void)
1066{
1067 ftrace_function_enabled = 0;
1068 unregister_ftrace_function(&trace_ops);
1069}
1070#endif
1071
1072enum trace_file_type {
1073 TRACE_FILE_LAT_FMT = 1,
1074};
1075
1076static struct trace_entry *
1077trace_entry_idx(struct trace_array *tr, struct trace_array_cpu *data,
1078 struct trace_iterator *iter, int cpu)
1079{
1080 struct page *page;
1081 struct trace_entry *array;
1082
1083 if (iter->next_idx[cpu] >= tr->entries ||
1084 iter->next_idx[cpu] >= data->trace_idx ||
1085 (data->trace_head == data->trace_tail &&
1086 data->trace_head_idx == data->trace_tail_idx))
1087 return NULL;
1088
1089 if (!iter->next_page[cpu]) {
1090 /* Initialize the iterator for this cpu trace buffer */
1091 WARN_ON(!data->trace_tail);
1092 page = virt_to_page(data->trace_tail);
1093 iter->next_page[cpu] = &page->lru;
1094 iter->next_page_idx[cpu] = data->trace_tail_idx;
1095 }
1096
1097 page = list_entry(iter->next_page[cpu], struct page, lru);
1098 BUG_ON(&data->trace_pages == &page->lru);
1099
1100 array = page_address(page);
1101
1102 WARN_ON(iter->next_page_idx[cpu] >= ENTRIES_PER_PAGE);
1103 return &array[iter->next_page_idx[cpu]];
1104}
1105
1106static struct trace_entry *
1107find_next_entry(struct trace_iterator *iter, int *ent_cpu)
1108{
1109 struct trace_array *tr = iter->tr;
1110 struct trace_entry *ent, *next = NULL;
1111 int next_cpu = -1;
1112 int cpu;
1113
1114 for_each_tracing_cpu(cpu) {
1115 if (!head_page(tr->data[cpu]))
1116 continue;
1117 ent = trace_entry_idx(tr, tr->data[cpu], iter, cpu);
1118 /*
1119 * Pick the entry with the smallest timestamp:
1120 */
1121 if (ent && (!next || ent->t < next->t)) {
1122 next = ent;
1123 next_cpu = cpu;
1124 }
1125 }
1126
1127 if (ent_cpu)
1128 *ent_cpu = next_cpu;
1129
1130 return next;
1131}
1132
1133static void trace_iterator_increment(struct trace_iterator *iter)
1134{
1135 iter->idx++;
1136 iter->next_idx[iter->cpu]++;
1137 iter->next_page_idx[iter->cpu]++;
1138
1139 if (iter->next_page_idx[iter->cpu] >= ENTRIES_PER_PAGE) {
1140 struct trace_array_cpu *data = iter->tr->data[iter->cpu];
1141
1142 iter->next_page_idx[iter->cpu] = 0;
1143 iter->next_page[iter->cpu] =
1144 trace_next_list(data, iter->next_page[iter->cpu]);
1145 }
1146}
1147
1148static void trace_consume(struct trace_iterator *iter)
1149{
1150 struct trace_array_cpu *data = iter->tr->data[iter->cpu];
1151
1152 data->trace_tail_idx++;
1153 if (data->trace_tail_idx >= ENTRIES_PER_PAGE) {
1154 data->trace_tail = trace_next_page(data, data->trace_tail);
1155 data->trace_tail_idx = 0;
1156 }
1157
1158 /* Check if we empty it, then reset the index */
1159 if (data->trace_head == data->trace_tail &&
1160 data->trace_head_idx == data->trace_tail_idx)
1161 data->trace_idx = 0;
1162}
1163
1164static void *find_next_entry_inc(struct trace_iterator *iter)
1165{
1166 struct trace_entry *next;
1167 int next_cpu = -1;
1168
1169 next = find_next_entry(iter, &next_cpu);
1170
1171 iter->prev_ent = iter->ent;
1172 iter->prev_cpu = iter->cpu;
1173
1174 iter->ent = next;
1175 iter->cpu = next_cpu;
1176
1177 if (next)
1178 trace_iterator_increment(iter);
1179
1180 return next ? iter : NULL;
1181}
1182
1183static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1184{
1185 struct trace_iterator *iter = m->private;
1186 int i = (int)*pos;
1187 void *ent;
1188
1189 (*pos)++;
1190
1191 /* can't go backwards */
1192 if (iter->idx > i)
1193 return NULL;
1194
1195 if (iter->idx < 0)
1196 ent = find_next_entry_inc(iter);
1197 else
1198 ent = iter;
1199
1200 while (ent && iter->idx < i)
1201 ent = find_next_entry_inc(iter);
1202
1203 iter->pos = *pos;
1204
1205 return ent;
1206}
1207
1208static void *s_start(struct seq_file *m, loff_t *pos)
1209{
1210 struct trace_iterator *iter = m->private;
1211 void *p = NULL;
1212 loff_t l = 0;
1213 int i;
1214
1215 mutex_lock(&trace_types_lock);
1216
1217 if (!current_trace || current_trace != iter->trace) {
1218 mutex_unlock(&trace_types_lock);
1219 return NULL;
1220 }
1221
1222 atomic_inc(&trace_record_cmdline_disabled);
1223
1224 /* let the tracer grab locks here if needed */
1225 if (current_trace->start)
1226 current_trace->start(iter);
1227
1228 if (*pos != iter->pos) {
1229 iter->ent = NULL;
1230 iter->cpu = 0;
1231 iter->idx = -1;
1232 iter->prev_ent = NULL;
1233 iter->prev_cpu = -1;
1234
1235 for_each_tracing_cpu(i) {
1236 iter->next_idx[i] = 0;
1237 iter->next_page[i] = NULL;
1238 }
1239
1240 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1241 ;
1242
1243 } else {
1244 l = *pos - 1;
1245 p = s_next(m, p, &l);
1246 }
1247
1248 return p;
1249}
1250
1251static void s_stop(struct seq_file *m, void *p)
1252{
1253 struct trace_iterator *iter = m->private;
1254
1255 atomic_dec(&trace_record_cmdline_disabled);
1256
1257 /* let the tracer release locks here if needed */
1258 if (current_trace && current_trace == iter->trace && iter->trace->stop)
1259 iter->trace->stop(iter);
1260
1261 mutex_unlock(&trace_types_lock);
1262}
1263
1264#define KRETPROBE_MSG "[unknown/kretprobe'd]"
1265
1266#ifdef CONFIG_KRETPROBES
1267static inline int kretprobed(unsigned long addr)
1268{
1269 return addr == (unsigned long)kretprobe_trampoline;
1270}
1271#else
1272static inline int kretprobed(unsigned long addr)
1273{
1274 return 0;
1275}
1276#endif /* CONFIG_KRETPROBES */
1277
1278static int
1279seq_print_sym_short(struct trace_seq *s, const char *fmt, unsigned long address)
1280{
1281#ifdef CONFIG_KALLSYMS
1282 char str[KSYM_SYMBOL_LEN];
1283
1284 kallsyms_lookup(address, NULL, NULL, NULL, str);
1285
1286 return trace_seq_printf(s, fmt, str);
1287#endif
1288 return 1;
1289}
1290
1291static int
1292seq_print_sym_offset(struct trace_seq *s, const char *fmt,
1293 unsigned long address)
1294{
1295#ifdef CONFIG_KALLSYMS
1296 char str[KSYM_SYMBOL_LEN];
1297
1298 sprint_symbol(str, address);
1299 return trace_seq_printf(s, fmt, str);
1300#endif
1301 return 1;
1302}
1303
1304#ifndef CONFIG_64BIT
1305# define IP_FMT "%08lx"
1306#else
1307# define IP_FMT "%016lx"
1308#endif
1309
1310static int
1311seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
1312{
1313 int ret;
1314
1315 if (!ip)
1316 return trace_seq_printf(s, "0");
1317
1318 if (sym_flags & TRACE_ITER_SYM_OFFSET)
1319 ret = seq_print_sym_offset(s, "%s", ip);
1320 else
1321 ret = seq_print_sym_short(s, "%s", ip);
1322
1323 if (!ret)
1324 return 0;
1325
1326 if (sym_flags & TRACE_ITER_SYM_ADDR)
1327 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
1328 return ret;
1329}
1330
1331static void print_lat_help_header(struct seq_file *m)
1332{
1333 seq_puts(m, "# _------=> CPU# \n");
1334 seq_puts(m, "# / _-----=> irqs-off \n");
1335 seq_puts(m, "# | / _----=> need-resched \n");
1336 seq_puts(m, "# || / _---=> hardirq/softirq \n");
1337 seq_puts(m, "# ||| / _--=> preempt-depth \n");
1338 seq_puts(m, "# |||| / \n");
1339 seq_puts(m, "# ||||| delay \n");
1340 seq_puts(m, "# cmd pid ||||| time | caller \n");
1341 seq_puts(m, "# \\ / ||||| \\ | / \n");
1342}
1343
1344static void print_func_help_header(struct seq_file *m)
1345{
1346 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
1347 seq_puts(m, "# | | | | |\n");
1348}
1349
1350
1351static void
1352print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1353{
1354 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1355 struct trace_array *tr = iter->tr;
1356 struct trace_array_cpu *data = tr->data[tr->cpu];
1357 struct tracer *type = current_trace;
1358 unsigned long total = 0;
1359 unsigned long entries = 0;
1360 int cpu;
1361 const char *name = "preemption";
1362
1363 if (type)
1364 name = type->name;
1365
1366 for_each_tracing_cpu(cpu) {
1367 if (head_page(tr->data[cpu])) {
1368 total += tr->data[cpu]->trace_idx;
1369 if (tr->data[cpu]->trace_idx > tr->entries)
1370 entries += tr->entries;
1371 else
1372 entries += tr->data[cpu]->trace_idx;
1373 }
1374 }
1375
1376 seq_printf(m, "%s latency trace v1.1.5 on %s\n",
1377 name, UTS_RELEASE);
1378 seq_puts(m, "-----------------------------------"
1379 "---------------------------------\n");
1380 seq_printf(m, " latency: %lu us, #%lu/%lu, CPU#%d |"
1381 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1382 nsecs_to_usecs(data->saved_latency),
1383 entries,
1384 total,
1385 tr->cpu,
1386#if defined(CONFIG_PREEMPT_NONE)
1387 "server",
1388#elif defined(CONFIG_PREEMPT_VOLUNTARY)
1389 "desktop",
1390#elif defined(CONFIG_PREEMPT)
1391 "preempt",
1392#else
1393 "unknown",
1394#endif
1395 /* These are reserved for later use */
1396 0, 0, 0, 0);
1397#ifdef CONFIG_SMP
1398 seq_printf(m, " #P:%d)\n", num_online_cpus());
1399#else
1400 seq_puts(m, ")\n");
1401#endif
1402 seq_puts(m, " -----------------\n");
1403 seq_printf(m, " | task: %.16s-%d "
1404 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1405 data->comm, data->pid, data->uid, data->nice,
1406 data->policy, data->rt_priority);
1407 seq_puts(m, " -----------------\n");
1408
1409 if (data->critical_start) {
1410 seq_puts(m, " => started at: ");
1411 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1412 trace_print_seq(m, &iter->seq);
1413 seq_puts(m, "\n => ended at: ");
1414 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1415 trace_print_seq(m, &iter->seq);
1416 seq_puts(m, "\n");
1417 }
1418
1419 seq_puts(m, "\n");
1420}
1421
1422static void
1423lat_print_generic(struct trace_seq *s, struct trace_entry *entry, int cpu)
1424{
1425 int hardirq, softirq;
1426 char *comm;
1427
1428 comm = trace_find_cmdline(entry->pid);
1429
1430 trace_seq_printf(s, "%8.8s-%-5d ", comm, entry->pid);
1431 trace_seq_printf(s, "%d", cpu);
1432 trace_seq_printf(s, "%c%c",
1433 (entry->flags & TRACE_FLAG_IRQS_OFF) ? 'd' : '.',
1434 ((entry->flags & TRACE_FLAG_NEED_RESCHED) ? 'N' : '.'));
1435
1436 hardirq = entry->flags & TRACE_FLAG_HARDIRQ;
1437 softirq = entry->flags & TRACE_FLAG_SOFTIRQ;
1438 if (hardirq && softirq) {
1439 trace_seq_putc(s, 'H');
1440 } else {
1441 if (hardirq) {
1442 trace_seq_putc(s, 'h');
1443 } else {
1444 if (softirq)
1445 trace_seq_putc(s, 's');
1446 else
1447 trace_seq_putc(s, '.');
1448 }
1449 }
1450
1451 if (entry->preempt_count)
1452 trace_seq_printf(s, "%x", entry->preempt_count);
1453 else
1454 trace_seq_puts(s, ".");
1455}
1456
1457unsigned long preempt_mark_thresh = 100;
1458
1459static void
1460lat_print_timestamp(struct trace_seq *s, unsigned long long abs_usecs,
1461 unsigned long rel_usecs)
1462{
1463 trace_seq_printf(s, " %4lldus", abs_usecs);
1464 if (rel_usecs > preempt_mark_thresh)
1465 trace_seq_puts(s, "!: ");
1466 else if (rel_usecs > 1)
1467 trace_seq_puts(s, "+: ");
1468 else
1469 trace_seq_puts(s, " : ");
1470}
1471
1472static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
1473
1474static int
1475print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1476{
1477 struct trace_seq *s = &iter->seq;
1478 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1479 struct trace_entry *next_entry = find_next_entry(iter, NULL);
1480 unsigned long verbose = (trace_flags & TRACE_ITER_VERBOSE);
1481 struct trace_entry *entry = iter->ent;
1482 unsigned long abs_usecs;
1483 unsigned long rel_usecs;
1484 char *comm;
1485 int S, T;
1486 int i;
1487 unsigned state;
1488
1489 if (!next_entry)
1490 next_entry = entry;
1491 rel_usecs = ns2usecs(next_entry->t - entry->t);
1492 abs_usecs = ns2usecs(entry->t - iter->tr->time_start);
1493
1494 if (verbose) {
1495 comm = trace_find_cmdline(entry->pid);
1496 trace_seq_printf(s, "%16s %5d %d %d %08x %08x [%08lx]"
1497 " %ld.%03ldms (+%ld.%03ldms): ",
1498 comm,
1499 entry->pid, cpu, entry->flags,
1500 entry->preempt_count, trace_idx,
1501 ns2usecs(entry->t),
1502 abs_usecs/1000,
1503 abs_usecs % 1000, rel_usecs/1000,
1504 rel_usecs % 1000);
1505 } else {
1506 lat_print_generic(s, entry, cpu);
1507 lat_print_timestamp(s, abs_usecs, rel_usecs);
1508 }
1509 switch (entry->type) {
1510 case TRACE_FN:
1511 seq_print_ip_sym(s, entry->fn.ip, sym_flags);
1512 trace_seq_puts(s, " (");
1513 if (kretprobed(entry->fn.parent_ip))
1514 trace_seq_puts(s, KRETPROBE_MSG);
1515 else
1516 seq_print_ip_sym(s, entry->fn.parent_ip, sym_flags);
1517 trace_seq_puts(s, ")\n");
1518 break;
1519 case TRACE_CTX:
1520 case TRACE_WAKE:
1521 T = entry->ctx.next_state < sizeof(state_to_char) ?
1522 state_to_char[entry->ctx.next_state] : 'X';
1523
1524 state = entry->ctx.prev_state ? __ffs(entry->ctx.prev_state) + 1 : 0;
1525 S = state < sizeof(state_to_char) - 1 ? state_to_char[state] : 'X';
1526 comm = trace_find_cmdline(entry->ctx.next_pid);
1527 trace_seq_printf(s, " %5d:%3d:%c %s %5d:%3d:%c %s\n",
1528 entry->ctx.prev_pid,
1529 entry->ctx.prev_prio,
1530 S, entry->type == TRACE_CTX ? "==>" : " +",
1531 entry->ctx.next_pid,
1532 entry->ctx.next_prio,
1533 T, comm);
1534 break;
1535 case TRACE_SPECIAL:
1536 trace_seq_printf(s, "# %ld %ld %ld\n",
1537 entry->special.arg1,
1538 entry->special.arg2,
1539 entry->special.arg3);
1540 break;
1541 case TRACE_STACK:
1542 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1543 if (i)
1544 trace_seq_puts(s, " <= ");
1545 seq_print_ip_sym(s, entry->stack.caller[i], sym_flags);
1546 }
1547 trace_seq_puts(s, "\n");
1548 break;
1549 default:
1550 trace_seq_printf(s, "Unknown type %d\n", entry->type);
1551 }
1552 return 1;
1553}
1554
1555static int print_trace_fmt(struct trace_iterator *iter)
1556{
1557 struct trace_seq *s = &iter->seq;
1558 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1559 struct trace_entry *entry;
1560 unsigned long usec_rem;
1561 unsigned long long t;
1562 unsigned long secs;
1563 char *comm;
1564 int ret;
1565 int S, T;
1566 int i;
1567
1568 entry = iter->ent;
1569
1570 comm = trace_find_cmdline(iter->ent->pid);
1571
1572 t = ns2usecs(entry->t);
1573 usec_rem = do_div(t, 1000000ULL);
1574 secs = (unsigned long)t;
1575
1576 ret = trace_seq_printf(s, "%16s-%-5d ", comm, entry->pid);
1577 if (!ret)
1578 return 0;
1579 ret = trace_seq_printf(s, "[%02d] ", iter->cpu);
1580 if (!ret)
1581 return 0;
1582 ret = trace_seq_printf(s, "%5lu.%06lu: ", secs, usec_rem);
1583 if (!ret)
1584 return 0;
1585
1586 switch (entry->type) {
1587 case TRACE_FN:
1588 ret = seq_print_ip_sym(s, entry->fn.ip, sym_flags);
1589 if (!ret)
1590 return 0;
1591 if ((sym_flags & TRACE_ITER_PRINT_PARENT) &&
1592 entry->fn.parent_ip) {
1593 ret = trace_seq_printf(s, " <-");
1594 if (!ret)
1595 return 0;
1596 if (kretprobed(entry->fn.parent_ip))
1597 ret = trace_seq_puts(s, KRETPROBE_MSG);
1598 else
1599 ret = seq_print_ip_sym(s, entry->fn.parent_ip,
1600 sym_flags);
1601 if (!ret)
1602 return 0;
1603 }
1604 ret = trace_seq_printf(s, "\n");
1605 if (!ret)
1606 return 0;
1607 break;
1608 case TRACE_CTX:
1609 case TRACE_WAKE:
1610 S = entry->ctx.prev_state < sizeof(state_to_char) ?
1611 state_to_char[entry->ctx.prev_state] : 'X';
1612 T = entry->ctx.next_state < sizeof(state_to_char) ?
1613 state_to_char[entry->ctx.next_state] : 'X';
1614 ret = trace_seq_printf(s, " %5d:%3d:%c %s %5d:%3d:%c\n",
1615 entry->ctx.prev_pid,
1616 entry->ctx.prev_prio,
1617 S,
1618 entry->type == TRACE_CTX ? "==>" : " +",
1619 entry->ctx.next_pid,
1620 entry->ctx.next_prio,
1621 T);
1622 if (!ret)
1623 return 0;
1624 break;
1625 case TRACE_SPECIAL:
1626 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
1627 entry->special.arg1,
1628 entry->special.arg2,
1629 entry->special.arg3);
1630 if (!ret)
1631 return 0;
1632 break;
1633 case TRACE_STACK:
1634 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1635 if (i) {
1636 ret = trace_seq_puts(s, " <= ");
1637 if (!ret)
1638 return 0;
1639 }
1640 ret = seq_print_ip_sym(s, entry->stack.caller[i],
1641 sym_flags);
1642 if (!ret)
1643 return 0;
1644 }
1645 ret = trace_seq_puts(s, "\n");
1646 if (!ret)
1647 return 0;
1648 break;
1649 }
1650 return 1;
1651}
1652
1653static int print_raw_fmt(struct trace_iterator *iter)
1654{
1655 struct trace_seq *s = &iter->seq;
1656 struct trace_entry *entry;
1657 int ret;
1658 int S, T;
1659
1660 entry = iter->ent;
1661
1662 ret = trace_seq_printf(s, "%d %d %llu ",
1663 entry->pid, iter->cpu, entry->t);
1664 if (!ret)
1665 return 0;
1666
1667 switch (entry->type) {
1668 case TRACE_FN:
1669 ret = trace_seq_printf(s, "%x %x\n",
1670 entry->fn.ip, entry->fn.parent_ip);
1671 if (!ret)
1672 return 0;
1673 break;
1674 case TRACE_CTX:
1675 case TRACE_WAKE:
1676 S = entry->ctx.prev_state < sizeof(state_to_char) ?
1677 state_to_char[entry->ctx.prev_state] : 'X';
1678 T = entry->ctx.next_state < sizeof(state_to_char) ?
1679 state_to_char[entry->ctx.next_state] : 'X';
1680 if (entry->type == TRACE_WAKE)
1681 S = '+';
1682 ret = trace_seq_printf(s, "%d %d %c %d %d %c\n",
1683 entry->ctx.prev_pid,
1684 entry->ctx.prev_prio,
1685 S,
1686 entry->ctx.next_pid,
1687 entry->ctx.next_prio,
1688 T);
1689 if (!ret)
1690 return 0;
1691 break;
1692 case TRACE_SPECIAL:
1693 case TRACE_STACK:
1694 ret = trace_seq_printf(s, "# %ld %ld %ld\n",
1695 entry->special.arg1,
1696 entry->special.arg2,
1697 entry->special.arg3);
1698 if (!ret)
1699 return 0;
1700 break;
1701 }
1702 return 1;
1703}
1704
1705#define SEQ_PUT_FIELD_RET(s, x) \
1706do { \
1707 if (!trace_seq_putmem(s, &(x), sizeof(x))) \
1708 return 0; \
1709} while (0)
1710
1711#define SEQ_PUT_HEX_FIELD_RET(s, x) \
1712do { \
1713 if (!trace_seq_putmem_hex(s, &(x), sizeof(x))) \
1714 return 0; \
1715} while (0)
1716
1717static int print_hex_fmt(struct trace_iterator *iter)
1718{
1719 struct trace_seq *s = &iter->seq;
1720 unsigned char newline = '\n';
1721 struct trace_entry *entry;
1722 int S, T;
1723
1724 entry = iter->ent;
1725
1726 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
1727 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
1728 SEQ_PUT_HEX_FIELD_RET(s, entry->t);
1729
1730 switch (entry->type) {
1731 case TRACE_FN:
1732 SEQ_PUT_HEX_FIELD_RET(s, entry->fn.ip);
1733 SEQ_PUT_HEX_FIELD_RET(s, entry->fn.parent_ip);
1734 break;
1735 case TRACE_CTX:
1736 case TRACE_WAKE:
1737 S = entry->ctx.prev_state < sizeof(state_to_char) ?
1738 state_to_char[entry->ctx.prev_state] : 'X';
1739 T = entry->ctx.next_state < sizeof(state_to_char) ?
1740 state_to_char[entry->ctx.next_state] : 'X';
1741 if (entry->type == TRACE_WAKE)
1742 S = '+';
1743 SEQ_PUT_HEX_FIELD_RET(s, entry->ctx.prev_pid);
1744 SEQ_PUT_HEX_FIELD_RET(s, entry->ctx.prev_prio);
1745 SEQ_PUT_HEX_FIELD_RET(s, S);
1746 SEQ_PUT_HEX_FIELD_RET(s, entry->ctx.next_pid);
1747 SEQ_PUT_HEX_FIELD_RET(s, entry->ctx.next_prio);
1748 SEQ_PUT_HEX_FIELD_RET(s, entry->fn.parent_ip);
1749 SEQ_PUT_HEX_FIELD_RET(s, T);
1750 break;
1751 case TRACE_SPECIAL:
1752 case TRACE_STACK:
1753 SEQ_PUT_HEX_FIELD_RET(s, entry->special.arg1);
1754 SEQ_PUT_HEX_FIELD_RET(s, entry->special.arg2);
1755 SEQ_PUT_HEX_FIELD_RET(s, entry->special.arg3);
1756 break;
1757 }
1758 SEQ_PUT_FIELD_RET(s, newline);
1759
1760 return 1;
1761}
1762
1763static int print_bin_fmt(struct trace_iterator *iter)
1764{
1765 struct trace_seq *s = &iter->seq;
1766 struct trace_entry *entry;
1767
1768 entry = iter->ent;
1769
1770 SEQ_PUT_FIELD_RET(s, entry->pid);
1771 SEQ_PUT_FIELD_RET(s, entry->cpu);
1772 SEQ_PUT_FIELD_RET(s, entry->t);
1773
1774 switch (entry->type) {
1775 case TRACE_FN:
1776 SEQ_PUT_FIELD_RET(s, entry->fn.ip);
1777 SEQ_PUT_FIELD_RET(s, entry->fn.parent_ip);
1778 break;
1779 case TRACE_CTX:
1780 SEQ_PUT_FIELD_RET(s, entry->ctx.prev_pid);
1781 SEQ_PUT_FIELD_RET(s, entry->ctx.prev_prio);
1782 SEQ_PUT_FIELD_RET(s, entry->ctx.prev_state);
1783 SEQ_PUT_FIELD_RET(s, entry->ctx.next_pid);
1784 SEQ_PUT_FIELD_RET(s, entry->ctx.next_prio);
1785 SEQ_PUT_FIELD_RET(s, entry->ctx.next_state);
1786 break;
1787 case TRACE_SPECIAL:
1788 case TRACE_STACK:
1789 SEQ_PUT_FIELD_RET(s, entry->special.arg1);
1790 SEQ_PUT_FIELD_RET(s, entry->special.arg2);
1791 SEQ_PUT_FIELD_RET(s, entry->special.arg3);
1792 break;
1793 }
1794 return 1;
1795}
1796
1797static int trace_empty(struct trace_iterator *iter)
1798{
1799 struct trace_array_cpu *data;
1800 int cpu;
1801
1802 for_each_tracing_cpu(cpu) {
1803 data = iter->tr->data[cpu];
1804
1805 if (head_page(data) && data->trace_idx &&
1806 (data->trace_tail != data->trace_head ||
1807 data->trace_tail_idx != data->trace_head_idx))
1808 return 0;
1809 }
1810 return 1;
1811}
1812
1813static int print_trace_line(struct trace_iterator *iter)
1814{
1815 if (iter->trace && iter->trace->print_line)
1816 return iter->trace->print_line(iter);
1817
1818 if (trace_flags & TRACE_ITER_BIN)
1819 return print_bin_fmt(iter);
1820
1821 if (trace_flags & TRACE_ITER_HEX)
1822 return print_hex_fmt(iter);
1823
1824 if (trace_flags & TRACE_ITER_RAW)
1825 return print_raw_fmt(iter);
1826
1827 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
1828 return print_lat_fmt(iter, iter->idx, iter->cpu);
1829
1830 return print_trace_fmt(iter);
1831}
1832
1833static int s_show(struct seq_file *m, void *v)
1834{
1835 struct trace_iterator *iter = v;
1836
1837 if (iter->ent == NULL) {
1838 if (iter->tr) {
1839 seq_printf(m, "# tracer: %s\n", iter->trace->name);
1840 seq_puts(m, "#\n");
1841 }
1842 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1843 /* print nothing if the buffers are empty */
1844 if (trace_empty(iter))
1845 return 0;
1846 print_trace_header(m, iter);
1847 if (!(trace_flags & TRACE_ITER_VERBOSE))
1848 print_lat_help_header(m);
1849 } else {
1850 if (!(trace_flags & TRACE_ITER_VERBOSE))
1851 print_func_help_header(m);
1852 }
1853 } else {
1854 print_trace_line(iter);
1855 trace_print_seq(m, &iter->seq);
1856 }
1857
1858 return 0;
1859}
1860
1861static struct seq_operations tracer_seq_ops = {
1862 .start = s_start,
1863 .next = s_next,
1864 .stop = s_stop,
1865 .show = s_show,
1866};
1867
1868static struct trace_iterator *
1869__tracing_open(struct inode *inode, struct file *file, int *ret)
1870{
1871 struct trace_iterator *iter;
1872
1873 if (tracing_disabled) {
1874 *ret = -ENODEV;
1875 return NULL;
1876 }
1877
1878 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1879 if (!iter) {
1880 *ret = -ENOMEM;
1881 goto out;
1882 }
1883
1884 mutex_lock(&trace_types_lock);
1885 if (current_trace && current_trace->print_max)
1886 iter->tr = &max_tr;
1887 else
1888 iter->tr = inode->i_private;
1889 iter->trace = current_trace;
1890 iter->pos = -1;
1891
1892 /* TODO stop tracer */
1893 *ret = seq_open(file, &tracer_seq_ops);
1894 if (!*ret) {
1895 struct seq_file *m = file->private_data;
1896 m->private = iter;
1897
1898 /* stop the trace while dumping */
1899 if (iter->tr->ctrl) {
1900 tracer_enabled = 0;
1901 ftrace_function_enabled = 0;
1902 }
1903
1904 if (iter->trace && iter->trace->open)
1905 iter->trace->open(iter);
1906 } else {
1907 kfree(iter);
1908 iter = NULL;
1909 }
1910 mutex_unlock(&trace_types_lock);
1911
1912 out:
1913 return iter;
1914}
1915
1916int tracing_open_generic(struct inode *inode, struct file *filp)
1917{
1918 if (tracing_disabled)
1919 return -ENODEV;
1920
1921 filp->private_data = inode->i_private;
1922 return 0;
1923}
1924
1925int tracing_release(struct inode *inode, struct file *file)
1926{
1927 struct seq_file *m = (struct seq_file *)file->private_data;
1928 struct trace_iterator *iter = m->private;
1929
1930 mutex_lock(&trace_types_lock);
1931 if (iter->trace && iter->trace->close)
1932 iter->trace->close(iter);
1933
1934 /* reenable tracing if it was previously enabled */
1935 if (iter->tr->ctrl) {
1936 tracer_enabled = 1;
1937 /*
1938 * It is safe to enable function tracing even if it
1939 * isn't used
1940 */
1941 ftrace_function_enabled = 1;
1942 }
1943 mutex_unlock(&trace_types_lock);
1944
1945 seq_release(inode, file);
1946 kfree(iter);
1947 return 0;
1948}
1949
1950static int tracing_open(struct inode *inode, struct file *file)
1951{
1952 int ret;
1953
1954 __tracing_open(inode, file, &ret);
1955
1956 return ret;
1957}
1958
1959static int tracing_lt_open(struct inode *inode, struct file *file)
1960{
1961 struct trace_iterator *iter;
1962 int ret;
1963
1964 iter = __tracing_open(inode, file, &ret);
1965
1966 if (!ret)
1967 iter->iter_flags |= TRACE_FILE_LAT_FMT;
1968
1969 return ret;
1970}
1971
1972
1973static void *
1974t_next(struct seq_file *m, void *v, loff_t *pos)
1975{
1976 struct tracer *t = m->private;
1977
1978 (*pos)++;
1979
1980 if (t)
1981 t = t->next;
1982
1983 m->private = t;
1984
1985 return t;
1986}
1987
1988static void *t_start(struct seq_file *m, loff_t *pos)
1989{
1990 struct tracer *t = m->private;
1991 loff_t l = 0;
1992
1993 mutex_lock(&trace_types_lock);
1994 for (; t && l < *pos; t = t_next(m, t, &l))
1995 ;
1996
1997 return t;
1998}
1999
2000static void t_stop(struct seq_file *m, void *p)
2001{
2002 mutex_unlock(&trace_types_lock);
2003}
2004
2005static int t_show(struct seq_file *m, void *v)
2006{
2007 struct tracer *t = v;
2008
2009 if (!t)
2010 return 0;
2011
2012 seq_printf(m, "%s", t->name);
2013 if (t->next)
2014 seq_putc(m, ' ');
2015 else
2016 seq_putc(m, '\n');
2017
2018 return 0;
2019}
2020
2021static struct seq_operations show_traces_seq_ops = {
2022 .start = t_start,
2023 .next = t_next,
2024 .stop = t_stop,
2025 .show = t_show,
2026};
2027
2028static int show_traces_open(struct inode *inode, struct file *file)
2029{
2030 int ret;
2031
2032 if (tracing_disabled)
2033 return -ENODEV;
2034
2035 ret = seq_open(file, &show_traces_seq_ops);
2036 if (!ret) {
2037 struct seq_file *m = file->private_data;
2038 m->private = trace_types;
2039 }
2040
2041 return ret;
2042}
2043
2044static struct file_operations tracing_fops = {
2045 .open = tracing_open,
2046 .read = seq_read,
2047 .llseek = seq_lseek,
2048 .release = tracing_release,
2049};
2050
2051static struct file_operations tracing_lt_fops = {
2052 .open = tracing_lt_open,
2053 .read = seq_read,
2054 .llseek = seq_lseek,
2055 .release = tracing_release,
2056};
2057
2058static struct file_operations show_traces_fops = {
2059 .open = show_traces_open,
2060 .read = seq_read,
2061 .release = seq_release,
2062};
2063
2064/*
2065 * Only trace on a CPU if the bitmask is set:
2066 */
2067static cpumask_t tracing_cpumask = CPU_MASK_ALL;
2068
2069/*
2070 * When tracing/tracing_cpu_mask is modified then this holds
2071 * the new bitmask we are about to install:
2072 */
2073static cpumask_t tracing_cpumask_new;
2074
2075/*
2076 * The tracer itself will not take this lock, but still we want
2077 * to provide a consistent cpumask to user-space:
2078 */
2079static DEFINE_MUTEX(tracing_cpumask_update_lock);
2080
2081/*
2082 * Temporary storage for the character representation of the
2083 * CPU bitmask (and one more byte for the newline):
2084 */
2085static char mask_str[NR_CPUS + 1];
2086
2087static ssize_t
2088tracing_cpumask_read(struct file *filp, char __user *ubuf,
2089 size_t count, loff_t *ppos)
2090{
2091 int len;
2092
2093 mutex_lock(&tracing_cpumask_update_lock);
2094
2095 len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2096 if (count - len < 2) {
2097 count = -EINVAL;
2098 goto out_err;
2099 }
2100 len += sprintf(mask_str + len, "\n");
2101 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2102
2103out_err:
2104 mutex_unlock(&tracing_cpumask_update_lock);
2105
2106 return count;
2107}
2108
2109static ssize_t
2110tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2111 size_t count, loff_t *ppos)
2112{
2113 int err, cpu;
2114
2115 mutex_lock(&tracing_cpumask_update_lock);
2116 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2117 if (err)
2118 goto err_unlock;
2119
2120 raw_local_irq_disable();
2121 __raw_spin_lock(&ftrace_max_lock);
2122 for_each_tracing_cpu(cpu) {
2123 /*
2124 * Increase/decrease the disabled counter if we are
2125 * about to flip a bit in the cpumask:
2126 */
2127 if (cpu_isset(cpu, tracing_cpumask) &&
2128 !cpu_isset(cpu, tracing_cpumask_new)) {
2129 atomic_inc(&global_trace.data[cpu]->disabled);
2130 }
2131 if (!cpu_isset(cpu, tracing_cpumask) &&
2132 cpu_isset(cpu, tracing_cpumask_new)) {
2133 atomic_dec(&global_trace.data[cpu]->disabled);
2134 }
2135 }
2136 __raw_spin_unlock(&ftrace_max_lock);
2137 raw_local_irq_enable();
2138
2139 tracing_cpumask = tracing_cpumask_new;
2140
2141 mutex_unlock(&tracing_cpumask_update_lock);
2142
2143 return count;
2144
2145err_unlock:
2146 mutex_unlock(&tracing_cpumask_update_lock);
2147
2148 return err;
2149}
2150
2151static struct file_operations tracing_cpumask_fops = {
2152 .open = tracing_open_generic,
2153 .read = tracing_cpumask_read,
2154 .write = tracing_cpumask_write,
2155};
2156
2157static ssize_t
2158tracing_iter_ctrl_read(struct file *filp, char __user *ubuf,
2159 size_t cnt, loff_t *ppos)
2160{
2161 char *buf;
2162 int r = 0;
2163 int len = 0;
2164 int i;
2165
2166 /* calulate max size */
2167 for (i = 0; trace_options[i]; i++) {
2168 len += strlen(trace_options[i]);
2169 len += 3; /* "no" and space */
2170 }
2171
2172 /* +2 for \n and \0 */
2173 buf = kmalloc(len + 2, GFP_KERNEL);
2174 if (!buf)
2175 return -ENOMEM;
2176
2177 for (i = 0; trace_options[i]; i++) {
2178 if (trace_flags & (1 << i))
2179 r += sprintf(buf + r, "%s ", trace_options[i]);
2180 else
2181 r += sprintf(buf + r, "no%s ", trace_options[i]);
2182 }
2183
2184 r += sprintf(buf + r, "\n");
2185 WARN_ON(r >= len + 2);
2186
2187 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2188
2189 kfree(buf);
2190
2191 return r;
2192}
2193
2194static ssize_t
2195tracing_iter_ctrl_write(struct file *filp, const char __user *ubuf,
2196 size_t cnt, loff_t *ppos)
2197{
2198 char buf[64];
2199 char *cmp = buf;
2200 int neg = 0;
2201 int i;
2202
2203 if (cnt >= sizeof(buf))
2204 return -EINVAL;
2205
2206 if (copy_from_user(&buf, ubuf, cnt))
2207 return -EFAULT;
2208
2209 buf[cnt] = 0;
2210
2211 if (strncmp(buf, "no", 2) == 0) {
2212 neg = 1;
2213 cmp += 2;
2214 }
2215
2216 for (i = 0; trace_options[i]; i++) {
2217 int len = strlen(trace_options[i]);
2218
2219 if (strncmp(cmp, trace_options[i], len) == 0) {
2220 if (neg)
2221 trace_flags &= ~(1 << i);
2222 else
2223 trace_flags |= (1 << i);
2224 break;
2225 }
2226 }
2227 /*
2228 * If no option could be set, return an error:
2229 */
2230 if (!trace_options[i])
2231 return -EINVAL;
2232
2233 filp->f_pos += cnt;
2234
2235 return cnt;
2236}
2237
2238static struct file_operations tracing_iter_fops = {
2239 .open = tracing_open_generic,
2240 .read = tracing_iter_ctrl_read,
2241 .write = tracing_iter_ctrl_write,
2242};
2243
2244static const char readme_msg[] =
2245 "tracing mini-HOWTO:\n\n"
2246 "# mkdir /debug\n"
2247 "# mount -t debugfs nodev /debug\n\n"
2248 "# cat /debug/tracing/available_tracers\n"
2249 "wakeup preemptirqsoff preemptoff irqsoff ftrace sched_switch none\n\n"
2250 "# cat /debug/tracing/current_tracer\n"
2251 "none\n"
2252 "# echo sched_switch > /debug/tracing/current_tracer\n"
2253 "# cat /debug/tracing/current_tracer\n"
2254 "sched_switch\n"
2255 "# cat /debug/tracing/iter_ctrl\n"
2256 "noprint-parent nosym-offset nosym-addr noverbose\n"
2257 "# echo print-parent > /debug/tracing/iter_ctrl\n"
2258 "# echo 1 > /debug/tracing/tracing_enabled\n"
2259 "# cat /debug/tracing/trace > /tmp/trace.txt\n"
2260 "echo 0 > /debug/tracing/tracing_enabled\n"
2261;
2262
2263static ssize_t
2264tracing_readme_read(struct file *filp, char __user *ubuf,
2265 size_t cnt, loff_t *ppos)
2266{
2267 return simple_read_from_buffer(ubuf, cnt, ppos,
2268 readme_msg, strlen(readme_msg));
2269}
2270
2271static struct file_operations tracing_readme_fops = {
2272 .open = tracing_open_generic,
2273 .read = tracing_readme_read,
2274};
2275
2276static ssize_t
2277tracing_ctrl_read(struct file *filp, char __user *ubuf,
2278 size_t cnt, loff_t *ppos)
2279{
2280 struct trace_array *tr = filp->private_data;
2281 char buf[64];
2282 int r;
2283
2284 r = sprintf(buf, "%ld\n", tr->ctrl);
2285 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2286}
2287
2288static ssize_t
2289tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2290 size_t cnt, loff_t *ppos)
2291{
2292 struct trace_array *tr = filp->private_data;
2293 char buf[64];
2294 long val;
2295 int ret;
2296
2297 if (cnt >= sizeof(buf))
2298 return -EINVAL;
2299
2300 if (copy_from_user(&buf, ubuf, cnt))
2301 return -EFAULT;
2302
2303 buf[cnt] = 0;
2304
2305 ret = strict_strtoul(buf, 10, &val);
2306 if (ret < 0)
2307 return ret;
2308
2309 val = !!val;
2310
2311 mutex_lock(&trace_types_lock);
2312 if (tr->ctrl ^ val) {
2313 if (val)
2314 tracer_enabled = 1;
2315 else
2316 tracer_enabled = 0;
2317
2318 tr->ctrl = val;
2319
2320 if (current_trace && current_trace->ctrl_update)
2321 current_trace->ctrl_update(tr);
2322 }
2323 mutex_unlock(&trace_types_lock);
2324
2325 filp->f_pos += cnt;
2326
2327 return cnt;
2328}
2329
2330static ssize_t
2331tracing_set_trace_read(struct file *filp, char __user *ubuf,
2332 size_t cnt, loff_t *ppos)
2333{
2334 char buf[max_tracer_type_len+2];
2335 int r;
2336
2337 mutex_lock(&trace_types_lock);
2338 if (current_trace)
2339 r = sprintf(buf, "%s\n", current_trace->name);
2340 else
2341 r = sprintf(buf, "\n");
2342 mutex_unlock(&trace_types_lock);
2343
2344 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2345}
2346
2347static ssize_t
2348tracing_set_trace_write(struct file *filp, const char __user *ubuf,
2349 size_t cnt, loff_t *ppos)
2350{
2351 struct trace_array *tr = &global_trace;
2352 struct tracer *t;
2353 char buf[max_tracer_type_len+1];
2354 int i;
2355
2356 if (cnt > max_tracer_type_len)
2357 cnt = max_tracer_type_len;
2358
2359 if (copy_from_user(&buf, ubuf, cnt))
2360 return -EFAULT;
2361
2362 buf[cnt] = 0;
2363
2364 /* strip ending whitespace. */
2365 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2366 buf[i] = 0;
2367
2368 mutex_lock(&trace_types_lock);
2369 for (t = trace_types; t; t = t->next) {
2370 if (strcmp(t->name, buf) == 0)
2371 break;
2372 }
2373 if (!t || t == current_trace)
2374 goto out;
2375
2376 if (current_trace && current_trace->reset)
2377 current_trace->reset(tr);
2378
2379 current_trace = t;
2380 if (t->init)
2381 t->init(tr);
2382
2383 out:
2384 mutex_unlock(&trace_types_lock);
2385
2386 filp->f_pos += cnt;
2387
2388 return cnt;
2389}
2390
2391static ssize_t
2392tracing_max_lat_read(struct file *filp, char __user *ubuf,
2393 size_t cnt, loff_t *ppos)
2394{
2395 unsigned long *ptr = filp->private_data;
2396 char buf[64];
2397 int r;
2398
2399 r = snprintf(buf, sizeof(buf), "%ld\n",
2400 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
2401 if (r > sizeof(buf))
2402 r = sizeof(buf);
2403 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2404}
2405
2406static ssize_t
2407tracing_max_lat_write(struct file *filp, const char __user *ubuf,
2408 size_t cnt, loff_t *ppos)
2409{
2410 long *ptr = filp->private_data;
2411 char buf[64];
2412 long val;
2413 int ret;
2414
2415 if (cnt >= sizeof(buf))
2416 return -EINVAL;
2417
2418 if (copy_from_user(&buf, ubuf, cnt))
2419 return -EFAULT;
2420
2421 buf[cnt] = 0;
2422
2423 ret = strict_strtoul(buf, 10, &val);
2424 if (ret < 0)
2425 return ret;
2426
2427 *ptr = val * 1000;
2428
2429 return cnt;
2430}
2431
2432static atomic_t tracing_reader;
2433
2434static int tracing_open_pipe(struct inode *inode, struct file *filp)
2435{
2436 struct trace_iterator *iter;
2437
2438 if (tracing_disabled)
2439 return -ENODEV;
2440
2441 /* We only allow for reader of the pipe */
2442 if (atomic_inc_return(&tracing_reader) != 1) {
2443 atomic_dec(&tracing_reader);
2444 return -EBUSY;
2445 }
2446
2447 /* create a buffer to store the information to pass to userspace */
2448 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2449 if (!iter)
2450 return -ENOMEM;
2451
2452 mutex_lock(&trace_types_lock);
2453 iter->tr = &global_trace;
2454 iter->trace = current_trace;
2455 filp->private_data = iter;
2456
2457 if (iter->trace->pipe_open)
2458 iter->trace->pipe_open(iter);
2459 mutex_unlock(&trace_types_lock);
2460
2461 return 0;
2462}
2463
2464static int tracing_release_pipe(struct inode *inode, struct file *file)
2465{
2466 struct trace_iterator *iter = file->private_data;
2467
2468 kfree(iter);
2469 atomic_dec(&tracing_reader);
2470
2471 return 0;
2472}
2473
2474static unsigned int
2475tracing_poll_pipe(struct file *filp, poll_table *poll_table)
2476{
2477 struct trace_iterator *iter = filp->private_data;
2478
2479 if (trace_flags & TRACE_ITER_BLOCK) {
2480 /*
2481 * Always select as readable when in blocking mode
2482 */
2483 return POLLIN | POLLRDNORM;
2484 } else {
2485 if (!trace_empty(iter))
2486 return POLLIN | POLLRDNORM;
2487 poll_wait(filp, &trace_wait, poll_table);
2488 if (!trace_empty(iter))
2489 return POLLIN | POLLRDNORM;
2490
2491 return 0;
2492 }
2493}
2494
2495/*
2496 * Consumer reader.
2497 */
2498static ssize_t
2499tracing_read_pipe(struct file *filp, char __user *ubuf,
2500 size_t cnt, loff_t *ppos)
2501{
2502 struct trace_iterator *iter = filp->private_data;
2503 struct trace_array_cpu *data;
2504 static cpumask_t mask;
2505 unsigned long flags;
2506#ifdef CONFIG_FTRACE
2507 int ftrace_save;
2508#endif
2509 int cpu;
2510 ssize_t sret;
2511
2512 /* return any leftover data */
2513 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2514 if (sret != -EBUSY)
2515 return sret;
2516 sret = 0;
2517
2518 trace_seq_reset(&iter->seq);
2519
2520 mutex_lock(&trace_types_lock);
2521 if (iter->trace->read) {
2522 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
2523 if (sret)
2524 goto out;
2525 }
2526
2527 while (trace_empty(iter)) {
2528
2529 if ((filp->f_flags & O_NONBLOCK)) {
2530 sret = -EAGAIN;
2531 goto out;
2532 }
2533
2534 /*
2535 * This is a make-shift waitqueue. The reason we don't use
2536 * an actual wait queue is because:
2537 * 1) we only ever have one waiter
2538 * 2) the tracing, traces all functions, we don't want
2539 * the overhead of calling wake_up and friends
2540 * (and tracing them too)
2541 * Anyway, this is really very primitive wakeup.
2542 */
2543 set_current_state(TASK_INTERRUPTIBLE);
2544 iter->tr->waiter = current;
2545
2546 mutex_unlock(&trace_types_lock);
2547
2548 /* sleep for 100 msecs, and try again. */
2549 schedule_timeout(HZ/10);
2550
2551 mutex_lock(&trace_types_lock);
2552
2553 iter->tr->waiter = NULL;
2554
2555 if (signal_pending(current)) {
2556 sret = -EINTR;
2557 goto out;
2558 }
2559
2560 if (iter->trace != current_trace)
2561 goto out;
2562
2563 /*
2564 * We block until we read something and tracing is disabled.
2565 * We still block if tracing is disabled, but we have never
2566 * read anything. This allows a user to cat this file, and
2567 * then enable tracing. But after we have read something,
2568 * we give an EOF when tracing is again disabled.
2569 *
2570 * iter->pos will be 0 if we haven't read anything.
2571 */
2572 if (!tracer_enabled && iter->pos)
2573 break;
2574
2575 continue;
2576 }
2577
2578 /* stop when tracing is finished */
2579 if (trace_empty(iter))
2580 goto out;
2581
2582 if (cnt >= PAGE_SIZE)
2583 cnt = PAGE_SIZE - 1;
2584
2585 /* reset all but tr, trace, and overruns */
2586 memset(&iter->seq, 0,
2587 sizeof(struct trace_iterator) -
2588 offsetof(struct trace_iterator, seq));
2589 iter->pos = -1;
2590
2591 /*
2592 * We need to stop all tracing on all CPUS to read the
2593 * the next buffer. This is a bit expensive, but is
2594 * not done often. We fill all what we can read,
2595 * and then release the locks again.
2596 */
2597
2598 cpus_clear(mask);
2599 local_irq_save(flags);
2600#ifdef CONFIG_FTRACE
2601 ftrace_save = ftrace_enabled;
2602 ftrace_enabled = 0;
2603#endif
2604 smp_wmb();
2605 for_each_tracing_cpu(cpu) {
2606 data = iter->tr->data[cpu];
2607
2608 if (!head_page(data) || !data->trace_idx)
2609 continue;
2610
2611 atomic_inc(&data->disabled);
2612 cpu_set(cpu, mask);
2613 }
2614
2615 for_each_cpu_mask(cpu, mask) {
2616 data = iter->tr->data[cpu];
2617 __raw_spin_lock(&data->lock);
2618
2619 if (data->overrun > iter->last_overrun[cpu])
2620 iter->overrun[cpu] +=
2621 data->overrun - iter->last_overrun[cpu];
2622 iter->last_overrun[cpu] = data->overrun;
2623 }
2624
2625 while (find_next_entry_inc(iter) != NULL) {
2626 int ret;
2627 int len = iter->seq.len;
2628
2629 ret = print_trace_line(iter);
2630 if (!ret) {
2631 /* don't print partial lines */
2632 iter->seq.len = len;
2633 break;
2634 }
2635
2636 trace_consume(iter);
2637
2638 if (iter->seq.len >= cnt)
2639 break;
2640 }
2641
2642 for_each_cpu_mask(cpu, mask) {
2643 data = iter->tr->data[cpu];
2644 __raw_spin_unlock(&data->lock);
2645 }
2646
2647 for_each_cpu_mask(cpu, mask) {
2648 data = iter->tr->data[cpu];
2649 atomic_dec(&data->disabled);
2650 }
2651#ifdef CONFIG_FTRACE
2652 ftrace_enabled = ftrace_save;
2653#endif
2654 local_irq_restore(flags);
2655
2656 /* Now copy what we have to the user */
2657 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2658 if (iter->seq.readpos >= iter->seq.len)
2659 trace_seq_reset(&iter->seq);
2660 if (sret == -EBUSY)
2661 sret = 0;
2662
2663out:
2664 mutex_unlock(&trace_types_lock);
2665
2666 return sret;
2667}
2668
2669static ssize_t
2670tracing_entries_read(struct file *filp, char __user *ubuf,
2671 size_t cnt, loff_t *ppos)
2672{
2673 struct trace_array *tr = filp->private_data;
2674 char buf[64];
2675 int r;
2676
2677 r = sprintf(buf, "%lu\n", tr->entries);
2678 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2679}
2680
2681static ssize_t
2682tracing_entries_write(struct file *filp, const char __user *ubuf,
2683 size_t cnt, loff_t *ppos)
2684{
2685 unsigned long val;
2686 char buf[64];
2687 int i, ret;
2688
2689 if (cnt >= sizeof(buf))
2690 return -EINVAL;
2691
2692 if (copy_from_user(&buf, ubuf, cnt))
2693 return -EFAULT;
2694
2695 buf[cnt] = 0;
2696
2697 ret = strict_strtoul(buf, 10, &val);
2698 if (ret < 0)
2699 return ret;
2700
2701 /* must have at least 1 entry */
2702 if (!val)
2703 return -EINVAL;
2704
2705 mutex_lock(&trace_types_lock);
2706
2707 if (current_trace != &no_tracer) {
2708 cnt = -EBUSY;
2709 pr_info("ftrace: set current_tracer to none"
2710 " before modifying buffer size\n");
2711 goto out;
2712 }
2713
2714 if (val > global_trace.entries) {
2715 long pages_requested;
2716 unsigned long freeable_pages;
2717
2718 /* make sure we have enough memory before mapping */
2719 pages_requested =
2720 (val + (ENTRIES_PER_PAGE-1)) / ENTRIES_PER_PAGE;
2721
2722 /* account for each buffer (and max_tr) */
2723 pages_requested *= tracing_nr_buffers * 2;
2724
2725 /* Check for overflow */
2726 if (pages_requested < 0) {
2727 cnt = -ENOMEM;
2728 goto out;
2729 }
2730
2731 freeable_pages = determine_dirtyable_memory();
2732
2733 /* we only allow to request 1/4 of useable memory */
2734 if (pages_requested >
2735 ((freeable_pages + tracing_pages_allocated) / 4)) {
2736 cnt = -ENOMEM;
2737 goto out;
2738 }
2739
2740 while (global_trace.entries < val) {
2741 if (trace_alloc_page()) {
2742 cnt = -ENOMEM;
2743 goto out;
2744 }
2745 /* double check that we don't go over the known pages */
2746 if (tracing_pages_allocated > pages_requested)
2747 break;
2748 }
2749
2750 } else {
2751 /* include the number of entries in val (inc of page entries) */
2752 while (global_trace.entries > val + (ENTRIES_PER_PAGE - 1))
2753 trace_free_page();
2754 }
2755
2756 /* check integrity */
2757 for_each_tracing_cpu(i)
2758 check_pages(global_trace.data[i]);
2759
2760 filp->f_pos += cnt;
2761
2762 /* If check pages failed, return ENOMEM */
2763 if (tracing_disabled)
2764 cnt = -ENOMEM;
2765 out:
2766 max_tr.entries = global_trace.entries;
2767 mutex_unlock(&trace_types_lock);
2768
2769 return cnt;
2770}
2771
2772static struct file_operations tracing_max_lat_fops = {
2773 .open = tracing_open_generic,
2774 .read = tracing_max_lat_read,
2775 .write = tracing_max_lat_write,
2776};
2777
2778static struct file_operations tracing_ctrl_fops = {
2779 .open = tracing_open_generic,
2780 .read = tracing_ctrl_read,
2781 .write = tracing_ctrl_write,
2782};
2783
2784static struct file_operations set_tracer_fops = {
2785 .open = tracing_open_generic,
2786 .read = tracing_set_trace_read,
2787 .write = tracing_set_trace_write,
2788};
2789
2790static struct file_operations tracing_pipe_fops = {
2791 .open = tracing_open_pipe,
2792 .poll = tracing_poll_pipe,
2793 .read = tracing_read_pipe,
2794 .release = tracing_release_pipe,
2795};
2796
2797static struct file_operations tracing_entries_fops = {
2798 .open = tracing_open_generic,
2799 .read = tracing_entries_read,
2800 .write = tracing_entries_write,
2801};
2802
2803#ifdef CONFIG_DYNAMIC_FTRACE
2804
2805static ssize_t
2806tracing_read_long(struct file *filp, char __user *ubuf,
2807 size_t cnt, loff_t *ppos)
2808{
2809 unsigned long *p = filp->private_data;
2810 char buf[64];
2811 int r;
2812
2813 r = sprintf(buf, "%ld\n", *p);
2814
2815 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2816}
2817
2818static struct file_operations tracing_read_long_fops = {
2819 .open = tracing_open_generic,
2820 .read = tracing_read_long,
2821};
2822#endif
2823
2824static struct dentry *d_tracer;
2825
2826struct dentry *tracing_init_dentry(void)
2827{
2828 static int once;
2829
2830 if (d_tracer)
2831 return d_tracer;
2832
2833 d_tracer = debugfs_create_dir("tracing", NULL);
2834
2835 if (!d_tracer && !once) {
2836 once = 1;
2837 pr_warning("Could not create debugfs directory 'tracing'\n");
2838 return NULL;
2839 }
2840
2841 return d_tracer;
2842}
2843
2844#ifdef CONFIG_FTRACE_SELFTEST
2845/* Let selftest have access to static functions in this file */
2846#include "trace_selftest.c"
2847#endif
2848
2849static __init void tracer_init_debugfs(void)
2850{
2851 struct dentry *d_tracer;
2852 struct dentry *entry;
2853
2854 d_tracer = tracing_init_dentry();
2855
2856 entry = debugfs_create_file("tracing_enabled", 0644, d_tracer,
2857 &global_trace, &tracing_ctrl_fops);
2858 if (!entry)
2859 pr_warning("Could not create debugfs 'tracing_enabled' entry\n");
2860
2861 entry = debugfs_create_file("iter_ctrl", 0644, d_tracer,
2862 NULL, &tracing_iter_fops);
2863 if (!entry)
2864 pr_warning("Could not create debugfs 'iter_ctrl' entry\n");
2865
2866 entry = debugfs_create_file("tracing_cpumask", 0644, d_tracer,
2867 NULL, &tracing_cpumask_fops);
2868 if (!entry)
2869 pr_warning("Could not create debugfs 'tracing_cpumask' entry\n");
2870
2871 entry = debugfs_create_file("latency_trace", 0444, d_tracer,
2872 &global_trace, &tracing_lt_fops);
2873 if (!entry)
2874 pr_warning("Could not create debugfs 'latency_trace' entry\n");
2875
2876 entry = debugfs_create_file("trace", 0444, d_tracer,
2877 &global_trace, &tracing_fops);
2878 if (!entry)
2879 pr_warning("Could not create debugfs 'trace' entry\n");
2880
2881 entry = debugfs_create_file("available_tracers", 0444, d_tracer,
2882 &global_trace, &show_traces_fops);
2883 if (!entry)
2884 pr_warning("Could not create debugfs 'trace' entry\n");
2885
2886 entry = debugfs_create_file("current_tracer", 0444, d_tracer,
2887 &global_trace, &set_tracer_fops);
2888 if (!entry)
2889 pr_warning("Could not create debugfs 'trace' entry\n");
2890
2891 entry = debugfs_create_file("tracing_max_latency", 0644, d_tracer,
2892 &tracing_max_latency,
2893 &tracing_max_lat_fops);
2894 if (!entry)
2895 pr_warning("Could not create debugfs "
2896 "'tracing_max_latency' entry\n");
2897
2898 entry = debugfs_create_file("tracing_thresh", 0644, d_tracer,
2899 &tracing_thresh, &tracing_max_lat_fops);
2900 if (!entry)
2901 pr_warning("Could not create debugfs "
2902 "'tracing_threash' entry\n");
2903 entry = debugfs_create_file("README", 0644, d_tracer,
2904 NULL, &tracing_readme_fops);
2905 if (!entry)
2906 pr_warning("Could not create debugfs 'README' entry\n");
2907
2908 entry = debugfs_create_file("trace_pipe", 0644, d_tracer,
2909 NULL, &tracing_pipe_fops);
2910 if (!entry)
2911 pr_warning("Could not create debugfs "
2912 "'tracing_threash' entry\n");
2913
2914 entry = debugfs_create_file("trace_entries", 0644, d_tracer,
2915 &global_trace, &tracing_entries_fops);
2916 if (!entry)
2917 pr_warning("Could not create debugfs "
2918 "'tracing_threash' entry\n");
2919
2920#ifdef CONFIG_DYNAMIC_FTRACE
2921 entry = debugfs_create_file("dyn_ftrace_total_info", 0444, d_tracer,
2922 &ftrace_update_tot_cnt,
2923 &tracing_read_long_fops);
2924 if (!entry)
2925 pr_warning("Could not create debugfs "
2926 "'dyn_ftrace_total_info' entry\n");
2927#endif
2928#ifdef CONFIG_SYSPROF_TRACER
2929 init_tracer_sysprof_debugfs(d_tracer);
2930#endif
2931}
2932
2933static int trace_alloc_page(void)
2934{
2935 struct trace_array_cpu *data;
2936 struct page *page, *tmp;
2937 LIST_HEAD(pages);
2938 void *array;
2939 unsigned pages_allocated = 0;
2940 int i;
2941
2942 /* first allocate a page for each CPU */
2943 for_each_tracing_cpu(i) {
2944 array = (void *)__get_free_page(GFP_KERNEL);
2945 if (array == NULL) {
2946 printk(KERN_ERR "tracer: failed to allocate page"
2947 "for trace buffer!\n");
2948 goto free_pages;
2949 }
2950
2951 pages_allocated++;
2952 page = virt_to_page(array);
2953 list_add(&page->lru, &pages);
2954
2955/* Only allocate if we are actually using the max trace */
2956#ifdef CONFIG_TRACER_MAX_TRACE
2957 array = (void *)__get_free_page(GFP_KERNEL);
2958 if (array == NULL) {
2959 printk(KERN_ERR "tracer: failed to allocate page"
2960 "for trace buffer!\n");
2961 goto free_pages;
2962 }
2963 pages_allocated++;
2964 page = virt_to_page(array);
2965 list_add(&page->lru, &pages);
2966#endif
2967 }
2968
2969 /* Now that we successfully allocate a page per CPU, add them */
2970 for_each_tracing_cpu(i) {
2971 data = global_trace.data[i];
2972 page = list_entry(pages.next, struct page, lru);
2973 list_del_init(&page->lru);
2974 list_add_tail(&page->lru, &data->trace_pages);
2975 ClearPageLRU(page);
2976
2977#ifdef CONFIG_TRACER_MAX_TRACE
2978 data = max_tr.data[i];
2979 page = list_entry(pages.next, struct page, lru);
2980 list_del_init(&page->lru);
2981 list_add_tail(&page->lru, &data->trace_pages);
2982 SetPageLRU(page);
2983#endif
2984 }
2985 tracing_pages_allocated += pages_allocated;
2986 global_trace.entries += ENTRIES_PER_PAGE;
2987
2988 return 0;
2989
2990 free_pages:
2991 list_for_each_entry_safe(page, tmp, &pages, lru) {
2992 list_del_init(&page->lru);
2993 __free_page(page);
2994 }
2995 return -ENOMEM;
2996}
2997
2998static int trace_free_page(void)
2999{
3000 struct trace_array_cpu *data;
3001 struct page *page;
3002 struct list_head *p;
3003 int i;
3004 int ret = 0;
3005
3006 /* free one page from each buffer */
3007 for_each_tracing_cpu(i) {
3008 data = global_trace.data[i];
3009 p = data->trace_pages.next;
3010 if (p == &data->trace_pages) {
3011 /* should never happen */
3012 WARN_ON(1);
3013 tracing_disabled = 1;
3014 ret = -1;
3015 break;
3016 }
3017 page = list_entry(p, struct page, lru);
3018 ClearPageLRU(page);
3019 list_del(&page->lru);
3020 tracing_pages_allocated--;
3021 tracing_pages_allocated--;
3022 __free_page(page);
3023
3024 tracing_reset(data);
3025
3026#ifdef CONFIG_TRACER_MAX_TRACE
3027 data = max_tr.data[i];
3028 p = data->trace_pages.next;
3029 if (p == &data->trace_pages) {
3030 /* should never happen */
3031 WARN_ON(1);
3032 tracing_disabled = 1;
3033 ret = -1;
3034 break;
3035 }
3036 page = list_entry(p, struct page, lru);
3037 ClearPageLRU(page);
3038 list_del(&page->lru);
3039 __free_page(page);
3040
3041 tracing_reset(data);
3042#endif
3043 }
3044 global_trace.entries -= ENTRIES_PER_PAGE;
3045
3046 return ret;
3047}
3048
3049__init static int tracer_alloc_buffers(void)
3050{
3051 struct trace_array_cpu *data;
3052 void *array;
3053 struct page *page;
3054 int pages = 0;
3055 int ret = -ENOMEM;
3056 int i;
3057
3058 /* TODO: make the number of buffers hot pluggable with CPUS */
3059 tracing_nr_buffers = num_possible_cpus();
3060 tracing_buffer_mask = cpu_possible_map;
3061
3062 /* Allocate the first page for all buffers */
3063 for_each_tracing_cpu(i) {
3064 data = global_trace.data[i] = &per_cpu(global_trace_cpu, i);
3065 max_tr.data[i] = &per_cpu(max_data, i);
3066
3067 array = (void *)__get_free_page(GFP_KERNEL);
3068 if (array == NULL) {
3069 printk(KERN_ERR "tracer: failed to allocate page"
3070 "for trace buffer!\n");
3071 goto free_buffers;
3072 }
3073
3074 /* set the array to the list */
3075 INIT_LIST_HEAD(&data->trace_pages);
3076 page = virt_to_page(array);
3077 list_add(&page->lru, &data->trace_pages);
3078 /* use the LRU flag to differentiate the two buffers */
3079 ClearPageLRU(page);
3080
3081 data->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
3082 max_tr.data[i]->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
3083
3084/* Only allocate if we are actually using the max trace */
3085#ifdef CONFIG_TRACER_MAX_TRACE
3086 array = (void *)__get_free_page(GFP_KERNEL);
3087 if (array == NULL) {
3088 printk(KERN_ERR "tracer: failed to allocate page"
3089 "for trace buffer!\n");
3090 goto free_buffers;
3091 }
3092
3093 INIT_LIST_HEAD(&max_tr.data[i]->trace_pages);
3094 page = virt_to_page(array);
3095 list_add(&page->lru, &max_tr.data[i]->trace_pages);
3096 SetPageLRU(page);
3097#endif
3098 }
3099
3100 /*
3101 * Since we allocate by orders of pages, we may be able to
3102 * round up a bit.
3103 */
3104 global_trace.entries = ENTRIES_PER_PAGE;
3105 pages++;
3106
3107 while (global_trace.entries < trace_nr_entries) {
3108 if (trace_alloc_page())
3109 break;
3110 pages++;
3111 }
3112 max_tr.entries = global_trace.entries;
3113
3114 pr_info("tracer: %d pages allocated for %ld entries of %ld bytes\n",
3115 pages, trace_nr_entries, (long)TRACE_ENTRY_SIZE);
3116 pr_info(" actual entries %ld\n", global_trace.entries);
3117
3118 tracer_init_debugfs();
3119
3120 trace_init_cmdlines();
3121
3122 register_tracer(&no_tracer);
3123 current_trace = &no_tracer;
3124
3125 /* All seems OK, enable tracing */
3126 global_trace.ctrl = tracer_enabled;
3127 tracing_disabled = 0;
3128
3129 return 0;
3130
3131 free_buffers:
3132 for (i-- ; i >= 0; i--) {
3133 struct page *page, *tmp;
3134 struct trace_array_cpu *data = global_trace.data[i];
3135
3136 if (data) {
3137 list_for_each_entry_safe(page, tmp,
3138 &data->trace_pages, lru) {
3139 list_del_init(&page->lru);
3140 __free_page(page);
3141 }
3142 }
3143
3144#ifdef CONFIG_TRACER_MAX_TRACE
3145 data = max_tr.data[i];
3146 if (data) {
3147 list_for_each_entry_safe(page, tmp,
3148 &data->trace_pages, lru) {
3149 list_del_init(&page->lru);
3150 __free_page(page);
3151 }
3152 }
3153#endif
3154 }
3155 return ret;
3156}
3157fs_initcall(tracer_alloc_buffers);
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
new file mode 100644
index 000000000000..f69f86788c2b
--- /dev/null
+++ b/kernel/trace/trace.h
@@ -0,0 +1,339 @@
1#ifndef _LINUX_KERNEL_TRACE_H
2#define _LINUX_KERNEL_TRACE_H
3
4#include <linux/fs.h>
5#include <asm/atomic.h>
6#include <linux/sched.h>
7#include <linux/clocksource.h>
8#include <linux/mmiotrace.h>
9
10enum trace_type {
11 __TRACE_FIRST_TYPE = 0,
12
13 TRACE_FN,
14 TRACE_CTX,
15 TRACE_WAKE,
16 TRACE_STACK,
17 TRACE_SPECIAL,
18 TRACE_MMIO_RW,
19 TRACE_MMIO_MAP,
20
21 __TRACE_LAST_TYPE
22};
23
24/*
25 * Function trace entry - function address and parent function addres:
26 */
27struct ftrace_entry {
28 unsigned long ip;
29 unsigned long parent_ip;
30};
31
32/*
33 * Context switch trace entry - which task (and prio) we switched from/to:
34 */
35struct ctx_switch_entry {
36 unsigned int prev_pid;
37 unsigned char prev_prio;
38 unsigned char prev_state;
39 unsigned int next_pid;
40 unsigned char next_prio;
41 unsigned char next_state;
42};
43
44/*
45 * Special (free-form) trace entry:
46 */
47struct special_entry {
48 unsigned long arg1;
49 unsigned long arg2;
50 unsigned long arg3;
51};
52
53/*
54 * Stack-trace entry:
55 */
56
57#define FTRACE_STACK_ENTRIES 8
58
59struct stack_entry {
60 unsigned long caller[FTRACE_STACK_ENTRIES];
61};
62
63/*
64 * The trace entry - the most basic unit of tracing. This is what
65 * is printed in the end as a single line in the trace output, such as:
66 *
67 * bash-15816 [01] 235.197585: idle_cpu <- irq_enter
68 */
69struct trace_entry {
70 char type;
71 char cpu;
72 char flags;
73 char preempt_count;
74 int pid;
75 cycle_t t;
76 union {
77 struct ftrace_entry fn;
78 struct ctx_switch_entry ctx;
79 struct special_entry special;
80 struct stack_entry stack;
81 struct mmiotrace_rw mmiorw;
82 struct mmiotrace_map mmiomap;
83 };
84};
85
86#define TRACE_ENTRY_SIZE sizeof(struct trace_entry)
87
88/*
89 * The CPU trace array - it consists of thousands of trace entries
90 * plus some other descriptor data: (for example which task started
91 * the trace, etc.)
92 */
93struct trace_array_cpu {
94 struct list_head trace_pages;
95 atomic_t disabled;
96 raw_spinlock_t lock;
97 struct lock_class_key lock_key;
98
99 /* these fields get copied into max-trace: */
100 unsigned trace_head_idx;
101 unsigned trace_tail_idx;
102 void *trace_head; /* producer */
103 void *trace_tail; /* consumer */
104 unsigned long trace_idx;
105 unsigned long overrun;
106 unsigned long saved_latency;
107 unsigned long critical_start;
108 unsigned long critical_end;
109 unsigned long critical_sequence;
110 unsigned long nice;
111 unsigned long policy;
112 unsigned long rt_priority;
113 cycle_t preempt_timestamp;
114 pid_t pid;
115 uid_t uid;
116 char comm[TASK_COMM_LEN];
117};
118
119struct trace_iterator;
120
121/*
122 * The trace array - an array of per-CPU trace arrays. This is the
123 * highest level data structure that individual tracers deal with.
124 * They have on/off state as well:
125 */
126struct trace_array {
127 unsigned long entries;
128 long ctrl;
129 int cpu;
130 cycle_t time_start;
131 struct task_struct *waiter;
132 struct trace_array_cpu *data[NR_CPUS];
133};
134
135/*
136 * A specific tracer, represented by methods that operate on a trace array:
137 */
138struct tracer {
139 const char *name;
140 void (*init)(struct trace_array *tr);
141 void (*reset)(struct trace_array *tr);
142 void (*open)(struct trace_iterator *iter);
143 void (*pipe_open)(struct trace_iterator *iter);
144 void (*close)(struct trace_iterator *iter);
145 void (*start)(struct trace_iterator *iter);
146 void (*stop)(struct trace_iterator *iter);
147 ssize_t (*read)(struct trace_iterator *iter,
148 struct file *filp, char __user *ubuf,
149 size_t cnt, loff_t *ppos);
150 void (*ctrl_update)(struct trace_array *tr);
151#ifdef CONFIG_FTRACE_STARTUP_TEST
152 int (*selftest)(struct tracer *trace,
153 struct trace_array *tr);
154#endif
155 int (*print_line)(struct trace_iterator *iter);
156 struct tracer *next;
157 int print_max;
158};
159
160struct trace_seq {
161 unsigned char buffer[PAGE_SIZE];
162 unsigned int len;
163 unsigned int readpos;
164};
165
166/*
167 * Trace iterator - used by printout routines who present trace
168 * results to users and which routines might sleep, etc:
169 */
170struct trace_iterator {
171 struct trace_array *tr;
172 struct tracer *trace;
173 void *private;
174 long last_overrun[NR_CPUS];
175 long overrun[NR_CPUS];
176
177 /* The below is zeroed out in pipe_read */
178 struct trace_seq seq;
179 struct trace_entry *ent;
180 int cpu;
181
182 struct trace_entry *prev_ent;
183 int prev_cpu;
184
185 unsigned long iter_flags;
186 loff_t pos;
187 unsigned long next_idx[NR_CPUS];
188 struct list_head *next_page[NR_CPUS];
189 unsigned next_page_idx[NR_CPUS];
190 long idx;
191};
192
193void tracing_reset(struct trace_array_cpu *data);
194int tracing_open_generic(struct inode *inode, struct file *filp);
195struct dentry *tracing_init_dentry(void);
196void init_tracer_sysprof_debugfs(struct dentry *d_tracer);
197
198void ftrace(struct trace_array *tr,
199 struct trace_array_cpu *data,
200 unsigned long ip,
201 unsigned long parent_ip,
202 unsigned long flags);
203void tracing_sched_switch_trace(struct trace_array *tr,
204 struct trace_array_cpu *data,
205 struct task_struct *prev,
206 struct task_struct *next,
207 unsigned long flags);
208void tracing_record_cmdline(struct task_struct *tsk);
209
210void tracing_sched_wakeup_trace(struct trace_array *tr,
211 struct trace_array_cpu *data,
212 struct task_struct *wakee,
213 struct task_struct *cur,
214 unsigned long flags);
215void trace_special(struct trace_array *tr,
216 struct trace_array_cpu *data,
217 unsigned long arg1,
218 unsigned long arg2,
219 unsigned long arg3);
220void trace_function(struct trace_array *tr,
221 struct trace_array_cpu *data,
222 unsigned long ip,
223 unsigned long parent_ip,
224 unsigned long flags);
225
226void tracing_start_cmdline_record(void);
227void tracing_stop_cmdline_record(void);
228int register_tracer(struct tracer *type);
229void unregister_tracer(struct tracer *type);
230
231extern unsigned long nsecs_to_usecs(unsigned long nsecs);
232
233extern unsigned long tracing_max_latency;
234extern unsigned long tracing_thresh;
235
236void update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu);
237void update_max_tr_single(struct trace_array *tr,
238 struct task_struct *tsk, int cpu);
239
240extern cycle_t ftrace_now(int cpu);
241
242#ifdef CONFIG_FTRACE
243void tracing_start_function_trace(void);
244void tracing_stop_function_trace(void);
245#else
246# define tracing_start_function_trace() do { } while (0)
247# define tracing_stop_function_trace() do { } while (0)
248#endif
249
250#ifdef CONFIG_CONTEXT_SWITCH_TRACER
251typedef void
252(*tracer_switch_func_t)(void *private,
253 void *__rq,
254 struct task_struct *prev,
255 struct task_struct *next);
256
257struct tracer_switch_ops {
258 tracer_switch_func_t func;
259 void *private;
260 struct tracer_switch_ops *next;
261};
262
263#endif /* CONFIG_CONTEXT_SWITCH_TRACER */
264
265#ifdef CONFIG_DYNAMIC_FTRACE
266extern unsigned long ftrace_update_tot_cnt;
267#define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func
268extern int DYN_FTRACE_TEST_NAME(void);
269#endif
270
271#ifdef CONFIG_MMIOTRACE
272extern void __trace_mmiotrace_rw(struct trace_array *tr,
273 struct trace_array_cpu *data,
274 struct mmiotrace_rw *rw);
275extern void __trace_mmiotrace_map(struct trace_array *tr,
276 struct trace_array_cpu *data,
277 struct mmiotrace_map *map);
278#endif
279
280#ifdef CONFIG_FTRACE_STARTUP_TEST
281#ifdef CONFIG_FTRACE
282extern int trace_selftest_startup_function(struct tracer *trace,
283 struct trace_array *tr);
284#endif
285#ifdef CONFIG_IRQSOFF_TRACER
286extern int trace_selftest_startup_irqsoff(struct tracer *trace,
287 struct trace_array *tr);
288#endif
289#ifdef CONFIG_PREEMPT_TRACER
290extern int trace_selftest_startup_preemptoff(struct tracer *trace,
291 struct trace_array *tr);
292#endif
293#if defined(CONFIG_IRQSOFF_TRACER) && defined(CONFIG_PREEMPT_TRACER)
294extern int trace_selftest_startup_preemptirqsoff(struct tracer *trace,
295 struct trace_array *tr);
296#endif
297#ifdef CONFIG_SCHED_TRACER
298extern int trace_selftest_startup_wakeup(struct tracer *trace,
299 struct trace_array *tr);
300#endif
301#ifdef CONFIG_CONTEXT_SWITCH_TRACER
302extern int trace_selftest_startup_sched_switch(struct tracer *trace,
303 struct trace_array *tr);
304#endif
305#ifdef CONFIG_SYSPROF_TRACER
306extern int trace_selftest_startup_sysprof(struct tracer *trace,
307 struct trace_array *tr);
308#endif
309#endif /* CONFIG_FTRACE_STARTUP_TEST */
310
311extern void *head_page(struct trace_array_cpu *data);
312extern int trace_seq_printf(struct trace_seq *s, const char *fmt, ...);
313extern ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf,
314 size_t cnt);
315extern long ns2usecs(cycle_t nsec);
316
317extern unsigned long trace_flags;
318
319/*
320 * trace_iterator_flags is an enumeration that defines bit
321 * positions into trace_flags that controls the output.
322 *
323 * NOTE: These bits must match the trace_options array in
324 * trace.c.
325 */
326enum trace_iterator_flags {
327 TRACE_ITER_PRINT_PARENT = 0x01,
328 TRACE_ITER_SYM_OFFSET = 0x02,
329 TRACE_ITER_SYM_ADDR = 0x04,
330 TRACE_ITER_VERBOSE = 0x08,
331 TRACE_ITER_RAW = 0x10,
332 TRACE_ITER_HEX = 0x20,
333 TRACE_ITER_BIN = 0x40,
334 TRACE_ITER_BLOCK = 0x80,
335 TRACE_ITER_STACKTRACE = 0x100,
336 TRACE_ITER_SCHED_TREE = 0x200,
337};
338
339#endif /* _LINUX_KERNEL_TRACE_H */
diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c
new file mode 100644
index 000000000000..312144897970
--- /dev/null
+++ b/kernel/trace/trace_functions.c
@@ -0,0 +1,81 @@
1/*
2 * ring buffer based function tracer
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Based on code from the latency_tracer, that is:
8 *
9 * Copyright (C) 2004-2006 Ingo Molnar
10 * Copyright (C) 2004 William Lee Irwin III
11 */
12#include <linux/debugfs.h>
13#include <linux/uaccess.h>
14#include <linux/ftrace.h>
15#include <linux/fs.h>
16
17#include "trace.h"
18
19static void function_reset(struct trace_array *tr)
20{
21 int cpu;
22
23 tr->time_start = ftrace_now(tr->cpu);
24
25 for_each_online_cpu(cpu)
26 tracing_reset(tr->data[cpu]);
27}
28
29static void start_function_trace(struct trace_array *tr)
30{
31 tr->cpu = get_cpu();
32 function_reset(tr);
33 put_cpu();
34
35 tracing_start_cmdline_record();
36 tracing_start_function_trace();
37}
38
39static void stop_function_trace(struct trace_array *tr)
40{
41 tracing_stop_function_trace();
42 tracing_stop_cmdline_record();
43}
44
45static void function_trace_init(struct trace_array *tr)
46{
47 if (tr->ctrl)
48 start_function_trace(tr);
49}
50
51static void function_trace_reset(struct trace_array *tr)
52{
53 if (tr->ctrl)
54 stop_function_trace(tr);
55}
56
57static void function_trace_ctrl_update(struct trace_array *tr)
58{
59 if (tr->ctrl)
60 start_function_trace(tr);
61 else
62 stop_function_trace(tr);
63}
64
65static struct tracer function_trace __read_mostly =
66{
67 .name = "ftrace",
68 .init = function_trace_init,
69 .reset = function_trace_reset,
70 .ctrl_update = function_trace_ctrl_update,
71#ifdef CONFIG_FTRACE_SELFTEST
72 .selftest = trace_selftest_startup_function,
73#endif
74};
75
76static __init int init_function_trace(void)
77{
78 return register_tracer(&function_trace);
79}
80
81device_initcall(init_function_trace);
diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c
new file mode 100644
index 000000000000..ece6cfb649fa
--- /dev/null
+++ b/kernel/trace/trace_irqsoff.c
@@ -0,0 +1,490 @@
1/*
2 * trace irqs off criticall timings
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * From code in the latency_tracer, that is:
8 *
9 * Copyright (C) 2004-2006 Ingo Molnar
10 * Copyright (C) 2004 William Lee Irwin III
11 */
12#include <linux/kallsyms.h>
13#include <linux/debugfs.h>
14#include <linux/uaccess.h>
15#include <linux/module.h>
16#include <linux/ftrace.h>
17#include <linux/fs.h>
18
19#include "trace.h"
20
21static struct trace_array *irqsoff_trace __read_mostly;
22static int tracer_enabled __read_mostly;
23
24static DEFINE_PER_CPU(int, tracing_cpu);
25
26static DEFINE_SPINLOCK(max_trace_lock);
27
28enum {
29 TRACER_IRQS_OFF = (1 << 1),
30 TRACER_PREEMPT_OFF = (1 << 2),
31};
32
33static int trace_type __read_mostly;
34
35#ifdef CONFIG_PREEMPT_TRACER
36static inline int
37preempt_trace(void)
38{
39 return ((trace_type & TRACER_PREEMPT_OFF) && preempt_count());
40}
41#else
42# define preempt_trace() (0)
43#endif
44
45#ifdef CONFIG_IRQSOFF_TRACER
46static inline int
47irq_trace(void)
48{
49 return ((trace_type & TRACER_IRQS_OFF) &&
50 irqs_disabled());
51}
52#else
53# define irq_trace() (0)
54#endif
55
56/*
57 * Sequence count - we record it when starting a measurement and
58 * skip the latency if the sequence has changed - some other section
59 * did a maximum and could disturb our measurement with serial console
60 * printouts, etc. Truly coinciding maximum latencies should be rare
61 * and what happens together happens separately as well, so this doesnt
62 * decrease the validity of the maximum found:
63 */
64static __cacheline_aligned_in_smp unsigned long max_sequence;
65
66#ifdef CONFIG_FTRACE
67/*
68 * irqsoff uses its own tracer function to keep the overhead down:
69 */
70static void
71irqsoff_tracer_call(unsigned long ip, unsigned long parent_ip)
72{
73 struct trace_array *tr = irqsoff_trace;
74 struct trace_array_cpu *data;
75 unsigned long flags;
76 long disabled;
77 int cpu;
78
79 /*
80 * Does not matter if we preempt. We test the flags
81 * afterward, to see if irqs are disabled or not.
82 * If we preempt and get a false positive, the flags
83 * test will fail.
84 */
85 cpu = raw_smp_processor_id();
86 if (likely(!per_cpu(tracing_cpu, cpu)))
87 return;
88
89 local_save_flags(flags);
90 /* slight chance to get a false positive on tracing_cpu */
91 if (!irqs_disabled_flags(flags))
92 return;
93
94 data = tr->data[cpu];
95 disabled = atomic_inc_return(&data->disabled);
96
97 if (likely(disabled == 1))
98 trace_function(tr, data, ip, parent_ip, flags);
99
100 atomic_dec(&data->disabled);
101}
102
103static struct ftrace_ops trace_ops __read_mostly =
104{
105 .func = irqsoff_tracer_call,
106};
107#endif /* CONFIG_FTRACE */
108
109/*
110 * Should this new latency be reported/recorded?
111 */
112static int report_latency(cycle_t delta)
113{
114 if (tracing_thresh) {
115 if (delta < tracing_thresh)
116 return 0;
117 } else {
118 if (delta <= tracing_max_latency)
119 return 0;
120 }
121 return 1;
122}
123
124static void
125check_critical_timing(struct trace_array *tr,
126 struct trace_array_cpu *data,
127 unsigned long parent_ip,
128 int cpu)
129{
130 unsigned long latency, t0, t1;
131 cycle_t T0, T1, delta;
132 unsigned long flags;
133
134 /*
135 * usecs conversion is slow so we try to delay the conversion
136 * as long as possible:
137 */
138 T0 = data->preempt_timestamp;
139 T1 = ftrace_now(cpu);
140 delta = T1-T0;
141
142 local_save_flags(flags);
143
144 if (!report_latency(delta))
145 goto out;
146
147 spin_lock_irqsave(&max_trace_lock, flags);
148
149 /* check if we are still the max latency */
150 if (!report_latency(delta))
151 goto out_unlock;
152
153 trace_function(tr, data, CALLER_ADDR0, parent_ip, flags);
154
155 latency = nsecs_to_usecs(delta);
156
157 if (data->critical_sequence != max_sequence)
158 goto out_unlock;
159
160 tracing_max_latency = delta;
161 t0 = nsecs_to_usecs(T0);
162 t1 = nsecs_to_usecs(T1);
163
164 data->critical_end = parent_ip;
165
166 update_max_tr_single(tr, current, cpu);
167
168 max_sequence++;
169
170out_unlock:
171 spin_unlock_irqrestore(&max_trace_lock, flags);
172
173out:
174 data->critical_sequence = max_sequence;
175 data->preempt_timestamp = ftrace_now(cpu);
176 tracing_reset(data);
177 trace_function(tr, data, CALLER_ADDR0, parent_ip, flags);
178}
179
180static inline void
181start_critical_timing(unsigned long ip, unsigned long parent_ip)
182{
183 int cpu;
184 struct trace_array *tr = irqsoff_trace;
185 struct trace_array_cpu *data;
186 unsigned long flags;
187
188 if (likely(!tracer_enabled))
189 return;
190
191 cpu = raw_smp_processor_id();
192
193 if (per_cpu(tracing_cpu, cpu))
194 return;
195
196 data = tr->data[cpu];
197
198 if (unlikely(!data) || atomic_read(&data->disabled))
199 return;
200
201 atomic_inc(&data->disabled);
202
203 data->critical_sequence = max_sequence;
204 data->preempt_timestamp = ftrace_now(cpu);
205 data->critical_start = parent_ip ? : ip;
206 tracing_reset(data);
207
208 local_save_flags(flags);
209
210 trace_function(tr, data, ip, parent_ip, flags);
211
212 per_cpu(tracing_cpu, cpu) = 1;
213
214 atomic_dec(&data->disabled);
215}
216
217static inline void
218stop_critical_timing(unsigned long ip, unsigned long parent_ip)
219{
220 int cpu;
221 struct trace_array *tr = irqsoff_trace;
222 struct trace_array_cpu *data;
223 unsigned long flags;
224
225 cpu = raw_smp_processor_id();
226 /* Always clear the tracing cpu on stopping the trace */
227 if (unlikely(per_cpu(tracing_cpu, cpu)))
228 per_cpu(tracing_cpu, cpu) = 0;
229 else
230 return;
231
232 if (!tracer_enabled)
233 return;
234
235 data = tr->data[cpu];
236
237 if (unlikely(!data) || unlikely(!head_page(data)) ||
238 !data->critical_start || atomic_read(&data->disabled))
239 return;
240
241 atomic_inc(&data->disabled);
242
243 local_save_flags(flags);
244 trace_function(tr, data, ip, parent_ip, flags);
245 check_critical_timing(tr, data, parent_ip ? : ip, cpu);
246 data->critical_start = 0;
247 atomic_dec(&data->disabled);
248}
249
250/* start and stop critical timings used to for stoppage (in idle) */
251void start_critical_timings(void)
252{
253 if (preempt_trace() || irq_trace())
254 start_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
255}
256EXPORT_SYMBOL_GPL(start_critical_timings);
257
258void stop_critical_timings(void)
259{
260 if (preempt_trace() || irq_trace())
261 stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
262}
263EXPORT_SYMBOL_GPL(stop_critical_timings);
264
265#ifdef CONFIG_IRQSOFF_TRACER
266#ifdef CONFIG_PROVE_LOCKING
267void time_hardirqs_on(unsigned long a0, unsigned long a1)
268{
269 if (!preempt_trace() && irq_trace())
270 stop_critical_timing(a0, a1);
271}
272
273void time_hardirqs_off(unsigned long a0, unsigned long a1)
274{
275 if (!preempt_trace() && irq_trace())
276 start_critical_timing(a0, a1);
277}
278
279#else /* !CONFIG_PROVE_LOCKING */
280
281/*
282 * Stubs:
283 */
284
285void early_boot_irqs_off(void)
286{
287}
288
289void early_boot_irqs_on(void)
290{
291}
292
293void trace_softirqs_on(unsigned long ip)
294{
295}
296
297void trace_softirqs_off(unsigned long ip)
298{
299}
300
301inline void print_irqtrace_events(struct task_struct *curr)
302{
303}
304
305/*
306 * We are only interested in hardirq on/off events:
307 */
308void trace_hardirqs_on(void)
309{
310 if (!preempt_trace() && irq_trace())
311 stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
312}
313EXPORT_SYMBOL(trace_hardirqs_on);
314
315void trace_hardirqs_off(void)
316{
317 if (!preempt_trace() && irq_trace())
318 start_critical_timing(CALLER_ADDR0, CALLER_ADDR1);
319}
320EXPORT_SYMBOL(trace_hardirqs_off);
321
322void trace_hardirqs_on_caller(unsigned long caller_addr)
323{
324 if (!preempt_trace() && irq_trace())
325 stop_critical_timing(CALLER_ADDR0, caller_addr);
326}
327EXPORT_SYMBOL(trace_hardirqs_on_caller);
328
329void trace_hardirqs_off_caller(unsigned long caller_addr)
330{
331 if (!preempt_trace() && irq_trace())
332 start_critical_timing(CALLER_ADDR0, caller_addr);
333}
334EXPORT_SYMBOL(trace_hardirqs_off_caller);
335
336#endif /* CONFIG_PROVE_LOCKING */
337#endif /* CONFIG_IRQSOFF_TRACER */
338
339#ifdef CONFIG_PREEMPT_TRACER
340void trace_preempt_on(unsigned long a0, unsigned long a1)
341{
342 if (preempt_trace())
343 stop_critical_timing(a0, a1);
344}
345
346void trace_preempt_off(unsigned long a0, unsigned long a1)
347{
348 if (preempt_trace())
349 start_critical_timing(a0, a1);
350}
351#endif /* CONFIG_PREEMPT_TRACER */
352
353static void start_irqsoff_tracer(struct trace_array *tr)
354{
355 register_ftrace_function(&trace_ops);
356 tracer_enabled = 1;
357}
358
359static void stop_irqsoff_tracer(struct trace_array *tr)
360{
361 tracer_enabled = 0;
362 unregister_ftrace_function(&trace_ops);
363}
364
365static void __irqsoff_tracer_init(struct trace_array *tr)
366{
367 irqsoff_trace = tr;
368 /* make sure that the tracer is visible */
369 smp_wmb();
370
371 if (tr->ctrl)
372 start_irqsoff_tracer(tr);
373}
374
375static void irqsoff_tracer_reset(struct trace_array *tr)
376{
377 if (tr->ctrl)
378 stop_irqsoff_tracer(tr);
379}
380
381static void irqsoff_tracer_ctrl_update(struct trace_array *tr)
382{
383 if (tr->ctrl)
384 start_irqsoff_tracer(tr);
385 else
386 stop_irqsoff_tracer(tr);
387}
388
389static void irqsoff_tracer_open(struct trace_iterator *iter)
390{
391 /* stop the trace while dumping */
392 if (iter->tr->ctrl)
393 stop_irqsoff_tracer(iter->tr);
394}
395
396static void irqsoff_tracer_close(struct trace_iterator *iter)
397{
398 if (iter->tr->ctrl)
399 start_irqsoff_tracer(iter->tr);
400}
401
402#ifdef CONFIG_IRQSOFF_TRACER
403static void irqsoff_tracer_init(struct trace_array *tr)
404{
405 trace_type = TRACER_IRQS_OFF;
406
407 __irqsoff_tracer_init(tr);
408}
409static struct tracer irqsoff_tracer __read_mostly =
410{
411 .name = "irqsoff",
412 .init = irqsoff_tracer_init,
413 .reset = irqsoff_tracer_reset,
414 .open = irqsoff_tracer_open,
415 .close = irqsoff_tracer_close,
416 .ctrl_update = irqsoff_tracer_ctrl_update,
417 .print_max = 1,
418#ifdef CONFIG_FTRACE_SELFTEST
419 .selftest = trace_selftest_startup_irqsoff,
420#endif
421};
422# define register_irqsoff(trace) register_tracer(&trace)
423#else
424# define register_irqsoff(trace) do { } while (0)
425#endif
426
427#ifdef CONFIG_PREEMPT_TRACER
428static void preemptoff_tracer_init(struct trace_array *tr)
429{
430 trace_type = TRACER_PREEMPT_OFF;
431
432 __irqsoff_tracer_init(tr);
433}
434
435static struct tracer preemptoff_tracer __read_mostly =
436{
437 .name = "preemptoff",
438 .init = preemptoff_tracer_init,
439 .reset = irqsoff_tracer_reset,
440 .open = irqsoff_tracer_open,
441 .close = irqsoff_tracer_close,
442 .ctrl_update = irqsoff_tracer_ctrl_update,
443 .print_max = 1,
444#ifdef CONFIG_FTRACE_SELFTEST
445 .selftest = trace_selftest_startup_preemptoff,
446#endif
447};
448# define register_preemptoff(trace) register_tracer(&trace)
449#else
450# define register_preemptoff(trace) do { } while (0)
451#endif
452
453#if defined(CONFIG_IRQSOFF_TRACER) && \
454 defined(CONFIG_PREEMPT_TRACER)
455
456static void preemptirqsoff_tracer_init(struct trace_array *tr)
457{
458 trace_type = TRACER_IRQS_OFF | TRACER_PREEMPT_OFF;
459
460 __irqsoff_tracer_init(tr);
461}
462
463static struct tracer preemptirqsoff_tracer __read_mostly =
464{
465 .name = "preemptirqsoff",
466 .init = preemptirqsoff_tracer_init,
467 .reset = irqsoff_tracer_reset,
468 .open = irqsoff_tracer_open,
469 .close = irqsoff_tracer_close,
470 .ctrl_update = irqsoff_tracer_ctrl_update,
471 .print_max = 1,
472#ifdef CONFIG_FTRACE_SELFTEST
473 .selftest = trace_selftest_startup_preemptirqsoff,
474#endif
475};
476
477# define register_preemptirqsoff(trace) register_tracer(&trace)
478#else
479# define register_preemptirqsoff(trace) do { } while (0)
480#endif
481
482__init static int init_irqsoff_tracer(void)
483{
484 register_irqsoff(irqsoff_tracer);
485 register_preemptoff(preemptoff_tracer);
486 register_preemptirqsoff(preemptirqsoff_tracer);
487
488 return 0;
489}
490device_initcall(init_irqsoff_tracer);
diff --git a/kernel/trace/trace_mmiotrace.c b/kernel/trace/trace_mmiotrace.c
new file mode 100644
index 000000000000..b13dc19dcbb4
--- /dev/null
+++ b/kernel/trace/trace_mmiotrace.c
@@ -0,0 +1,295 @@
1/*
2 * Memory mapped I/O tracing
3 *
4 * Copyright (C) 2008 Pekka Paalanen <pq@iki.fi>
5 */
6
7#define DEBUG 1
8
9#include <linux/kernel.h>
10#include <linux/mmiotrace.h>
11#include <linux/pci.h>
12
13#include "trace.h"
14
15struct header_iter {
16 struct pci_dev *dev;
17};
18
19static struct trace_array *mmio_trace_array;
20static bool overrun_detected;
21
22static void mmio_reset_data(struct trace_array *tr)
23{
24 int cpu;
25
26 overrun_detected = false;
27 tr->time_start = ftrace_now(tr->cpu);
28
29 for_each_online_cpu(cpu)
30 tracing_reset(tr->data[cpu]);
31}
32
33static void mmio_trace_init(struct trace_array *tr)
34{
35 pr_debug("in %s\n", __func__);
36 mmio_trace_array = tr;
37 if (tr->ctrl) {
38 mmio_reset_data(tr);
39 enable_mmiotrace();
40 }
41}
42
43static void mmio_trace_reset(struct trace_array *tr)
44{
45 pr_debug("in %s\n", __func__);
46 if (tr->ctrl)
47 disable_mmiotrace();
48 mmio_reset_data(tr);
49 mmio_trace_array = NULL;
50}
51
52static void mmio_trace_ctrl_update(struct trace_array *tr)
53{
54 pr_debug("in %s\n", __func__);
55 if (tr->ctrl) {
56 mmio_reset_data(tr);
57 enable_mmiotrace();
58 } else {
59 disable_mmiotrace();
60 }
61}
62
63static int mmio_print_pcidev(struct trace_seq *s, const struct pci_dev *dev)
64{
65 int ret = 0;
66 int i;
67 resource_size_t start, end;
68 const struct pci_driver *drv = pci_dev_driver(dev);
69
70 /* XXX: incomplete checks for trace_seq_printf() return value */
71 ret += trace_seq_printf(s, "PCIDEV %02x%02x %04x%04x %x",
72 dev->bus->number, dev->devfn,
73 dev->vendor, dev->device, dev->irq);
74 /*
75 * XXX: is pci_resource_to_user() appropriate, since we are
76 * supposed to interpret the __ioremap() phys_addr argument based on
77 * these printed values?
78 */
79 for (i = 0; i < 7; i++) {
80 pci_resource_to_user(dev, i, &dev->resource[i], &start, &end);
81 ret += trace_seq_printf(s, " %llx",
82 (unsigned long long)(start |
83 (dev->resource[i].flags & PCI_REGION_FLAG_MASK)));
84 }
85 for (i = 0; i < 7; i++) {
86 pci_resource_to_user(dev, i, &dev->resource[i], &start, &end);
87 ret += trace_seq_printf(s, " %llx",
88 dev->resource[i].start < dev->resource[i].end ?
89 (unsigned long long)(end - start) + 1 : 0);
90 }
91 if (drv)
92 ret += trace_seq_printf(s, " %s\n", drv->name);
93 else
94 ret += trace_seq_printf(s, " \n");
95 return ret;
96}
97
98static void destroy_header_iter(struct header_iter *hiter)
99{
100 if (!hiter)
101 return;
102 pci_dev_put(hiter->dev);
103 kfree(hiter);
104}
105
106static void mmio_pipe_open(struct trace_iterator *iter)
107{
108 struct header_iter *hiter;
109 struct trace_seq *s = &iter->seq;
110
111 trace_seq_printf(s, "VERSION 20070824\n");
112
113 hiter = kzalloc(sizeof(*hiter), GFP_KERNEL);
114 if (!hiter)
115 return;
116
117 hiter->dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, NULL);
118 iter->private = hiter;
119}
120
121/* XXX: This is not called when the pipe is closed! */
122static void mmio_close(struct trace_iterator *iter)
123{
124 struct header_iter *hiter = iter->private;
125 destroy_header_iter(hiter);
126 iter->private = NULL;
127}
128
129static unsigned long count_overruns(struct trace_iterator *iter)
130{
131 int cpu;
132 unsigned long cnt = 0;
133 for_each_online_cpu(cpu) {
134 cnt += iter->overrun[cpu];
135 iter->overrun[cpu] = 0;
136 }
137 return cnt;
138}
139
140static ssize_t mmio_read(struct trace_iterator *iter, struct file *filp,
141 char __user *ubuf, size_t cnt, loff_t *ppos)
142{
143 ssize_t ret;
144 struct header_iter *hiter = iter->private;
145 struct trace_seq *s = &iter->seq;
146 unsigned long n;
147
148 n = count_overruns(iter);
149 if (n) {
150 /* XXX: This is later than where events were lost. */
151 trace_seq_printf(s, "MARK 0.000000 Lost %lu events.\n", n);
152 if (!overrun_detected)
153 pr_warning("mmiotrace has lost events.\n");
154 overrun_detected = true;
155 goto print_out;
156 }
157
158 if (!hiter)
159 return 0;
160
161 mmio_print_pcidev(s, hiter->dev);
162 hiter->dev = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, hiter->dev);
163
164 if (!hiter->dev) {
165 destroy_header_iter(hiter);
166 iter->private = NULL;
167 }
168
169print_out:
170 ret = trace_seq_to_user(s, ubuf, cnt);
171 return (ret == -EBUSY) ? 0 : ret;
172}
173
174static int mmio_print_rw(struct trace_iterator *iter)
175{
176 struct trace_entry *entry = iter->ent;
177 struct mmiotrace_rw *rw = &entry->mmiorw;
178 struct trace_seq *s = &iter->seq;
179 unsigned long long t = ns2usecs(entry->t);
180 unsigned long usec_rem = do_div(t, 1000000ULL);
181 unsigned secs = (unsigned long)t;
182 int ret = 1;
183
184 switch (entry->mmiorw.opcode) {
185 case MMIO_READ:
186 ret = trace_seq_printf(s,
187 "R %d %lu.%06lu %d 0x%llx 0x%lx 0x%lx %d\n",
188 rw->width, secs, usec_rem, rw->map_id,
189 (unsigned long long)rw->phys,
190 rw->value, rw->pc, 0);
191 break;
192 case MMIO_WRITE:
193 ret = trace_seq_printf(s,
194 "W %d %lu.%06lu %d 0x%llx 0x%lx 0x%lx %d\n",
195 rw->width, secs, usec_rem, rw->map_id,
196 (unsigned long long)rw->phys,
197 rw->value, rw->pc, 0);
198 break;
199 case MMIO_UNKNOWN_OP:
200 ret = trace_seq_printf(s,
201 "UNKNOWN %lu.%06lu %d 0x%llx %02x,%02x,%02x 0x%lx %d\n",
202 secs, usec_rem, rw->map_id,
203 (unsigned long long)rw->phys,
204 (rw->value >> 16) & 0xff, (rw->value >> 8) & 0xff,
205 (rw->value >> 0) & 0xff, rw->pc, 0);
206 break;
207 default:
208 ret = trace_seq_printf(s, "rw what?\n");
209 break;
210 }
211 if (ret)
212 return 1;
213 return 0;
214}
215
216static int mmio_print_map(struct trace_iterator *iter)
217{
218 struct trace_entry *entry = iter->ent;
219 struct mmiotrace_map *m = &entry->mmiomap;
220 struct trace_seq *s = &iter->seq;
221 unsigned long long t = ns2usecs(entry->t);
222 unsigned long usec_rem = do_div(t, 1000000ULL);
223 unsigned secs = (unsigned long)t;
224 int ret = 1;
225
226 switch (entry->mmiorw.opcode) {
227 case MMIO_PROBE:
228 ret = trace_seq_printf(s,
229 "MAP %lu.%06lu %d 0x%llx 0x%lx 0x%lx 0x%lx %d\n",
230 secs, usec_rem, m->map_id,
231 (unsigned long long)m->phys, m->virt, m->len,
232 0UL, 0);
233 break;
234 case MMIO_UNPROBE:
235 ret = trace_seq_printf(s,
236 "UNMAP %lu.%06lu %d 0x%lx %d\n",
237 secs, usec_rem, m->map_id, 0UL, 0);
238 break;
239 default:
240 ret = trace_seq_printf(s, "map what?\n");
241 break;
242 }
243 if (ret)
244 return 1;
245 return 0;
246}
247
248/* return 0 to abort printing without consuming current entry in pipe mode */
249static int mmio_print_line(struct trace_iterator *iter)
250{
251 switch (iter->ent->type) {
252 case TRACE_MMIO_RW:
253 return mmio_print_rw(iter);
254 case TRACE_MMIO_MAP:
255 return mmio_print_map(iter);
256 default:
257 return 1; /* ignore unknown entries */
258 }
259}
260
261static struct tracer mmio_tracer __read_mostly =
262{
263 .name = "mmiotrace",
264 .init = mmio_trace_init,
265 .reset = mmio_trace_reset,
266 .pipe_open = mmio_pipe_open,
267 .close = mmio_close,
268 .read = mmio_read,
269 .ctrl_update = mmio_trace_ctrl_update,
270 .print_line = mmio_print_line,
271};
272
273__init static int init_mmio_trace(void)
274{
275 return register_tracer(&mmio_tracer);
276}
277device_initcall(init_mmio_trace);
278
279void mmio_trace_rw(struct mmiotrace_rw *rw)
280{
281 struct trace_array *tr = mmio_trace_array;
282 struct trace_array_cpu *data = tr->data[smp_processor_id()];
283 __trace_mmiotrace_rw(tr, data, rw);
284}
285
286void mmio_trace_mapping(struct mmiotrace_map *map)
287{
288 struct trace_array *tr = mmio_trace_array;
289 struct trace_array_cpu *data;
290
291 preempt_disable();
292 data = tr->data[smp_processor_id()];
293 __trace_mmiotrace_map(tr, data, map);
294 preempt_enable();
295}
diff --git a/kernel/trace/trace_sched_switch.c b/kernel/trace/trace_sched_switch.c
new file mode 100644
index 000000000000..cb817a209aa0
--- /dev/null
+++ b/kernel/trace/trace_sched_switch.c
@@ -0,0 +1,286 @@
1/*
2 * trace context switch
3 *
4 * Copyright (C) 2007 Steven Rostedt <srostedt@redhat.com>
5 *
6 */
7#include <linux/module.h>
8#include <linux/fs.h>
9#include <linux/debugfs.h>
10#include <linux/kallsyms.h>
11#include <linux/uaccess.h>
12#include <linux/marker.h>
13#include <linux/ftrace.h>
14
15#include "trace.h"
16
17static struct trace_array *ctx_trace;
18static int __read_mostly tracer_enabled;
19static atomic_t sched_ref;
20
21static void
22sched_switch_func(void *private, void *__rq, struct task_struct *prev,
23 struct task_struct *next)
24{
25 struct trace_array **ptr = private;
26 struct trace_array *tr = *ptr;
27 struct trace_array_cpu *data;
28 unsigned long flags;
29 long disabled;
30 int cpu;
31
32 tracing_record_cmdline(prev);
33 tracing_record_cmdline(next);
34
35 if (!tracer_enabled)
36 return;
37
38 local_irq_save(flags);
39 cpu = raw_smp_processor_id();
40 data = tr->data[cpu];
41 disabled = atomic_inc_return(&data->disabled);
42
43 if (likely(disabled == 1))
44 tracing_sched_switch_trace(tr, data, prev, next, flags);
45
46 atomic_dec(&data->disabled);
47 local_irq_restore(flags);
48}
49
50static notrace void
51sched_switch_callback(void *probe_data, void *call_data,
52 const char *format, va_list *args)
53{
54 struct task_struct *prev;
55 struct task_struct *next;
56 struct rq *__rq;
57
58 if (!atomic_read(&sched_ref))
59 return;
60
61 /* skip prev_pid %d next_pid %d prev_state %ld */
62 (void)va_arg(*args, int);
63 (void)va_arg(*args, int);
64 (void)va_arg(*args, long);
65 __rq = va_arg(*args, typeof(__rq));
66 prev = va_arg(*args, typeof(prev));
67 next = va_arg(*args, typeof(next));
68
69 /*
70 * If tracer_switch_func only points to the local
71 * switch func, it still needs the ptr passed to it.
72 */
73 sched_switch_func(probe_data, __rq, prev, next);
74}
75
76static void
77wakeup_func(void *private, void *__rq, struct task_struct *wakee, struct
78 task_struct *curr)
79{
80 struct trace_array **ptr = private;
81 struct trace_array *tr = *ptr;
82 struct trace_array_cpu *data;
83 unsigned long flags;
84 long disabled;
85 int cpu;
86
87 if (!tracer_enabled)
88 return;
89
90 tracing_record_cmdline(curr);
91
92 local_irq_save(flags);
93 cpu = raw_smp_processor_id();
94 data = tr->data[cpu];
95 disabled = atomic_inc_return(&data->disabled);
96
97 if (likely(disabled == 1))
98 tracing_sched_wakeup_trace(tr, data, wakee, curr, flags);
99
100 atomic_dec(&data->disabled);
101 local_irq_restore(flags);
102}
103
104static notrace void
105wake_up_callback(void *probe_data, void *call_data,
106 const char *format, va_list *args)
107{
108 struct task_struct *curr;
109 struct task_struct *task;
110 struct rq *__rq;
111
112 if (likely(!tracer_enabled))
113 return;
114
115 /* Skip pid %d state %ld */
116 (void)va_arg(*args, int);
117 (void)va_arg(*args, long);
118 /* now get the meat: "rq %p task %p rq->curr %p" */
119 __rq = va_arg(*args, typeof(__rq));
120 task = va_arg(*args, typeof(task));
121 curr = va_arg(*args, typeof(curr));
122
123 tracing_record_cmdline(task);
124 tracing_record_cmdline(curr);
125
126 wakeup_func(probe_data, __rq, task, curr);
127}
128
129static void sched_switch_reset(struct trace_array *tr)
130{
131 int cpu;
132
133 tr->time_start = ftrace_now(tr->cpu);
134
135 for_each_online_cpu(cpu)
136 tracing_reset(tr->data[cpu]);
137}
138
139static int tracing_sched_register(void)
140{
141 int ret;
142
143 ret = marker_probe_register("kernel_sched_wakeup",
144 "pid %d state %ld ## rq %p task %p rq->curr %p",
145 wake_up_callback,
146 &ctx_trace);
147 if (ret) {
148 pr_info("wakeup trace: Couldn't add marker"
149 " probe to kernel_sched_wakeup\n");
150 return ret;
151 }
152
153 ret = marker_probe_register("kernel_sched_wakeup_new",
154 "pid %d state %ld ## rq %p task %p rq->curr %p",
155 wake_up_callback,
156 &ctx_trace);
157 if (ret) {
158 pr_info("wakeup trace: Couldn't add marker"
159 " probe to kernel_sched_wakeup_new\n");
160 goto fail_deprobe;
161 }
162
163 ret = marker_probe_register("kernel_sched_schedule",
164 "prev_pid %d next_pid %d prev_state %ld "
165 "## rq %p prev %p next %p",
166 sched_switch_callback,
167 &ctx_trace);
168 if (ret) {
169 pr_info("sched trace: Couldn't add marker"
170 " probe to kernel_sched_schedule\n");
171 goto fail_deprobe_wake_new;
172 }
173
174 return ret;
175fail_deprobe_wake_new:
176 marker_probe_unregister("kernel_sched_wakeup_new",
177 wake_up_callback,
178 &ctx_trace);
179fail_deprobe:
180 marker_probe_unregister("kernel_sched_wakeup",
181 wake_up_callback,
182 &ctx_trace);
183 return ret;
184}
185
186static void tracing_sched_unregister(void)
187{
188 marker_probe_unregister("kernel_sched_schedule",
189 sched_switch_callback,
190 &ctx_trace);
191 marker_probe_unregister("kernel_sched_wakeup_new",
192 wake_up_callback,
193 &ctx_trace);
194 marker_probe_unregister("kernel_sched_wakeup",
195 wake_up_callback,
196 &ctx_trace);
197}
198
199static void tracing_start_sched_switch(void)
200{
201 long ref;
202
203 ref = atomic_inc_return(&sched_ref);
204 if (ref == 1)
205 tracing_sched_register();
206}
207
208static void tracing_stop_sched_switch(void)
209{
210 long ref;
211
212 ref = atomic_dec_and_test(&sched_ref);
213 if (ref)
214 tracing_sched_unregister();
215}
216
217void tracing_start_cmdline_record(void)
218{
219 tracing_start_sched_switch();
220}
221
222void tracing_stop_cmdline_record(void)
223{
224 tracing_stop_sched_switch();
225}
226
227static void start_sched_trace(struct trace_array *tr)
228{
229 sched_switch_reset(tr);
230 tracing_start_cmdline_record();
231 tracer_enabled = 1;
232}
233
234static void stop_sched_trace(struct trace_array *tr)
235{
236 tracer_enabled = 0;
237 tracing_stop_cmdline_record();
238}
239
240static void sched_switch_trace_init(struct trace_array *tr)
241{
242 ctx_trace = tr;
243
244 if (tr->ctrl)
245 start_sched_trace(tr);
246}
247
248static void sched_switch_trace_reset(struct trace_array *tr)
249{
250 if (tr->ctrl)
251 stop_sched_trace(tr);
252}
253
254static void sched_switch_trace_ctrl_update(struct trace_array *tr)
255{
256 /* When starting a new trace, reset the buffers */
257 if (tr->ctrl)
258 start_sched_trace(tr);
259 else
260 stop_sched_trace(tr);
261}
262
263static struct tracer sched_switch_trace __read_mostly =
264{
265 .name = "sched_switch",
266 .init = sched_switch_trace_init,
267 .reset = sched_switch_trace_reset,
268 .ctrl_update = sched_switch_trace_ctrl_update,
269#ifdef CONFIG_FTRACE_SELFTEST
270 .selftest = trace_selftest_startup_sched_switch,
271#endif
272};
273
274__init static int init_sched_switch_trace(void)
275{
276 int ret = 0;
277
278 if (atomic_read(&sched_ref))
279 ret = tracing_sched_register();
280 if (ret) {
281 pr_info("error registering scheduler trace\n");
282 return ret;
283 }
284 return register_tracer(&sched_switch_trace);
285}
286device_initcall(init_sched_switch_trace);
diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c
new file mode 100644
index 000000000000..e303ccb62cdf
--- /dev/null
+++ b/kernel/trace/trace_sched_wakeup.c
@@ -0,0 +1,453 @@
1/*
2 * trace task wakeup timings
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Based on code from the latency_tracer, that is:
8 *
9 * Copyright (C) 2004-2006 Ingo Molnar
10 * Copyright (C) 2004 William Lee Irwin III
11 */
12#include <linux/module.h>
13#include <linux/fs.h>
14#include <linux/debugfs.h>
15#include <linux/kallsyms.h>
16#include <linux/uaccess.h>
17#include <linux/ftrace.h>
18#include <linux/marker.h>
19
20#include "trace.h"
21
22static struct trace_array *wakeup_trace;
23static int __read_mostly tracer_enabled;
24
25static struct task_struct *wakeup_task;
26static int wakeup_cpu;
27static unsigned wakeup_prio = -1;
28
29static raw_spinlock_t wakeup_lock =
30 (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
31
32static void __wakeup_reset(struct trace_array *tr);
33
34#ifdef CONFIG_FTRACE
35/*
36 * irqsoff uses its own tracer function to keep the overhead down:
37 */
38static void
39wakeup_tracer_call(unsigned long ip, unsigned long parent_ip)
40{
41 struct trace_array *tr = wakeup_trace;
42 struct trace_array_cpu *data;
43 unsigned long flags;
44 long disabled;
45 int resched;
46 int cpu;
47
48 if (likely(!wakeup_task))
49 return;
50
51 resched = need_resched();
52 preempt_disable_notrace();
53
54 cpu = raw_smp_processor_id();
55 data = tr->data[cpu];
56 disabled = atomic_inc_return(&data->disabled);
57 if (unlikely(disabled != 1))
58 goto out;
59
60 local_irq_save(flags);
61 __raw_spin_lock(&wakeup_lock);
62
63 if (unlikely(!wakeup_task))
64 goto unlock;
65
66 /*
67 * The task can't disappear because it needs to
68 * wake up first, and we have the wakeup_lock.
69 */
70 if (task_cpu(wakeup_task) != cpu)
71 goto unlock;
72
73 trace_function(tr, data, ip, parent_ip, flags);
74
75 unlock:
76 __raw_spin_unlock(&wakeup_lock);
77 local_irq_restore(flags);
78
79 out:
80 atomic_dec(&data->disabled);
81
82 /*
83 * To prevent recursion from the scheduler, if the
84 * resched flag was set before we entered, then
85 * don't reschedule.
86 */
87 if (resched)
88 preempt_enable_no_resched_notrace();
89 else
90 preempt_enable_notrace();
91}
92
93static struct ftrace_ops trace_ops __read_mostly =
94{
95 .func = wakeup_tracer_call,
96};
97#endif /* CONFIG_FTRACE */
98
99/*
100 * Should this new latency be reported/recorded?
101 */
102static int report_latency(cycle_t delta)
103{
104 if (tracing_thresh) {
105 if (delta < tracing_thresh)
106 return 0;
107 } else {
108 if (delta <= tracing_max_latency)
109 return 0;
110 }
111 return 1;
112}
113
114static void notrace
115wakeup_sched_switch(void *private, void *rq, struct task_struct *prev,
116 struct task_struct *next)
117{
118 unsigned long latency = 0, t0 = 0, t1 = 0;
119 struct trace_array **ptr = private;
120 struct trace_array *tr = *ptr;
121 struct trace_array_cpu *data;
122 cycle_t T0, T1, delta;
123 unsigned long flags;
124 long disabled;
125 int cpu;
126
127 if (unlikely(!tracer_enabled))
128 return;
129
130 /*
131 * When we start a new trace, we set wakeup_task to NULL
132 * and then set tracer_enabled = 1. We want to make sure
133 * that another CPU does not see the tracer_enabled = 1
134 * and the wakeup_task with an older task, that might
135 * actually be the same as next.
136 */
137 smp_rmb();
138
139 if (next != wakeup_task)
140 return;
141
142 /* The task we are waiting for is waking up */
143 data = tr->data[wakeup_cpu];
144
145 /* disable local data, not wakeup_cpu data */
146 cpu = raw_smp_processor_id();
147 disabled = atomic_inc_return(&tr->data[cpu]->disabled);
148 if (likely(disabled != 1))
149 goto out;
150
151 local_irq_save(flags);
152 __raw_spin_lock(&wakeup_lock);
153
154 /* We could race with grabbing wakeup_lock */
155 if (unlikely(!tracer_enabled || next != wakeup_task))
156 goto out_unlock;
157
158 trace_function(tr, data, CALLER_ADDR1, CALLER_ADDR2, flags);
159
160 /*
161 * usecs conversion is slow so we try to delay the conversion
162 * as long as possible:
163 */
164 T0 = data->preempt_timestamp;
165 T1 = ftrace_now(cpu);
166 delta = T1-T0;
167
168 if (!report_latency(delta))
169 goto out_unlock;
170
171 latency = nsecs_to_usecs(delta);
172
173 tracing_max_latency = delta;
174 t0 = nsecs_to_usecs(T0);
175 t1 = nsecs_to_usecs(T1);
176
177 update_max_tr(tr, wakeup_task, wakeup_cpu);
178
179out_unlock:
180 __wakeup_reset(tr);
181 __raw_spin_unlock(&wakeup_lock);
182 local_irq_restore(flags);
183out:
184 atomic_dec(&tr->data[cpu]->disabled);
185}
186
187static notrace void
188sched_switch_callback(void *probe_data, void *call_data,
189 const char *format, va_list *args)
190{
191 struct task_struct *prev;
192 struct task_struct *next;
193 struct rq *__rq;
194
195 /* skip prev_pid %d next_pid %d prev_state %ld */
196 (void)va_arg(*args, int);
197 (void)va_arg(*args, int);
198 (void)va_arg(*args, long);
199 __rq = va_arg(*args, typeof(__rq));
200 prev = va_arg(*args, typeof(prev));
201 next = va_arg(*args, typeof(next));
202
203 tracing_record_cmdline(prev);
204
205 /*
206 * If tracer_switch_func only points to the local
207 * switch func, it still needs the ptr passed to it.
208 */
209 wakeup_sched_switch(probe_data, __rq, prev, next);
210}
211
212static void __wakeup_reset(struct trace_array *tr)
213{
214 struct trace_array_cpu *data;
215 int cpu;
216
217 for_each_possible_cpu(cpu) {
218 data = tr->data[cpu];
219 tracing_reset(data);
220 }
221
222 wakeup_cpu = -1;
223 wakeup_prio = -1;
224
225 if (wakeup_task)
226 put_task_struct(wakeup_task);
227
228 wakeup_task = NULL;
229}
230
231static void wakeup_reset(struct trace_array *tr)
232{
233 unsigned long flags;
234
235 local_irq_save(flags);
236 __raw_spin_lock(&wakeup_lock);
237 __wakeup_reset(tr);
238 __raw_spin_unlock(&wakeup_lock);
239 local_irq_restore(flags);
240}
241
242static void
243wakeup_check_start(struct trace_array *tr, struct task_struct *p,
244 struct task_struct *curr)
245{
246 int cpu = smp_processor_id();
247 unsigned long flags;
248 long disabled;
249
250 if (likely(!rt_task(p)) ||
251 p->prio >= wakeup_prio ||
252 p->prio >= curr->prio)
253 return;
254
255 disabled = atomic_inc_return(&tr->data[cpu]->disabled);
256 if (unlikely(disabled != 1))
257 goto out;
258
259 /* interrupts should be off from try_to_wake_up */
260 __raw_spin_lock(&wakeup_lock);
261
262 /* check for races. */
263 if (!tracer_enabled || p->prio >= wakeup_prio)
264 goto out_locked;
265
266 /* reset the trace */
267 __wakeup_reset(tr);
268
269 wakeup_cpu = task_cpu(p);
270 wakeup_prio = p->prio;
271
272 wakeup_task = p;
273 get_task_struct(wakeup_task);
274
275 local_save_flags(flags);
276
277 tr->data[wakeup_cpu]->preempt_timestamp = ftrace_now(cpu);
278 trace_function(tr, tr->data[wakeup_cpu],
279 CALLER_ADDR1, CALLER_ADDR2, flags);
280
281out_locked:
282 __raw_spin_unlock(&wakeup_lock);
283out:
284 atomic_dec(&tr->data[cpu]->disabled);
285}
286
287static notrace void
288wake_up_callback(void *probe_data, void *call_data,
289 const char *format, va_list *args)
290{
291 struct trace_array **ptr = probe_data;
292 struct trace_array *tr = *ptr;
293 struct task_struct *curr;
294 struct task_struct *task;
295 struct rq *__rq;
296
297 if (likely(!tracer_enabled))
298 return;
299
300 /* Skip pid %d state %ld */
301 (void)va_arg(*args, int);
302 (void)va_arg(*args, long);
303 /* now get the meat: "rq %p task %p rq->curr %p" */
304 __rq = va_arg(*args, typeof(__rq));
305 task = va_arg(*args, typeof(task));
306 curr = va_arg(*args, typeof(curr));
307
308 tracing_record_cmdline(task);
309 tracing_record_cmdline(curr);
310
311 wakeup_check_start(tr, task, curr);
312}
313
314static void start_wakeup_tracer(struct trace_array *tr)
315{
316 int ret;
317
318 ret = marker_probe_register("kernel_sched_wakeup",
319 "pid %d state %ld ## rq %p task %p rq->curr %p",
320 wake_up_callback,
321 &wakeup_trace);
322 if (ret) {
323 pr_info("wakeup trace: Couldn't add marker"
324 " probe to kernel_sched_wakeup\n");
325 return;
326 }
327
328 ret = marker_probe_register("kernel_sched_wakeup_new",
329 "pid %d state %ld ## rq %p task %p rq->curr %p",
330 wake_up_callback,
331 &wakeup_trace);
332 if (ret) {
333 pr_info("wakeup trace: Couldn't add marker"
334 " probe to kernel_sched_wakeup_new\n");
335 goto fail_deprobe;
336 }
337
338 ret = marker_probe_register("kernel_sched_schedule",
339 "prev_pid %d next_pid %d prev_state %ld "
340 "## rq %p prev %p next %p",
341 sched_switch_callback,
342 &wakeup_trace);
343 if (ret) {
344 pr_info("sched trace: Couldn't add marker"
345 " probe to kernel_sched_schedule\n");
346 goto fail_deprobe_wake_new;
347 }
348
349 wakeup_reset(tr);
350
351 /*
352 * Don't let the tracer_enabled = 1 show up before
353 * the wakeup_task is reset. This may be overkill since
354 * wakeup_reset does a spin_unlock after setting the
355 * wakeup_task to NULL, but I want to be safe.
356 * This is a slow path anyway.
357 */
358 smp_wmb();
359
360 register_ftrace_function(&trace_ops);
361
362 tracer_enabled = 1;
363
364 return;
365fail_deprobe_wake_new:
366 marker_probe_unregister("kernel_sched_wakeup_new",
367 wake_up_callback,
368 &wakeup_trace);
369fail_deprobe:
370 marker_probe_unregister("kernel_sched_wakeup",
371 wake_up_callback,
372 &wakeup_trace);
373}
374
375static void stop_wakeup_tracer(struct trace_array *tr)
376{
377 tracer_enabled = 0;
378 unregister_ftrace_function(&trace_ops);
379 marker_probe_unregister("kernel_sched_schedule",
380 sched_switch_callback,
381 &wakeup_trace);
382 marker_probe_unregister("kernel_sched_wakeup_new",
383 wake_up_callback,
384 &wakeup_trace);
385 marker_probe_unregister("kernel_sched_wakeup",
386 wake_up_callback,
387 &wakeup_trace);
388}
389
390static void wakeup_tracer_init(struct trace_array *tr)
391{
392 wakeup_trace = tr;
393
394 if (tr->ctrl)
395 start_wakeup_tracer(tr);
396}
397
398static void wakeup_tracer_reset(struct trace_array *tr)
399{
400 if (tr->ctrl) {
401 stop_wakeup_tracer(tr);
402 /* make sure we put back any tasks we are tracing */
403 wakeup_reset(tr);
404 }
405}
406
407static void wakeup_tracer_ctrl_update(struct trace_array *tr)
408{
409 if (tr->ctrl)
410 start_wakeup_tracer(tr);
411 else
412 stop_wakeup_tracer(tr);
413}
414
415static void wakeup_tracer_open(struct trace_iterator *iter)
416{
417 /* stop the trace while dumping */
418 if (iter->tr->ctrl)
419 stop_wakeup_tracer(iter->tr);
420}
421
422static void wakeup_tracer_close(struct trace_iterator *iter)
423{
424 /* forget about any processes we were recording */
425 if (iter->tr->ctrl)
426 start_wakeup_tracer(iter->tr);
427}
428
429static struct tracer wakeup_tracer __read_mostly =
430{
431 .name = "wakeup",
432 .init = wakeup_tracer_init,
433 .reset = wakeup_tracer_reset,
434 .open = wakeup_tracer_open,
435 .close = wakeup_tracer_close,
436 .ctrl_update = wakeup_tracer_ctrl_update,
437 .print_max = 1,
438#ifdef CONFIG_FTRACE_SELFTEST
439 .selftest = trace_selftest_startup_wakeup,
440#endif
441};
442
443__init static int init_wakeup_tracer(void)
444{
445 int ret;
446
447 ret = register_tracer(&wakeup_tracer);
448 if (ret)
449 return ret;
450
451 return 0;
452}
453device_initcall(init_wakeup_tracer);
diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c
new file mode 100644
index 000000000000..0911b7e073bf
--- /dev/null
+++ b/kernel/trace/trace_selftest.c
@@ -0,0 +1,563 @@
1/* Include in trace.c */
2
3#include <linux/kthread.h>
4#include <linux/delay.h>
5
6static inline int trace_valid_entry(struct trace_entry *entry)
7{
8 switch (entry->type) {
9 case TRACE_FN:
10 case TRACE_CTX:
11 case TRACE_WAKE:
12 case TRACE_STACK:
13 case TRACE_SPECIAL:
14 return 1;
15 }
16 return 0;
17}
18
19static int
20trace_test_buffer_cpu(struct trace_array *tr, struct trace_array_cpu *data)
21{
22 struct trace_entry *entries;
23 struct page *page;
24 int idx = 0;
25 int i;
26
27 BUG_ON(list_empty(&data->trace_pages));
28 page = list_entry(data->trace_pages.next, struct page, lru);
29 entries = page_address(page);
30
31 check_pages(data);
32 if (head_page(data) != entries)
33 goto failed;
34
35 /*
36 * The starting trace buffer always has valid elements,
37 * if any element exists.
38 */
39 entries = head_page(data);
40
41 for (i = 0; i < tr->entries; i++) {
42
43 if (i < data->trace_idx && !trace_valid_entry(&entries[idx])) {
44 printk(KERN_CONT ".. invalid entry %d ",
45 entries[idx].type);
46 goto failed;
47 }
48
49 idx++;
50 if (idx >= ENTRIES_PER_PAGE) {
51 page = virt_to_page(entries);
52 if (page->lru.next == &data->trace_pages) {
53 if (i != tr->entries - 1) {
54 printk(KERN_CONT ".. entries buffer mismatch");
55 goto failed;
56 }
57 } else {
58 page = list_entry(page->lru.next, struct page, lru);
59 entries = page_address(page);
60 }
61 idx = 0;
62 }
63 }
64
65 page = virt_to_page(entries);
66 if (page->lru.next != &data->trace_pages) {
67 printk(KERN_CONT ".. too many entries");
68 goto failed;
69 }
70
71 return 0;
72
73 failed:
74 /* disable tracing */
75 tracing_disabled = 1;
76 printk(KERN_CONT ".. corrupted trace buffer .. ");
77 return -1;
78}
79
80/*
81 * Test the trace buffer to see if all the elements
82 * are still sane.
83 */
84static int trace_test_buffer(struct trace_array *tr, unsigned long *count)
85{
86 unsigned long flags, cnt = 0;
87 int cpu, ret = 0;
88
89 /* Don't allow flipping of max traces now */
90 raw_local_irq_save(flags);
91 __raw_spin_lock(&ftrace_max_lock);
92 for_each_possible_cpu(cpu) {
93 if (!head_page(tr->data[cpu]))
94 continue;
95
96 cnt += tr->data[cpu]->trace_idx;
97
98 ret = trace_test_buffer_cpu(tr, tr->data[cpu]);
99 if (ret)
100 break;
101 }
102 __raw_spin_unlock(&ftrace_max_lock);
103 raw_local_irq_restore(flags);
104
105 if (count)
106 *count = cnt;
107
108 return ret;
109}
110
111#ifdef CONFIG_FTRACE
112
113#ifdef CONFIG_DYNAMIC_FTRACE
114
115#define __STR(x) #x
116#define STR(x) __STR(x)
117
118/* Test dynamic code modification and ftrace filters */
119int trace_selftest_startup_dynamic_tracing(struct tracer *trace,
120 struct trace_array *tr,
121 int (*func)(void))
122{
123 unsigned long count;
124 int ret;
125 int save_ftrace_enabled = ftrace_enabled;
126 int save_tracer_enabled = tracer_enabled;
127 char *func_name;
128
129 /* The ftrace test PASSED */
130 printk(KERN_CONT "PASSED\n");
131 pr_info("Testing dynamic ftrace: ");
132
133 /* enable tracing, and record the filter function */
134 ftrace_enabled = 1;
135 tracer_enabled = 1;
136
137 /* passed in by parameter to fool gcc from optimizing */
138 func();
139
140 /* update the records */
141 ret = ftrace_force_update();
142 if (ret) {
143 printk(KERN_CONT ".. ftraced failed .. ");
144 return ret;
145 }
146
147 /*
148 * Some archs *cough*PowerPC*cough* add charachters to the
149 * start of the function names. We simply put a '*' to
150 * accomodate them.
151 */
152 func_name = "*" STR(DYN_FTRACE_TEST_NAME);
153
154 /* filter only on our function */
155 ftrace_set_filter(func_name, strlen(func_name), 1);
156
157 /* enable tracing */
158 tr->ctrl = 1;
159 trace->init(tr);
160 /* Sleep for a 1/10 of a second */
161 msleep(100);
162
163 /* we should have nothing in the buffer */
164 ret = trace_test_buffer(tr, &count);
165 if (ret)
166 goto out;
167
168 if (count) {
169 ret = -1;
170 printk(KERN_CONT ".. filter did not filter .. ");
171 goto out;
172 }
173
174 /* call our function again */
175 func();
176
177 /* sleep again */
178 msleep(100);
179
180 /* stop the tracing. */
181 tr->ctrl = 0;
182 trace->ctrl_update(tr);
183 ftrace_enabled = 0;
184
185 /* check the trace buffer */
186 ret = trace_test_buffer(tr, &count);
187 trace->reset(tr);
188
189 /* we should only have one item */
190 if (!ret && count != 1) {
191 printk(KERN_CONT ".. filter failed count=%ld ..", count);
192 ret = -1;
193 goto out;
194 }
195 out:
196 ftrace_enabled = save_ftrace_enabled;
197 tracer_enabled = save_tracer_enabled;
198
199 /* Enable tracing on all functions again */
200 ftrace_set_filter(NULL, 0, 1);
201
202 return ret;
203}
204#else
205# define trace_selftest_startup_dynamic_tracing(trace, tr, func) ({ 0; })
206#endif /* CONFIG_DYNAMIC_FTRACE */
207/*
208 * Simple verification test of ftrace function tracer.
209 * Enable ftrace, sleep 1/10 second, and then read the trace
210 * buffer to see if all is in order.
211 */
212int
213trace_selftest_startup_function(struct tracer *trace, struct trace_array *tr)
214{
215 unsigned long count;
216 int ret;
217 int save_ftrace_enabled = ftrace_enabled;
218 int save_tracer_enabled = tracer_enabled;
219
220 /* make sure msleep has been recorded */
221 msleep(1);
222
223 /* force the recorded functions to be traced */
224 ret = ftrace_force_update();
225 if (ret) {
226 printk(KERN_CONT ".. ftraced failed .. ");
227 return ret;
228 }
229
230 /* start the tracing */
231 ftrace_enabled = 1;
232 tracer_enabled = 1;
233
234 tr->ctrl = 1;
235 trace->init(tr);
236 /* Sleep for a 1/10 of a second */
237 msleep(100);
238 /* stop the tracing. */
239 tr->ctrl = 0;
240 trace->ctrl_update(tr);
241 ftrace_enabled = 0;
242
243 /* check the trace buffer */
244 ret = trace_test_buffer(tr, &count);
245 trace->reset(tr);
246
247 if (!ret && !count) {
248 printk(KERN_CONT ".. no entries found ..");
249 ret = -1;
250 goto out;
251 }
252
253 ret = trace_selftest_startup_dynamic_tracing(trace, tr,
254 DYN_FTRACE_TEST_NAME);
255
256 out:
257 ftrace_enabled = save_ftrace_enabled;
258 tracer_enabled = save_tracer_enabled;
259
260 /* kill ftrace totally if we failed */
261 if (ret)
262 ftrace_kill();
263
264 return ret;
265}
266#endif /* CONFIG_FTRACE */
267
268#ifdef CONFIG_IRQSOFF_TRACER
269int
270trace_selftest_startup_irqsoff(struct tracer *trace, struct trace_array *tr)
271{
272 unsigned long save_max = tracing_max_latency;
273 unsigned long count;
274 int ret;
275
276 /* start the tracing */
277 tr->ctrl = 1;
278 trace->init(tr);
279 /* reset the max latency */
280 tracing_max_latency = 0;
281 /* disable interrupts for a bit */
282 local_irq_disable();
283 udelay(100);
284 local_irq_enable();
285 /* stop the tracing. */
286 tr->ctrl = 0;
287 trace->ctrl_update(tr);
288 /* check both trace buffers */
289 ret = trace_test_buffer(tr, NULL);
290 if (!ret)
291 ret = trace_test_buffer(&max_tr, &count);
292 trace->reset(tr);
293
294 if (!ret && !count) {
295 printk(KERN_CONT ".. no entries found ..");
296 ret = -1;
297 }
298
299 tracing_max_latency = save_max;
300
301 return ret;
302}
303#endif /* CONFIG_IRQSOFF_TRACER */
304
305#ifdef CONFIG_PREEMPT_TRACER
306int
307trace_selftest_startup_preemptoff(struct tracer *trace, struct trace_array *tr)
308{
309 unsigned long save_max = tracing_max_latency;
310 unsigned long count;
311 int ret;
312
313 /* start the tracing */
314 tr->ctrl = 1;
315 trace->init(tr);
316 /* reset the max latency */
317 tracing_max_latency = 0;
318 /* disable preemption for a bit */
319 preempt_disable();
320 udelay(100);
321 preempt_enable();
322 /* stop the tracing. */
323 tr->ctrl = 0;
324 trace->ctrl_update(tr);
325 /* check both trace buffers */
326 ret = trace_test_buffer(tr, NULL);
327 if (!ret)
328 ret = trace_test_buffer(&max_tr, &count);
329 trace->reset(tr);
330
331 if (!ret && !count) {
332 printk(KERN_CONT ".. no entries found ..");
333 ret = -1;
334 }
335
336 tracing_max_latency = save_max;
337
338 return ret;
339}
340#endif /* CONFIG_PREEMPT_TRACER */
341
342#if defined(CONFIG_IRQSOFF_TRACER) && defined(CONFIG_PREEMPT_TRACER)
343int
344trace_selftest_startup_preemptirqsoff(struct tracer *trace, struct trace_array *tr)
345{
346 unsigned long save_max = tracing_max_latency;
347 unsigned long count;
348 int ret;
349
350 /* start the tracing */
351 tr->ctrl = 1;
352 trace->init(tr);
353
354 /* reset the max latency */
355 tracing_max_latency = 0;
356
357 /* disable preemption and interrupts for a bit */
358 preempt_disable();
359 local_irq_disable();
360 udelay(100);
361 preempt_enable();
362 /* reverse the order of preempt vs irqs */
363 local_irq_enable();
364
365 /* stop the tracing. */
366 tr->ctrl = 0;
367 trace->ctrl_update(tr);
368 /* check both trace buffers */
369 ret = trace_test_buffer(tr, NULL);
370 if (ret)
371 goto out;
372
373 ret = trace_test_buffer(&max_tr, &count);
374 if (ret)
375 goto out;
376
377 if (!ret && !count) {
378 printk(KERN_CONT ".. no entries found ..");
379 ret = -1;
380 goto out;
381 }
382
383 /* do the test by disabling interrupts first this time */
384 tracing_max_latency = 0;
385 tr->ctrl = 1;
386 trace->ctrl_update(tr);
387 preempt_disable();
388 local_irq_disable();
389 udelay(100);
390 preempt_enable();
391 /* reverse the order of preempt vs irqs */
392 local_irq_enable();
393
394 /* stop the tracing. */
395 tr->ctrl = 0;
396 trace->ctrl_update(tr);
397 /* check both trace buffers */
398 ret = trace_test_buffer(tr, NULL);
399 if (ret)
400 goto out;
401
402 ret = trace_test_buffer(&max_tr, &count);
403
404 if (!ret && !count) {
405 printk(KERN_CONT ".. no entries found ..");
406 ret = -1;
407 goto out;
408 }
409
410 out:
411 trace->reset(tr);
412 tracing_max_latency = save_max;
413
414 return ret;
415}
416#endif /* CONFIG_IRQSOFF_TRACER && CONFIG_PREEMPT_TRACER */
417
418#ifdef CONFIG_SCHED_TRACER
419static int trace_wakeup_test_thread(void *data)
420{
421 /* Make this a RT thread, doesn't need to be too high */
422 struct sched_param param = { .sched_priority = 5 };
423 struct completion *x = data;
424
425 sched_setscheduler(current, SCHED_FIFO, &param);
426
427 /* Make it know we have a new prio */
428 complete(x);
429
430 /* now go to sleep and let the test wake us up */
431 set_current_state(TASK_INTERRUPTIBLE);
432 schedule();
433
434 /* we are awake, now wait to disappear */
435 while (!kthread_should_stop()) {
436 /*
437 * This is an RT task, do short sleeps to let
438 * others run.
439 */
440 msleep(100);
441 }
442
443 return 0;
444}
445
446int
447trace_selftest_startup_wakeup(struct tracer *trace, struct trace_array *tr)
448{
449 unsigned long save_max = tracing_max_latency;
450 struct task_struct *p;
451 struct completion isrt;
452 unsigned long count;
453 int ret;
454
455 init_completion(&isrt);
456
457 /* create a high prio thread */
458 p = kthread_run(trace_wakeup_test_thread, &isrt, "ftrace-test");
459 if (IS_ERR(p)) {
460 printk(KERN_CONT "Failed to create ftrace wakeup test thread ");
461 return -1;
462 }
463
464 /* make sure the thread is running at an RT prio */
465 wait_for_completion(&isrt);
466
467 /* start the tracing */
468 tr->ctrl = 1;
469 trace->init(tr);
470 /* reset the max latency */
471 tracing_max_latency = 0;
472
473 /* sleep to let the RT thread sleep too */
474 msleep(100);
475
476 /*
477 * Yes this is slightly racy. It is possible that for some
478 * strange reason that the RT thread we created, did not
479 * call schedule for 100ms after doing the completion,
480 * and we do a wakeup on a task that already is awake.
481 * But that is extremely unlikely, and the worst thing that
482 * happens in such a case, is that we disable tracing.
483 * Honestly, if this race does happen something is horrible
484 * wrong with the system.
485 */
486
487 wake_up_process(p);
488
489 /* stop the tracing. */
490 tr->ctrl = 0;
491 trace->ctrl_update(tr);
492 /* check both trace buffers */
493 ret = trace_test_buffer(tr, NULL);
494 if (!ret)
495 ret = trace_test_buffer(&max_tr, &count);
496
497
498 trace->reset(tr);
499
500 tracing_max_latency = save_max;
501
502 /* kill the thread */
503 kthread_stop(p);
504
505 if (!ret && !count) {
506 printk(KERN_CONT ".. no entries found ..");
507 ret = -1;
508 }
509
510 return ret;
511}
512#endif /* CONFIG_SCHED_TRACER */
513
514#ifdef CONFIG_CONTEXT_SWITCH_TRACER
515int
516trace_selftest_startup_sched_switch(struct tracer *trace, struct trace_array *tr)
517{
518 unsigned long count;
519 int ret;
520
521 /* start the tracing */
522 tr->ctrl = 1;
523 trace->init(tr);
524 /* Sleep for a 1/10 of a second */
525 msleep(100);
526 /* stop the tracing. */
527 tr->ctrl = 0;
528 trace->ctrl_update(tr);
529 /* check the trace buffer */
530 ret = trace_test_buffer(tr, &count);
531 trace->reset(tr);
532
533 if (!ret && !count) {
534 printk(KERN_CONT ".. no entries found ..");
535 ret = -1;
536 }
537
538 return ret;
539}
540#endif /* CONFIG_CONTEXT_SWITCH_TRACER */
541
542#ifdef CONFIG_SYSPROF_TRACER
543int
544trace_selftest_startup_sysprof(struct tracer *trace, struct trace_array *tr)
545{
546 unsigned long count;
547 int ret;
548
549 /* start the tracing */
550 tr->ctrl = 1;
551 trace->init(tr);
552 /* Sleep for a 1/10 of a second */
553 msleep(100);
554 /* stop the tracing. */
555 tr->ctrl = 0;
556 trace->ctrl_update(tr);
557 /* check the trace buffer */
558 ret = trace_test_buffer(tr, &count);
559 trace->reset(tr);
560
561 return ret;
562}
563#endif /* CONFIG_SYSPROF_TRACER */
diff --git a/kernel/trace/trace_selftest_dynamic.c b/kernel/trace/trace_selftest_dynamic.c
new file mode 100644
index 000000000000..54dd77cce5bf
--- /dev/null
+++ b/kernel/trace/trace_selftest_dynamic.c
@@ -0,0 +1,7 @@
1#include "trace.h"
2
3int DYN_FTRACE_TEST_NAME(void)
4{
5 /* used to call mcount */
6 return 0;
7}
diff --git a/kernel/trace/trace_sysprof.c b/kernel/trace/trace_sysprof.c
new file mode 100644
index 000000000000..ce2d723c10e1
--- /dev/null
+++ b/kernel/trace/trace_sysprof.c
@@ -0,0 +1,365 @@
1/*
2 * trace stack traces
3 *
4 * Copyright (C) 2004-2008, Soeren Sandmann
5 * Copyright (C) 2007 Steven Rostedt <srostedt@redhat.com>
6 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 */
8#include <linux/kallsyms.h>
9#include <linux/debugfs.h>
10#include <linux/hrtimer.h>
11#include <linux/uaccess.h>
12#include <linux/ftrace.h>
13#include <linux/module.h>
14#include <linux/irq.h>
15#include <linux/fs.h>
16
17#include <asm/stacktrace.h>
18
19#include "trace.h"
20
21static struct trace_array *sysprof_trace;
22static int __read_mostly tracer_enabled;
23
24/*
25 * 1 msec sample interval by default:
26 */
27static unsigned long sample_period = 1000000;
28static const unsigned int sample_max_depth = 512;
29
30static DEFINE_MUTEX(sample_timer_lock);
31/*
32 * Per CPU hrtimers that do the profiling:
33 */
34static DEFINE_PER_CPU(struct hrtimer, stack_trace_hrtimer);
35
36struct stack_frame {
37 const void __user *next_fp;
38 unsigned long return_address;
39};
40
41static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
42{
43 int ret;
44
45 if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
46 return 0;
47
48 ret = 1;
49 pagefault_disable();
50 if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
51 ret = 0;
52 pagefault_enable();
53
54 return ret;
55}
56
57struct backtrace_info {
58 struct trace_array_cpu *data;
59 struct trace_array *tr;
60 int pos;
61};
62
63static void
64backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
65{
66 /* Ignore warnings */
67}
68
69static void backtrace_warning(void *data, char *msg)
70{
71 /* Ignore warnings */
72}
73
74static int backtrace_stack(void *data, char *name)
75{
76 /* Don't bother with IRQ stacks for now */
77 return -1;
78}
79
80static void backtrace_address(void *data, unsigned long addr, int reliable)
81{
82 struct backtrace_info *info = data;
83
84 if (info->pos < sample_max_depth && reliable) {
85 __trace_special(info->tr, info->data, 1, addr, 0);
86
87 info->pos++;
88 }
89}
90
91const static struct stacktrace_ops backtrace_ops = {
92 .warning = backtrace_warning,
93 .warning_symbol = backtrace_warning_symbol,
94 .stack = backtrace_stack,
95 .address = backtrace_address,
96};
97
98static int
99trace_kernel(struct pt_regs *regs, struct trace_array *tr,
100 struct trace_array_cpu *data)
101{
102 struct backtrace_info info;
103 unsigned long bp;
104 char *stack;
105
106 info.tr = tr;
107 info.data = data;
108 info.pos = 1;
109
110 __trace_special(info.tr, info.data, 1, regs->ip, 0);
111
112 stack = ((char *)regs + sizeof(struct pt_regs));
113#ifdef CONFIG_FRAME_POINTER
114 bp = regs->bp;
115#else
116 bp = 0;
117#endif
118
119 dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, &info);
120
121 return info.pos;
122}
123
124static void timer_notify(struct pt_regs *regs, int cpu)
125{
126 struct trace_array_cpu *data;
127 struct stack_frame frame;
128 struct trace_array *tr;
129 const void __user *fp;
130 int is_user;
131 int i;
132
133 if (!regs)
134 return;
135
136 tr = sysprof_trace;
137 data = tr->data[cpu];
138 is_user = user_mode(regs);
139
140 if (!current || current->pid == 0)
141 return;
142
143 if (is_user && current->state != TASK_RUNNING)
144 return;
145
146 __trace_special(tr, data, 0, 0, current->pid);
147
148 if (!is_user)
149 i = trace_kernel(regs, tr, data);
150 else
151 i = 0;
152
153 /*
154 * Trace user stack if we are not a kernel thread
155 */
156 if (current->mm && i < sample_max_depth) {
157 regs = (struct pt_regs *)current->thread.sp0 - 1;
158
159 fp = (void __user *)regs->bp;
160
161 __trace_special(tr, data, 2, regs->ip, 0);
162
163 while (i < sample_max_depth) {
164 frame.next_fp = NULL;
165 frame.return_address = 0;
166 if (!copy_stack_frame(fp, &frame))
167 break;
168 if ((unsigned long)fp < regs->sp)
169 break;
170
171 __trace_special(tr, data, 2, frame.return_address,
172 (unsigned long)fp);
173 fp = frame.next_fp;
174
175 i++;
176 }
177
178 }
179
180 /*
181 * Special trace entry if we overflow the max depth:
182 */
183 if (i == sample_max_depth)
184 __trace_special(tr, data, -1, -1, -1);
185
186 __trace_special(tr, data, 3, current->pid, i);
187}
188
189static enum hrtimer_restart stack_trace_timer_fn(struct hrtimer *hrtimer)
190{
191 /* trace here */
192 timer_notify(get_irq_regs(), smp_processor_id());
193
194 hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));
195
196 return HRTIMER_RESTART;
197}
198
199static void start_stack_timer(int cpu)
200{
201 struct hrtimer *hrtimer = &per_cpu(stack_trace_hrtimer, cpu);
202
203 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
204 hrtimer->function = stack_trace_timer_fn;
205 hrtimer->cb_mode = HRTIMER_CB_IRQSAFE_NO_SOFTIRQ;
206
207 hrtimer_start(hrtimer, ns_to_ktime(sample_period), HRTIMER_MODE_REL);
208}
209
210static void start_stack_timers(void)
211{
212 cpumask_t saved_mask = current->cpus_allowed;
213 int cpu;
214
215 for_each_online_cpu(cpu) {
216 cpumask_of_cpu_ptr(new_mask, cpu);
217
218 set_cpus_allowed_ptr(current, new_mask);
219 start_stack_timer(cpu);
220 }
221 set_cpus_allowed_ptr(current, &saved_mask);
222}
223
224static void stop_stack_timer(int cpu)
225{
226 struct hrtimer *hrtimer = &per_cpu(stack_trace_hrtimer, cpu);
227
228 hrtimer_cancel(hrtimer);
229}
230
231static void stop_stack_timers(void)
232{
233 int cpu;
234
235 for_each_online_cpu(cpu)
236 stop_stack_timer(cpu);
237}
238
239static void stack_reset(struct trace_array *tr)
240{
241 int cpu;
242
243 tr->time_start = ftrace_now(tr->cpu);
244
245 for_each_online_cpu(cpu)
246 tracing_reset(tr->data[cpu]);
247}
248
249static void start_stack_trace(struct trace_array *tr)
250{
251 mutex_lock(&sample_timer_lock);
252 stack_reset(tr);
253 start_stack_timers();
254 tracer_enabled = 1;
255 mutex_unlock(&sample_timer_lock);
256}
257
258static void stop_stack_trace(struct trace_array *tr)
259{
260 mutex_lock(&sample_timer_lock);
261 stop_stack_timers();
262 tracer_enabled = 0;
263 mutex_unlock(&sample_timer_lock);
264}
265
266static void stack_trace_init(struct trace_array *tr)
267{
268 sysprof_trace = tr;
269
270 if (tr->ctrl)
271 start_stack_trace(tr);
272}
273
274static void stack_trace_reset(struct trace_array *tr)
275{
276 if (tr->ctrl)
277 stop_stack_trace(tr);
278}
279
280static void stack_trace_ctrl_update(struct trace_array *tr)
281{
282 /* When starting a new trace, reset the buffers */
283 if (tr->ctrl)
284 start_stack_trace(tr);
285 else
286 stop_stack_trace(tr);
287}
288
289static struct tracer stack_trace __read_mostly =
290{
291 .name = "sysprof",
292 .init = stack_trace_init,
293 .reset = stack_trace_reset,
294 .ctrl_update = stack_trace_ctrl_update,
295#ifdef CONFIG_FTRACE_SELFTEST
296 .selftest = trace_selftest_startup_sysprof,
297#endif
298};
299
300__init static int init_stack_trace(void)
301{
302 return register_tracer(&stack_trace);
303}
304device_initcall(init_stack_trace);
305
306#define MAX_LONG_DIGITS 22
307
308static ssize_t
309sysprof_sample_read(struct file *filp, char __user *ubuf,
310 size_t cnt, loff_t *ppos)
311{
312 char buf[MAX_LONG_DIGITS];
313 int r;
314
315 r = sprintf(buf, "%ld\n", nsecs_to_usecs(sample_period));
316
317 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
318}
319
320static ssize_t
321sysprof_sample_write(struct file *filp, const char __user *ubuf,
322 size_t cnt, loff_t *ppos)
323{
324 char buf[MAX_LONG_DIGITS];
325 unsigned long val;
326
327 if (cnt > MAX_LONG_DIGITS-1)
328 cnt = MAX_LONG_DIGITS-1;
329
330 if (copy_from_user(&buf, ubuf, cnt))
331 return -EFAULT;
332
333 buf[cnt] = 0;
334
335 val = simple_strtoul(buf, NULL, 10);
336 /*
337 * Enforce a minimum sample period of 100 usecs:
338 */
339 if (val < 100)
340 val = 100;
341
342 mutex_lock(&sample_timer_lock);
343 stop_stack_timers();
344 sample_period = val * 1000;
345 start_stack_timers();
346 mutex_unlock(&sample_timer_lock);
347
348 return cnt;
349}
350
351static struct file_operations sysprof_sample_fops = {
352 .read = sysprof_sample_read,
353 .write = sysprof_sample_write,
354};
355
356void init_tracer_sysprof_debugfs(struct dentry *d_tracer)
357{
358 struct dentry *entry;
359
360 entry = debugfs_create_file("sysprof_sample_period", 0644,
361 d_tracer, NULL, &sysprof_sample_fops);
362 if (entry)
363 return;
364 pr_warning("Could not create debugfs 'dyn_ftrace_total_info' entry\n");
365}
diff --git a/kernel/tsacct.c b/kernel/tsacct.c
index 4ab1b584961b..f9cd2561689c 100644
--- a/kernel/tsacct.c
+++ b/kernel/tsacct.c
@@ -28,14 +28,14 @@
28void bacct_add_tsk(struct taskstats *stats, struct task_struct *tsk) 28void bacct_add_tsk(struct taskstats *stats, struct task_struct *tsk)
29{ 29{
30 struct timespec uptime, ts; 30 struct timespec uptime, ts;
31 s64 ac_etime; 31 u64 ac_etime;
32 32
33 BUILD_BUG_ON(TS_COMM_LEN < TASK_COMM_LEN); 33 BUILD_BUG_ON(TS_COMM_LEN < TASK_COMM_LEN);
34 34
35 /* calculate task elapsed time in timespec */ 35 /* calculate task elapsed time in timespec */
36 do_posix_clock_monotonic_gettime(&uptime); 36 do_posix_clock_monotonic_gettime(&uptime);
37 ts = timespec_sub(uptime, tsk->start_time); 37 ts = timespec_sub(uptime, tsk->start_time);
38 /* rebase elapsed time to usec */ 38 /* rebase elapsed time to usec (should never be negative) */
39 ac_etime = timespec_to_ns(&ts); 39 ac_etime = timespec_to_ns(&ts);
40 do_div(ac_etime, NSEC_PER_USEC); 40 do_div(ac_etime, NSEC_PER_USEC);
41 stats->ac_etime = ac_etime; 41 stats->ac_etime = ac_etime;
@@ -84,9 +84,9 @@ void xacct_add_tsk(struct taskstats *stats, struct task_struct *p)
84{ 84{
85 struct mm_struct *mm; 85 struct mm_struct *mm;
86 86
87 /* convert pages-jiffies to Mbyte-usec */ 87 /* convert pages-usec to Mbyte-usec */
88 stats->coremem = jiffies_to_usecs(p->acct_rss_mem1) * PAGE_SIZE / MB; 88 stats->coremem = p->acct_rss_mem1 * PAGE_SIZE / MB;
89 stats->virtmem = jiffies_to_usecs(p->acct_vm_mem1) * PAGE_SIZE / MB; 89 stats->virtmem = p->acct_vm_mem1 * PAGE_SIZE / MB;
90 mm = get_task_mm(p); 90 mm = get_task_mm(p);
91 if (mm) { 91 if (mm) {
92 /* adjust to KB unit */ 92 /* adjust to KB unit */
@@ -94,14 +94,14 @@ void xacct_add_tsk(struct taskstats *stats, struct task_struct *p)
94 stats->hiwater_vm = mm->hiwater_vm * PAGE_SIZE / KB; 94 stats->hiwater_vm = mm->hiwater_vm * PAGE_SIZE / KB;
95 mmput(mm); 95 mmput(mm);
96 } 96 }
97 stats->read_char = p->rchar; 97 stats->read_char = p->ioac.chr.rchar;
98 stats->write_char = p->wchar; 98 stats->write_char = p->ioac.chr.wchar;
99 stats->read_syscalls = p->syscr; 99 stats->read_syscalls = p->ioac.chr.syscr;
100 stats->write_syscalls = p->syscw; 100 stats->write_syscalls = p->ioac.chr.syscw;
101#ifdef CONFIG_TASK_IO_ACCOUNTING 101#ifdef CONFIG_TASK_IO_ACCOUNTING
102 stats->read_bytes = p->ioac.read_bytes; 102 stats->read_bytes = p->ioac.blk.read_bytes;
103 stats->write_bytes = p->ioac.write_bytes; 103 stats->write_bytes = p->ioac.blk.write_bytes;
104 stats->cancelled_write_bytes = p->ioac.cancelled_write_bytes; 104 stats->cancelled_write_bytes = p->ioac.blk.cancelled_write_bytes;
105#else 105#else
106 stats->read_bytes = 0; 106 stats->read_bytes = 0;
107 stats->write_bytes = 0; 107 stats->write_bytes = 0;
@@ -118,12 +118,19 @@ void xacct_add_tsk(struct taskstats *stats, struct task_struct *p)
118void acct_update_integrals(struct task_struct *tsk) 118void acct_update_integrals(struct task_struct *tsk)
119{ 119{
120 if (likely(tsk->mm)) { 120 if (likely(tsk->mm)) {
121 long delta = cputime_to_jiffies( 121 cputime_t time, dtime;
122 cputime_sub(tsk->stime, tsk->acct_stimexpd)); 122 struct timeval value;
123 u64 delta;
124
125 time = tsk->stime + tsk->utime;
126 dtime = cputime_sub(time, tsk->acct_timexpd);
127 jiffies_to_timeval(cputime_to_jiffies(dtime), &value);
128 delta = value.tv_sec;
129 delta = delta * USEC_PER_SEC + value.tv_usec;
123 130
124 if (delta == 0) 131 if (delta == 0)
125 return; 132 return;
126 tsk->acct_stimexpd = tsk->stime; 133 tsk->acct_timexpd = time;
127 tsk->acct_rss_mem1 += delta * get_mm_rss(tsk->mm); 134 tsk->acct_rss_mem1 += delta * get_mm_rss(tsk->mm);
128 tsk->acct_vm_mem1 += delta * tsk->mm->total_vm; 135 tsk->acct_vm_mem1 += delta * tsk->mm->total_vm;
129 } 136 }
@@ -135,7 +142,7 @@ void acct_update_integrals(struct task_struct *tsk)
135 */ 142 */
136void acct_clear_integrals(struct task_struct *tsk) 143void acct_clear_integrals(struct task_struct *tsk)
137{ 144{
138 tsk->acct_stimexpd = 0; 145 tsk->acct_timexpd = 0;
139 tsk->acct_rss_mem1 = 0; 146 tsk->acct_rss_mem1 = 0;
140 tsk->acct_vm_mem1 = 0; 147 tsk->acct_vm_mem1 = 0;
141} 148}
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index ce7799540c91..ec7e4f62aaff 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -125,7 +125,7 @@ struct cpu_workqueue_struct *get_wq_data(struct work_struct *work)
125} 125}
126 126
127static void insert_work(struct cpu_workqueue_struct *cwq, 127static void insert_work(struct cpu_workqueue_struct *cwq,
128 struct work_struct *work, int tail) 128 struct work_struct *work, struct list_head *head)
129{ 129{
130 set_wq_data(work, cwq); 130 set_wq_data(work, cwq);
131 /* 131 /*
@@ -133,21 +133,17 @@ static void insert_work(struct cpu_workqueue_struct *cwq,
133 * result of list_add() below, see try_to_grab_pending(). 133 * result of list_add() below, see try_to_grab_pending().
134 */ 134 */
135 smp_wmb(); 135 smp_wmb();
136 if (tail) 136 list_add_tail(&work->entry, head);
137 list_add_tail(&work->entry, &cwq->worklist);
138 else
139 list_add(&work->entry, &cwq->worklist);
140 wake_up(&cwq->more_work); 137 wake_up(&cwq->more_work);
141} 138}
142 139
143/* Preempt must be disabled. */
144static void __queue_work(struct cpu_workqueue_struct *cwq, 140static void __queue_work(struct cpu_workqueue_struct *cwq,
145 struct work_struct *work) 141 struct work_struct *work)
146{ 142{
147 unsigned long flags; 143 unsigned long flags;
148 144
149 spin_lock_irqsave(&cwq->lock, flags); 145 spin_lock_irqsave(&cwq->lock, flags);
150 insert_work(cwq, work, 1); 146 insert_work(cwq, work, &cwq->worklist);
151 spin_unlock_irqrestore(&cwq->lock, flags); 147 spin_unlock_irqrestore(&cwq->lock, flags);
152} 148}
153 149
@@ -163,17 +159,39 @@ static void __queue_work(struct cpu_workqueue_struct *cwq,
163 */ 159 */
164int queue_work(struct workqueue_struct *wq, struct work_struct *work) 160int queue_work(struct workqueue_struct *wq, struct work_struct *work)
165{ 161{
162 int ret;
163
164 ret = queue_work_on(get_cpu(), wq, work);
165 put_cpu();
166
167 return ret;
168}
169EXPORT_SYMBOL_GPL(queue_work);
170
171/**
172 * queue_work_on - queue work on specific cpu
173 * @cpu: CPU number to execute work on
174 * @wq: workqueue to use
175 * @work: work to queue
176 *
177 * Returns 0 if @work was already on a queue, non-zero otherwise.
178 *
179 * We queue the work to a specific CPU, the caller must ensure it
180 * can't go away.
181 */
182int
183queue_work_on(int cpu, struct workqueue_struct *wq, struct work_struct *work)
184{
166 int ret = 0; 185 int ret = 0;
167 186
168 if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) { 187 if (!test_and_set_bit(WORK_STRUCT_PENDING, work_data_bits(work))) {
169 BUG_ON(!list_empty(&work->entry)); 188 BUG_ON(!list_empty(&work->entry));
170 __queue_work(wq_per_cpu(wq, get_cpu()), work); 189 __queue_work(wq_per_cpu(wq, cpu), work);
171 put_cpu();
172 ret = 1; 190 ret = 1;
173 } 191 }
174 return ret; 192 return ret;
175} 193}
176EXPORT_SYMBOL_GPL(queue_work); 194EXPORT_SYMBOL_GPL(queue_work_on);
177 195
178static void delayed_work_timer_fn(unsigned long __data) 196static void delayed_work_timer_fn(unsigned long __data)
179{ 197{
@@ -337,14 +355,14 @@ static void wq_barrier_func(struct work_struct *work)
337} 355}
338 356
339static void insert_wq_barrier(struct cpu_workqueue_struct *cwq, 357static void insert_wq_barrier(struct cpu_workqueue_struct *cwq,
340 struct wq_barrier *barr, int tail) 358 struct wq_barrier *barr, struct list_head *head)
341{ 359{
342 INIT_WORK(&barr->work, wq_barrier_func); 360 INIT_WORK(&barr->work, wq_barrier_func);
343 __set_bit(WORK_STRUCT_PENDING, work_data_bits(&barr->work)); 361 __set_bit(WORK_STRUCT_PENDING, work_data_bits(&barr->work));
344 362
345 init_completion(&barr->done); 363 init_completion(&barr->done);
346 364
347 insert_work(cwq, &barr->work, tail); 365 insert_work(cwq, &barr->work, head);
348} 366}
349 367
350static int flush_cpu_workqueue(struct cpu_workqueue_struct *cwq) 368static int flush_cpu_workqueue(struct cpu_workqueue_struct *cwq)
@@ -364,7 +382,7 @@ static int flush_cpu_workqueue(struct cpu_workqueue_struct *cwq)
364 active = 0; 382 active = 0;
365 spin_lock_irq(&cwq->lock); 383 spin_lock_irq(&cwq->lock);
366 if (!list_empty(&cwq->worklist) || cwq->current_work != NULL) { 384 if (!list_empty(&cwq->worklist) || cwq->current_work != NULL) {
367 insert_wq_barrier(cwq, &barr, 1); 385 insert_wq_barrier(cwq, &barr, &cwq->worklist);
368 active = 1; 386 active = 1;
369 } 387 }
370 spin_unlock_irq(&cwq->lock); 388 spin_unlock_irq(&cwq->lock);
@@ -397,11 +415,62 @@ void flush_workqueue(struct workqueue_struct *wq)
397 might_sleep(); 415 might_sleep();
398 lock_acquire(&wq->lockdep_map, 0, 0, 0, 2, _THIS_IP_); 416 lock_acquire(&wq->lockdep_map, 0, 0, 0, 2, _THIS_IP_);
399 lock_release(&wq->lockdep_map, 1, _THIS_IP_); 417 lock_release(&wq->lockdep_map, 1, _THIS_IP_);
400 for_each_cpu_mask(cpu, *cpu_map) 418 for_each_cpu_mask_nr(cpu, *cpu_map)
401 flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, cpu)); 419 flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, cpu));
402} 420}
403EXPORT_SYMBOL_GPL(flush_workqueue); 421EXPORT_SYMBOL_GPL(flush_workqueue);
404 422
423/**
424 * flush_work - block until a work_struct's callback has terminated
425 * @work: the work which is to be flushed
426 *
427 * Returns false if @work has already terminated.
428 *
429 * It is expected that, prior to calling flush_work(), the caller has
430 * arranged for the work to not be requeued, otherwise it doesn't make
431 * sense to use this function.
432 */
433int flush_work(struct work_struct *work)
434{
435 struct cpu_workqueue_struct *cwq;
436 struct list_head *prev;
437 struct wq_barrier barr;
438
439 might_sleep();
440 cwq = get_wq_data(work);
441 if (!cwq)
442 return 0;
443
444 lock_acquire(&cwq->wq->lockdep_map, 0, 0, 0, 2, _THIS_IP_);
445 lock_release(&cwq->wq->lockdep_map, 1, _THIS_IP_);
446
447 prev = NULL;
448 spin_lock_irq(&cwq->lock);
449 if (!list_empty(&work->entry)) {
450 /*
451 * See the comment near try_to_grab_pending()->smp_rmb().
452 * If it was re-queued under us we are not going to wait.
453 */
454 smp_rmb();
455 if (unlikely(cwq != get_wq_data(work)))
456 goto out;
457 prev = &work->entry;
458 } else {
459 if (cwq->current_work != work)
460 goto out;
461 prev = &cwq->worklist;
462 }
463 insert_wq_barrier(cwq, &barr, prev->next);
464out:
465 spin_unlock_irq(&cwq->lock);
466 if (!prev)
467 return 0;
468
469 wait_for_completion(&barr.done);
470 return 1;
471}
472EXPORT_SYMBOL_GPL(flush_work);
473
405/* 474/*
406 * Upon a successful return (>= 0), the caller "owns" WORK_STRUCT_PENDING bit, 475 * Upon a successful return (>= 0), the caller "owns" WORK_STRUCT_PENDING bit,
407 * so this work can't be re-armed in any way. 476 * so this work can't be re-armed in any way.
@@ -449,7 +518,7 @@ static void wait_on_cpu_work(struct cpu_workqueue_struct *cwq,
449 518
450 spin_lock_irq(&cwq->lock); 519 spin_lock_irq(&cwq->lock);
451 if (unlikely(cwq->current_work == work)) { 520 if (unlikely(cwq->current_work == work)) {
452 insert_wq_barrier(cwq, &barr, 0); 521 insert_wq_barrier(cwq, &barr, cwq->worklist.next);
453 running = 1; 522 running = 1;
454 } 523 }
455 spin_unlock_irq(&cwq->lock); 524 spin_unlock_irq(&cwq->lock);
@@ -477,7 +546,7 @@ static void wait_on_work(struct work_struct *work)
477 wq = cwq->wq; 546 wq = cwq->wq;
478 cpu_map = wq_cpu_map(wq); 547 cpu_map = wq_cpu_map(wq);
479 548
480 for_each_cpu_mask(cpu, *cpu_map) 549 for_each_cpu_mask_nr(cpu, *cpu_map)
481 wait_on_cpu_work(per_cpu_ptr(wq->cpu_wq, cpu), work); 550 wait_on_cpu_work(per_cpu_ptr(wq->cpu_wq, cpu), work);
482} 551}
483 552
@@ -553,6 +622,19 @@ int schedule_work(struct work_struct *work)
553} 622}
554EXPORT_SYMBOL(schedule_work); 623EXPORT_SYMBOL(schedule_work);
555 624
625/*
626 * schedule_work_on - put work task on a specific cpu
627 * @cpu: cpu to put the work task on
628 * @work: job to be done
629 *
630 * This puts a job on a specific cpu
631 */
632int schedule_work_on(int cpu, struct work_struct *work)
633{
634 return queue_work_on(cpu, keventd_wq, work);
635}
636EXPORT_SYMBOL(schedule_work_on);
637
556/** 638/**
557 * schedule_delayed_work - put work task in global workqueue after delay 639 * schedule_delayed_work - put work task in global workqueue after delay
558 * @dwork: job to be done 640 * @dwork: job to be done
@@ -607,10 +689,10 @@ int schedule_on_each_cpu(work_func_t func)
607 struct work_struct *work = per_cpu_ptr(works, cpu); 689 struct work_struct *work = per_cpu_ptr(works, cpu);
608 690
609 INIT_WORK(work, func); 691 INIT_WORK(work, func);
610 set_bit(WORK_STRUCT_PENDING, work_data_bits(work)); 692 schedule_work_on(cpu, work);
611 __queue_work(per_cpu_ptr(keventd_wq->cpu_wq, cpu), work);
612 } 693 }
613 flush_workqueue(keventd_wq); 694 for_each_online_cpu(cpu)
695 flush_work(per_cpu_ptr(works, cpu));
614 put_online_cpus(); 696 put_online_cpus();
615 free_percpu(works); 697 free_percpu(works);
616 return 0; 698 return 0;
@@ -747,7 +829,7 @@ struct workqueue_struct *__create_workqueue_key(const char *name,
747 err = create_workqueue_thread(cwq, singlethread_cpu); 829 err = create_workqueue_thread(cwq, singlethread_cpu);
748 start_workqueue_thread(cwq, -1); 830 start_workqueue_thread(cwq, -1);
749 } else { 831 } else {
750 get_online_cpus(); 832 cpu_maps_update_begin();
751 spin_lock(&workqueue_lock); 833 spin_lock(&workqueue_lock);
752 list_add(&wq->list, &workqueues); 834 list_add(&wq->list, &workqueues);
753 spin_unlock(&workqueue_lock); 835 spin_unlock(&workqueue_lock);
@@ -759,7 +841,7 @@ struct workqueue_struct *__create_workqueue_key(const char *name,
759 err = create_workqueue_thread(cwq, cpu); 841 err = create_workqueue_thread(cwq, cpu);
760 start_workqueue_thread(cwq, cpu); 842 start_workqueue_thread(cwq, cpu);
761 } 843 }
762 put_online_cpus(); 844 cpu_maps_update_done();
763 } 845 }
764 846
765 if (err) { 847 if (err) {
@@ -773,8 +855,8 @@ EXPORT_SYMBOL_GPL(__create_workqueue_key);
773static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq) 855static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq)
774{ 856{
775 /* 857 /*
776 * Our caller is either destroy_workqueue() or CPU_DEAD, 858 * Our caller is either destroy_workqueue() or CPU_POST_DEAD,
777 * get_online_cpus() protects cwq->thread. 859 * cpu_add_remove_lock protects cwq->thread.
778 */ 860 */
779 if (cwq->thread == NULL) 861 if (cwq->thread == NULL)
780 return; 862 return;
@@ -784,7 +866,7 @@ static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq)
784 866
785 flush_cpu_workqueue(cwq); 867 flush_cpu_workqueue(cwq);
786 /* 868 /*
787 * If the caller is CPU_DEAD and cwq->worklist was not empty, 869 * If the caller is CPU_POST_DEAD and cwq->worklist was not empty,
788 * a concurrent flush_workqueue() can insert a barrier after us. 870 * a concurrent flush_workqueue() can insert a barrier after us.
789 * However, in that case run_workqueue() won't return and check 871 * However, in that case run_workqueue() won't return and check
790 * kthread_should_stop() until it flushes all work_struct's. 872 * kthread_should_stop() until it flushes all work_struct's.
@@ -808,14 +890,14 @@ void destroy_workqueue(struct workqueue_struct *wq)
808 const cpumask_t *cpu_map = wq_cpu_map(wq); 890 const cpumask_t *cpu_map = wq_cpu_map(wq);
809 int cpu; 891 int cpu;
810 892
811 get_online_cpus(); 893 cpu_maps_update_begin();
812 spin_lock(&workqueue_lock); 894 spin_lock(&workqueue_lock);
813 list_del(&wq->list); 895 list_del(&wq->list);
814 spin_unlock(&workqueue_lock); 896 spin_unlock(&workqueue_lock);
815 897
816 for_each_cpu_mask(cpu, *cpu_map) 898 for_each_cpu_mask_nr(cpu, *cpu_map)
817 cleanup_workqueue_thread(per_cpu_ptr(wq->cpu_wq, cpu)); 899 cleanup_workqueue_thread(per_cpu_ptr(wq->cpu_wq, cpu));
818 put_online_cpus(); 900 cpu_maps_update_done();
819 901
820 free_percpu(wq->cpu_wq); 902 free_percpu(wq->cpu_wq);
821 kfree(wq); 903 kfree(wq);
@@ -829,6 +911,7 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb,
829 unsigned int cpu = (unsigned long)hcpu; 911 unsigned int cpu = (unsigned long)hcpu;
830 struct cpu_workqueue_struct *cwq; 912 struct cpu_workqueue_struct *cwq;
831 struct workqueue_struct *wq; 913 struct workqueue_struct *wq;
914 int ret = NOTIFY_OK;
832 915
833 action &= ~CPU_TASKS_FROZEN; 916 action &= ~CPU_TASKS_FROZEN;
834 917
@@ -836,7 +919,7 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb,
836 case CPU_UP_PREPARE: 919 case CPU_UP_PREPARE:
837 cpu_set(cpu, cpu_populated_map); 920 cpu_set(cpu, cpu_populated_map);
838 } 921 }
839 922undo:
840 list_for_each_entry(wq, &workqueues, list) { 923 list_for_each_entry(wq, &workqueues, list) {
841 cwq = per_cpu_ptr(wq->cpu_wq, cpu); 924 cwq = per_cpu_ptr(wq->cpu_wq, cpu);
842 925
@@ -846,7 +929,9 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb,
846 break; 929 break;
847 printk(KERN_ERR "workqueue [%s] for %i failed\n", 930 printk(KERN_ERR "workqueue [%s] for %i failed\n",
848 wq->name, cpu); 931 wq->name, cpu);
849 return NOTIFY_BAD; 932 action = CPU_UP_CANCELED;
933 ret = NOTIFY_BAD;
934 goto undo;
850 935
851 case CPU_ONLINE: 936 case CPU_ONLINE:
852 start_workqueue_thread(cwq, cpu); 937 start_workqueue_thread(cwq, cpu);
@@ -854,7 +939,7 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb,
854 939
855 case CPU_UP_CANCELED: 940 case CPU_UP_CANCELED:
856 start_workqueue_thread(cwq, -1); 941 start_workqueue_thread(cwq, -1);
857 case CPU_DEAD: 942 case CPU_POST_DEAD:
858 cleanup_workqueue_thread(cwq); 943 cleanup_workqueue_thread(cwq);
859 break; 944 break;
860 } 945 }
@@ -862,11 +947,11 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb,
862 947
863 switch (action) { 948 switch (action) {
864 case CPU_UP_CANCELED: 949 case CPU_UP_CANCELED:
865 case CPU_DEAD: 950 case CPU_POST_DEAD:
866 cpu_clear(cpu, cpu_populated_map); 951 cpu_clear(cpu, cpu_populated_map);
867 } 952 }
868 953
869 return NOTIFY_OK; 954 return ret;
870} 955}
871 956
872void __init init_workqueues(void) 957void __init init_workqueues(void)
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-09 05:42:04 -0500